2026-02-18T08:04:55.057 INFO:root:teuthology version: 1.2.4.dev9+gfa17720d0 2026-02-18T08:04:55.072 DEBUG:teuthology.report:Pushing job info to https://paddles-paddles.apps.pok.os.sepia.ceph.com 2026-02-18T08:04:55.150 INFO:teuthology.run:Config: archive_path: /home/teuthworker/mnt/teuthology/yuriw-2026-02-17_21:26:40-rbd-20.2.1_2.17.26-baseline_2-distro-default-trial/54948 branch: 20.2.1_2.17.26-baseline_2 description: rbd/mirror-thrash/{base/install clients/mirror cluster/{2-node} conf/{disable-pool-app} msgr-failures/few objectstore/bluestore-hybrid policy/simple rbd-mirror/four-per-cluster supported-random-distro$/{ubuntu_latest} workloads/rbd-mirror-journal-stress-workunit} email: yweinste@redhat.com first_in_suite: false flavor: default job_id: '54948' kernel: branch: distro kdb: true sha1: distro ktype: distro last_in_suite: false machine_type: trial meta: - desc: run two ceph clusters and install rbd-mirror - desc: configure the permissions for client.mirror - desc: 2 ceph clusters with 1 mon and 3 osds each - desc: run four rbd-mirror daemons per cluster - desc: run the rbd_mirror_stress.sh workunit to test the rbd-mirror daemon name: yuriw-2026-02-17_21:26:40-rbd-20.2.1_2.17.26-baseline_2-distro-default-trial no_nested_subset: false os_type: ubuntu os_version: '22.04' overrides: admin_socket: branch: 20.2.1_2.17.26-baseline_2 ceph: conf: client: debug rbd: 20 debug rbd_mirror: 15 log to stderr: false rbd default features: 125 rbd mirror image policy type: simple client.mirror.0: admin socket: /var/run/ceph/rbd-mirror.$cluster-$name.asok pid file: /var/run/ceph/rbd-mirror.$cluster-$name.pid client.mirror.1: admin socket: /var/run/ceph/rbd-mirror.$cluster-$name.asok pid file: /var/run/ceph/rbd-mirror.$cluster-$name.pid client.mirror.2: admin socket: /var/run/ceph/rbd-mirror.$cluster-$name.asok pid file: /var/run/ceph/rbd-mirror.$cluster-$name.pid client.mirror.3: admin socket: /var/run/ceph/rbd-mirror.$cluster-$name.asok pid file: /var/run/ceph/rbd-mirror.$cluster-$name.pid global: mon client directed command retry: 5 mon warn on pool no app: false ms inject socket failures: 5000 mgr: debug mgr: 20 debug ms: 1 mon: debug mon: 20 debug ms: 1 debug paxos: 20 osd: bluefs allocator: hybrid bluestore allocator: hybrid bluestore block size: 96636764160 bluestore fsck on mount: true debug bluefs: 1/20 debug bluestore: 1/20 debug ms: 1 debug osd: 20 debug rocksdb: 4/10 mon osd backfillfull_ratio: 0.85 mon osd full ratio: 0.9 mon osd nearfull ratio: 0.8 osd failsafe full ratio: 0.95 osd objectstore: bluestore flavor: default fs: xfs log-ignorelist: - \(MDS_ALL_DOWN\) - \(MDS_UP_LESS_THAN_MAX\) - \(OSD_SLOW_PING_TIME sha1: adc1923395f2bcd6ae89b5d331d278364d1848ed ceph-deploy: bluestore: true conf: client: log file: /var/log/ceph/ceph-$name.$pid.log mon: {} osd: bluestore block size: 96636764160 bluestore fsck on mount: true debug bluefs: 1/20 debug bluestore: 1/20 debug rocksdb: 4/10 mon osd backfillfull_ratio: 0.85 mon osd full ratio: 0.9 mon osd nearfull ratio: 0.8 osd failsafe full ratio: 0.95 osd objectstore: bluestore fs: xfs install: ceph: flavor: default sha1: adc1923395f2bcd6ae89b5d331d278364d1848ed thrashosds: bdev_inject_crash: 2 bdev_inject_crash_probability: 0.5 workunit: branch: 20.2.1_2.17.26-baseline_2 sha1: adc1923395f2bcd6ae89b5d331d278364d1848ed owner: scheduled_yuriw@soko04.front.sepia.ceph.com priority: 69 repo: https://github.com/ceph/ceph-ci.git roles: - - cluster1.mon.a - cluster1.mgr.x - cluster2.mgr.x - cluster1.osd.0 - cluster1.osd.1 - cluster1.osd.2 - cluster1.client.0 - cluster2.client.0 - - cluster2.mon.a - cluster2.osd.0 - cluster2.osd.1 - cluster2.osd.2 - cluster1.client.mirror - cluster1.client.mirror.0 - cluster1.client.mirror.1 - cluster1.client.mirror.2 - cluster1.client.mirror.3 - cluster1.client.mirror.4 - cluster1.client.mirror.5 - cluster1.client.mirror.6 - cluster2.client.mirror - cluster2.client.mirror.0 - cluster2.client.mirror.1 - cluster2.client.mirror.2 - cluster2.client.mirror.3 - cluster2.client.mirror.4 - cluster2.client.mirror.5 - cluster2.client.mirror.6 seed: 7878 sha1: adc1923395f2bcd6ae89b5d331d278364d1848ed sleep_before_teardown: 0 subset: 111/120000 suite: rbd suite_branch: 20.2.1_2.17.26-baseline_2 suite_path: /home/teuthworker/src/github.com_ceph_ceph-c_adc1923395f2bcd6ae89b5d331d278364d1848ed/qa suite_relpath: qa suite_repo: https://github.com/ceph/ceph-ci.git suite_sha1: adc1923395f2bcd6ae89b5d331d278364d1848ed targets: trial034.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBF0BoVKPHOg0QKqwwjlPElLLlbn535mlxpDa0ESvNojoG4w+irzEhqHqNKGlXw1d/sbAP/gj/AqXlTteMMmvQPo= trial163.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBLTZW5+I/rQuc56SPLMku4v11X/kldvg5rnpQDqR+Im5nfwRzHmq3wJFTgZ2+NqmphkPILaOoEHK845hjsXPPN8= tasks: - install: extra_packages: - rbd-mirror - ceph: cluster: cluster1 - ceph: cluster: cluster2 - exec: cluster1.client.mirror.0: - sudo ceph --cluster cluster1 auth caps client.mirror mon 'profile rbd-mirror-peer' osd 'profile rbd' - sudo ceph --cluster cluster1 auth caps client.mirror.0 mon 'profile rbd-mirror' osd 'profile rbd' - sudo ceph --cluster cluster1 auth caps client.mirror.1 mon 'profile rbd-mirror' osd 'profile rbd' - sudo ceph --cluster cluster1 auth caps client.mirror.2 mon 'profile rbd-mirror' osd 'profile rbd' - sudo ceph --cluster cluster1 auth caps client.mirror.3 mon 'profile rbd-mirror' osd 'profile rbd' - sudo ceph --cluster cluster2 auth caps client.mirror mon 'profile rbd-mirror-peer' osd 'profile rbd' - sudo ceph --cluster cluster2 auth caps client.mirror.0 mon 'profile rbd-mirror' osd 'profile rbd' - sudo ceph --cluster cluster2 auth caps client.mirror.1 mon 'profile rbd-mirror' osd 'profile rbd' - sudo ceph --cluster cluster2 auth caps client.mirror.2 mon 'profile rbd-mirror' osd 'profile rbd' - sudo ceph --cluster cluster2 auth caps client.mirror.3 mon 'profile rbd-mirror' osd 'profile rbd' - rbd-mirror: client: cluster1.client.mirror.0 thrash: true - rbd-mirror: client: cluster1.client.mirror.1 thrash: true - rbd-mirror: client: cluster1.client.mirror.2 thrash: true - rbd-mirror: client: cluster1.client.mirror.3 thrash: true - rbd-mirror: client: cluster2.client.mirror.0 thrash: true - rbd-mirror: client: cluster2.client.mirror.1 thrash: true - rbd-mirror: client: cluster2.client.mirror.2 thrash: true - rbd-mirror: client: cluster2.client.mirror.3 thrash: true - rbd-mirror-thrash: cluster: cluster1 - rbd-mirror-thrash: cluster: cluster2 - workunit: clients: cluster1.client.mirror: - rbd/rbd_mirror_stress.sh env: CEPH_ARGS: '' MIRROR_POOL_MODE: pool RBD_MIRROR_INSTANCES: '4' RBD_MIRROR_MODE: journal RBD_MIRROR_USE_EXISTING_CLUSTER: '1' RBD_MIRROR_USE_RBD_MIRROR: '1' timeout: 6h teuthology: fragments_dropped: [] meta: {} postmerge: [] teuthology_branch: main teuthology_sha1: fa17720d0088c3ac28e473468bfc79eeaff5cd38 timestamp: 2026-02-17_21:26:40 tube: trial user: yuriw verbose: true worker_log: /home/teuthworker/mnt/teuthology/worker_logs/dispatcher.trial.3682 2026-02-18T08:04:55.150 INFO:teuthology.run:suite_path is set to /home/teuthworker/src/github.com_ceph_ceph-c_adc1923395f2bcd6ae89b5d331d278364d1848ed/qa; will attempt to use it 2026-02-18T08:04:55.151 INFO:teuthology.run:Found tasks at /home/teuthworker/src/github.com_ceph_ceph-c_adc1923395f2bcd6ae89b5d331d278364d1848ed/qa/tasks 2026-02-18T08:04:55.151 INFO:teuthology.run_tasks:Running task internal.check_packages... 2026-02-18T08:04:55.152 INFO:teuthology.task.internal:Checking packages... 2026-02-18T08:04:55.153 INFO:teuthology.task.internal:Checking packages for os_type 'ubuntu', flavor 'default' and ceph hash 'adc1923395f2bcd6ae89b5d331d278364d1848ed' 2026-02-18T08:04:55.153 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2026-02-18T08:04:55.153 INFO:teuthology.packaging:ref: None 2026-02-18T08:04:55.153 INFO:teuthology.packaging:tag: None 2026-02-18T08:04:55.153 INFO:teuthology.packaging:branch: 20.2.1_2.17.26-baseline_2 2026-02-18T08:04:55.153 INFO:teuthology.packaging:sha1: adc1923395f2bcd6ae89b5d331d278364d1848ed 2026-02-18T08:04:55.153 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=ubuntu%2F22.04%2Fx86_64&ref=20.2.1_2.17.26-baseline_2 2026-02-18T08:04:55.265 INFO:teuthology.task.internal:Found packages for ceph version 20.2.0-670-gadc19233-1jammy 2026-02-18T08:04:55.266 INFO:teuthology.run_tasks:Running task internal.buildpackages_prep... 2026-02-18T08:04:55.279 INFO:teuthology.task.internal:no buildpackages task found 2026-02-18T08:04:55.279 INFO:teuthology.run_tasks:Running task internal.save_config... 2026-02-18T08:04:55.293 INFO:teuthology.task.internal:Saving configuration 2026-02-18T08:04:55.301 INFO:teuthology.run_tasks:Running task internal.check_lock... 2026-02-18T08:04:55.312 INFO:teuthology.task.internal.check_lock:Checking locks... 2026-02-18T08:04:55.368 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'trial034.front.sepia.ceph.com', 'description': '/home/teuthworker/mnt/teuthology/yuriw-2026-02-17_21:26:40-rbd-20.2.1_2.17.26-baseline_2-distro-default-trial/54948', 'up': True, 'machine_type': 'trial', 'is_vm': False, 'vm_host': None, 'os_type': 'ubuntu', 'os_version': '22.04', 'arch': 'x86_64', 'locked': True, 'locked_since': '2026-02-18 08:01:24.931286', 'locked_by': 'scheduled_yuriw@soko04.front.sepia.ceph.com', 'mac_address': None, 'ssh_pub_key': 'ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIKbl5hQG3kwOeG4lidGTqBOFCeVjGOfr8TKiejJkMVMd'} 2026-02-18T08:04:55.555 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'trial163.front.sepia.ceph.com', 'description': '/home/teuthworker/mnt/teuthology/yuriw-2026-02-17_21:26:40-rbd-20.2.1_2.17.26-baseline_2-distro-default-trial/54948', 'up': True, 'machine_type': 'trial', 'is_vm': False, 'vm_host': None, 'os_type': 'ubuntu', 'os_version': '22.04', 'arch': 'x86_64', 'locked': True, 'locked_since': '2026-02-18 08:01:24.930603', 'locked_by': 'scheduled_yuriw@soko04.front.sepia.ceph.com', 'mac_address': None, 'ssh_pub_key': 'ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIC9LnqtzWJt3s38unzD/C+0SQngUJl5ivZehFRjIV0+w'} 2026-02-18T08:04:55.555 INFO:teuthology.run_tasks:Running task internal.add_remotes... 2026-02-18T08:04:55.567 INFO:teuthology.task.internal:roles: ubuntu@trial034.front.sepia.ceph.com - ['cluster1.mon.a', 'cluster1.mgr.x', 'cluster2.mgr.x', 'cluster1.osd.0', 'cluster1.osd.1', 'cluster1.osd.2', 'cluster1.client.0', 'cluster2.client.0'] 2026-02-18T08:04:55.567 INFO:teuthology.task.internal:roles: ubuntu@trial163.front.sepia.ceph.com - ['cluster2.mon.a', 'cluster2.osd.0', 'cluster2.osd.1', 'cluster2.osd.2', 'cluster1.client.mirror', 'cluster1.client.mirror.0', 'cluster1.client.mirror.1', 'cluster1.client.mirror.2', 'cluster1.client.mirror.3', 'cluster1.client.mirror.4', 'cluster1.client.mirror.5', 'cluster1.client.mirror.6', 'cluster2.client.mirror', 'cluster2.client.mirror.0', 'cluster2.client.mirror.1', 'cluster2.client.mirror.2', 'cluster2.client.mirror.3', 'cluster2.client.mirror.4', 'cluster2.client.mirror.5', 'cluster2.client.mirror.6'] 2026-02-18T08:04:55.567 INFO:teuthology.run_tasks:Running task console_log... 2026-02-18T08:04:55.696 DEBUG:teuthology.exit:Installing handler: Handler(exiter=, func=.kill_console_loggers at 0x75e6187d7880>, signals=[15]) 2026-02-18T08:04:55.697 INFO:teuthology.run_tasks:Running task internal.connect... 2026-02-18T08:04:55.709 INFO:teuthology.task.internal:Opening connections... 2026-02-18T08:04:55.709 DEBUG:teuthology.task.internal:connecting to ubuntu@trial034.front.sepia.ceph.com 2026-02-18T08:04:55.710 DEBUG:teuthology.orchestra.connection:{'hostname': 'trial034.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2026-02-18T08:04:55.771 DEBUG:teuthology.task.internal:connecting to ubuntu@trial163.front.sepia.ceph.com 2026-02-18T08:04:55.772 DEBUG:teuthology.orchestra.connection:{'hostname': 'trial163.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2026-02-18T08:04:55.833 INFO:teuthology.run_tasks:Running task internal.push_inventory... 2026-02-18T08:04:55.844 DEBUG:teuthology.orchestra.run.trial034:> uname -m 2026-02-18T08:04:55.848 INFO:teuthology.orchestra.run.trial034.stdout:x86_64 2026-02-18T08:04:55.848 DEBUG:teuthology.orchestra.run.trial034:> cat /etc/os-release 2026-02-18T08:04:55.894 INFO:teuthology.orchestra.run.trial034.stdout:PRETTY_NAME="Ubuntu 22.04.5 LTS" 2026-02-18T08:04:55.894 INFO:teuthology.orchestra.run.trial034.stdout:NAME="Ubuntu" 2026-02-18T08:04:55.894 INFO:teuthology.orchestra.run.trial034.stdout:VERSION_ID="22.04" 2026-02-18T08:04:55.895 INFO:teuthology.orchestra.run.trial034.stdout:VERSION="22.04.5 LTS (Jammy Jellyfish)" 2026-02-18T08:04:55.895 INFO:teuthology.orchestra.run.trial034.stdout:VERSION_CODENAME=jammy 2026-02-18T08:04:55.895 INFO:teuthology.orchestra.run.trial034.stdout:ID=ubuntu 2026-02-18T08:04:55.895 INFO:teuthology.orchestra.run.trial034.stdout:ID_LIKE=debian 2026-02-18T08:04:55.895 INFO:teuthology.orchestra.run.trial034.stdout:HOME_URL="https://www.ubuntu.com/" 2026-02-18T08:04:55.895 INFO:teuthology.orchestra.run.trial034.stdout:SUPPORT_URL="https://help.ubuntu.com/" 2026-02-18T08:04:55.895 INFO:teuthology.orchestra.run.trial034.stdout:BUG_REPORT_URL="https://bugs.launchpad.net/ubuntu/" 2026-02-18T08:04:55.895 INFO:teuthology.orchestra.run.trial034.stdout:PRIVACY_POLICY_URL="https://www.ubuntu.com/legal/terms-and-policies/privacy-policy" 2026-02-18T08:04:55.895 INFO:teuthology.orchestra.run.trial034.stdout:UBUNTU_CODENAME=jammy 2026-02-18T08:04:55.896 INFO:teuthology.lock.ops:Updating trial034.front.sepia.ceph.com on lock server 2026-02-18T08:04:55.961 DEBUG:teuthology.orchestra.run.trial163:> uname -m 2026-02-18T08:04:55.965 INFO:teuthology.orchestra.run.trial163.stdout:x86_64 2026-02-18T08:04:55.965 DEBUG:teuthology.orchestra.run.trial163:> cat /etc/os-release 2026-02-18T08:04:56.015 INFO:teuthology.orchestra.run.trial163.stdout:PRETTY_NAME="Ubuntu 22.04.5 LTS" 2026-02-18T08:04:56.015 INFO:teuthology.orchestra.run.trial163.stdout:NAME="Ubuntu" 2026-02-18T08:04:56.015 INFO:teuthology.orchestra.run.trial163.stdout:VERSION_ID="22.04" 2026-02-18T08:04:56.015 INFO:teuthology.orchestra.run.trial163.stdout:VERSION="22.04.5 LTS (Jammy Jellyfish)" 2026-02-18T08:04:56.015 INFO:teuthology.orchestra.run.trial163.stdout:VERSION_CODENAME=jammy 2026-02-18T08:04:56.015 INFO:teuthology.orchestra.run.trial163.stdout:ID=ubuntu 2026-02-18T08:04:56.015 INFO:teuthology.orchestra.run.trial163.stdout:ID_LIKE=debian 2026-02-18T08:04:56.015 INFO:teuthology.orchestra.run.trial163.stdout:HOME_URL="https://www.ubuntu.com/" 2026-02-18T08:04:56.015 INFO:teuthology.orchestra.run.trial163.stdout:SUPPORT_URL="https://help.ubuntu.com/" 2026-02-18T08:04:56.015 INFO:teuthology.orchestra.run.trial163.stdout:BUG_REPORT_URL="https://bugs.launchpad.net/ubuntu/" 2026-02-18T08:04:56.016 INFO:teuthology.orchestra.run.trial163.stdout:PRIVACY_POLICY_URL="https://www.ubuntu.com/legal/terms-and-policies/privacy-policy" 2026-02-18T08:04:56.016 INFO:teuthology.orchestra.run.trial163.stdout:UBUNTU_CODENAME=jammy 2026-02-18T08:04:56.016 INFO:teuthology.lock.ops:Updating trial163.front.sepia.ceph.com on lock server 2026-02-18T08:04:56.084 INFO:teuthology.run_tasks:Running task internal.serialize_remote_roles... 2026-02-18T08:04:56.097 INFO:teuthology.run_tasks:Running task internal.check_conflict... 2026-02-18T08:04:56.109 INFO:teuthology.task.internal:Checking for old test directory... 2026-02-18T08:04:56.109 DEBUG:teuthology.orchestra.run.trial034:> test '!' -e /home/ubuntu/cephtest 2026-02-18T08:04:56.110 DEBUG:teuthology.orchestra.run.trial163:> test '!' -e /home/ubuntu/cephtest 2026-02-18T08:04:56.113 INFO:teuthology.run_tasks:Running task internal.check_ceph_data... 2026-02-18T08:04:56.124 INFO:teuthology.task.internal:Checking for non-empty /var/lib/ceph... 2026-02-18T08:04:56.125 DEBUG:teuthology.orchestra.run.trial034:> test -z $(ls -A /var/lib/ceph) 2026-02-18T08:04:56.156 DEBUG:teuthology.orchestra.run.trial163:> test -z $(ls -A /var/lib/ceph) 2026-02-18T08:04:56.160 INFO:teuthology.orchestra.run.trial034.stderr:ls: cannot access '/var/lib/ceph': No such file or directory 2026-02-18T08:04:56.161 INFO:teuthology.orchestra.run.trial163.stderr:ls: cannot access '/var/lib/ceph': No such file or directory 2026-02-18T08:04:56.161 INFO:teuthology.run_tasks:Running task internal.vm_setup... 2026-02-18T08:04:56.378 INFO:teuthology.run_tasks:Running task kernel... 2026-02-18T08:04:56.397 INFO:teuthology.task.kernel:normalize config orig: {'branch': 'distro', 'kdb': True, 'sha1': 'distro'} 2026-02-18T08:04:56.397 INFO:teuthology.task.kernel:config {'cluster1.mon.a': {'branch': 'distro', 'kdb': True, 'sha1': 'distro'}, 'cluster2.mon.a': {'branch': 'distro', 'kdb': True, 'sha1': 'distro'}}, timeout 300 2026-02-18T08:04:56.397 DEBUG:teuthology.orchestra.run.trial034:> test -f /run/.containerenv -o -f /.dockerenv 2026-02-18T08:04:56.398 DEBUG:teuthology.orchestra.run.trial163:> test -f /run/.containerenv -o -f /.dockerenv 2026-02-18T08:04:56.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2026-02-18T08:04:56.401 DEBUG:teuthology.orchestra.run.trial034:> uname -r 2026-02-18T08:04:56.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2026-02-18T08:04:56.401 DEBUG:teuthology.orchestra.run.trial163:> uname -r 2026-02-18T08:04:56.446 INFO:teuthology.orchestra.run.trial034.stdout:5.15.0-164-generic 2026-02-18T08:04:56.447 INFO:teuthology.task.kernel:Running kernel on trial034: 5.15.0-164-generic 2026-02-18T08:04:56.447 DEBUG:teuthology.orchestra.run.trial034:> sudo apt-get clean 2026-02-18T08:04:56.451 INFO:teuthology.orchestra.run.trial163.stdout:5.15.0-164-generic 2026-02-18T08:04:56.451 INFO:teuthology.task.kernel:Running kernel on trial163: 5.15.0-164-generic 2026-02-18T08:04:56.451 DEBUG:teuthology.orchestra.run.trial163:> sudo apt-get clean 2026-02-18T08:04:56.508 DEBUG:teuthology.orchestra.run.trial034:> sudo apt-get update 2026-02-18T08:04:56.518 DEBUG:teuthology.orchestra.run.trial163:> sudo apt-get update 2026-02-18T08:04:56.942 INFO:teuthology.orchestra.run.trial034.stdout:Hit:1 https://archive.ubuntu.com/ubuntu jammy InRelease 2026-02-18T08:04:57.015 INFO:teuthology.orchestra.run.trial163.stdout:Hit:1 https://archive.ubuntu.com/ubuntu jammy InRelease 2026-02-18T08:04:57.015 INFO:teuthology.orchestra.run.trial034.stdout:Get:2 https://archive.ubuntu.com/ubuntu jammy-updates InRelease [128 kB] 2026-02-18T08:04:57.104 INFO:teuthology.orchestra.run.trial163.stdout:Get:2 https://archive.ubuntu.com/ubuntu jammy-updates InRelease [128 kB] 2026-02-18T08:04:57.290 INFO:teuthology.orchestra.run.trial034.stdout:Get:3 https://archive.ubuntu.com/ubuntu jammy-security InRelease [129 kB] 2026-02-18T08:04:57.412 INFO:teuthology.orchestra.run.trial034.stdout:Get:4 https://archive.ubuntu.com/ubuntu jammy-backports InRelease [127 kB] 2026-02-18T08:04:57.438 INFO:teuthology.orchestra.run.trial163.stdout:Get:3 https://archive.ubuntu.com/ubuntu jammy-security InRelease [129 kB] 2026-02-18T08:04:57.527 INFO:teuthology.orchestra.run.trial034.stdout:Get:5 https://archive.ubuntu.com/ubuntu jammy-updates/main amd64 Packages [3240 kB] 2026-02-18T08:04:57.587 INFO:teuthology.orchestra.run.trial163.stdout:Get:4 https://archive.ubuntu.com/ubuntu jammy-backports InRelease [127 kB] 2026-02-18T08:04:57.727 INFO:teuthology.orchestra.run.trial163.stdout:Get:5 https://archive.ubuntu.com/ubuntu jammy-updates/main amd64 Packages [3240 kB] 2026-02-18T08:04:57.857 INFO:teuthology.orchestra.run.trial034.stdout:Get:6 https://archive.ubuntu.com/ubuntu jammy-updates/main Translation-en [494 kB] 2026-02-18T08:04:57.865 INFO:teuthology.orchestra.run.trial034.stdout:Get:7 https://archive.ubuntu.com/ubuntu jammy-updates/main amd64 c-n-f Metadata [19.2 kB] 2026-02-18T08:04:57.865 INFO:teuthology.orchestra.run.trial034.stdout:Get:8 https://archive.ubuntu.com/ubuntu jammy-updates/universe amd64 Packages [1256 kB] 2026-02-18T08:04:57.893 INFO:teuthology.orchestra.run.trial034.stdout:Get:9 https://archive.ubuntu.com/ubuntu jammy-updates/universe Translation-en [314 kB] 2026-02-18T08:04:57.899 INFO:teuthology.orchestra.run.trial034.stdout:Get:10 https://archive.ubuntu.com/ubuntu jammy-updates/restricted amd64 Packages [5295 kB] 2026-02-18T08:04:58.028 INFO:teuthology.orchestra.run.trial034.stdout:Get:11 https://archive.ubuntu.com/ubuntu jammy-updates/restricted Translation-en [1001 kB] 2026-02-18T08:04:58.046 INFO:teuthology.orchestra.run.trial034.stdout:Get:12 https://archive.ubuntu.com/ubuntu jammy-security/main amd64 Packages [2975 kB] 2026-02-18T08:04:58.117 INFO:teuthology.orchestra.run.trial034.stdout:Get:13 https://archive.ubuntu.com/ubuntu jammy-security/main amd64 c-n-f Metadata [14.1 kB] 2026-02-18T08:04:58.117 INFO:teuthology.orchestra.run.trial034.stdout:Get:14 https://archive.ubuntu.com/ubuntu jammy-security/universe amd64 Packages [1017 kB] 2026-02-18T08:04:58.127 INFO:teuthology.orchestra.run.trial163.stdout:Get:6 https://archive.ubuntu.com/ubuntu jammy-updates/main Translation-en [494 kB] 2026-02-18T08:04:58.138 INFO:teuthology.orchestra.run.trial163.stdout:Get:7 https://archive.ubuntu.com/ubuntu jammy-updates/main amd64 c-n-f Metadata [19.2 kB] 2026-02-18T08:04:58.138 INFO:teuthology.orchestra.run.trial163.stdout:Get:8 https://archive.ubuntu.com/ubuntu jammy-updates/universe amd64 Packages [1256 kB] 2026-02-18T08:04:58.173 INFO:teuthology.orchestra.run.trial163.stdout:Get:9 https://archive.ubuntu.com/ubuntu jammy-updates/universe Translation-en [314 kB] 2026-02-18T08:04:58.180 INFO:teuthology.orchestra.run.trial163.stdout:Get:10 https://archive.ubuntu.com/ubuntu jammy-updates/restricted amd64 Packages [5295 kB] 2026-02-18T08:04:58.336 INFO:teuthology.orchestra.run.trial163.stdout:Get:11 https://archive.ubuntu.com/ubuntu jammy-updates/restricted Translation-en [1001 kB] 2026-02-18T08:04:58.357 INFO:teuthology.orchestra.run.trial163.stdout:Get:12 https://archive.ubuntu.com/ubuntu jammy-security/main amd64 Packages [2975 kB] 2026-02-18T08:04:58.445 INFO:teuthology.orchestra.run.trial163.stdout:Get:13 https://archive.ubuntu.com/ubuntu jammy-security/main amd64 c-n-f Metadata [14.1 kB] 2026-02-18T08:04:58.445 INFO:teuthology.orchestra.run.trial163.stdout:Get:14 https://archive.ubuntu.com/ubuntu jammy-security/universe amd64 Packages [1017 kB] 2026-02-18T08:05:01.242 INFO:teuthology.orchestra.run.trial034.stdout:Fetched 16.0 MB in 2s (7311 kB/s) 2026-02-18T08:05:01.542 INFO:teuthology.orchestra.run.trial163.stdout:Fetched 16.0 MB in 2s (6554 kB/s) 2026-02-18T08:05:01.763 INFO:teuthology.orchestra.run.trial034.stdout:Reading package lists... 2026-02-18T08:05:01.777 DEBUG:teuthology.orchestra.run.trial034:> sudo DEBIAN_FRONTEND=noninteractive apt-get -y install linux-image-generic 2026-02-18T08:05:01.851 INFO:teuthology.orchestra.run.trial034.stdout:Reading package lists... 2026-02-18T08:05:01.957 INFO:teuthology.orchestra.run.trial034.stdout:Building dependency tree... 2026-02-18T08:05:01.958 INFO:teuthology.orchestra.run.trial034.stdout:Reading state information... 2026-02-18T08:05:02.044 INFO:teuthology.orchestra.run.trial034.stdout:The following packages were automatically installed and are no longer required: 2026-02-18T08:05:02.044 INFO:teuthology.orchestra.run.trial034.stdout: kpartx libboost-iostreams1.74.0 libboost-thread1.74.0 libgfapi0 libgfrpc0 2026-02-18T08:05:02.044 INFO:teuthology.orchestra.run.trial034.stdout: libgfxdr0 libglusterfs0 libiscsi7 libpmemobj1 libsgutils2-2 sg3-utils 2026-02-18T08:05:02.044 INFO:teuthology.orchestra.run.trial034.stdout: sg3-utils-udev 2026-02-18T08:05:02.044 INFO:teuthology.orchestra.run.trial034.stdout:Use 'sudo apt autoremove' to remove them. 2026-02-18T08:05:02.044 INFO:teuthology.orchestra.run.trial034.stdout:The following additional packages will be installed: 2026-02-18T08:05:02.044 INFO:teuthology.orchestra.run.trial034.stdout: linux-generic linux-headers-5.15.0-170 linux-headers-5.15.0-170-generic 2026-02-18T08:05:02.044 INFO:teuthology.orchestra.run.trial034.stdout: linux-headers-generic linux-image-5.15.0-170-generic 2026-02-18T08:05:02.044 INFO:teuthology.orchestra.run.trial034.stdout: linux-modules-5.15.0-170-generic linux-modules-extra-5.15.0-170-generic 2026-02-18T08:05:02.045 INFO:teuthology.orchestra.run.trial034.stdout:Suggested packages: 2026-02-18T08:05:02.045 INFO:teuthology.orchestra.run.trial034.stdout: fdutils linux-doc | linux-source-5.15.0 linux-tools 2026-02-18T08:05:02.057 INFO:teuthology.orchestra.run.trial034.stdout:The following NEW packages will be installed: 2026-02-18T08:05:02.058 INFO:teuthology.orchestra.run.trial034.stdout: linux-headers-5.15.0-170 linux-headers-5.15.0-170-generic 2026-02-18T08:05:02.058 INFO:teuthology.orchestra.run.trial034.stdout: linux-image-5.15.0-170-generic linux-modules-5.15.0-170-generic 2026-02-18T08:05:02.058 INFO:teuthology.orchestra.run.trial034.stdout: linux-modules-extra-5.15.0-170-generic 2026-02-18T08:05:02.058 INFO:teuthology.orchestra.run.trial034.stdout:The following packages will be upgraded: 2026-02-18T08:05:02.058 INFO:teuthology.orchestra.run.trial034.stdout: linux-generic linux-headers-generic linux-image-generic 2026-02-18T08:05:02.062 INFO:teuthology.orchestra.run.trial163.stdout:Reading package lists... 2026-02-18T08:05:02.075 DEBUG:teuthology.orchestra.run.trial163:> sudo DEBIAN_FRONTEND=noninteractive apt-get -y install linux-image-generic 2026-02-18T08:05:02.108 INFO:teuthology.orchestra.run.trial163.stdout:Reading package lists... 2026-02-18T08:05:02.214 INFO:teuthology.orchestra.run.trial163.stdout:Building dependency tree... 2026-02-18T08:05:02.214 INFO:teuthology.orchestra.run.trial163.stdout:Reading state information... 2026-02-18T08:05:02.300 INFO:teuthology.orchestra.run.trial163.stdout:The following packages were automatically installed and are no longer required: 2026-02-18T08:05:02.300 INFO:teuthology.orchestra.run.trial163.stdout: kpartx libboost-iostreams1.74.0 libboost-thread1.74.0 libgfapi0 libgfrpc0 2026-02-18T08:05:02.300 INFO:teuthology.orchestra.run.trial163.stdout: libgfxdr0 libglusterfs0 libiscsi7 libpmemobj1 libsgutils2-2 sg3-utils 2026-02-18T08:05:02.300 INFO:teuthology.orchestra.run.trial163.stdout: sg3-utils-udev 2026-02-18T08:05:02.300 INFO:teuthology.orchestra.run.trial163.stdout:Use 'sudo apt autoremove' to remove them. 2026-02-18T08:05:02.300 INFO:teuthology.orchestra.run.trial163.stdout:The following additional packages will be installed: 2026-02-18T08:05:02.301 INFO:teuthology.orchestra.run.trial163.stdout: linux-generic linux-headers-5.15.0-170 linux-headers-5.15.0-170-generic 2026-02-18T08:05:02.301 INFO:teuthology.orchestra.run.trial163.stdout: linux-headers-generic linux-image-5.15.0-170-generic 2026-02-18T08:05:02.301 INFO:teuthology.orchestra.run.trial163.stdout: linux-modules-5.15.0-170-generic linux-modules-extra-5.15.0-170-generic 2026-02-18T08:05:02.301 INFO:teuthology.orchestra.run.trial163.stdout:Suggested packages: 2026-02-18T08:05:02.301 INFO:teuthology.orchestra.run.trial163.stdout: fdutils linux-doc | linux-source-5.15.0 linux-tools 2026-02-18T08:05:02.314 INFO:teuthology.orchestra.run.trial163.stdout:The following NEW packages will be installed: 2026-02-18T08:05:02.314 INFO:teuthology.orchestra.run.trial163.stdout: linux-headers-5.15.0-170 linux-headers-5.15.0-170-generic 2026-02-18T08:05:02.314 INFO:teuthology.orchestra.run.trial163.stdout: linux-image-5.15.0-170-generic linux-modules-5.15.0-170-generic 2026-02-18T08:05:02.314 INFO:teuthology.orchestra.run.trial163.stdout: linux-modules-extra-5.15.0-170-generic 2026-02-18T08:05:02.315 INFO:teuthology.orchestra.run.trial163.stdout:The following packages will be upgraded: 2026-02-18T08:05:02.315 INFO:teuthology.orchestra.run.trial163.stdout: linux-generic linux-headers-generic linux-image-generic 2026-02-18T08:05:02.401 INFO:teuthology.orchestra.run.trial034.stdout:3 upgraded, 5 newly installed, 0 to remove and 53 not upgraded. 2026-02-18T08:05:02.401 INFO:teuthology.orchestra.run.trial034.stdout:Need to get 113 MB of archives. 2026-02-18T08:05:02.401 INFO:teuthology.orchestra.run.trial034.stdout:After this operation, 585 MB of additional disk space will be used. 2026-02-18T08:05:02.401 INFO:teuthology.orchestra.run.trial034.stdout:Get:1 https://archive.ubuntu.com/ubuntu jammy-updates/main amd64 linux-modules-5.15.0-170-generic amd64 5.15.0-170.180 [22.7 MB] 2026-02-18T08:05:02.409 INFO:teuthology.orchestra.run.trial163.stdout:3 upgraded, 5 newly installed, 0 to remove and 53 not upgraded. 2026-02-18T08:05:02.409 INFO:teuthology.orchestra.run.trial163.stdout:Need to get 113 MB of archives. 2026-02-18T08:05:02.409 INFO:teuthology.orchestra.run.trial163.stdout:After this operation, 585 MB of additional disk space will be used. 2026-02-18T08:05:02.409 INFO:teuthology.orchestra.run.trial163.stdout:Get:1 https://archive.ubuntu.com/ubuntu jammy-updates/main amd64 linux-modules-5.15.0-170-generic amd64 5.15.0-170.180 [22.7 MB] 2026-02-18T08:05:02.573 INFO:teuthology.orchestra.run.trial163.stdout:Get:2 https://archive.ubuntu.com/ubuntu jammy-updates/main amd64 linux-image-5.15.0-170-generic amd64 5.15.0-170.180 [11.6 MB] 2026-02-18T08:05:02.616 INFO:teuthology.orchestra.run.trial163.stdout:Get:3 https://archive.ubuntu.com/ubuntu jammy-updates/main amd64 linux-modules-extra-5.15.0-170-generic amd64 5.15.0-170.180 [64.0 MB] 2026-02-18T08:05:02.851 INFO:teuthology.orchestra.run.trial163.stdout:Get:4 https://archive.ubuntu.com/ubuntu jammy-updates/main amd64 linux-generic amd64 5.15.0.170.159 [1654 B] 2026-02-18T08:05:02.852 INFO:teuthology.orchestra.run.trial163.stdout:Get:5 https://archive.ubuntu.com/ubuntu jammy-updates/main amd64 linux-image-generic amd64 5.15.0.170.159 [2474 B] 2026-02-18T08:05:02.852 INFO:teuthology.orchestra.run.trial163.stdout:Get:6 https://archive.ubuntu.com/ubuntu jammy-updates/main amd64 linux-headers-5.15.0-170 all 5.15.0-170.180 [12.4 MB] 2026-02-18T08:05:02.902 INFO:teuthology.orchestra.run.trial163.stdout:Get:7 https://archive.ubuntu.com/ubuntu jammy-updates/main amd64 linux-headers-5.15.0-170-generic amd64 5.15.0-170.180 [2828 kB] 2026-02-18T08:05:02.913 INFO:teuthology.orchestra.run.trial163.stdout:Get:8 https://archive.ubuntu.com/ubuntu jammy-updates/main amd64 linux-headers-generic amd64 5.15.0.170.159 [2316 B] 2026-02-18T08:05:03.113 INFO:teuthology.orchestra.run.trial163.stdout:Fetched 113 MB in 1s (190 MB/s) 2026-02-18T08:05:03.133 INFO:teuthology.orchestra.run.trial163.stdout:Selecting previously unselected package linux-modules-5.15.0-170-generic. 2026-02-18T08:05:03.297 INFO:teuthology.orchestra.run.trial163.stdout:(Reading database ... (Reading database ... 5% (Reading database ... 10% (Reading database ... 15% (Reading database ... 20% (Reading database ... 25% (Reading database ... 30% (Reading database ... 35% (Reading database ... 40% (Reading database ... 45% (Reading database ... 50% (Reading database ... 55% (Reading database ... 60% (Reading database ... 65% (Reading database ... 70% (Reading database ... 75% (Reading database ... 80% (Reading database ... 85% (Reading database ... 90% (Reading database ... 95% (Reading database ... 100% (Reading database ... 131954 files and directories currently installed.) 2026-02-18T08:05:03.298 INFO:teuthology.orchestra.run.trial163.stdout:Preparing to unpack .../0-linux-modules-5.15.0-170-generic_5.15.0-170.180_amd64.deb ... 2026-02-18T08:05:03.300 INFO:teuthology.orchestra.run.trial163.stdout:Unpacking linux-modules-5.15.0-170-generic (5.15.0-170.180) ... 2026-02-18T08:05:03.494 INFO:teuthology.orchestra.run.trial034.stdout:Get:2 https://archive.ubuntu.com/ubuntu jammy-updates/main amd64 linux-image-5.15.0-170-generic amd64 5.15.0-170.180 [11.6 MB] 2026-02-18T08:05:03.646 INFO:teuthology.orchestra.run.trial163.stdout:Selecting previously unselected package linux-image-5.15.0-170-generic. 2026-02-18T08:05:03.652 INFO:teuthology.orchestra.run.trial163.stdout:Preparing to unpack .../1-linux-image-5.15.0-170-generic_5.15.0-170.180_amd64.deb ... 2026-02-18T08:05:03.673 INFO:teuthology.orchestra.run.trial163.stdout:Unpacking linux-image-5.15.0-170-generic (5.15.0-170.180) ... 2026-02-18T08:05:03.712 INFO:teuthology.orchestra.run.trial163.stdout:Selecting previously unselected package linux-modules-extra-5.15.0-170-generic. 2026-02-18T08:05:03.719 INFO:teuthology.orchestra.run.trial163.stdout:Preparing to unpack .../2-linux-modules-extra-5.15.0-170-generic_5.15.0-170.180_amd64.deb ... 2026-02-18T08:05:03.719 INFO:teuthology.orchestra.run.trial163.stdout:Unpacking linux-modules-extra-5.15.0-170-generic (5.15.0-170.180) ... 2026-02-18T08:05:03.764 INFO:teuthology.orchestra.run.trial034.stdout:Get:3 https://archive.ubuntu.com/ubuntu jammy-updates/main amd64 linux-modules-extra-5.15.0-170-generic amd64 5.15.0-170.180 [64.0 MB] 2026-02-18T08:05:04.806 INFO:teuthology.orchestra.run.trial163.stdout:Preparing to unpack .../3-linux-generic_5.15.0.170.159_amd64.deb ... 2026-02-18T08:05:04.807 INFO:teuthology.orchestra.run.trial163.stdout:Unpacking linux-generic (5.15.0.170.159) over (5.15.0.164.159) ... 2026-02-18T08:05:04.828 INFO:teuthology.orchestra.run.trial163.stdout:Preparing to unpack .../4-linux-image-generic_5.15.0.170.159_amd64.deb ... 2026-02-18T08:05:04.829 INFO:teuthology.orchestra.run.trial163.stdout:Unpacking linux-image-generic (5.15.0.170.159) over (5.15.0.164.159) ... 2026-02-18T08:05:04.845 INFO:teuthology.orchestra.run.trial163.stdout:Selecting previously unselected package linux-headers-5.15.0-170. 2026-02-18T08:05:04.852 INFO:teuthology.orchestra.run.trial163.stdout:Preparing to unpack .../5-linux-headers-5.15.0-170_5.15.0-170.180_all.deb ... 2026-02-18T08:05:04.852 INFO:teuthology.orchestra.run.trial163.stdout:Unpacking linux-headers-5.15.0-170 (5.15.0-170.180) ... 2026-02-18T08:05:05.259 INFO:teuthology.orchestra.run.trial034.stdout:Get:4 https://archive.ubuntu.com/ubuntu jammy-updates/main amd64 linux-generic amd64 5.15.0.170.159 [1654 B] 2026-02-18T08:05:05.259 INFO:teuthology.orchestra.run.trial034.stdout:Get:5 https://archive.ubuntu.com/ubuntu jammy-updates/main amd64 linux-image-generic amd64 5.15.0.170.159 [2474 B] 2026-02-18T08:05:05.259 INFO:teuthology.orchestra.run.trial034.stdout:Get:6 https://archive.ubuntu.com/ubuntu jammy-updates/main amd64 linux-headers-5.15.0-170 all 5.15.0-170.180 [12.4 MB] 2026-02-18T08:05:05.536 INFO:teuthology.orchestra.run.trial034.stdout:Get:7 https://archive.ubuntu.com/ubuntu jammy-updates/main amd64 linux-headers-5.15.0-170-generic amd64 5.15.0-170.180 [2828 kB] 2026-02-18T08:05:05.599 INFO:teuthology.orchestra.run.trial034.stdout:Get:8 https://archive.ubuntu.com/ubuntu jammy-updates/main amd64 linux-headers-generic amd64 5.15.0.170.159 [2316 B] 2026-02-18T08:05:05.800 INFO:teuthology.orchestra.run.trial034.stdout:Fetched 113 MB in 4s (32.0 MB/s) 2026-02-18T08:05:05.821 INFO:teuthology.orchestra.run.trial034.stdout:Selecting previously unselected package linux-modules-5.15.0-170-generic. 2026-02-18T08:05:05.982 INFO:teuthology.orchestra.run.trial034.stdout:(Reading database ... (Reading database ... 5% (Reading database ... 10% (Reading database ... 15% (Reading database ... 20% (Reading database ... 25% (Reading database ... 30% (Reading database ... 35% (Reading database ... 40% (Reading database ... 45% (Reading database ... 50% (Reading database ... 55% (Reading database ... 60% (Reading database ... 65% (Reading database ... 70% (Reading database ... 75% (Reading database ... 80% (Reading database ... 85% (Reading database ... 90% (Reading database ... 95% (Reading database ... 100% (Reading database ... 131954 files and directories currently installed.) 2026-02-18T08:05:05.983 INFO:teuthology.orchestra.run.trial034.stdout:Preparing to unpack .../0-linux-modules-5.15.0-170-generic_5.15.0-170.180_amd64.deb ... 2026-02-18T08:05:05.984 INFO:teuthology.orchestra.run.trial034.stdout:Unpacking linux-modules-5.15.0-170-generic (5.15.0-170.180) ... 2026-02-18T08:05:06.237 INFO:teuthology.orchestra.run.trial163.stdout:Selecting previously unselected package linux-headers-5.15.0-170-generic. 2026-02-18T08:05:06.249 INFO:teuthology.orchestra.run.trial163.stdout:Preparing to unpack .../6-linux-headers-5.15.0-170-generic_5.15.0-170.180_amd64.deb ... 2026-02-18T08:05:06.249 INFO:teuthology.orchestra.run.trial163.stdout:Unpacking linux-headers-5.15.0-170-generic (5.15.0-170.180) ... 2026-02-18T08:05:06.332 INFO:teuthology.orchestra.run.trial034.stdout:Selecting previously unselected package linux-image-5.15.0-170-generic. 2026-02-18T08:05:06.339 INFO:teuthology.orchestra.run.trial034.stdout:Preparing to unpack .../1-linux-image-5.15.0-170-generic_5.15.0-170.180_amd64.deb ... 2026-02-18T08:05:06.355 INFO:teuthology.orchestra.run.trial034.stdout:Unpacking linux-image-5.15.0-170-generic (5.15.0-170.180) ... 2026-02-18T08:05:06.395 INFO:teuthology.orchestra.run.trial034.stdout:Selecting previously unselected package linux-modules-extra-5.15.0-170-generic. 2026-02-18T08:05:06.401 INFO:teuthology.orchestra.run.trial034.stdout:Preparing to unpack .../2-linux-modules-extra-5.15.0-170-generic_5.15.0-170.180_amd64.deb ... 2026-02-18T08:05:06.402 INFO:teuthology.orchestra.run.trial034.stdout:Unpacking linux-modules-extra-5.15.0-170-generic (5.15.0-170.180) ... 2026-02-18T08:05:06.847 INFO:teuthology.orchestra.run.trial163.stdout:Preparing to unpack .../7-linux-headers-generic_5.15.0.170.159_amd64.deb ... 2026-02-18T08:05:06.847 INFO:teuthology.orchestra.run.trial163.stdout:Unpacking linux-headers-generic (5.15.0.170.159) over (5.15.0.164.159) ... 2026-02-18T08:05:06.873 INFO:teuthology.orchestra.run.trial163.stdout:Setting up linux-headers-5.15.0-170 (5.15.0-170.180) ... 2026-02-18T08:05:06.874 INFO:teuthology.orchestra.run.trial163.stdout:Setting up linux-headers-5.15.0-170-generic (5.15.0-170.180) ... 2026-02-18T08:05:06.876 INFO:teuthology.orchestra.run.trial163.stdout:Setting up linux-headers-generic (5.15.0.170.159) ... 2026-02-18T08:05:06.877 INFO:teuthology.orchestra.run.trial163.stdout:Setting up linux-modules-5.15.0-170-generic (5.15.0-170.180) ... 2026-02-18T08:05:07.502 INFO:teuthology.orchestra.run.trial034.stdout:Preparing to unpack .../3-linux-generic_5.15.0.170.159_amd64.deb ... 2026-02-18T08:05:07.502 INFO:teuthology.orchestra.run.trial034.stdout:Unpacking linux-generic (5.15.0.170.159) over (5.15.0.164.159) ... 2026-02-18T08:05:07.524 INFO:teuthology.orchestra.run.trial034.stdout:Preparing to unpack .../4-linux-image-generic_5.15.0.170.159_amd64.deb ... 2026-02-18T08:05:07.525 INFO:teuthology.orchestra.run.trial034.stdout:Unpacking linux-image-generic (5.15.0.170.159) over (5.15.0.164.159) ... 2026-02-18T08:05:07.541 INFO:teuthology.orchestra.run.trial034.stdout:Selecting previously unselected package linux-headers-5.15.0-170. 2026-02-18T08:05:07.548 INFO:teuthology.orchestra.run.trial034.stdout:Preparing to unpack .../5-linux-headers-5.15.0-170_5.15.0-170.180_all.deb ... 2026-02-18T08:05:07.549 INFO:teuthology.orchestra.run.trial034.stdout:Unpacking linux-headers-5.15.0-170 (5.15.0-170.180) ... 2026-02-18T08:05:07.651 INFO:teuthology.orchestra.run.trial163.stdout:Setting up linux-image-5.15.0-170-generic (5.15.0-170.180) ... 2026-02-18T08:05:08.469 INFO:teuthology.orchestra.run.trial163.stdout:I: /boot/vmlinuz is now a symlink to vmlinuz-5.15.0-170-generic 2026-02-18T08:05:08.469 INFO:teuthology.orchestra.run.trial163.stdout:I: /boot/initrd.img is now a symlink to initrd.img-5.15.0-170-generic 2026-02-18T08:05:08.476 INFO:teuthology.orchestra.run.trial163.stdout:Setting up linux-modules-extra-5.15.0-170-generic (5.15.0-170.180) ... 2026-02-18T08:05:08.898 INFO:teuthology.orchestra.run.trial034.stdout:Selecting previously unselected package linux-headers-5.15.0-170-generic. 2026-02-18T08:05:08.909 INFO:teuthology.orchestra.run.trial034.stdout:Preparing to unpack .../6-linux-headers-5.15.0-170-generic_5.15.0-170.180_amd64.deb ... 2026-02-18T08:05:08.909 INFO:teuthology.orchestra.run.trial034.stdout:Unpacking linux-headers-5.15.0-170-generic (5.15.0-170.180) ... 2026-02-18T08:05:09.266 INFO:teuthology.orchestra.run.trial163.stdout:Setting up linux-image-generic (5.15.0.170.159) ... 2026-02-18T08:05:09.267 INFO:teuthology.orchestra.run.trial163.stdout:Setting up linux-generic (5.15.0.170.159) ... 2026-02-18T08:05:09.268 INFO:teuthology.orchestra.run.trial163.stdout:Processing triggers for linux-image-5.15.0-170-generic (5.15.0-170.180) ... 2026-02-18T08:05:09.274 INFO:teuthology.orchestra.run.trial163.stdout:/etc/kernel/postinst.d/initramfs-tools: 2026-02-18T08:05:09.274 INFO:teuthology.orchestra.run.trial163.stdout:update-initramfs: Generating /boot/initrd.img-5.15.0-170-generic 2026-02-18T08:05:09.502 INFO:teuthology.orchestra.run.trial034.stdout:Preparing to unpack .../7-linux-headers-generic_5.15.0.170.159_amd64.deb ... 2026-02-18T08:05:09.503 INFO:teuthology.orchestra.run.trial034.stdout:Unpacking linux-headers-generic (5.15.0.170.159) over (5.15.0.164.159) ... 2026-02-18T08:05:09.526 INFO:teuthology.orchestra.run.trial034.stdout:Setting up linux-headers-5.15.0-170 (5.15.0-170.180) ... 2026-02-18T08:05:09.528 INFO:teuthology.orchestra.run.trial034.stdout:Setting up linux-headers-5.15.0-170-generic (5.15.0-170.180) ... 2026-02-18T08:05:09.529 INFO:teuthology.orchestra.run.trial034.stdout:Setting up linux-headers-generic (5.15.0.170.159) ... 2026-02-18T08:05:09.530 INFO:teuthology.orchestra.run.trial034.stdout:Setting up linux-modules-5.15.0-170-generic (5.15.0-170.180) ... 2026-02-18T08:05:10.302 INFO:teuthology.orchestra.run.trial034.stdout:Setting up linux-image-5.15.0-170-generic (5.15.0-170.180) ... 2026-02-18T08:05:11.122 INFO:teuthology.orchestra.run.trial034.stdout:I: /boot/vmlinuz is now a symlink to vmlinuz-5.15.0-170-generic 2026-02-18T08:05:11.123 INFO:teuthology.orchestra.run.trial034.stdout:I: /boot/initrd.img is now a symlink to initrd.img-5.15.0-170-generic 2026-02-18T08:05:11.128 INFO:teuthology.orchestra.run.trial034.stdout:Setting up linux-modules-extra-5.15.0-170-generic (5.15.0-170.180) ... 2026-02-18T08:05:11.911 INFO:teuthology.orchestra.run.trial034.stdout:Setting up linux-image-generic (5.15.0.170.159) ... 2026-02-18T08:05:11.911 INFO:teuthology.orchestra.run.trial034.stdout:Setting up linux-generic (5.15.0.170.159) ... 2026-02-18T08:05:11.913 INFO:teuthology.orchestra.run.trial034.stdout:Processing triggers for linux-image-5.15.0-170-generic (5.15.0-170.180) ... 2026-02-18T08:05:11.918 INFO:teuthology.orchestra.run.trial034.stdout:/etc/kernel/postinst.d/initramfs-tools: 2026-02-18T08:05:11.918 INFO:teuthology.orchestra.run.trial034.stdout:update-initramfs: Generating /boot/initrd.img-5.15.0-170-generic 2026-02-18T08:05:16.341 INFO:teuthology.orchestra.run.trial163.stdout:W: Possible missing firmware /lib/firmware/ast_dp501_fw.bin for module ast 2026-02-18T08:05:19.015 INFO:teuthology.orchestra.run.trial034.stdout:W: Possible missing firmware /lib/firmware/ast_dp501_fw.bin for module ast 2026-02-18T08:05:22.930 INFO:teuthology.orchestra.run.trial163.stdout:/etc/kernel/postinst.d/zz-update-grub: 2026-02-18T08:05:22.931 INFO:teuthology.orchestra.run.trial163.stdout:Sourcing file `/etc/default/grub' 2026-02-18T08:05:22.950 INFO:teuthology.orchestra.run.trial163.stdout:Sourcing file `/etc/default/grub.d/50-curtin-settings.cfg' 2026-02-18T08:05:22.951 INFO:teuthology.orchestra.run.trial163.stdout:Sourcing file `/etc/default/grub.d/init-select.cfg' 2026-02-18T08:05:22.953 INFO:teuthology.orchestra.run.trial163.stdout:Generating grub configuration file ... 2026-02-18T08:05:23.032 INFO:teuthology.orchestra.run.trial163.stdout:Found linux image: /boot/vmlinuz-5.15.0-170-generic 2026-02-18T08:05:23.039 INFO:teuthology.orchestra.run.trial163.stdout:Found initrd image: /boot/initrd.img-5.15.0-170-generic 2026-02-18T08:05:23.210 INFO:teuthology.orchestra.run.trial163.stdout:Found linux image: /boot/vmlinuz-5.15.0-164-generic 2026-02-18T08:05:23.216 INFO:teuthology.orchestra.run.trial163.stdout:Found initrd image: /boot/initrd.img-5.15.0-164-generic 2026-02-18T08:05:23.289 INFO:teuthology.orchestra.run.trial163.stdout:Warning: os-prober will not be executed to detect other bootable partitions. 2026-02-18T08:05:23.289 INFO:teuthology.orchestra.run.trial163.stdout:Systems on them will not be added to the GRUB boot configuration. 2026-02-18T08:05:23.289 INFO:teuthology.orchestra.run.trial163.stdout:Check GRUB_DISABLE_OS_PROBER documentation entry. 2026-02-18T08:05:23.291 INFO:teuthology.orchestra.run.trial163.stdout:Adding boot menu entry for UEFI Firmware Settings ... 2026-02-18T08:05:23.300 INFO:teuthology.orchestra.run.trial163.stdout:done 2026-02-18T08:05:23.689 INFO:teuthology.orchestra.run.trial163.stdout: 2026-02-18T08:05:23.689 INFO:teuthology.orchestra.run.trial163.stdout:Pending kernel upgrade! 2026-02-18T08:05:23.689 INFO:teuthology.orchestra.run.trial163.stdout: 2026-02-18T08:05:23.690 INFO:teuthology.orchestra.run.trial163.stdout:Running kernel version: 2026-02-18T08:05:23.690 INFO:teuthology.orchestra.run.trial163.stdout: 5.15.0-164-generic 2026-02-18T08:05:23.690 INFO:teuthology.orchestra.run.trial163.stdout: 2026-02-18T08:05:23.690 INFO:teuthology.orchestra.run.trial163.stdout:Diagnostics: 2026-02-18T08:05:23.690 INFO:teuthology.orchestra.run.trial163.stdout: The currently running kernel version is not the expected kernel version 5.15.0-170-generic. 2026-02-18T08:05:23.690 INFO:teuthology.orchestra.run.trial163.stdout: 2026-02-18T08:05:23.690 INFO:teuthology.orchestra.run.trial163.stdout:Restarting the system to load the new kernel will not be handled automatically, so you should consider rebooting. [Return] 2026-02-18T08:05:23.690 INFO:teuthology.orchestra.run.trial163.stdout: 2026-02-18T08:05:23.690 INFO:teuthology.orchestra.run.trial163.stdout:The processor microcode seems to be up-to-date. 2026-02-18T08:05:23.690 INFO:teuthology.orchestra.run.trial163.stdout: 2026-02-18T08:05:23.690 INFO:teuthology.orchestra.run.trial163.stdout:No services need to be restarted. 2026-02-18T08:05:23.690 INFO:teuthology.orchestra.run.trial163.stdout: 2026-02-18T08:05:23.691 INFO:teuthology.orchestra.run.trial163.stdout:No containers need to be restarted. 2026-02-18T08:05:23.691 INFO:teuthology.orchestra.run.trial163.stdout: 2026-02-18T08:05:23.691 INFO:teuthology.orchestra.run.trial163.stdout:No user sessions are running outdated binaries. 2026-02-18T08:05:23.691 INFO:teuthology.orchestra.run.trial163.stdout: 2026-02-18T08:05:23.691 INFO:teuthology.orchestra.run.trial163.stdout:No VM guests are running outdated hypervisor (qemu) binaries on this host. 2026-02-18T08:05:24.334 DEBUG:teuthology.orchestra.run.trial163:> dpkg -s linux-image-generic 2026-02-18T08:05:24.344 INFO:teuthology.orchestra.run.trial163.stdout:Package: linux-image-generic 2026-02-18T08:05:24.344 INFO:teuthology.orchestra.run.trial163.stdout:Status: install ok installed 2026-02-18T08:05:24.344 INFO:teuthology.orchestra.run.trial163.stdout:Priority: optional 2026-02-18T08:05:24.344 INFO:teuthology.orchestra.run.trial163.stdout:Section: kernel 2026-02-18T08:05:24.344 INFO:teuthology.orchestra.run.trial163.stdout:Installed-Size: 21 2026-02-18T08:05:24.344 INFO:teuthology.orchestra.run.trial163.stdout:Maintainer: Ubuntu Kernel Team 2026-02-18T08:05:24.344 INFO:teuthology.orchestra.run.trial163.stdout:Architecture: amd64 2026-02-18T08:05:24.345 INFO:teuthology.orchestra.run.trial163.stdout:Source: linux-meta 2026-02-18T08:05:24.345 INFO:teuthology.orchestra.run.trial163.stdout:Version: 5.15.0.170.159 2026-02-18T08:05:24.345 INFO:teuthology.orchestra.run.trial163.stdout:Provides: spl-modules (= 2.1.5-1ubuntu6~22.04.6), v4l2loopback-modules (= 0.12.7-2ubuntu2~22.04.1), virtualbox-guest-modules (= 5.15.0-170), wireguard-modules (= 1.0.0), zfs-modules (= 2.1.5-1ubuntu6~22.04.6) 2026-02-18T08:05:24.345 INFO:teuthology.orchestra.run.trial163.stdout:Depends: linux-image-5.15.0-170-generic, linux-modules-extra-5.15.0-170-generic, linux-firmware, intel-microcode, amd64-microcode 2026-02-18T08:05:24.345 INFO:teuthology.orchestra.run.trial163.stdout:Recommends: thermald 2026-02-18T08:05:24.345 INFO:teuthology.orchestra.run.trial163.stdout:Description: Generic Linux kernel image 2026-02-18T08:05:24.345 INFO:teuthology.orchestra.run.trial163.stdout: This package will always depend on the latest generic kernel image 2026-02-18T08:05:24.345 INFO:teuthology.orchestra.run.trial163.stdout: available. 2026-02-18T08:05:24.346 INFO:teuthology.task.kernel:Not newest distro kernel. Current: 5.15.0-164-generic Expected: 5.15.0-170-generic 2026-02-18T08:05:24.347 INFO:teuthology.task.kernel:Skipping firmware on distro kernel 2026-02-18T08:05:24.347 DEBUG:teuthology.task.kernel:src is distro, skipping download 2026-02-18T08:05:24.347 INFO:teuthology.task.kernel:Installing distro kernel on cluster2.mon.a... 2026-02-18T08:05:24.347 DEBUG:teuthology.task.kernel:install_kernel(remote=ubuntu@trial163.front.sepia.ceph.com, path=None, version=distro) 2026-02-18T08:05:24.347 DEBUG:teuthology.orchestra.run.trial163:> sudo apt-get clean 2026-02-18T08:05:24.427 DEBUG:teuthology.orchestra.run.trial163:> sudo apt-get update 2026-02-18T08:05:24.616 INFO:teuthology.orchestra.run.trial163.stdout:Hit:1 https://archive.ubuntu.com/ubuntu jammy InRelease 2026-02-18T08:05:24.619 INFO:teuthology.orchestra.run.trial163.stdout:Hit:2 https://archive.ubuntu.com/ubuntu jammy-updates InRelease 2026-02-18T08:05:24.628 INFO:teuthology.orchestra.run.trial163.stdout:Hit:3 https://archive.ubuntu.com/ubuntu jammy-security InRelease 2026-02-18T08:05:24.637 INFO:teuthology.orchestra.run.trial163.stdout:Hit:4 https://archive.ubuntu.com/ubuntu jammy-backports InRelease 2026-02-18T08:05:25.482 INFO:teuthology.orchestra.run.trial163.stdout:Reading package lists... 2026-02-18T08:05:25.497 DEBUG:teuthology.orchestra.run.trial163:> sudo DEBIAN_FRONTEND=noninteractive apt-get -y install linux-image-generic 2026-02-18T08:05:25.531 INFO:teuthology.orchestra.run.trial163.stdout:Reading package lists... 2026-02-18T08:05:25.586 INFO:teuthology.orchestra.run.trial034.stdout:/etc/kernel/postinst.d/zz-update-grub: 2026-02-18T08:05:25.586 INFO:teuthology.orchestra.run.trial034.stdout:Sourcing file `/etc/default/grub' 2026-02-18T08:05:25.605 INFO:teuthology.orchestra.run.trial034.stdout:Sourcing file `/etc/default/grub.d/50-curtin-settings.cfg' 2026-02-18T08:05:25.606 INFO:teuthology.orchestra.run.trial034.stdout:Sourcing file `/etc/default/grub.d/init-select.cfg' 2026-02-18T08:05:25.608 INFO:teuthology.orchestra.run.trial034.stdout:Generating grub configuration file ... 2026-02-18T08:05:25.650 INFO:teuthology.orchestra.run.trial163.stdout:Building dependency tree... 2026-02-18T08:05:25.651 INFO:teuthology.orchestra.run.trial163.stdout:Reading state information... 2026-02-18T08:05:25.689 INFO:teuthology.orchestra.run.trial034.stdout:Found linux image: /boot/vmlinuz-5.15.0-170-generic 2026-02-18T08:05:25.696 INFO:teuthology.orchestra.run.trial034.stdout:Found initrd image: /boot/initrd.img-5.15.0-170-generic 2026-02-18T08:05:25.739 INFO:teuthology.orchestra.run.trial163.stdout:linux-image-generic is already the newest version (5.15.0.170.159). 2026-02-18T08:05:25.739 INFO:teuthology.orchestra.run.trial163.stdout:linux-image-generic set to manually installed. 2026-02-18T08:05:25.739 INFO:teuthology.orchestra.run.trial163.stdout:The following packages were automatically installed and are no longer required: 2026-02-18T08:05:25.739 INFO:teuthology.orchestra.run.trial163.stdout: kpartx libboost-iostreams1.74.0 libboost-thread1.74.0 libgfapi0 libgfrpc0 2026-02-18T08:05:25.739 INFO:teuthology.orchestra.run.trial163.stdout: libgfxdr0 libglusterfs0 libiscsi7 libpmemobj1 libsgutils2-2 sg3-utils 2026-02-18T08:05:25.740 INFO:teuthology.orchestra.run.trial163.stdout: sg3-utils-udev 2026-02-18T08:05:25.740 INFO:teuthology.orchestra.run.trial163.stdout:Use 'sudo apt autoremove' to remove them. 2026-02-18T08:05:25.760 INFO:teuthology.orchestra.run.trial163.stdout:0 upgraded, 0 newly installed, 0 to remove and 53 not upgraded. 2026-02-18T08:05:25.762 DEBUG:teuthology.orchestra.run.trial163:> dpkg -s linux-image-generic 2026-02-18T08:05:25.812 INFO:teuthology.orchestra.run.trial163.stdout:Package: linux-image-generic 2026-02-18T08:05:25.812 INFO:teuthology.orchestra.run.trial163.stdout:Status: install ok installed 2026-02-18T08:05:25.812 INFO:teuthology.orchestra.run.trial163.stdout:Priority: optional 2026-02-18T08:05:25.813 INFO:teuthology.orchestra.run.trial163.stdout:Section: kernel 2026-02-18T08:05:25.813 INFO:teuthology.orchestra.run.trial163.stdout:Installed-Size: 21 2026-02-18T08:05:25.813 INFO:teuthology.orchestra.run.trial163.stdout:Maintainer: Ubuntu Kernel Team 2026-02-18T08:05:25.813 INFO:teuthology.orchestra.run.trial163.stdout:Architecture: amd64 2026-02-18T08:05:25.813 INFO:teuthology.orchestra.run.trial163.stdout:Source: linux-meta 2026-02-18T08:05:25.813 INFO:teuthology.orchestra.run.trial163.stdout:Version: 5.15.0.170.159 2026-02-18T08:05:25.813 INFO:teuthology.orchestra.run.trial163.stdout:Provides: spl-modules (= 2.1.5-1ubuntu6~22.04.6), v4l2loopback-modules (= 0.12.7-2ubuntu2~22.04.1), virtualbox-guest-modules (= 5.15.0-170), wireguard-modules (= 1.0.0), zfs-modules (= 2.1.5-1ubuntu6~22.04.6) 2026-02-18T08:05:25.813 INFO:teuthology.orchestra.run.trial163.stdout:Depends: linux-image-5.15.0-170-generic, linux-modules-extra-5.15.0-170-generic, linux-firmware, intel-microcode, amd64-microcode 2026-02-18T08:05:25.813 INFO:teuthology.orchestra.run.trial163.stdout:Recommends: thermald 2026-02-18T08:05:25.813 INFO:teuthology.orchestra.run.trial163.stdout:Description: Generic Linux kernel image 2026-02-18T08:05:25.813 INFO:teuthology.orchestra.run.trial163.stdout: This package will always depend on the latest generic kernel image 2026-02-18T08:05:25.813 INFO:teuthology.orchestra.run.trial163.stdout: available. 2026-02-18T08:05:25.815 DEBUG:teuthology.orchestra.run.trial163:> mktemp 2026-02-18T08:05:25.859 INFO:teuthology.orchestra.run.trial163.stdout:/tmp/tmp.LZtvc62Rqq 2026-02-18T08:05:25.860 DEBUG:teuthology.orchestra.run.trial163:> sudo cp /boot/grub/grub.cfg /tmp/tmp.LZtvc62Rqq 2026-02-18T08:05:25.872 INFO:teuthology.orchestra.run.trial034.stdout:Found linux image: /boot/vmlinuz-5.15.0-164-generic 2026-02-18T08:05:25.878 INFO:teuthology.orchestra.run.trial034.stdout:Found initrd image: /boot/initrd.img-5.15.0-164-generic 2026-02-18T08:05:25.908 DEBUG:teuthology.orchestra.run.trial163:> sudo chmod 0666 /tmp/tmp.LZtvc62Rqq 2026-02-18T08:05:25.951 INFO:teuthology.orchestra.run.trial034.stdout:Warning: os-prober will not be executed to detect other bootable partitions. 2026-02-18T08:05:25.952 INFO:teuthology.orchestra.run.trial034.stdout:Systems on them will not be added to the GRUB boot configuration. 2026-02-18T08:05:25.952 INFO:teuthology.orchestra.run.trial034.stdout:Check GRUB_DISABLE_OS_PROBER documentation entry. 2026-02-18T08:05:25.954 INFO:teuthology.orchestra.run.trial034.stdout:Adding boot menu entry for UEFI Firmware Settings ... 2026-02-18T08:05:25.963 INFO:teuthology.orchestra.run.trial034.stdout:done 2026-02-18T08:05:26.006 DEBUG:teuthology.orchestra.remote:trial163:/tmp/tmp.LZtvc62Rqq is 7KB 2026-02-18T08:05:26.013 DEBUG:teuthology.orchestra.run.trial163:> rm -fr /tmp/tmp.LZtvc62Rqq 2026-02-18T08:05:26.018 DEBUG:teuthology.orchestra.run.trial163:> sudo rm -f -- /etc/grub.d/01_ceph_kernel 2026-02-18T08:05:26.071 DEBUG:teuthology.orchestra.run.trial163:> set -ex 2026-02-18T08:05:26.071 DEBUG:teuthology.orchestra.run.trial163:> sudo dd of=/etc/grub.d/01_ceph_kernel 2026-02-18T08:05:26.071 DEBUG:teuthology.orchestra.run.trial163:> sudo chmod 755 /etc/grub.d/01_ceph_kernel 2026-02-18T08:05:26.125 INFO:teuthology.task.kernel:Distro Kernel Version: 5.15.0-170-generic 2026-02-18T08:05:26.126 DEBUG:teuthology.orchestra.run.trial163:> sudo update-grub 2026-02-18T08:05:26.340 INFO:teuthology.orchestra.run.trial034.stdout: 2026-02-18T08:05:26.341 INFO:teuthology.orchestra.run.trial034.stdout:Pending kernel upgrade! 2026-02-18T08:05:26.341 INFO:teuthology.orchestra.run.trial034.stdout: 2026-02-18T08:05:26.341 INFO:teuthology.orchestra.run.trial034.stdout:Running kernel version: 2026-02-18T08:05:26.341 INFO:teuthology.orchestra.run.trial034.stdout: 5.15.0-164-generic 2026-02-18T08:05:26.341 INFO:teuthology.orchestra.run.trial034.stdout: 2026-02-18T08:05:26.341 INFO:teuthology.orchestra.run.trial034.stdout:Diagnostics: 2026-02-18T08:05:26.341 INFO:teuthology.orchestra.run.trial034.stdout: The currently running kernel version is not the expected kernel version 5.15.0-170-generic. 2026-02-18T08:05:26.341 INFO:teuthology.orchestra.run.trial034.stdout: 2026-02-18T08:05:26.341 INFO:teuthology.orchestra.run.trial034.stdout:Restarting the system to load the new kernel will not be handled automatically, so you should consider rebooting. [Return] 2026-02-18T08:05:26.341 INFO:teuthology.orchestra.run.trial034.stdout: 2026-02-18T08:05:26.341 INFO:teuthology.orchestra.run.trial034.stdout:The processor microcode seems to be up-to-date. 2026-02-18T08:05:26.342 INFO:teuthology.orchestra.run.trial034.stdout: 2026-02-18T08:05:26.342 INFO:teuthology.orchestra.run.trial034.stdout:No services need to be restarted. 2026-02-18T08:05:26.342 INFO:teuthology.orchestra.run.trial034.stdout: 2026-02-18T08:05:26.342 INFO:teuthology.orchestra.run.trial034.stdout:No containers need to be restarted. 2026-02-18T08:05:26.342 INFO:teuthology.orchestra.run.trial034.stdout: 2026-02-18T08:05:26.342 INFO:teuthology.orchestra.run.trial034.stdout:No user sessions are running outdated binaries. 2026-02-18T08:05:26.342 INFO:teuthology.orchestra.run.trial034.stdout: 2026-02-18T08:05:26.342 INFO:teuthology.orchestra.run.trial034.stdout:No VM guests are running outdated hypervisor (qemu) binaries on this host. 2026-02-18T08:05:26.367 INFO:teuthology.orchestra.run.trial163.stderr:Sourcing file `/etc/default/grub' 2026-02-18T08:05:26.384 INFO:teuthology.orchestra.run.trial163.stderr:Sourcing file `/etc/default/grub.d/50-curtin-settings.cfg' 2026-02-18T08:05:26.385 INFO:teuthology.orchestra.run.trial163.stderr:Sourcing file `/etc/default/grub.d/init-select.cfg' 2026-02-18T08:05:26.387 INFO:teuthology.orchestra.run.trial163.stderr:Generating grub configuration file ... 2026-02-18T08:05:26.467 INFO:teuthology.orchestra.run.trial163.stderr:Found linux image: /boot/vmlinuz-5.15.0-170-generic 2026-02-18T08:05:26.473 INFO:teuthology.orchestra.run.trial163.stderr:Found initrd image: /boot/initrd.img-5.15.0-170-generic 2026-02-18T08:05:26.644 INFO:teuthology.orchestra.run.trial163.stderr:Found linux image: /boot/vmlinuz-5.15.0-164-generic 2026-02-18T08:05:26.649 INFO:teuthology.orchestra.run.trial163.stderr:Found initrd image: /boot/initrd.img-5.15.0-164-generic 2026-02-18T08:05:26.719 INFO:teuthology.orchestra.run.trial163.stderr:Warning: os-prober will not be executed to detect other bootable partitions. 2026-02-18T08:05:26.720 INFO:teuthology.orchestra.run.trial163.stderr:Systems on them will not be added to the GRUB boot configuration. 2026-02-18T08:05:26.720 INFO:teuthology.orchestra.run.trial163.stderr:Check GRUB_DISABLE_OS_PROBER documentation entry. 2026-02-18T08:05:26.722 INFO:teuthology.orchestra.run.trial163.stderr:Adding boot menu entry for UEFI Firmware Settings ... 2026-02-18T08:05:26.729 INFO:teuthology.orchestra.run.trial163.stderr:done 2026-02-18T08:05:26.731 INFO:teuthology.orchestra.remote:Triggering safe hard reboot via SysRq on trial163 2026-02-18T08:05:26.731 DEBUG:teuthology.orchestra.run.trial163:> sudo bash -c ' 2026-02-18T08:05:26.732 DEBUG:teuthology.orchestra.run.trial163:> echo pci > /sys/kernel/reboot/type && echo 1 > /sys/kernel/reboot/force && sync && echo s > /proc/sysrq-trigger && echo u > /proc/sysrq-trigger && sleep 2 && echo b > /proc/sysrq-trigger 2026-02-18T08:05:26.732 DEBUG:teuthology.orchestra.run.trial163:> ' 2026-02-18T08:05:26.991 DEBUG:teuthology.orchestra.run.trial034:> dpkg -s linux-image-generic 2026-02-18T08:05:27.000 INFO:teuthology.orchestra.run.trial034.stdout:Package: linux-image-generic 2026-02-18T08:05:27.001 INFO:teuthology.orchestra.run.trial034.stdout:Status: install ok installed 2026-02-18T08:05:27.001 INFO:teuthology.orchestra.run.trial034.stdout:Priority: optional 2026-02-18T08:05:27.001 INFO:teuthology.orchestra.run.trial034.stdout:Section: kernel 2026-02-18T08:05:27.001 INFO:teuthology.orchestra.run.trial034.stdout:Installed-Size: 21 2026-02-18T08:05:27.001 INFO:teuthology.orchestra.run.trial034.stdout:Maintainer: Ubuntu Kernel Team 2026-02-18T08:05:27.001 INFO:teuthology.orchestra.run.trial034.stdout:Architecture: amd64 2026-02-18T08:05:27.001 INFO:teuthology.orchestra.run.trial034.stdout:Source: linux-meta 2026-02-18T08:05:27.001 INFO:teuthology.orchestra.run.trial034.stdout:Version: 5.15.0.170.159 2026-02-18T08:05:27.001 INFO:teuthology.orchestra.run.trial034.stdout:Provides: spl-modules (= 2.1.5-1ubuntu6~22.04.6), v4l2loopback-modules (= 0.12.7-2ubuntu2~22.04.1), virtualbox-guest-modules (= 5.15.0-170), wireguard-modules (= 1.0.0), zfs-modules (= 2.1.5-1ubuntu6~22.04.6) 2026-02-18T08:05:27.002 INFO:teuthology.orchestra.run.trial034.stdout:Depends: linux-image-5.15.0-170-generic, linux-modules-extra-5.15.0-170-generic, linux-firmware, intel-microcode, amd64-microcode 2026-02-18T08:05:27.002 INFO:teuthology.orchestra.run.trial034.stdout:Recommends: thermald 2026-02-18T08:05:27.002 INFO:teuthology.orchestra.run.trial034.stdout:Description: Generic Linux kernel image 2026-02-18T08:05:27.002 INFO:teuthology.orchestra.run.trial034.stdout: This package will always depend on the latest generic kernel image 2026-02-18T08:05:27.002 INFO:teuthology.orchestra.run.trial034.stdout: available. 2026-02-18T08:05:27.002 INFO:teuthology.task.kernel:Not newest distro kernel. Current: 5.15.0-164-generic Expected: 5.15.0-170-generic 2026-02-18T08:05:27.002 INFO:teuthology.task.kernel:Skipping firmware on distro kernel 2026-02-18T08:05:27.002 DEBUG:teuthology.task.kernel:src is distro, skipping download 2026-02-18T08:05:27.003 INFO:teuthology.task.kernel:Installing distro kernel on cluster1.mon.a... 2026-02-18T08:05:27.003 DEBUG:teuthology.task.kernel:install_kernel(remote=ubuntu@trial034.front.sepia.ceph.com, path=None, version=distro) 2026-02-18T08:05:27.003 DEBUG:teuthology.orchestra.run.trial034:> sudo apt-get clean 2026-02-18T08:05:27.083 DEBUG:teuthology.orchestra.run.trial034:> sudo apt-get update 2026-02-18T08:05:27.514 INFO:teuthology.orchestra.run.trial034.stdout:Hit:1 https://archive.ubuntu.com/ubuntu jammy InRelease 2026-02-18T08:05:27.581 INFO:teuthology.orchestra.run.trial034.stdout:Hit:2 https://archive.ubuntu.com/ubuntu jammy-updates InRelease 2026-02-18T08:05:27.654 INFO:teuthology.orchestra.run.trial034.stdout:Hit:3 https://archive.ubuntu.com/ubuntu jammy-security InRelease 2026-02-18T08:05:27.726 INFO:teuthology.orchestra.run.trial034.stdout:Hit:4 https://archive.ubuntu.com/ubuntu jammy-backports InRelease 2026-02-18T08:05:28.385 INFO:teuthology.orchestra.run.trial034.stdout:Reading package lists... 2026-02-18T08:05:28.399 DEBUG:teuthology.orchestra.run.trial034:> sudo DEBIAN_FRONTEND=noninteractive apt-get -y install linux-image-generic 2026-02-18T08:05:28.432 INFO:teuthology.orchestra.run.trial034.stdout:Reading package lists... 2026-02-18T08:05:28.542 INFO:teuthology.orchestra.run.trial034.stdout:Building dependency tree... 2026-02-18T08:05:28.542 INFO:teuthology.orchestra.run.trial034.stdout:Reading state information... 2026-02-18T08:05:28.629 INFO:teuthology.orchestra.run.trial034.stdout:linux-image-generic is already the newest version (5.15.0.170.159). 2026-02-18T08:05:28.629 INFO:teuthology.orchestra.run.trial034.stdout:linux-image-generic set to manually installed. 2026-02-18T08:05:28.629 INFO:teuthology.orchestra.run.trial034.stdout:The following packages were automatically installed and are no longer required: 2026-02-18T08:05:28.629 INFO:teuthology.orchestra.run.trial034.stdout: kpartx libboost-iostreams1.74.0 libboost-thread1.74.0 libgfapi0 libgfrpc0 2026-02-18T08:05:28.629 INFO:teuthology.orchestra.run.trial034.stdout: libgfxdr0 libglusterfs0 libiscsi7 libpmemobj1 libsgutils2-2 sg3-utils 2026-02-18T08:05:28.630 INFO:teuthology.orchestra.run.trial034.stdout: sg3-utils-udev 2026-02-18T08:05:28.630 INFO:teuthology.orchestra.run.trial034.stdout:Use 'sudo apt autoremove' to remove them. 2026-02-18T08:05:28.649 INFO:teuthology.orchestra.run.trial034.stdout:0 upgraded, 0 newly installed, 0 to remove and 53 not upgraded. 2026-02-18T08:05:28.651 DEBUG:teuthology.orchestra.run.trial034:> dpkg -s linux-image-generic 2026-02-18T08:05:28.700 INFO:teuthology.orchestra.run.trial034.stdout:Package: linux-image-generic 2026-02-18T08:05:28.701 INFO:teuthology.orchestra.run.trial034.stdout:Status: install ok installed 2026-02-18T08:05:28.701 INFO:teuthology.orchestra.run.trial034.stdout:Priority: optional 2026-02-18T08:05:28.701 INFO:teuthology.orchestra.run.trial034.stdout:Section: kernel 2026-02-18T08:05:28.701 INFO:teuthology.orchestra.run.trial034.stdout:Installed-Size: 21 2026-02-18T08:05:28.701 INFO:teuthology.orchestra.run.trial034.stdout:Maintainer: Ubuntu Kernel Team 2026-02-18T08:05:28.701 INFO:teuthology.orchestra.run.trial034.stdout:Architecture: amd64 2026-02-18T08:05:28.701 INFO:teuthology.orchestra.run.trial034.stdout:Source: linux-meta 2026-02-18T08:05:28.701 INFO:teuthology.orchestra.run.trial034.stdout:Version: 5.15.0.170.159 2026-02-18T08:05:28.701 INFO:teuthology.orchestra.run.trial034.stdout:Provides: spl-modules (= 2.1.5-1ubuntu6~22.04.6), v4l2loopback-modules (= 0.12.7-2ubuntu2~22.04.1), virtualbox-guest-modules (= 5.15.0-170), wireguard-modules (= 1.0.0), zfs-modules (= 2.1.5-1ubuntu6~22.04.6) 2026-02-18T08:05:28.701 INFO:teuthology.orchestra.run.trial034.stdout:Depends: linux-image-5.15.0-170-generic, linux-modules-extra-5.15.0-170-generic, linux-firmware, intel-microcode, amd64-microcode 2026-02-18T08:05:28.702 INFO:teuthology.orchestra.run.trial034.stdout:Recommends: thermald 2026-02-18T08:05:28.702 INFO:teuthology.orchestra.run.trial034.stdout:Description: Generic Linux kernel image 2026-02-18T08:05:28.702 INFO:teuthology.orchestra.run.trial034.stdout: This package will always depend on the latest generic kernel image 2026-02-18T08:05:28.702 INFO:teuthology.orchestra.run.trial034.stdout: available. 2026-02-18T08:05:28.702 DEBUG:teuthology.orchestra.run.trial034:> mktemp 2026-02-18T08:05:28.748 INFO:teuthology.orchestra.run.trial034.stdout:/tmp/tmp.rC03ugwtIR 2026-02-18T08:05:28.748 DEBUG:teuthology.orchestra.run.trial034:> sudo cp /boot/grub/grub.cfg /tmp/tmp.rC03ugwtIR 2026-02-18T08:05:28.800 DEBUG:teuthology.orchestra.run.trial034:> sudo chmod 0666 /tmp/tmp.rC03ugwtIR 2026-02-18T08:05:28.898 DEBUG:teuthology.orchestra.remote:trial034:/tmp/tmp.rC03ugwtIR is 7KB 2026-02-18T08:05:28.905 DEBUG:teuthology.orchestra.run.trial034:> rm -fr /tmp/tmp.rC03ugwtIR 2026-02-18T08:05:28.909 DEBUG:teuthology.orchestra.run.trial034:> sudo rm -f -- /etc/grub.d/01_ceph_kernel 2026-02-18T08:05:28.963 DEBUG:teuthology.orchestra.run.trial034:> set -ex 2026-02-18T08:05:28.963 DEBUG:teuthology.orchestra.run.trial034:> sudo dd of=/etc/grub.d/01_ceph_kernel 2026-02-18T08:05:28.964 DEBUG:teuthology.orchestra.run.trial034:> sudo chmod 755 /etc/grub.d/01_ceph_kernel 2026-02-18T08:05:29.017 INFO:teuthology.task.kernel:Distro Kernel Version: 5.15.0-170-generic 2026-02-18T08:05:29.018 DEBUG:teuthology.orchestra.run.trial034:> sudo update-grub 2026-02-18T08:05:29.261 INFO:teuthology.orchestra.run.trial034.stderr:Sourcing file `/etc/default/grub' 2026-02-18T08:05:29.278 INFO:teuthology.orchestra.run.trial034.stderr:Sourcing file `/etc/default/grub.d/50-curtin-settings.cfg' 2026-02-18T08:05:29.279 INFO:teuthology.orchestra.run.trial034.stderr:Sourcing file `/etc/default/grub.d/init-select.cfg' 2026-02-18T08:05:29.281 INFO:teuthology.orchestra.run.trial034.stderr:Generating grub configuration file ... 2026-02-18T08:05:29.363 INFO:teuthology.orchestra.run.trial034.stderr:Found linux image: /boot/vmlinuz-5.15.0-170-generic 2026-02-18T08:05:29.369 INFO:teuthology.orchestra.run.trial034.stderr:Found initrd image: /boot/initrd.img-5.15.0-170-generic 2026-02-18T08:05:29.542 INFO:teuthology.orchestra.run.trial034.stderr:Found linux image: /boot/vmlinuz-5.15.0-164-generic 2026-02-18T08:05:29.548 INFO:teuthology.orchestra.run.trial034.stderr:Found initrd image: /boot/initrd.img-5.15.0-164-generic 2026-02-18T08:05:29.616 INFO:teuthology.orchestra.run.trial034.stderr:Warning: os-prober will not be executed to detect other bootable partitions. 2026-02-18T08:05:29.616 INFO:teuthology.orchestra.run.trial034.stderr:Systems on them will not be added to the GRUB boot configuration. 2026-02-18T08:05:29.617 INFO:teuthology.orchestra.run.trial034.stderr:Check GRUB_DISABLE_OS_PROBER documentation entry. 2026-02-18T08:05:29.619 INFO:teuthology.orchestra.run.trial034.stderr:Adding boot menu entry for UEFI Firmware Settings ... 2026-02-18T08:05:29.626 INFO:teuthology.orchestra.run.trial034.stderr:done 2026-02-18T08:05:29.628 INFO:teuthology.orchestra.remote:Triggering safe hard reboot via SysRq on trial034 2026-02-18T08:05:29.628 DEBUG:teuthology.orchestra.run.trial034:> sudo bash -c ' 2026-02-18T08:05:29.628 DEBUG:teuthology.orchestra.run.trial034:> echo pci > /sys/kernel/reboot/type && echo 1 > /sys/kernel/reboot/force && sync && echo s > /proc/sysrq-trigger && echo u > /proc/sysrq-trigger && sleep 2 && echo b > /proc/sysrq-trigger 2026-02-18T08:05:29.628 DEBUG:teuthology.orchestra.run.trial034:> ' 2026-02-18T08:05:56.734 INFO:teuthology.task.kernel:Checking client cluster2.mon.a for new kernel version... 2026-02-18T08:05:56.735 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@trial163.front.sepia.ceph.com' 2026-02-18T08:05:56.736 DEBUG:teuthology.orchestra.connection:{'hostname': 'trial163.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2026-02-18T08:05:59.631 INFO:teuthology.task.kernel:Checking client cluster1.mon.a for new kernel version... 2026-02-18T08:05:59.631 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@trial034.front.sepia.ceph.com' 2026-02-18T08:05:59.632 DEBUG:teuthology.orchestra.connection:{'hostname': 'trial034.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2026-02-18T08:06:56.795 DEBUG:teuthology.orchestra.remote:timed out 2026-02-18T08:06:59.637 DEBUG:teuthology.orchestra.remote:timed out 2026-02-18T08:07:05.802 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@trial163.front.sepia.ceph.com' 2026-02-18T08:07:05.803 DEBUG:teuthology.orchestra.connection:{'hostname': 'trial163.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2026-02-18T08:07:05.907 DEBUG:teuthology.orchestra.run.trial163:> true 2026-02-18T08:07:06.295 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@trial163.front.sepia.ceph.com' 2026-02-18T08:07:06.296 INFO:teuthology.task.kernel:Checking kernel version of cluster2.mon.a, want "5.15.0-170-generic"... 2026-02-18T08:07:06.296 DEBUG:teuthology.orchestra.run.trial163:> uname -r 2026-02-18T08:07:06.344 INFO:teuthology.orchestra.run.trial163.stdout:5.15.0-170-generic 2026-02-18T08:07:06.344 DEBUG:teuthology.task.kernel:current kernel version is 5.15.0-170-generic vs 5.15.0-170-generic 2026-02-18T08:07:06.345 DEBUG:teuthology.task.kernel:utsrelease strings match, do not need to install 2026-02-18T08:07:06.345 DEBUG:teuthology.task.kernel:Distro of this test job: ubuntu 2026-02-18T08:07:07.345 INFO:teuthology.task.kernel:Enabling kdb on cluster2.mon.a... 2026-02-18T08:07:07.346 DEBUG:teuthology.orchestra.run.trial163:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2026-02-18T08:07:07.359 INFO:teuthology.orchestra.run.trial163.stdout:ttyS1 2026-02-18T08:07:07.384 DEBUG:teuthology.parallel:result is None 2026-02-18T08:07:08.638 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@trial034.front.sepia.ceph.com' 2026-02-18T08:07:08.639 DEBUG:teuthology.orchestra.connection:{'hostname': 'trial034.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2026-02-18T08:07:08.761 DEBUG:teuthology.orchestra.run.trial034:> true 2026-02-18T08:07:09.156 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@trial034.front.sepia.ceph.com' 2026-02-18T08:07:09.157 INFO:teuthology.task.kernel:Checking kernel version of cluster1.mon.a, want "5.15.0-170-generic"... 2026-02-18T08:07:09.157 DEBUG:teuthology.orchestra.run.trial034:> uname -r 2026-02-18T08:07:09.203 INFO:teuthology.orchestra.run.trial034.stdout:5.15.0-170-generic 2026-02-18T08:07:09.203 DEBUG:teuthology.task.kernel:current kernel version is 5.15.0-170-generic vs 5.15.0-170-generic 2026-02-18T08:07:09.204 DEBUG:teuthology.task.kernel:utsrelease strings match, do not need to install 2026-02-18T08:07:09.204 DEBUG:teuthology.task.kernel:Distro of this test job: ubuntu 2026-02-18T08:07:10.204 INFO:teuthology.task.kernel:Enabling kdb on cluster1.mon.a... 2026-02-18T08:07:10.204 DEBUG:teuthology.orchestra.run.trial034:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2026-02-18T08:07:10.216 INFO:teuthology.orchestra.run.trial034.stdout:ttyS1 2026-02-18T08:07:10.241 DEBUG:teuthology.parallel:result is None 2026-02-18T08:07:10.241 INFO:teuthology.run_tasks:Running task internal.base... 2026-02-18T08:07:10.252 INFO:teuthology.task.internal:Creating test directory... 2026-02-18T08:07:10.253 DEBUG:teuthology.orchestra.run.trial034:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2026-02-18T08:07:10.284 DEBUG:teuthology.orchestra.run.trial163:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2026-02-18T08:07:10.288 INFO:teuthology.run_tasks:Running task internal.archive_upload... 2026-02-18T08:07:10.300 INFO:teuthology.run_tasks:Running task internal.archive... 2026-02-18T08:07:10.315 INFO:teuthology.task.internal:Creating archive directory... 2026-02-18T08:07:10.315 DEBUG:teuthology.orchestra.run.trial034:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2026-02-18T08:07:10.328 DEBUG:teuthology.orchestra.run.trial163:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2026-02-18T08:07:10.338 INFO:teuthology.run_tasks:Running task internal.coredump... 2026-02-18T08:07:10.349 INFO:teuthology.task.internal:Enabling coredump saving... 2026-02-18T08:07:10.349 DEBUG:teuthology.orchestra.run.trial034:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2026-02-18T08:07:10.373 DEBUG:teuthology.orchestra.run.trial163:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2026-02-18T08:07:10.381 INFO:teuthology.orchestra.run.trial034.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2026-02-18T08:07:10.381 INFO:teuthology.orchestra.run.trial163.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2026-02-18T08:07:10.387 INFO:teuthology.orchestra.run.trial034.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2026-02-18T08:07:10.387 INFO:teuthology.orchestra.run.trial163.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2026-02-18T08:07:10.388 INFO:teuthology.run_tasks:Running task internal.sudo... 2026-02-18T08:07:10.399 INFO:teuthology.task.internal:Configuring sudo... 2026-02-18T08:07:10.400 DEBUG:teuthology.orchestra.run.trial034:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2026-02-18T08:07:10.432 DEBUG:teuthology.orchestra.run.trial163:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2026-02-18T08:07:10.440 INFO:teuthology.run_tasks:Running task internal.syslog... 2026-02-18T08:07:10.454 INFO:teuthology.task.internal.syslog:Starting syslog monitoring... 2026-02-18T08:07:10.454 DEBUG:teuthology.orchestra.run.trial034:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2026-02-18T08:07:10.480 DEBUG:teuthology.orchestra.run.trial163:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2026-02-18T08:07:10.484 DEBUG:teuthology.orchestra.run.trial034:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2026-02-18T08:07:10.527 DEBUG:teuthology.orchestra.run.trial034:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2026-02-18T08:07:10.571 DEBUG:teuthology.orchestra.run.trial034:> set -ex 2026-02-18T08:07:10.571 DEBUG:teuthology.orchestra.run.trial034:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2026-02-18T08:07:10.621 DEBUG:teuthology.orchestra.run.trial163:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2026-02-18T08:07:10.624 DEBUG:teuthology.orchestra.run.trial163:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2026-02-18T08:07:10.672 DEBUG:teuthology.orchestra.run.trial163:> set -ex 2026-02-18T08:07:10.672 DEBUG:teuthology.orchestra.run.trial163:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2026-02-18T08:07:10.722 DEBUG:teuthology.orchestra.run.trial034:> sudo service rsyslog restart 2026-02-18T08:07:10.723 DEBUG:teuthology.orchestra.run.trial163:> sudo service rsyslog restart 2026-02-18T08:07:10.782 INFO:teuthology.run_tasks:Running task internal.timer... 2026-02-18T08:07:10.793 INFO:teuthology.task.internal:Starting timer... 2026-02-18T08:07:10.793 INFO:teuthology.run_tasks:Running task pcp... 2026-02-18T08:07:10.810 INFO:teuthology.run_tasks:Running task selinux... 2026-02-18T08:07:10.822 DEBUG:teuthology.task.selinux:Excluding trial034: OS 'ubuntu' does not support SELinux 2026-02-18T08:07:10.822 DEBUG:teuthology.task.selinux:Excluding trial163: OS 'ubuntu' does not support SELinux 2026-02-18T08:07:10.822 DEBUG:teuthology.task.selinux:Getting current SELinux state 2026-02-18T08:07:10.822 DEBUG:teuthology.task.selinux:Existing SELinux modes: {} 2026-02-18T08:07:10.823 INFO:teuthology.task.selinux:Putting SELinux into permissive mode 2026-02-18T08:07:10.823 INFO:teuthology.run_tasks:Running task ansible.cephlab... 2026-02-18T08:07:10.836 DEBUG:teuthology.repo_utils:Setting repo remote to https://git.ceph.com/ceph-cm-ansible.git 2026-02-18T08:07:10.838 INFO:teuthology.repo_utils:Fetching git.ceph.com_ceph-cm-ansible_main from origin 2026-02-18T08:07:10.967 DEBUG:teuthology.repo_utils:Resetting repo at /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main to origin/main 2026-02-18T08:07:10.974 INFO:teuthology.task.ansible:Playbook: [{'import_playbook': 'ansible_managed.yml'}, {'import_playbook': 'teuthology.yml'}, {'hosts': 'testnodes', 'tasks': [{'set_fact': {'ran_from_cephlab_playbook': True}}]}, {'import_playbook': 'testnodes.yml'}, {'import_playbook': 'container-host.yml'}, {'import_playbook': 'cobbler.yml'}, {'import_playbook': 'paddles.yml'}, {'import_playbook': 'pulpito.yml'}, {'hosts': 'testnodes', 'become': True, 'tasks': [{'name': 'Touch /ceph-qa-ready', 'file': {'path': '/ceph-qa-ready', 'state': 'touch'}, 'when': 'ran_from_cephlab_playbook|bool'}]}] 2026-02-18T08:07:10.974 DEBUG:teuthology.task.ansible:Running ansible-playbook -v --extra-vars '{"ansible_ssh_user": "ubuntu"}' -i /etc/ansible/hosts --limit trial034.front.sepia.ceph.com,trial163.front.sepia.ceph.com /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main/cephlab.yml 2026-02-18T08:08:14.113 DEBUG:teuthology.task.ansible:Reconnecting to [Remote(name='ubuntu@trial034.front.sepia.ceph.com'), Remote(name='ubuntu@trial163.front.sepia.ceph.com')] 2026-02-18T08:08:14.114 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@trial034.front.sepia.ceph.com' 2026-02-18T08:08:14.115 DEBUG:teuthology.orchestra.connection:{'hostname': 'trial034.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2026-02-18T08:08:14.225 DEBUG:teuthology.orchestra.run.trial034:> true 2026-02-18T08:08:14.475 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@trial034.front.sepia.ceph.com' 2026-02-18T08:08:14.476 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@trial163.front.sepia.ceph.com' 2026-02-18T08:08:14.476 DEBUG:teuthology.orchestra.connection:{'hostname': 'trial163.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2026-02-18T08:08:14.585 DEBUG:teuthology.orchestra.run.trial163:> true 2026-02-18T08:08:14.832 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@trial163.front.sepia.ceph.com' 2026-02-18T08:08:14.832 INFO:teuthology.run_tasks:Running task clock... 2026-02-18T08:08:14.848 INFO:teuthology.task.clock:Syncing clocks and checking initial clock skew... 2026-02-18T08:08:14.848 INFO:teuthology.orchestra.run:Running command with timeout 360 2026-02-18T08:08:14.849 DEBUG:teuthology.orchestra.run.trial034:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2026-02-18T08:08:14.851 INFO:teuthology.orchestra.run:Running command with timeout 360 2026-02-18T08:08:14.851 DEBUG:teuthology.orchestra.run.trial163:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2026-02-18T08:08:14.870 INFO:teuthology.orchestra.run.trial034.stdout:18 Feb 08:08:15 ntpd[4908]: ntpd 4.2.8p15@1.3728-o Wed Feb 16 17:13:02 UTC 2022 (1): Starting 2026-02-18T08:08:14.870 INFO:teuthology.orchestra.run.trial034.stdout:18 Feb 08:08:15 ntpd[4908]: Command line: ntpd -gq 2026-02-18T08:08:14.871 INFO:teuthology.orchestra.run.trial034.stdout:18 Feb 08:08:15 ntpd[4908]: ---------------------------------------------------- 2026-02-18T08:08:14.871 INFO:teuthology.orchestra.run.trial034.stdout:18 Feb 08:08:15 ntpd[4908]: ntp-4 is maintained by Network Time Foundation, 2026-02-18T08:08:14.871 INFO:teuthology.orchestra.run.trial034.stdout:18 Feb 08:08:15 ntpd[4908]: Inc. (NTF), a non-profit 501(c)(3) public-benefit 2026-02-18T08:08:14.871 INFO:teuthology.orchestra.run.trial034.stdout:18 Feb 08:08:15 ntpd[4908]: corporation. Support and training for ntp-4 are 2026-02-18T08:08:14.871 INFO:teuthology.orchestra.run.trial034.stdout:18 Feb 08:08:15 ntpd[4908]: available at https://www.nwtime.org/support 2026-02-18T08:08:14.871 INFO:teuthology.orchestra.run.trial034.stdout:18 Feb 08:08:15 ntpd[4908]: ---------------------------------------------------- 2026-02-18T08:08:14.871 INFO:teuthology.orchestra.run.trial034.stdout:18 Feb 08:08:15 ntpd[4908]: proto: precision = 0.020 usec (-25) 2026-02-18T08:08:14.871 INFO:teuthology.orchestra.run.trial034.stdout:18 Feb 08:08:15 ntpd[4908]: basedate set to 2022-02-04 2026-02-18T08:08:14.871 INFO:teuthology.orchestra.run.trial034.stdout:18 Feb 08:08:15 ntpd[4908]: gps base set to 2022-02-06 (week 2196) 2026-02-18T08:08:14.872 INFO:teuthology.orchestra.run.trial034.stderr:restrict 0.0.0.0: KOD does nothing without LIMITED. 2026-02-18T08:08:14.872 INFO:teuthology.orchestra.run.trial034.stdout:18 Feb 08:08:15 ntpd[4908]: restrict 0.0.0.0: KOD does nothing without LIMITED. 2026-02-18T08:08:14.872 INFO:teuthology.orchestra.run.trial034.stdout:18 Feb 08:08:15 ntpd[4908]: restrict ::: KOD does nothing without LIMITED. 2026-02-18T08:08:14.873 INFO:teuthology.orchestra.run.trial034.stderr:restrict ::: KOD does nothing without LIMITED. 2026-02-18T08:08:14.873 INFO:teuthology.orchestra.run.trial034.stdout:18 Feb 08:08:15 ntpd[4908]: Listen and drop on 0 v6wildcard [::]:123 2026-02-18T08:08:14.874 INFO:teuthology.orchestra.run.trial034.stdout:18 Feb 08:08:15 ntpd[4908]: Listen and drop on 1 v4wildcard 0.0.0.0:123 2026-02-18T08:08:14.874 INFO:teuthology.orchestra.run.trial034.stdout:18 Feb 08:08:15 ntpd[4908]: Listen normally on 2 lo 127.0.0.1:123 2026-02-18T08:08:14.874 INFO:teuthology.orchestra.run.trial034.stdout:18 Feb 08:08:15 ntpd[4908]: Listen normally on 3 enp1s0f0 10.20.193.34:123 2026-02-18T08:08:14.874 INFO:teuthology.orchestra.run.trial034.stdout:18 Feb 08:08:15 ntpd[4908]: Listen normally on 4 lo [::1]:123 2026-02-18T08:08:14.874 INFO:teuthology.orchestra.run.trial034.stdout:18 Feb 08:08:15 ntpd[4908]: Listen normally on 5 enp1s0f0 [fe80::925a:8ff:fe77:6032%2]:123 2026-02-18T08:08:14.874 INFO:teuthology.orchestra.run.trial034.stdout:18 Feb 08:08:15 ntpd[4908]: Listening on routing socket on fd #22 for interface updates 2026-02-18T08:08:14.891 INFO:teuthology.orchestra.run.trial163.stdout:18 Feb 08:08:14 ntpd[4922]: ntpd 4.2.8p15@1.3728-o Wed Feb 16 17:13:02 UTC 2022 (1): Starting 2026-02-18T08:08:14.891 INFO:teuthology.orchestra.run.trial163.stdout:18 Feb 08:08:14 ntpd[4922]: Command line: ntpd -gq 2026-02-18T08:08:14.891 INFO:teuthology.orchestra.run.trial163.stdout:18 Feb 08:08:14 ntpd[4922]: ---------------------------------------------------- 2026-02-18T08:08:14.891 INFO:teuthology.orchestra.run.trial163.stdout:18 Feb 08:08:14 ntpd[4922]: ntp-4 is maintained by Network Time Foundation, 2026-02-18T08:08:14.891 INFO:teuthology.orchestra.run.trial163.stdout:18 Feb 08:08:14 ntpd[4922]: Inc. (NTF), a non-profit 501(c)(3) public-benefit 2026-02-18T08:08:14.891 INFO:teuthology.orchestra.run.trial163.stdout:18 Feb 08:08:14 ntpd[4922]: corporation. Support and training for ntp-4 are 2026-02-18T08:08:14.891 INFO:teuthology.orchestra.run.trial163.stdout:18 Feb 08:08:14 ntpd[4922]: available at https://www.nwtime.org/support 2026-02-18T08:08:14.892 INFO:teuthology.orchestra.run.trial163.stdout:18 Feb 08:08:14 ntpd[4922]: ---------------------------------------------------- 2026-02-18T08:08:14.892 INFO:teuthology.orchestra.run.trial163.stdout:18 Feb 08:08:14 ntpd[4922]: proto: precision = 0.020 usec (-25) 2026-02-18T08:08:14.892 INFO:teuthology.orchestra.run.trial163.stdout:18 Feb 08:08:14 ntpd[4922]: basedate set to 2022-02-04 2026-02-18T08:08:14.892 INFO:teuthology.orchestra.run.trial163.stdout:18 Feb 08:08:14 ntpd[4922]: gps base set to 2022-02-06 (week 2196) 2026-02-18T08:08:14.892 INFO:teuthology.orchestra.run.trial163.stdout:18 Feb 08:08:14 ntpd[4922]: restrict 0.0.0.0: KOD does nothing without LIMITED. 2026-02-18T08:08:14.892 INFO:teuthology.orchestra.run.trial163.stderr:restrict 0.0.0.0: KOD does nothing without LIMITED. 2026-02-18T08:08:14.892 INFO:teuthology.orchestra.run.trial163.stderr:restrict ::: KOD does nothing without LIMITED. 2026-02-18T08:08:14.893 INFO:teuthology.orchestra.run.trial163.stdout:18 Feb 08:08:14 ntpd[4922]: restrict ::: KOD does nothing without LIMITED. 2026-02-18T08:08:14.894 INFO:teuthology.orchestra.run.trial163.stdout:18 Feb 08:08:14 ntpd[4922]: Listen and drop on 0 v6wildcard [::]:123 2026-02-18T08:08:14.894 INFO:teuthology.orchestra.run.trial163.stdout:18 Feb 08:08:14 ntpd[4922]: Listen and drop on 1 v4wildcard 0.0.0.0:123 2026-02-18T08:08:14.894 INFO:teuthology.orchestra.run.trial163.stdout:18 Feb 08:08:14 ntpd[4922]: Listen normally on 2 lo 127.0.0.1:123 2026-02-18T08:08:14.894 INFO:teuthology.orchestra.run.trial163.stdout:18 Feb 08:08:14 ntpd[4922]: Listen normally on 3 enp1s0f0 10.20.193.163:123 2026-02-18T08:08:14.894 INFO:teuthology.orchestra.run.trial163.stdout:18 Feb 08:08:14 ntpd[4922]: Listen normally on 4 lo [::1]:123 2026-02-18T08:08:14.894 INFO:teuthology.orchestra.run.trial163.stdout:18 Feb 08:08:14 ntpd[4922]: Listen normally on 5 enp1s0f0 [fe80::925a:8ff:fe77:6374%2]:123 2026-02-18T08:08:14.894 INFO:teuthology.orchestra.run.trial163.stdout:18 Feb 08:08:14 ntpd[4922]: Listening on routing socket on fd #22 for interface updates 2026-02-18T08:08:15.871 INFO:teuthology.orchestra.run.trial034.stderr:18 Feb 08:08:16 ntpd[4908]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2026-02-18T08:08:15.871 INFO:teuthology.orchestra.run.trial034.stderr:18 Feb 08:08:16 ntpd[4908]: can't open /var/log/ntpstats/rawstats.20260218: Permission denied 2026-02-18T08:08:15.871 INFO:teuthology.orchestra.run.trial034.stderr:18 Feb 08:08:16 ntpd[4908]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2026-02-18T08:08:15.871 INFO:teuthology.orchestra.run.trial034.stderr:18 Feb 08:08:16 ntpd[4908]: can't open /var/log/ntpstats/rawstats.20260218: Permission denied 2026-02-18T08:08:15.871 INFO:teuthology.orchestra.run.trial034.stderr:18 Feb 08:08:16 ntpd[4908]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2026-02-18T08:08:15.871 INFO:teuthology.orchestra.run.trial034.stderr:18 Feb 08:08:16 ntpd[4908]: can't open /var/log/ntpstats/rawstats.20260218: Permission denied 2026-02-18T08:08:15.891 INFO:teuthology.orchestra.run.trial163.stderr:18 Feb 08:08:15 ntpd[4922]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2026-02-18T08:08:15.891 INFO:teuthology.orchestra.run.trial163.stderr:18 Feb 08:08:15 ntpd[4922]: can't open /var/log/ntpstats/rawstats.20260218: Permission denied 2026-02-18T08:08:15.891 INFO:teuthology.orchestra.run.trial163.stderr:18 Feb 08:08:15 ntpd[4922]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2026-02-18T08:08:15.891 INFO:teuthology.orchestra.run.trial163.stderr:18 Feb 08:08:15 ntpd[4922]: can't open /var/log/ntpstats/rawstats.20260218: Permission denied 2026-02-18T08:08:15.921 INFO:teuthology.orchestra.run.trial163.stderr:18 Feb 08:08:15 ntpd[4922]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2026-02-18T08:08:15.921 INFO:teuthology.orchestra.run.trial163.stderr:18 Feb 08:08:15 ntpd[4922]: can't open /var/log/ntpstats/rawstats.20260218: Permission denied 2026-02-18T08:08:15.921 INFO:teuthology.orchestra.run.trial163.stderr:18 Feb 08:08:15 ntpd[4922]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2026-02-18T08:08:15.921 INFO:teuthology.orchestra.run.trial163.stderr:18 Feb 08:08:15 ntpd[4922]: can't open /var/log/ntpstats/peerstats.20260218: Permission denied 2026-02-18T08:08:15.927 INFO:teuthology.orchestra.run.trial163.stderr:18 Feb 08:08:15 ntpd[4922]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2026-02-18T08:08:15.928 INFO:teuthology.orchestra.run.trial163.stderr:18 Feb 08:08:15 ntpd[4922]: can't open /var/log/ntpstats/rawstats.20260218: Permission denied 2026-02-18T08:08:15.928 INFO:teuthology.orchestra.run.trial163.stderr:18 Feb 08:08:15 ntpd[4922]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2026-02-18T08:08:15.928 INFO:teuthology.orchestra.run.trial163.stderr:18 Feb 08:08:15 ntpd[4922]: can't open /var/log/ntpstats/peerstats.20260218: Permission denied 2026-02-18T08:08:15.929 INFO:teuthology.orchestra.run.trial034.stderr:18 Feb 08:08:16 ntpd[4908]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2026-02-18T08:08:15.929 INFO:teuthology.orchestra.run.trial034.stderr:18 Feb 08:08:16 ntpd[4908]: can't open /var/log/ntpstats/rawstats.20260218: Permission denied 2026-02-18T08:08:15.929 INFO:teuthology.orchestra.run.trial034.stderr:18 Feb 08:08:16 ntpd[4908]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2026-02-18T08:08:15.929 INFO:teuthology.orchestra.run.trial034.stderr:18 Feb 08:08:16 ntpd[4908]: can't open /var/log/ntpstats/peerstats.20260218: Permission denied 2026-02-18T08:08:15.938 INFO:teuthology.orchestra.run.trial034.stderr:18 Feb 08:08:16 ntpd[4908]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2026-02-18T08:08:15.938 INFO:teuthology.orchestra.run.trial034.stderr:18 Feb 08:08:16 ntpd[4908]: can't open /var/log/ntpstats/rawstats.20260218: Permission denied 2026-02-18T08:08:15.938 INFO:teuthology.orchestra.run.trial034.stderr:18 Feb 08:08:16 ntpd[4908]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2026-02-18T08:08:15.938 INFO:teuthology.orchestra.run.trial034.stderr:18 Feb 08:08:16 ntpd[4908]: can't open /var/log/ntpstats/peerstats.20260218: Permission denied 2026-02-18T08:08:16.008 INFO:teuthology.orchestra.run.trial034.stderr:18 Feb 08:08:16 ntpd[4908]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2026-02-18T08:08:16.009 INFO:teuthology.orchestra.run.trial034.stderr:18 Feb 08:08:16 ntpd[4908]: can't open /var/log/ntpstats/rawstats.20260218: Permission denied 2026-02-18T08:08:16.009 INFO:teuthology.orchestra.run.trial034.stderr:18 Feb 08:08:16 ntpd[4908]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2026-02-18T08:08:16.009 INFO:teuthology.orchestra.run.trial034.stderr:18 Feb 08:08:16 ntpd[4908]: can't open /var/log/ntpstats/peerstats.20260218: Permission denied 2026-02-18T08:08:17.871 INFO:teuthology.orchestra.run.trial034.stderr:18 Feb 08:08:18 ntpd[4908]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2026-02-18T08:08:17.871 INFO:teuthology.orchestra.run.trial034.stderr:18 Feb 08:08:18 ntpd[4908]: can't open /var/log/ntpstats/rawstats.20260218: Permission denied 2026-02-18T08:08:17.871 INFO:teuthology.orchestra.run.trial034.stderr:18 Feb 08:08:18 ntpd[4908]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2026-02-18T08:08:17.871 INFO:teuthology.orchestra.run.trial034.stderr:18 Feb 08:08:18 ntpd[4908]: can't open /var/log/ntpstats/rawstats.20260218: Permission denied 2026-02-18T08:08:17.871 INFO:teuthology.orchestra.run.trial034.stderr:18 Feb 08:08:18 ntpd[4908]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2026-02-18T08:08:17.871 INFO:teuthology.orchestra.run.trial034.stderr:18 Feb 08:08:18 ntpd[4908]: can't open /var/log/ntpstats/rawstats.20260218: Permission denied 2026-02-18T08:08:17.871 INFO:teuthology.orchestra.run.trial034.stderr:18 Feb 08:08:18 ntpd[4908]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2026-02-18T08:08:17.872 INFO:teuthology.orchestra.run.trial034.stderr:18 Feb 08:08:18 ntpd[4908]: can't open /var/log/ntpstats/rawstats.20260218: Permission denied 2026-02-18T08:08:17.891 INFO:teuthology.orchestra.run.trial163.stderr:18 Feb 08:08:17 ntpd[4922]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2026-02-18T08:08:17.891 INFO:teuthology.orchestra.run.trial163.stderr:18 Feb 08:08:17 ntpd[4922]: can't open /var/log/ntpstats/rawstats.20260218: Permission denied 2026-02-18T08:08:17.891 INFO:teuthology.orchestra.run.trial163.stderr:18 Feb 08:08:17 ntpd[4922]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2026-02-18T08:08:17.891 INFO:teuthology.orchestra.run.trial163.stderr:18 Feb 08:08:17 ntpd[4922]: can't open /var/log/ntpstats/rawstats.20260218: Permission denied 2026-02-18T08:08:17.920 INFO:teuthology.orchestra.run.trial163.stderr:18 Feb 08:08:17 ntpd[4922]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2026-02-18T08:08:17.921 INFO:teuthology.orchestra.run.trial163.stderr:18 Feb 08:08:17 ntpd[4922]: can't open /var/log/ntpstats/rawstats.20260218: Permission denied 2026-02-18T08:08:17.921 INFO:teuthology.orchestra.run.trial163.stderr:18 Feb 08:08:17 ntpd[4922]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2026-02-18T08:08:17.921 INFO:teuthology.orchestra.run.trial163.stderr:18 Feb 08:08:17 ntpd[4922]: can't open /var/log/ntpstats/peerstats.20260218: Permission denied 2026-02-18T08:08:17.927 INFO:teuthology.orchestra.run.trial163.stderr:18 Feb 08:08:17 ntpd[4922]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2026-02-18T08:08:17.927 INFO:teuthology.orchestra.run.trial163.stderr:18 Feb 08:08:17 ntpd[4922]: can't open /var/log/ntpstats/rawstats.20260218: Permission denied 2026-02-18T08:08:17.928 INFO:teuthology.orchestra.run.trial163.stderr:18 Feb 08:08:17 ntpd[4922]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2026-02-18T08:08:17.928 INFO:teuthology.orchestra.run.trial163.stderr:18 Feb 08:08:17 ntpd[4922]: can't open /var/log/ntpstats/peerstats.20260218: Permission denied 2026-02-18T08:08:17.928 INFO:teuthology.orchestra.run.trial034.stderr:18 Feb 08:08:18 ntpd[4908]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2026-02-18T08:08:17.929 INFO:teuthology.orchestra.run.trial034.stderr:18 Feb 08:08:18 ntpd[4908]: can't open /var/log/ntpstats/rawstats.20260218: Permission denied 2026-02-18T08:08:17.929 INFO:teuthology.orchestra.run.trial034.stderr:18 Feb 08:08:18 ntpd[4908]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2026-02-18T08:08:17.929 INFO:teuthology.orchestra.run.trial034.stderr:18 Feb 08:08:18 ntpd[4908]: can't open /var/log/ntpstats/peerstats.20260218: Permission denied 2026-02-18T08:08:17.936 INFO:teuthology.orchestra.run.trial034.stderr:18 Feb 08:08:18 ntpd[4908]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2026-02-18T08:08:17.937 INFO:teuthology.orchestra.run.trial034.stderr:18 Feb 08:08:18 ntpd[4908]: can't open /var/log/ntpstats/rawstats.20260218: Permission denied 2026-02-18T08:08:17.937 INFO:teuthology.orchestra.run.trial034.stderr:18 Feb 08:08:18 ntpd[4908]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2026-02-18T08:08:17.937 INFO:teuthology.orchestra.run.trial034.stderr:18 Feb 08:08:18 ntpd[4908]: can't open /var/log/ntpstats/peerstats.20260218: Permission denied 2026-02-18T08:08:17.940 INFO:teuthology.orchestra.run.trial034.stderr:18 Feb 08:08:18 ntpd[4908]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2026-02-18T08:08:17.940 INFO:teuthology.orchestra.run.trial034.stderr:18 Feb 08:08:18 ntpd[4908]: can't open /var/log/ntpstats/rawstats.20260218: Permission denied 2026-02-18T08:08:17.940 INFO:teuthology.orchestra.run.trial034.stderr:18 Feb 08:08:18 ntpd[4908]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2026-02-18T08:08:17.940 INFO:teuthology.orchestra.run.trial034.stderr:18 Feb 08:08:18 ntpd[4908]: can't open /var/log/ntpstats/peerstats.20260218: Permission denied 2026-02-18T08:08:18.008 INFO:teuthology.orchestra.run.trial034.stderr:18 Feb 08:08:18 ntpd[4908]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2026-02-18T08:08:18.008 INFO:teuthology.orchestra.run.trial034.stderr:18 Feb 08:08:18 ntpd[4908]: can't open /var/log/ntpstats/rawstats.20260218: Permission denied 2026-02-18T08:08:18.008 INFO:teuthology.orchestra.run.trial034.stderr:18 Feb 08:08:18 ntpd[4908]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2026-02-18T08:08:18.008 INFO:teuthology.orchestra.run.trial034.stderr:18 Feb 08:08:18 ntpd[4908]: can't open /var/log/ntpstats/peerstats.20260218: Permission denied 2026-02-18T08:08:19.871 INFO:teuthology.orchestra.run.trial034.stderr:18 Feb 08:08:20 ntpd[4908]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2026-02-18T08:08:19.871 INFO:teuthology.orchestra.run.trial034.stderr:18 Feb 08:08:20 ntpd[4908]: can't open /var/log/ntpstats/rawstats.20260218: Permission denied 2026-02-18T08:08:19.871 INFO:teuthology.orchestra.run.trial034.stderr:18 Feb 08:08:20 ntpd[4908]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2026-02-18T08:08:19.871 INFO:teuthology.orchestra.run.trial034.stderr:18 Feb 08:08:20 ntpd[4908]: can't open /var/log/ntpstats/rawstats.20260218: Permission denied 2026-02-18T08:08:19.871 INFO:teuthology.orchestra.run.trial034.stderr:18 Feb 08:08:20 ntpd[4908]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2026-02-18T08:08:19.871 INFO:teuthology.orchestra.run.trial034.stderr:18 Feb 08:08:20 ntpd[4908]: can't open /var/log/ntpstats/rawstats.20260218: Permission denied 2026-02-18T08:08:19.871 INFO:teuthology.orchestra.run.trial034.stderr:18 Feb 08:08:20 ntpd[4908]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2026-02-18T08:08:19.872 INFO:teuthology.orchestra.run.trial034.stderr:18 Feb 08:08:20 ntpd[4908]: can't open /var/log/ntpstats/rawstats.20260218: Permission denied 2026-02-18T08:08:19.891 INFO:teuthology.orchestra.run.trial163.stderr:18 Feb 08:08:19 ntpd[4922]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2026-02-18T08:08:19.891 INFO:teuthology.orchestra.run.trial163.stderr:18 Feb 08:08:19 ntpd[4922]: can't open /var/log/ntpstats/rawstats.20260218: Permission denied 2026-02-18T08:08:19.891 INFO:teuthology.orchestra.run.trial163.stderr:18 Feb 08:08:19 ntpd[4922]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2026-02-18T08:08:19.891 INFO:teuthology.orchestra.run.trial163.stderr:18 Feb 08:08:19 ntpd[4922]: can't open /var/log/ntpstats/rawstats.20260218: Permission denied 2026-02-18T08:08:19.891 INFO:teuthology.orchestra.run.trial163.stderr:18 Feb 08:08:19 ntpd[4922]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2026-02-18T08:08:19.891 INFO:teuthology.orchestra.run.trial163.stderr:18 Feb 08:08:19 ntpd[4922]: can't open /var/log/ntpstats/rawstats.20260218: Permission denied 2026-02-18T08:08:19.892 INFO:teuthology.orchestra.run.trial163.stderr:18 Feb 08:08:19 ntpd[4922]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2026-02-18T08:08:19.892 INFO:teuthology.orchestra.run.trial163.stderr:18 Feb 08:08:19 ntpd[4922]: can't open /var/log/ntpstats/rawstats.20260218: Permission denied 2026-02-18T08:08:19.921 INFO:teuthology.orchestra.run.trial163.stderr:18 Feb 08:08:19 ntpd[4922]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2026-02-18T08:08:19.921 INFO:teuthology.orchestra.run.trial163.stderr:18 Feb 08:08:19 ntpd[4922]: can't open /var/log/ntpstats/rawstats.20260218: Permission denied 2026-02-18T08:08:19.921 INFO:teuthology.orchestra.run.trial163.stderr:18 Feb 08:08:19 ntpd[4922]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2026-02-18T08:08:19.921 INFO:teuthology.orchestra.run.trial163.stderr:18 Feb 08:08:19 ntpd[4922]: can't open /var/log/ntpstats/peerstats.20260218: Permission denied 2026-02-18T08:08:19.927 INFO:teuthology.orchestra.run.trial163.stderr:18 Feb 08:08:19 ntpd[4922]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2026-02-18T08:08:19.928 INFO:teuthology.orchestra.run.trial163.stderr:18 Feb 08:08:19 ntpd[4922]: can't open /var/log/ntpstats/rawstats.20260218: Permission denied 2026-02-18T08:08:19.928 INFO:teuthology.orchestra.run.trial163.stderr:18 Feb 08:08:19 ntpd[4922]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2026-02-18T08:08:19.928 INFO:teuthology.orchestra.run.trial163.stderr:18 Feb 08:08:19 ntpd[4922]: can't open /var/log/ntpstats/peerstats.20260218: Permission denied 2026-02-18T08:08:19.929 INFO:teuthology.orchestra.run.trial034.stderr:18 Feb 08:08:20 ntpd[4908]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2026-02-18T08:08:19.929 INFO:teuthology.orchestra.run.trial034.stderr:18 Feb 08:08:20 ntpd[4908]: can't open /var/log/ntpstats/rawstats.20260218: Permission denied 2026-02-18T08:08:19.930 INFO:teuthology.orchestra.run.trial034.stderr:18 Feb 08:08:20 ntpd[4908]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2026-02-18T08:08:19.930 INFO:teuthology.orchestra.run.trial034.stderr:18 Feb 08:08:20 ntpd[4908]: can't open /var/log/ntpstats/peerstats.20260218: Permission denied 2026-02-18T08:08:19.937 INFO:teuthology.orchestra.run.trial034.stderr:18 Feb 08:08:20 ntpd[4908]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2026-02-18T08:08:19.937 INFO:teuthology.orchestra.run.trial034.stderr:18 Feb 08:08:20 ntpd[4908]: can't open /var/log/ntpstats/rawstats.20260218: Permission denied 2026-02-18T08:08:19.937 INFO:teuthology.orchestra.run.trial034.stderr:18 Feb 08:08:20 ntpd[4908]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2026-02-18T08:08:19.937 INFO:teuthology.orchestra.run.trial034.stderr:18 Feb 08:08:20 ntpd[4908]: can't open /var/log/ntpstats/peerstats.20260218: Permission denied 2026-02-18T08:08:19.940 INFO:teuthology.orchestra.run.trial034.stderr:18 Feb 08:08:20 ntpd[4908]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2026-02-18T08:08:19.940 INFO:teuthology.orchestra.run.trial034.stderr:18 Feb 08:08:20 ntpd[4908]: can't open /var/log/ntpstats/rawstats.20260218: Permission denied 2026-02-18T08:08:19.940 INFO:teuthology.orchestra.run.trial034.stderr:18 Feb 08:08:20 ntpd[4908]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2026-02-18T08:08:19.940 INFO:teuthology.orchestra.run.trial034.stderr:18 Feb 08:08:20 ntpd[4908]: can't open /var/log/ntpstats/peerstats.20260218: Permission denied 2026-02-18T08:08:19.951 INFO:teuthology.orchestra.run.trial163.stderr:18 Feb 08:08:19 ntpd[4922]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2026-02-18T08:08:19.951 INFO:teuthology.orchestra.run.trial163.stderr:18 Feb 08:08:19 ntpd[4922]: can't open /var/log/ntpstats/rawstats.20260218: Permission denied 2026-02-18T08:08:19.951 INFO:teuthology.orchestra.run.trial163.stderr:18 Feb 08:08:19 ntpd[4922]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2026-02-18T08:08:19.951 INFO:teuthology.orchestra.run.trial163.stderr:18 Feb 08:08:19 ntpd[4922]: can't open /var/log/ntpstats/peerstats.20260218: Permission denied 2026-02-18T08:08:20.008 INFO:teuthology.orchestra.run.trial034.stderr:18 Feb 08:08:20 ntpd[4908]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2026-02-18T08:08:20.008 INFO:teuthology.orchestra.run.trial034.stderr:18 Feb 08:08:20 ntpd[4908]: can't open /var/log/ntpstats/rawstats.20260218: Permission denied 2026-02-18T08:08:20.008 INFO:teuthology.orchestra.run.trial034.stderr:18 Feb 08:08:20 ntpd[4908]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2026-02-18T08:08:20.008 INFO:teuthology.orchestra.run.trial034.stderr:18 Feb 08:08:20 ntpd[4908]: can't open /var/log/ntpstats/peerstats.20260218: Permission denied 2026-02-18T08:08:20.025 INFO:teuthology.orchestra.run.trial163.stderr:18 Feb 08:08:19 ntpd[4922]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2026-02-18T08:08:20.025 INFO:teuthology.orchestra.run.trial163.stderr:18 Feb 08:08:19 ntpd[4922]: can't open /var/log/ntpstats/rawstats.20260218: Permission denied 2026-02-18T08:08:20.025 INFO:teuthology.orchestra.run.trial163.stderr:18 Feb 08:08:19 ntpd[4922]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2026-02-18T08:08:20.025 INFO:teuthology.orchestra.run.trial163.stderr:18 Feb 08:08:19 ntpd[4922]: can't open /var/log/ntpstats/peerstats.20260218: Permission denied 2026-02-18T08:08:21.871 INFO:teuthology.orchestra.run.trial034.stderr:18 Feb 08:08:22 ntpd[4908]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2026-02-18T08:08:21.871 INFO:teuthology.orchestra.run.trial034.stderr:18 Feb 08:08:22 ntpd[4908]: can't open /var/log/ntpstats/rawstats.20260218: Permission denied 2026-02-18T08:08:21.871 INFO:teuthology.orchestra.run.trial034.stderr:18 Feb 08:08:22 ntpd[4908]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2026-02-18T08:08:21.871 INFO:teuthology.orchestra.run.trial034.stderr:18 Feb 08:08:22 ntpd[4908]: can't open /var/log/ntpstats/rawstats.20260218: Permission denied 2026-02-18T08:08:21.871 INFO:teuthology.orchestra.run.trial034.stderr:18 Feb 08:08:22 ntpd[4908]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2026-02-18T08:08:21.871 INFO:teuthology.orchestra.run.trial034.stderr:18 Feb 08:08:22 ntpd[4908]: can't open /var/log/ntpstats/rawstats.20260218: Permission denied 2026-02-18T08:08:21.871 INFO:teuthology.orchestra.run.trial034.stderr:18 Feb 08:08:22 ntpd[4908]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2026-02-18T08:08:21.871 INFO:teuthology.orchestra.run.trial034.stderr:18 Feb 08:08:22 ntpd[4908]: can't open /var/log/ntpstats/rawstats.20260218: Permission denied 2026-02-18T08:08:21.891 INFO:teuthology.orchestra.run.trial163.stderr:18 Feb 08:08:21 ntpd[4922]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2026-02-18T08:08:21.891 INFO:teuthology.orchestra.run.trial163.stderr:18 Feb 08:08:21 ntpd[4922]: can't open /var/log/ntpstats/rawstats.20260218: Permission denied 2026-02-18T08:08:21.891 INFO:teuthology.orchestra.run.trial163.stderr:18 Feb 08:08:21 ntpd[4922]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2026-02-18T08:08:21.891 INFO:teuthology.orchestra.run.trial163.stderr:18 Feb 08:08:21 ntpd[4922]: can't open /var/log/ntpstats/rawstats.20260218: Permission denied 2026-02-18T08:08:21.891 INFO:teuthology.orchestra.run.trial163.stderr:18 Feb 08:08:21 ntpd[4922]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2026-02-18T08:08:21.892 INFO:teuthology.orchestra.run.trial163.stderr:18 Feb 08:08:21 ntpd[4922]: can't open /var/log/ntpstats/rawstats.20260218: Permission denied 2026-02-18T08:08:21.892 INFO:teuthology.orchestra.run.trial163.stderr:18 Feb 08:08:21 ntpd[4922]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2026-02-18T08:08:21.892 INFO:teuthology.orchestra.run.trial163.stderr:18 Feb 08:08:21 ntpd[4922]: can't open /var/log/ntpstats/rawstats.20260218: Permission denied 2026-02-18T08:08:21.920 INFO:teuthology.orchestra.run.trial163.stderr:18 Feb 08:08:21 ntpd[4922]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2026-02-18T08:08:21.921 INFO:teuthology.orchestra.run.trial163.stderr:18 Feb 08:08:21 ntpd[4922]: can't open /var/log/ntpstats/rawstats.20260218: Permission denied 2026-02-18T08:08:21.921 INFO:teuthology.orchestra.run.trial163.stderr:18 Feb 08:08:21 ntpd[4922]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2026-02-18T08:08:21.921 INFO:teuthology.orchestra.run.trial163.stderr:18 Feb 08:08:21 ntpd[4922]: can't open /var/log/ntpstats/peerstats.20260218: Permission denied 2026-02-18T08:08:21.921 INFO:teuthology.orchestra.run.trial163.stdout:18 Feb 08:08:21 ntpd[4922]: ntpd: time set +0.425534 s 2026-02-18T08:08:21.921 INFO:teuthology.orchestra.run.trial163.stdout:ntpd: time set +0.425534s 2026-02-18T08:08:21.922 INFO:teuthology.orchestra.run.trial163.stderr:18 Feb 08:08:21 ntpd[4922]: couldn't unlink /var/log/ntpstats/loopstats: Permission denied 2026-02-18T08:08:21.922 INFO:teuthology.orchestra.run.trial163.stderr:18 Feb 08:08:21 ntpd[4922]: can't open /var/log/ntpstats/loopstats.20260218: Permission denied 2026-02-18T08:08:21.928 INFO:teuthology.orchestra.run.trial034.stderr:18 Feb 08:08:22 ntpd[4908]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2026-02-18T08:08:21.928 INFO:teuthology.orchestra.run.trial034.stderr:18 Feb 08:08:22 ntpd[4908]: can't open /var/log/ntpstats/rawstats.20260218: Permission denied 2026-02-18T08:08:21.929 INFO:teuthology.orchestra.run.trial034.stderr:18 Feb 08:08:22 ntpd[4908]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2026-02-18T08:08:21.929 INFO:teuthology.orchestra.run.trial034.stderr:18 Feb 08:08:22 ntpd[4908]: can't open /var/log/ntpstats/peerstats.20260218: Permission denied 2026-02-18T08:08:21.929 INFO:teuthology.orchestra.run.trial034.stdout:18 Feb 08:08:21 ntpd[4908]: ntpd: time set -0.606934 s 2026-02-18T08:08:21.929 INFO:teuthology.orchestra.run.trial034.stdout:ntpd: time set -0.606934s 2026-02-18T08:08:21.929 INFO:teuthology.orchestra.run.trial034.stderr:18 Feb 08:08:21 ntpd[4908]: couldn't unlink /var/log/ntpstats/loopstats: Permission denied 2026-02-18T08:08:21.930 INFO:teuthology.orchestra.run.trial034.stderr:18 Feb 08:08:21 ntpd[4908]: can't open /var/log/ntpstats/loopstats.20260218: Permission denied 2026-02-18T08:08:21.987 INFO:teuthology.orchestra.run.trial163.stdout: remote refid st t when poll reach delay offset jitter 2026-02-18T08:08:21.987 INFO:teuthology.orchestra.run.trial163.stdout:============================================================================== 2026-02-18T08:08:21.988 INFO:teuthology.orchestra.run.trial163.stdout: 172-234-44-141. .INIT. 16 u - 64 0 0.000 +0.000 0.000 2026-02-18T08:08:21.988 INFO:teuthology.orchestra.run.trial163.stdout: h96-60-247-29.m .INIT. 16 u - 64 0 0.000 +0.000 0.000 2026-02-18T08:08:21.988 INFO:teuthology.orchestra.run.trial163.stdout: time.ecansol.ne .INIT. 16 u - 64 0 0.000 +0.000 0.000 2026-02-18T08:08:21.988 INFO:teuthology.orchestra.run.trial163.stdout: dns-e.ns4v.icu .INIT. 16 u - 64 0 0.000 +0.000 0.000 2026-02-18T08:08:22.004 INFO:teuthology.orchestra.run.trial034.stdout: remote refid st t when poll reach delay offset jitter 2026-02-18T08:08:22.004 INFO:teuthology.orchestra.run.trial034.stdout:============================================================================== 2026-02-18T08:08:22.004 INFO:teuthology.orchestra.run.trial034.stdout: as393746.mci.tr .INIT. 16 u - 64 0 0.000 +0.000 0.000 2026-02-18T08:08:22.005 INFO:teuthology.orchestra.run.trial034.stdout: dns-e.ns4v.icu .INIT. 16 u - 64 0 0.000 +0.000 0.000 2026-02-18T08:08:22.005 INFO:teuthology.orchestra.run.trial034.stdout: clock.xmission. .INIT. 16 u - 64 0 0.000 +0.000 0.000 2026-02-18T08:08:22.005 INFO:teuthology.orchestra.run.trial034.stdout: 172-235-154-118 .INIT. 16 u - 64 0 0.000 +0.000 0.000 2026-02-18T08:08:22.006 INFO:teuthology.run_tasks:Running task install... 2026-02-18T08:08:22.021 DEBUG:teuthology.task.install:project ceph 2026-02-18T08:08:22.021 DEBUG:teuthology.task.install:INSTALL overrides: {'ceph': {'flavor': 'default', 'sha1': 'adc1923395f2bcd6ae89b5d331d278364d1848ed'}} 2026-02-18T08:08:22.021 DEBUG:teuthology.task.install:config {'extra_packages': ['rbd-mirror'], 'flavor': 'default', 'sha1': 'adc1923395f2bcd6ae89b5d331d278364d1848ed'} 2026-02-18T08:08:22.021 INFO:teuthology.task.install:Using flavor: default 2026-02-18T08:08:22.029 DEBUG:teuthology.task.install:Package list is: {'deb': ['ceph', 'cephadm', 'ceph-mds', 'ceph-mgr', 'ceph-common', 'ceph-fuse', 'ceph-test', 'ceph-volume', 'radosgw', 'python3-rados', 'python3-rgw', 'python3-cephfs', 'python3-rbd', 'libcephfs2', 'libcephfs-dev', 'librados2', 'librbd1', 'rbd-fuse'], 'rpm': ['ceph-radosgw', 'ceph-test', 'ceph', 'ceph-base', 'cephadm', 'ceph-immutable-object-cache', 'ceph-mgr', 'ceph-mgr-dashboard', 'ceph-mgr-diskprediction-local', 'ceph-mgr-rook', 'ceph-mgr-cephadm', 'ceph-fuse', 'ceph-volume', 'librados-devel', 'libcephfs2', 'libcephfs-devel', 'librados2', 'librbd1', 'python3-rados', 'python3-rgw', 'python3-cephfs', 'python3-rbd', 'rbd-fuse', 'rbd-mirror', 'rbd-nbd']} 2026-02-18T08:08:22.029 INFO:teuthology.task.install:extra packages: ['rbd-mirror'] 2026-02-18T08:08:22.030 DEBUG:teuthology.orchestra.run.trial034:> sudo apt-key list | grep Ceph 2026-02-18T08:08:22.030 DEBUG:teuthology.orchestra.run.trial163:> sudo apt-key list | grep Ceph 2026-02-18T08:08:22.079 INFO:teuthology.orchestra.run.trial163.stderr:Warning: apt-key is deprecated. Manage keyring files in trusted.gpg.d instead (see apt-key(8)). 2026-02-18T08:08:22.093 INFO:teuthology.orchestra.run.trial034.stderr:Warning: apt-key is deprecated. Manage keyring files in trusted.gpg.d instead (see apt-key(8)). 2026-02-18T08:08:22.109 INFO:teuthology.orchestra.run.trial163.stdout:uid [ unknown] Ceph automated package build (Ceph automated package build) 2026-02-18T08:08:22.109 INFO:teuthology.orchestra.run.trial163.stdout:uid [ unknown] Ceph.com (release key) 2026-02-18T08:08:22.110 INFO:teuthology.task.install.deb:Installing packages: ceph, cephadm, ceph-mds, ceph-mgr, ceph-common, ceph-fuse, ceph-test, ceph-volume, radosgw, python3-rados, python3-rgw, python3-cephfs, python3-rbd, libcephfs2, libcephfs-dev, librados2, librbd1, rbd-fuse, rbd-mirror on remote deb x86_64 2026-02-18T08:08:22.110 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=ubuntu%2F22.04%2Fx86_64&sha1=adc1923395f2bcd6ae89b5d331d278364d1848ed 2026-02-18T08:08:22.121 INFO:teuthology.orchestra.run.trial034.stdout:uid [ unknown] Ceph automated package build (Ceph automated package build) 2026-02-18T08:08:22.121 INFO:teuthology.orchestra.run.trial034.stdout:uid [ unknown] Ceph.com (release key) 2026-02-18T08:08:22.122 INFO:teuthology.task.install.deb:Installing packages: ceph, cephadm, ceph-mds, ceph-mgr, ceph-common, ceph-fuse, ceph-test, ceph-volume, radosgw, python3-rados, python3-rgw, python3-cephfs, python3-rbd, libcephfs2, libcephfs-dev, librados2, librbd1, rbd-fuse, rbd-mirror on remote deb x86_64 2026-02-18T08:08:22.123 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=ubuntu%2F22.04%2Fx86_64&sha1=adc1923395f2bcd6ae89b5d331d278364d1848ed 2026-02-18T08:08:22.297 INFO:teuthology.task.install.deb:Pulling from https://2.chacra.ceph.com/r/ceph/20.2.1_2.17.26-baseline_2/adc1923395f2bcd6ae89b5d331d278364d1848ed/ubuntu/jammy/flavors/default/ 2026-02-18T08:08:22.297 INFO:teuthology.task.install.deb:Package version is 20.2.0-670-gadc19233-1jammy 2026-02-18T08:08:22.312 INFO:teuthology.task.install.deb:Pulling from https://2.chacra.ceph.com/r/ceph/20.2.1_2.17.26-baseline_2/adc1923395f2bcd6ae89b5d331d278364d1848ed/ubuntu/jammy/flavors/default/ 2026-02-18T08:08:22.312 INFO:teuthology.task.install.deb:Package version is 20.2.0-670-gadc19233-1jammy 2026-02-18T08:08:22.468 DEBUG:teuthology.orchestra.run.trial163:> set -ex 2026-02-18T08:08:22.469 DEBUG:teuthology.orchestra.run.trial163:> sudo dd of=/etc/apt/sources.list.d/ceph.list 2026-02-18T08:08:22.477 DEBUG:teuthology.orchestra.run.trial034:> set -ex 2026-02-18T08:08:22.477 DEBUG:teuthology.orchestra.run.trial034:> sudo dd of=/etc/apt/sources.list.d/ceph.list 2026-02-18T08:08:22.477 DEBUG:teuthology.orchestra.run.trial163:> sudo apt-get update 2026-02-18T08:08:22.487 DEBUG:teuthology.orchestra.run.trial034:> sudo apt-get update 2026-02-18T08:08:22.643 INFO:teuthology.orchestra.run.trial163.stdout:Ign:1 https://2.chacra.ceph.com/r/ceph/20.2.1_2.17.26-baseline_2/adc1923395f2bcd6ae89b5d331d278364d1848ed/ubuntu/jammy/flavors/default jammy InRelease 2026-02-18T08:08:22.643 INFO:teuthology.orchestra.run.trial034.stdout:Ign:1 https://2.chacra.ceph.com/r/ceph/20.2.1_2.17.26-baseline_2/adc1923395f2bcd6ae89b5d331d278364d1848ed/ubuntu/jammy/flavors/default jammy InRelease 2026-02-18T08:08:22.655 INFO:teuthology.orchestra.run.trial163.stdout:Get:2 https://2.chacra.ceph.com/r/ceph/20.2.1_2.17.26-baseline_2/adc1923395f2bcd6ae89b5d331d278364d1848ed/ubuntu/jammy/flavors/default jammy Release [7680 B] 2026-02-18T08:08:22.656 INFO:teuthology.orchestra.run.trial034.stdout:Get:2 https://2.chacra.ceph.com/r/ceph/20.2.1_2.17.26-baseline_2/adc1923395f2bcd6ae89b5d331d278364d1848ed/ubuntu/jammy/flavors/default jammy Release [7680 B] 2026-02-18T08:08:22.666 INFO:teuthology.orchestra.run.trial163.stdout:Ign:3 https://2.chacra.ceph.com/r/ceph/20.2.1_2.17.26-baseline_2/adc1923395f2bcd6ae89b5d331d278364d1848ed/ubuntu/jammy/flavors/default jammy Release.gpg 2026-02-18T08:08:22.667 INFO:teuthology.orchestra.run.trial034.stdout:Ign:3 https://2.chacra.ceph.com/r/ceph/20.2.1_2.17.26-baseline_2/adc1923395f2bcd6ae89b5d331d278364d1848ed/ubuntu/jammy/flavors/default jammy Release.gpg 2026-02-18T08:08:22.681 INFO:teuthology.orchestra.run.trial163.stdout:Hit:4 https://archive.ubuntu.com/ubuntu jammy InRelease 2026-02-18T08:08:22.681 INFO:teuthology.orchestra.run.trial034.stdout:Hit:4 https://archive.ubuntu.com/ubuntu jammy InRelease 2026-02-18T08:08:22.685 INFO:teuthology.orchestra.run.trial163.stdout:Hit:5 https://archive.ubuntu.com/ubuntu jammy-updates InRelease 2026-02-18T08:08:22.685 INFO:teuthology.orchestra.run.trial034.stdout:Hit:5 https://archive.ubuntu.com/ubuntu jammy-updates InRelease 2026-02-18T08:08:22.691 INFO:teuthology.orchestra.run.trial163.stdout:Get:6 https://2.chacra.ceph.com/r/ceph/20.2.1_2.17.26-baseline_2/adc1923395f2bcd6ae89b5d331d278364d1848ed/ubuntu/jammy/flavors/default jammy/main amd64 Packages [18.8 kB] 2026-02-18T08:08:22.691 INFO:teuthology.orchestra.run.trial034.stdout:Get:6 https://2.chacra.ceph.com/r/ceph/20.2.1_2.17.26-baseline_2/adc1923395f2bcd6ae89b5d331d278364d1848ed/ubuntu/jammy/flavors/default jammy/main amd64 Packages [18.8 kB] 2026-02-18T08:08:22.695 INFO:teuthology.orchestra.run.trial034.stdout:Hit:7 https://archive.ubuntu.com/ubuntu jammy-security InRelease 2026-02-18T08:08:22.696 INFO:teuthology.orchestra.run.trial163.stdout:Hit:7 https://archive.ubuntu.com/ubuntu jammy-security InRelease 2026-02-18T08:08:22.704 INFO:teuthology.orchestra.run.trial163.stdout:Hit:8 https://archive.ubuntu.com/ubuntu jammy-backports InRelease 2026-02-18T08:08:22.706 INFO:teuthology.orchestra.run.trial034.stdout:Hit:8 https://archive.ubuntu.com/ubuntu jammy-backports InRelease 2026-02-18T08:08:23.019 INFO:teuthology.orchestra.run.trial034.stdout:Fetched 26.5 kB in 0s (70.9 kB/s) 2026-02-18T08:08:23.020 INFO:teuthology.orchestra.run.trial163.stdout:Fetched 26.5 kB in 0s (69.5 kB/s) 2026-02-18T08:08:23.539 INFO:teuthology.orchestra.run.trial034.stdout:Reading package lists... 2026-02-18T08:08:23.552 DEBUG:teuthology.orchestra.run.trial034:> sudo DEBIAN_FRONTEND=noninteractive apt-get -y --force-yes -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" install ceph=20.2.0-670-gadc19233-1jammy cephadm=20.2.0-670-gadc19233-1jammy ceph-mds=20.2.0-670-gadc19233-1jammy ceph-mgr=20.2.0-670-gadc19233-1jammy ceph-common=20.2.0-670-gadc19233-1jammy ceph-fuse=20.2.0-670-gadc19233-1jammy ceph-test=20.2.0-670-gadc19233-1jammy ceph-volume=20.2.0-670-gadc19233-1jammy radosgw=20.2.0-670-gadc19233-1jammy python3-rados=20.2.0-670-gadc19233-1jammy python3-rgw=20.2.0-670-gadc19233-1jammy python3-cephfs=20.2.0-670-gadc19233-1jammy python3-rbd=20.2.0-670-gadc19233-1jammy libcephfs2=20.2.0-670-gadc19233-1jammy libcephfs-dev=20.2.0-670-gadc19233-1jammy librados2=20.2.0-670-gadc19233-1jammy librbd1=20.2.0-670-gadc19233-1jammy rbd-fuse=20.2.0-670-gadc19233-1jammy rbd-mirror=20.2.0-670-gadc19233-1jammy 2026-02-18T08:08:23.566 INFO:teuthology.orchestra.run.trial163.stdout:Reading package lists... 2026-02-18T08:08:23.580 DEBUG:teuthology.orchestra.run.trial163:> sudo DEBIAN_FRONTEND=noninteractive apt-get -y --force-yes -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" install ceph=20.2.0-670-gadc19233-1jammy cephadm=20.2.0-670-gadc19233-1jammy ceph-mds=20.2.0-670-gadc19233-1jammy ceph-mgr=20.2.0-670-gadc19233-1jammy ceph-common=20.2.0-670-gadc19233-1jammy ceph-fuse=20.2.0-670-gadc19233-1jammy ceph-test=20.2.0-670-gadc19233-1jammy ceph-volume=20.2.0-670-gadc19233-1jammy radosgw=20.2.0-670-gadc19233-1jammy python3-rados=20.2.0-670-gadc19233-1jammy python3-rgw=20.2.0-670-gadc19233-1jammy python3-cephfs=20.2.0-670-gadc19233-1jammy python3-rbd=20.2.0-670-gadc19233-1jammy libcephfs2=20.2.0-670-gadc19233-1jammy libcephfs-dev=20.2.0-670-gadc19233-1jammy librados2=20.2.0-670-gadc19233-1jammy librbd1=20.2.0-670-gadc19233-1jammy rbd-fuse=20.2.0-670-gadc19233-1jammy rbd-mirror=20.2.0-670-gadc19233-1jammy 2026-02-18T08:08:23.585 INFO:teuthology.orchestra.run.trial034.stdout:Reading package lists... 2026-02-18T08:08:23.613 INFO:teuthology.orchestra.run.trial163.stdout:Reading package lists... 2026-02-18T08:08:23.692 INFO:teuthology.orchestra.run.trial034.stdout:Building dependency tree... 2026-02-18T08:08:23.692 INFO:teuthology.orchestra.run.trial034.stdout:Reading state information... 2026-02-18T08:08:23.726 INFO:teuthology.orchestra.run.trial163.stdout:Building dependency tree... 2026-02-18T08:08:23.726 INFO:teuthology.orchestra.run.trial163.stdout:Reading state information... 2026-02-18T08:08:23.781 INFO:teuthology.orchestra.run.trial034.stdout:The following packages were automatically installed and are no longer required: 2026-02-18T08:08:23.781 INFO:teuthology.orchestra.run.trial034.stdout: kpartx libboost-iostreams1.74.0 libboost-thread1.74.0 libgfapi0 libgfrpc0 2026-02-18T08:08:23.781 INFO:teuthology.orchestra.run.trial034.stdout: libgfxdr0 libglusterfs0 libiscsi7 libpmemobj1 libsgutils2-2 sg3-utils 2026-02-18T08:08:23.781 INFO:teuthology.orchestra.run.trial034.stdout: sg3-utils-udev 2026-02-18T08:08:23.781 INFO:teuthology.orchestra.run.trial034.stdout:Use 'sudo apt autoremove' to remove them. 2026-02-18T08:08:23.782 INFO:teuthology.orchestra.run.trial034.stdout:The following additional packages will be installed: 2026-02-18T08:08:23.782 INFO:teuthology.orchestra.run.trial034.stdout: ceph-base ceph-mgr-cephadm ceph-mgr-dashboard ceph-mgr-diskprediction-local 2026-02-18T08:08:23.782 INFO:teuthology.orchestra.run.trial034.stdout: ceph-mgr-k8sevents ceph-mgr-modules-core ceph-mon ceph-osd jq 2026-02-18T08:08:23.782 INFO:teuthology.orchestra.run.trial034.stdout: libcephfs-daemon libcephfs-proxy2 libdouble-conversion3 libfuse2 libjq1 2026-02-18T08:08:23.782 INFO:teuthology.orchestra.run.trial034.stdout: liblttng-ust1 libnbd0 liboath0 libonig5 libpcre2-16-0 libqt5core5a 2026-02-18T08:08:23.782 INFO:teuthology.orchestra.run.trial034.stdout: libqt5dbus5 libqt5network5 libradosstriper1 librdkafka1 librgw2 2026-02-18T08:08:23.782 INFO:teuthology.orchestra.run.trial034.stdout: libsqlite3-mod-ceph libthrift-0.16.0 nvme-cli python-asyncssh-doc 2026-02-18T08:08:23.782 INFO:teuthology.orchestra.run.trial034.stdout: python3-asyncssh python3-cachetools python3-ceph-argparse 2026-02-18T08:08:23.782 INFO:teuthology.orchestra.run.trial034.stdout: python3-ceph-common python3-cheroot python3-cherrypy3 python3-google-auth 2026-02-18T08:08:23.782 INFO:teuthology.orchestra.run.trial034.stdout: python3-iniconfig python3-jaraco.classes python3-jaraco.collections 2026-02-18T08:08:23.782 INFO:teuthology.orchestra.run.trial034.stdout: python3-jaraco.functools python3-jaraco.text python3-joblib 2026-02-18T08:08:23.782 INFO:teuthology.orchestra.run.trial034.stdout: python3-kubernetes python3-natsort python3-pluggy python3-portend 2026-02-18T08:08:23.782 INFO:teuthology.orchestra.run.trial034.stdout: python3-prettytable python3-psutil python3-py python3-pygments 2026-02-18T08:08:23.783 INFO:teuthology.orchestra.run.trial034.stdout: python3-pytest python3-repoze.lru python3-requests-oauthlib python3-routes 2026-02-18T08:08:23.783 INFO:teuthology.orchestra.run.trial034.stdout: python3-rsa python3-simplejson python3-sklearn python3-sklearn-lib 2026-02-18T08:08:23.783 INFO:teuthology.orchestra.run.trial034.stdout: python3-tempora python3-threadpoolctl python3-toml python3-wcwidth 2026-02-18T08:08:23.783 INFO:teuthology.orchestra.run.trial034.stdout: python3-webob python3-websocket python3-zc.lockfile qttranslations5-l10n 2026-02-18T08:08:23.783 INFO:teuthology.orchestra.run.trial034.stdout: socat xmlstarlet 2026-02-18T08:08:23.783 INFO:teuthology.orchestra.run.trial034.stdout:Suggested packages: 2026-02-18T08:08:23.783 INFO:teuthology.orchestra.run.trial034.stdout: python3-influxdb liblua5.3-dev luarocks python-natsort-doc python-psutil-doc 2026-02-18T08:08:23.783 INFO:teuthology.orchestra.run.trial034.stdout: subversion python-pygments-doc ttf-bitstream-vera python3-paste python3-dap 2026-02-18T08:08:23.783 INFO:teuthology.orchestra.run.trial034.stdout: python-sklearn-doc ipython3 python-webob-doc 2026-02-18T08:08:23.783 INFO:teuthology.orchestra.run.trial034.stdout:Recommended packages: 2026-02-18T08:08:23.783 INFO:teuthology.orchestra.run.trial034.stdout: btrfs-tools 2026-02-18T08:08:23.815 INFO:teuthology.orchestra.run.trial034.stdout:The following NEW packages will be installed: 2026-02-18T08:08:23.816 INFO:teuthology.orchestra.run.trial034.stdout: ceph ceph-base ceph-common ceph-fuse ceph-mds ceph-mgr ceph-mgr-cephadm 2026-02-18T08:08:23.816 INFO:teuthology.orchestra.run.trial034.stdout: ceph-mgr-dashboard ceph-mgr-diskprediction-local ceph-mgr-k8sevents 2026-02-18T08:08:23.816 INFO:teuthology.orchestra.run.trial034.stdout: ceph-mgr-modules-core ceph-mon ceph-osd ceph-test ceph-volume cephadm jq 2026-02-18T08:08:23.816 INFO:teuthology.orchestra.run.trial034.stdout: libcephfs-daemon libcephfs-dev libcephfs-proxy2 libcephfs2 2026-02-18T08:08:23.816 INFO:teuthology.orchestra.run.trial034.stdout: libdouble-conversion3 libfuse2 libjq1 liblttng-ust1 libnbd0 liboath0 2026-02-18T08:08:23.816 INFO:teuthology.orchestra.run.trial034.stdout: libonig5 libpcre2-16-0 libqt5core5a libqt5dbus5 libqt5network5 librados2 2026-02-18T08:08:23.816 INFO:teuthology.orchestra.run.trial034.stdout: libradosstriper1 librbd1 librdkafka1 librgw2 libsqlite3-mod-ceph 2026-02-18T08:08:23.816 INFO:teuthology.orchestra.run.trial034.stdout: libthrift-0.16.0 nvme-cli python-asyncssh-doc python3-asyncssh 2026-02-18T08:08:23.816 INFO:teuthology.orchestra.run.trial034.stdout: python3-cachetools python3-ceph-argparse python3-ceph-common python3-cephfs 2026-02-18T08:08:23.816 INFO:teuthology.orchestra.run.trial034.stdout: python3-cheroot python3-cherrypy3 python3-google-auth python3-iniconfig 2026-02-18T08:08:23.816 INFO:teuthology.orchestra.run.trial034.stdout: python3-jaraco.classes python3-jaraco.collections python3-jaraco.functools 2026-02-18T08:08:23.817 INFO:teuthology.orchestra.run.trial034.stdout: python3-jaraco.text python3-joblib python3-kubernetes python3-natsort 2026-02-18T08:08:23.817 INFO:teuthology.orchestra.run.trial034.stdout: python3-pluggy python3-portend python3-prettytable python3-psutil python3-py 2026-02-18T08:08:23.817 INFO:teuthology.orchestra.run.trial034.stdout: python3-pygments python3-pytest python3-rados python3-rbd python3-repoze.lru 2026-02-18T08:08:23.817 INFO:teuthology.orchestra.run.trial034.stdout: python3-requests-oauthlib python3-rgw python3-routes python3-rsa 2026-02-18T08:08:23.817 INFO:teuthology.orchestra.run.trial034.stdout: python3-simplejson python3-sklearn python3-sklearn-lib python3-tempora 2026-02-18T08:08:23.817 INFO:teuthology.orchestra.run.trial034.stdout: python3-threadpoolctl python3-toml python3-wcwidth python3-webob 2026-02-18T08:08:23.817 INFO:teuthology.orchestra.run.trial034.stdout: python3-websocket python3-zc.lockfile qttranslations5-l10n radosgw rbd-fuse 2026-02-18T08:08:23.817 INFO:teuthology.orchestra.run.trial034.stdout: rbd-mirror socat xmlstarlet 2026-02-18T08:08:23.819 INFO:teuthology.orchestra.run.trial163.stdout:The following packages were automatically installed and are no longer required: 2026-02-18T08:08:23.819 INFO:teuthology.orchestra.run.trial163.stdout: kpartx libboost-iostreams1.74.0 libboost-thread1.74.0 libgfapi0 libgfrpc0 2026-02-18T08:08:23.819 INFO:teuthology.orchestra.run.trial163.stdout: libgfxdr0 libglusterfs0 libiscsi7 libpmemobj1 libsgutils2-2 sg3-utils 2026-02-18T08:08:23.819 INFO:teuthology.orchestra.run.trial163.stdout: sg3-utils-udev 2026-02-18T08:08:23.819 INFO:teuthology.orchestra.run.trial163.stdout:Use 'sudo apt autoremove' to remove them. 2026-02-18T08:08:23.820 INFO:teuthology.orchestra.run.trial163.stdout:The following additional packages will be installed: 2026-02-18T08:08:23.820 INFO:teuthology.orchestra.run.trial163.stdout: ceph-base ceph-mgr-cephadm ceph-mgr-dashboard ceph-mgr-diskprediction-local 2026-02-18T08:08:23.820 INFO:teuthology.orchestra.run.trial163.stdout: ceph-mgr-k8sevents ceph-mgr-modules-core ceph-mon ceph-osd jq 2026-02-18T08:08:23.820 INFO:teuthology.orchestra.run.trial163.stdout: libcephfs-daemon libcephfs-proxy2 libdouble-conversion3 libfuse2 libjq1 2026-02-18T08:08:23.820 INFO:teuthology.orchestra.run.trial163.stdout: liblttng-ust1 libnbd0 liboath0 libonig5 libpcre2-16-0 libqt5core5a 2026-02-18T08:08:23.820 INFO:teuthology.orchestra.run.trial163.stdout: libqt5dbus5 libqt5network5 libradosstriper1 librdkafka1 librgw2 2026-02-18T08:08:23.820 INFO:teuthology.orchestra.run.trial163.stdout: libsqlite3-mod-ceph libthrift-0.16.0 nvme-cli python-asyncssh-doc 2026-02-18T08:08:23.820 INFO:teuthology.orchestra.run.trial163.stdout: python3-asyncssh python3-cachetools python3-ceph-argparse 2026-02-18T08:08:23.820 INFO:teuthology.orchestra.run.trial163.stdout: python3-ceph-common python3-cheroot python3-cherrypy3 python3-google-auth 2026-02-18T08:08:23.820 INFO:teuthology.orchestra.run.trial163.stdout: python3-iniconfig python3-jaraco.classes python3-jaraco.collections 2026-02-18T08:08:23.820 INFO:teuthology.orchestra.run.trial163.stdout: python3-jaraco.functools python3-jaraco.text python3-joblib 2026-02-18T08:08:23.820 INFO:teuthology.orchestra.run.trial163.stdout: python3-kubernetes python3-natsort python3-pluggy python3-portend 2026-02-18T08:08:23.820 INFO:teuthology.orchestra.run.trial163.stdout: python3-prettytable python3-psutil python3-py python3-pygments 2026-02-18T08:08:23.821 INFO:teuthology.orchestra.run.trial163.stdout: python3-pytest python3-repoze.lru python3-requests-oauthlib python3-routes 2026-02-18T08:08:23.821 INFO:teuthology.orchestra.run.trial163.stdout: python3-rsa python3-simplejson python3-sklearn python3-sklearn-lib 2026-02-18T08:08:23.821 INFO:teuthology.orchestra.run.trial163.stdout: python3-tempora python3-threadpoolctl python3-toml python3-wcwidth 2026-02-18T08:08:23.821 INFO:teuthology.orchestra.run.trial163.stdout: python3-webob python3-websocket python3-zc.lockfile qttranslations5-l10n 2026-02-18T08:08:23.821 INFO:teuthology.orchestra.run.trial163.stdout: socat xmlstarlet 2026-02-18T08:08:23.821 INFO:teuthology.orchestra.run.trial163.stdout:Suggested packages: 2026-02-18T08:08:23.821 INFO:teuthology.orchestra.run.trial163.stdout: python3-influxdb liblua5.3-dev luarocks python-natsort-doc python-psutil-doc 2026-02-18T08:08:23.821 INFO:teuthology.orchestra.run.trial163.stdout: subversion python-pygments-doc ttf-bitstream-vera python3-paste python3-dap 2026-02-18T08:08:23.821 INFO:teuthology.orchestra.run.trial163.stdout: python-sklearn-doc ipython3 python-webob-doc 2026-02-18T08:08:23.821 INFO:teuthology.orchestra.run.trial163.stdout:Recommended packages: 2026-02-18T08:08:23.821 INFO:teuthology.orchestra.run.trial163.stdout: btrfs-tools 2026-02-18T08:08:23.853 INFO:teuthology.orchestra.run.trial163.stdout:The following NEW packages will be installed: 2026-02-18T08:08:23.853 INFO:teuthology.orchestra.run.trial163.stdout: ceph ceph-base ceph-common ceph-fuse ceph-mds ceph-mgr ceph-mgr-cephadm 2026-02-18T08:08:23.854 INFO:teuthology.orchestra.run.trial163.stdout: ceph-mgr-dashboard ceph-mgr-diskprediction-local ceph-mgr-k8sevents 2026-02-18T08:08:23.854 INFO:teuthology.orchestra.run.trial163.stdout: ceph-mgr-modules-core ceph-mon ceph-osd ceph-test ceph-volume cephadm jq 2026-02-18T08:08:23.854 INFO:teuthology.orchestra.run.trial163.stdout: libcephfs-daemon libcephfs-dev libcephfs-proxy2 libcephfs2 2026-02-18T08:08:23.854 INFO:teuthology.orchestra.run.trial163.stdout: libdouble-conversion3 libfuse2 libjq1 liblttng-ust1 libnbd0 liboath0 2026-02-18T08:08:23.854 INFO:teuthology.orchestra.run.trial163.stdout: libonig5 libpcre2-16-0 libqt5core5a libqt5dbus5 libqt5network5 librados2 2026-02-18T08:08:23.854 INFO:teuthology.orchestra.run.trial163.stdout: libradosstriper1 librbd1 librdkafka1 librgw2 libsqlite3-mod-ceph 2026-02-18T08:08:23.854 INFO:teuthology.orchestra.run.trial163.stdout: libthrift-0.16.0 nvme-cli python-asyncssh-doc python3-asyncssh 2026-02-18T08:08:23.854 INFO:teuthology.orchestra.run.trial163.stdout: python3-cachetools python3-ceph-argparse python3-ceph-common python3-cephfs 2026-02-18T08:08:23.854 INFO:teuthology.orchestra.run.trial163.stdout: python3-cheroot python3-cherrypy3 python3-google-auth python3-iniconfig 2026-02-18T08:08:23.854 INFO:teuthology.orchestra.run.trial163.stdout: python3-jaraco.classes python3-jaraco.collections python3-jaraco.functools 2026-02-18T08:08:23.854 INFO:teuthology.orchestra.run.trial163.stdout: python3-jaraco.text python3-joblib python3-kubernetes python3-natsort 2026-02-18T08:08:23.854 INFO:teuthology.orchestra.run.trial163.stdout: python3-pluggy python3-portend python3-prettytable python3-psutil python3-py 2026-02-18T08:08:23.855 INFO:teuthology.orchestra.run.trial163.stdout: python3-pygments python3-pytest python3-rados python3-rbd python3-repoze.lru 2026-02-18T08:08:23.855 INFO:teuthology.orchestra.run.trial163.stdout: python3-requests-oauthlib python3-rgw python3-routes python3-rsa 2026-02-18T08:08:23.855 INFO:teuthology.orchestra.run.trial163.stdout: python3-simplejson python3-sklearn python3-sklearn-lib python3-tempora 2026-02-18T08:08:23.855 INFO:teuthology.orchestra.run.trial163.stdout: python3-threadpoolctl python3-toml python3-wcwidth python3-webob 2026-02-18T08:08:23.855 INFO:teuthology.orchestra.run.trial163.stdout: python3-websocket python3-zc.lockfile qttranslations5-l10n radosgw rbd-fuse 2026-02-18T08:08:23.855 INFO:teuthology.orchestra.run.trial163.stdout: rbd-mirror socat xmlstarlet 2026-02-18T08:08:23.907 INFO:teuthology.orchestra.run.trial034.stdout:0 upgraded, 87 newly installed, 0 to remove and 53 not upgraded. 2026-02-18T08:08:23.907 INFO:teuthology.orchestra.run.trial034.stdout:Need to get 283 MB of archives. 2026-02-18T08:08:23.907 INFO:teuthology.orchestra.run.trial034.stdout:After this operation, 1130 MB of additional disk space will be used. 2026-02-18T08:08:23.907 INFO:teuthology.orchestra.run.trial034.stdout:Get:1 https://2.chacra.ceph.com/r/ceph/20.2.1_2.17.26-baseline_2/adc1923395f2bcd6ae89b5d331d278364d1848ed/ubuntu/jammy/flavors/default jammy/main amd64 librados2 amd64 20.2.0-670-gadc19233-1jammy [3561 kB] 2026-02-18T08:08:23.930 INFO:teuthology.orchestra.run.trial163.stdout:0 upgraded, 87 newly installed, 0 to remove and 53 not upgraded. 2026-02-18T08:08:23.930 INFO:teuthology.orchestra.run.trial163.stdout:Need to get 283 MB of archives. 2026-02-18T08:08:23.930 INFO:teuthology.orchestra.run.trial163.stdout:After this operation, 1130 MB of additional disk space will be used. 2026-02-18T08:08:23.931 INFO:teuthology.orchestra.run.trial163.stdout:Get:1 https://2.chacra.ceph.com/r/ceph/20.2.1_2.17.26-baseline_2/adc1923395f2bcd6ae89b5d331d278364d1848ed/ubuntu/jammy/flavors/default jammy/main amd64 librados2 amd64 20.2.0-670-gadc19233-1jammy [3561 kB] 2026-02-18T08:08:24.030 INFO:teuthology.orchestra.run.trial034.stdout:Get:2 https://2.chacra.ceph.com/r/ceph/20.2.1_2.17.26-baseline_2/adc1923395f2bcd6ae89b5d331d278364d1848ed/ubuntu/jammy/flavors/default jammy/main amd64 librbd1 amd64 20.2.0-670-gadc19233-1jammy [2867 kB] 2026-02-18T08:08:24.097 INFO:teuthology.orchestra.run.trial034.stdout:Get:3 https://2.chacra.ceph.com/r/ceph/20.2.1_2.17.26-baseline_2/adc1923395f2bcd6ae89b5d331d278364d1848ed/ubuntu/jammy/flavors/default jammy/main amd64 libcephfs2 amd64 20.2.0-670-gadc19233-1jammy [833 kB] 2026-02-18T08:08:24.119 INFO:teuthology.orchestra.run.trial034.stdout:Get:4 https://2.chacra.ceph.com/r/ceph/20.2.1_2.17.26-baseline_2/adc1923395f2bcd6ae89b5d331d278364d1848ed/ubuntu/jammy/flavors/default jammy/main amd64 python3-rados amd64 20.2.0-670-gadc19233-1jammy [364 kB] 2026-02-18T08:08:24.132 INFO:teuthology.orchestra.run.trial034.stdout:Get:5 https://2.chacra.ceph.com/r/ceph/20.2.1_2.17.26-baseline_2/adc1923395f2bcd6ae89b5d331d278364d1848ed/ubuntu/jammy/flavors/default jammy/main amd64 python3-ceph-argparse all 20.2.0-670-gadc19233-1jammy [32.8 kB] 2026-02-18T08:08:24.138 INFO:teuthology.orchestra.run.trial163.stdout:Get:2 https://2.chacra.ceph.com/r/ceph/20.2.1_2.17.26-baseline_2/adc1923395f2bcd6ae89b5d331d278364d1848ed/ubuntu/jammy/flavors/default jammy/main amd64 librbd1 amd64 20.2.0-670-gadc19233-1jammy [2867 kB] 2026-02-18T08:08:24.151 INFO:teuthology.orchestra.run.trial034.stdout:Get:6 https://2.chacra.ceph.com/r/ceph/20.2.1_2.17.26-baseline_2/adc1923395f2bcd6ae89b5d331d278364d1848ed/ubuntu/jammy/flavors/default jammy/main amd64 python3-cephfs amd64 20.2.0-670-gadc19233-1jammy [184 kB] 2026-02-18T08:08:24.169 INFO:teuthology.orchestra.run.trial034.stdout:Get:7 https://archive.ubuntu.com/ubuntu jammy/main amd64 liblttng-ust1 amd64 2.13.1-1ubuntu1 [190 kB] 2026-02-18T08:08:24.194 INFO:teuthology.orchestra.run.trial163.stdout:Get:3 https://2.chacra.ceph.com/r/ceph/20.2.1_2.17.26-baseline_2/adc1923395f2bcd6ae89b5d331d278364d1848ed/ubuntu/jammy/flavors/default jammy/main amd64 libcephfs2 amd64 20.2.0-670-gadc19233-1jammy [833 kB] 2026-02-18T08:08:24.205 INFO:teuthology.orchestra.run.trial163.stdout:Get:4 https://2.chacra.ceph.com/r/ceph/20.2.1_2.17.26-baseline_2/adc1923395f2bcd6ae89b5d331d278364d1848ed/ubuntu/jammy/flavors/default jammy/main amd64 python3-rados amd64 20.2.0-670-gadc19233-1jammy [364 kB] 2026-02-18T08:08:24.215 INFO:teuthology.orchestra.run.trial163.stdout:Get:5 https://2.chacra.ceph.com/r/ceph/20.2.1_2.17.26-baseline_2/adc1923395f2bcd6ae89b5d331d278364d1848ed/ubuntu/jammy/flavors/default jammy/main amd64 python3-ceph-argparse all 20.2.0-670-gadc19233-1jammy [32.8 kB] 2026-02-18T08:08:24.216 INFO:teuthology.orchestra.run.trial163.stdout:Get:6 https://2.chacra.ceph.com/r/ceph/20.2.1_2.17.26-baseline_2/adc1923395f2bcd6ae89b5d331d278364d1848ed/ubuntu/jammy/flavors/default jammy/main amd64 python3-cephfs amd64 20.2.0-670-gadc19233-1jammy [184 kB] 2026-02-18T08:08:24.230 INFO:teuthology.orchestra.run.trial163.stdout:Get:7 https://2.chacra.ceph.com/r/ceph/20.2.1_2.17.26-baseline_2/adc1923395f2bcd6ae89b5d331d278364d1848ed/ubuntu/jammy/flavors/default jammy/main amd64 python3-ceph-common all 20.2.0-670-gadc19233-1jammy [83.9 kB] 2026-02-18T08:08:24.230 INFO:teuthology.orchestra.run.trial034.stdout:Get:8 https://2.chacra.ceph.com/r/ceph/20.2.1_2.17.26-baseline_2/adc1923395f2bcd6ae89b5d331d278364d1848ed/ubuntu/jammy/flavors/default jammy/main amd64 python3-ceph-common all 20.2.0-670-gadc19233-1jammy [83.9 kB] 2026-02-18T08:08:24.244 INFO:teuthology.orchestra.run.trial163.stdout:Get:8 https://2.chacra.ceph.com/r/ceph/20.2.1_2.17.26-baseline_2/adc1923395f2bcd6ae89b5d331d278364d1848ed/ubuntu/jammy/flavors/default jammy/main amd64 python3-rbd amd64 20.2.0-670-gadc19233-1jammy [341 kB] 2026-02-18T08:08:24.245 INFO:teuthology.orchestra.run.trial034.stdout:Get:9 https://2.chacra.ceph.com/r/ceph/20.2.1_2.17.26-baseline_2/adc1923395f2bcd6ae89b5d331d278364d1848ed/ubuntu/jammy/flavors/default jammy/main amd64 python3-rbd amd64 20.2.0-670-gadc19233-1jammy [341 kB] 2026-02-18T08:08:24.267 INFO:teuthology.orchestra.run.trial163.stdout:Get:9 https://archive.ubuntu.com/ubuntu jammy/main amd64 liblttng-ust1 amd64 2.13.1-1ubuntu1 [190 kB] 2026-02-18T08:08:24.288 INFO:teuthology.orchestra.run.trial034.stdout:Get:10 https://2.chacra.ceph.com/r/ceph/20.2.1_2.17.26-baseline_2/adc1923395f2bcd6ae89b5d331d278364d1848ed/ubuntu/jammy/flavors/default jammy/main amd64 librgw2 amd64 20.2.0-670-gadc19233-1jammy [8687 kB] 2026-02-18T08:08:24.289 INFO:teuthology.orchestra.run.trial163.stdout:Get:10 https://2.chacra.ceph.com/r/ceph/20.2.1_2.17.26-baseline_2/adc1923395f2bcd6ae89b5d331d278364d1848ed/ubuntu/jammy/flavors/default jammy/main amd64 librgw2 amd64 20.2.0-670-gadc19233-1jammy [8687 kB] 2026-02-18T08:08:24.505 INFO:teuthology.orchestra.run.trial034.stdout:Get:11 https://archive.ubuntu.com/ubuntu jammy/universe amd64 libdouble-conversion3 amd64 3.1.7-4 [39.0 kB] 2026-02-18T08:08:24.516 INFO:teuthology.orchestra.run.trial034.stdout:Get:12 https://archive.ubuntu.com/ubuntu jammy-updates/main amd64 libpcre2-16-0 amd64 10.39-3ubuntu0.1 [203 kB] 2026-02-18T08:08:24.523 INFO:teuthology.orchestra.run.trial034.stdout:Get:13 https://2.chacra.ceph.com/r/ceph/20.2.1_2.17.26-baseline_2/adc1923395f2bcd6ae89b5d331d278364d1848ed/ubuntu/jammy/flavors/default jammy/main amd64 python3-rgw amd64 20.2.0-670-gadc19233-1jammy [112 kB] 2026-02-18T08:08:24.585 INFO:teuthology.orchestra.run.trial034.stdout:Get:14 https://archive.ubuntu.com/ubuntu jammy-updates/universe amd64 libqt5core5a amd64 5.15.3+dfsg-2ubuntu0.2 [2006 kB] 2026-02-18T08:08:24.604 INFO:teuthology.orchestra.run.trial163.stdout:Get:11 https://2.chacra.ceph.com/r/ceph/20.2.1_2.17.26-baseline_2/adc1923395f2bcd6ae89b5d331d278364d1848ed/ubuntu/jammy/flavors/default jammy/main amd64 python3-rgw amd64 20.2.0-670-gadc19233-1jammy [112 kB] 2026-02-18T08:08:24.677 INFO:teuthology.orchestra.run.trial163.stdout:Get:12 https://archive.ubuntu.com/ubuntu jammy/universe amd64 libdouble-conversion3 amd64 3.1.7-4 [39.0 kB] 2026-02-18T08:08:24.690 INFO:teuthology.orchestra.run.trial163.stdout:Get:13 https://archive.ubuntu.com/ubuntu jammy-updates/main amd64 libpcre2-16-0 amd64 10.39-3ubuntu0.1 [203 kB] 2026-02-18T08:08:24.775 INFO:teuthology.orchestra.run.trial163.stdout:Get:14 https://archive.ubuntu.com/ubuntu jammy-updates/universe amd64 libqt5core5a amd64 5.15.3+dfsg-2ubuntu0.2 [2006 kB] 2026-02-18T08:08:24.785 INFO:teuthology.orchestra.run.trial034.stdout:Get:15 https://archive.ubuntu.com/ubuntu jammy-updates/universe amd64 libqt5dbus5 amd64 5.15.3+dfsg-2ubuntu0.2 [222 kB] 2026-02-18T08:08:24.788 INFO:teuthology.orchestra.run.trial034.stdout:Get:16 https://2.chacra.ceph.com/r/ceph/20.2.1_2.17.26-baseline_2/adc1923395f2bcd6ae89b5d331d278364d1848ed/ubuntu/jammy/flavors/default jammy/main amd64 libradosstriper1 amd64 20.2.0-670-gadc19233-1jammy [261 kB] 2026-02-18T08:08:24.788 INFO:teuthology.orchestra.run.trial163.stdout:Get:15 https://2.chacra.ceph.com/r/ceph/20.2.1_2.17.26-baseline_2/adc1923395f2bcd6ae89b5d331d278364d1848ed/ubuntu/jammy/flavors/default jammy/main amd64 libradosstriper1 amd64 20.2.0-670-gadc19233-1jammy [261 kB] 2026-02-18T08:08:24.795 INFO:teuthology.orchestra.run.trial034.stdout:Get:17 https://archive.ubuntu.com/ubuntu jammy-updates/universe amd64 libqt5network5 amd64 5.15.3+dfsg-2ubuntu0.2 [731 kB] 2026-02-18T08:08:24.822 INFO:teuthology.orchestra.run.trial034.stdout:Get:18 https://archive.ubuntu.com/ubuntu jammy/universe amd64 libthrift-0.16.0 amd64 0.16.0-2 [267 kB] 2026-02-18T08:08:24.829 INFO:teuthology.orchestra.run.trial034.stdout:Get:19 https://2.chacra.ceph.com/r/ceph/20.2.1_2.17.26-baseline_2/adc1923395f2bcd6ae89b5d331d278364d1848ed/ubuntu/jammy/flavors/default jammy/main amd64 ceph-common amd64 20.2.0-670-gadc19233-1jammy [29.3 MB] 2026-02-18T08:08:24.829 INFO:teuthology.orchestra.run.trial163.stdout:Get:16 https://2.chacra.ceph.com/r/ceph/20.2.1_2.17.26-baseline_2/adc1923395f2bcd6ae89b5d331d278364d1848ed/ubuntu/jammy/flavors/default jammy/main amd64 ceph-common amd64 20.2.0-670-gadc19233-1jammy [29.3 MB] 2026-02-18T08:08:24.830 INFO:teuthology.orchestra.run.trial034.stdout:Get:20 https://archive.ubuntu.com/ubuntu jammy/universe amd64 libnbd0 amd64 1.10.5-1 [71.3 kB] 2026-02-18T08:08:24.832 INFO:teuthology.orchestra.run.trial034.stdout:Get:21 https://archive.ubuntu.com/ubuntu jammy/main amd64 python3-wcwidth all 0.2.5+dfsg1-1 [21.9 kB] 2026-02-18T08:08:24.833 INFO:teuthology.orchestra.run.trial034.stdout:Get:22 https://archive.ubuntu.com/ubuntu jammy/main amd64 python3-prettytable all 2.5.0-2 [31.3 kB] 2026-02-18T08:08:24.834 INFO:teuthology.orchestra.run.trial034.stdout:Get:23 https://archive.ubuntu.com/ubuntu jammy/universe amd64 librdkafka1 amd64 1.8.0-1build1 [633 kB] 2026-02-18T08:08:24.851 INFO:teuthology.orchestra.run.trial034.stdout:Get:24 https://archive.ubuntu.com/ubuntu jammy-updates/main amd64 liboath0 amd64 2.6.7-3ubuntu0.1 [41.3 kB] 2026-02-18T08:08:24.852 INFO:teuthology.orchestra.run.trial034.stdout:Get:25 https://archive.ubuntu.com/ubuntu jammy/main amd64 python3-jaraco.functools all 3.4.0-2 [9030 B] 2026-02-18T08:08:24.857 INFO:teuthology.orchestra.run.trial034.stdout:Get:26 https://archive.ubuntu.com/ubuntu jammy-updates/main amd64 python3-cheroot all 8.5.2+ds1-1ubuntu3.1 [71.1 kB] 2026-02-18T08:08:24.924 INFO:teuthology.orchestra.run.trial034.stdout:Get:27 https://archive.ubuntu.com/ubuntu jammy/main amd64 python3-jaraco.classes all 3.2.1-3 [6452 B] 2026-02-18T08:08:24.924 INFO:teuthology.orchestra.run.trial034.stdout:Get:28 https://archive.ubuntu.com/ubuntu jammy/main amd64 python3-jaraco.text all 3.6.0-2 [8716 B] 2026-02-18T08:08:24.924 INFO:teuthology.orchestra.run.trial034.stdout:Get:29 https://archive.ubuntu.com/ubuntu jammy/main amd64 python3-jaraco.collections all 3.4.0-2 [11.4 kB] 2026-02-18T08:08:24.924 INFO:teuthology.orchestra.run.trial034.stdout:Get:30 https://archive.ubuntu.com/ubuntu jammy/main amd64 python3-tempora all 4.1.2-1 [14.8 kB] 2026-02-18T08:08:24.925 INFO:teuthology.orchestra.run.trial034.stdout:Get:31 https://archive.ubuntu.com/ubuntu jammy/main amd64 python3-portend all 3.0.0-1 [7240 B] 2026-02-18T08:08:24.925 INFO:teuthology.orchestra.run.trial034.stdout:Get:32 https://archive.ubuntu.com/ubuntu jammy/main amd64 python3-zc.lockfile all 2.0-1 [8980 B] 2026-02-18T08:08:24.925 INFO:teuthology.orchestra.run.trial034.stdout:Get:33 https://archive.ubuntu.com/ubuntu jammy/main amd64 python3-cherrypy3 all 18.6.1-4 [208 kB] 2026-02-18T08:08:24.929 INFO:teuthology.orchestra.run.trial034.stdout:Get:34 https://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-natsort all 8.0.2-1 [35.3 kB] 2026-02-18T08:08:24.930 INFO:teuthology.orchestra.run.trial034.stdout:Get:35 https://archive.ubuntu.com/ubuntu jammy/universe amd64 libfuse2 amd64 2.9.9-5ubuntu3 [90.3 kB] 2026-02-18T08:08:24.999 INFO:teuthology.orchestra.run.trial034.stdout:Get:36 https://archive.ubuntu.com/ubuntu jammy-updates/universe amd64 python3-asyncssh all 2.5.0-1ubuntu0.1 [189 kB] 2026-02-18T08:08:25.001 INFO:teuthology.orchestra.run.trial034.stdout:Get:37 https://archive.ubuntu.com/ubuntu jammy/main amd64 python3-repoze.lru all 0.7-2 [12.1 kB] 2026-02-18T08:08:25.001 INFO:teuthology.orchestra.run.trial034.stdout:Get:38 https://archive.ubuntu.com/ubuntu jammy/main amd64 python3-routes all 2.5.1-1ubuntu1 [89.0 kB] 2026-02-18T08:08:25.002 INFO:teuthology.orchestra.run.trial034.stdout:Get:39 https://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-sklearn-lib amd64 0.23.2-5ubuntu6 [2058 kB] 2026-02-18T08:08:25.019 INFO:teuthology.orchestra.run.trial163.stdout:Get:17 https://archive.ubuntu.com/ubuntu jammy-updates/universe amd64 libqt5dbus5 amd64 5.15.3+dfsg-2ubuntu0.2 [222 kB] 2026-02-18T08:08:25.027 INFO:teuthology.orchestra.run.trial034.stdout:Get:40 https://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-joblib all 0.17.0-4ubuntu1 [204 kB] 2026-02-18T08:08:25.029 INFO:teuthology.orchestra.run.trial034.stdout:Get:41 https://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-threadpoolctl all 3.1.0-1 [21.3 kB] 2026-02-18T08:08:25.029 INFO:teuthology.orchestra.run.trial034.stdout:Get:42 https://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-sklearn all 0.23.2-5ubuntu6 [1829 kB] 2026-02-18T08:08:25.032 INFO:teuthology.orchestra.run.trial163.stdout:Get:18 https://archive.ubuntu.com/ubuntu jammy-updates/universe amd64 libqt5network5 amd64 5.15.3+dfsg-2ubuntu0.2 [731 kB] 2026-02-18T08:08:25.064 INFO:teuthology.orchestra.run.trial163.stdout:Get:19 https://archive.ubuntu.com/ubuntu jammy/universe amd64 libthrift-0.16.0 amd64 0.16.0-2 [267 kB] 2026-02-18T08:08:25.074 INFO:teuthology.orchestra.run.trial163.stdout:Get:20 https://archive.ubuntu.com/ubuntu jammy/universe amd64 libnbd0 amd64 1.10.5-1 [71.3 kB] 2026-02-18T08:08:25.076 INFO:teuthology.orchestra.run.trial163.stdout:Get:21 https://archive.ubuntu.com/ubuntu jammy/main amd64 python3-wcwidth all 0.2.5+dfsg1-1 [21.9 kB] 2026-02-18T08:08:25.078 INFO:teuthology.orchestra.run.trial163.stdout:Get:22 https://archive.ubuntu.com/ubuntu jammy/main amd64 python3-prettytable all 2.5.0-2 [31.3 kB] 2026-02-18T08:08:25.079 INFO:teuthology.orchestra.run.trial163.stdout:Get:23 https://archive.ubuntu.com/ubuntu jammy/universe amd64 librdkafka1 amd64 1.8.0-1build1 [633 kB] 2026-02-18T08:08:25.086 INFO:teuthology.orchestra.run.trial034.stdout:Get:43 https://archive.ubuntu.com/ubuntu jammy/main amd64 python3-cachetools all 5.0.0-1 [9722 B] 2026-02-18T08:08:25.087 INFO:teuthology.orchestra.run.trial034.stdout:Get:44 https://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-rsa all 4.8-1 [28.4 kB] 2026-02-18T08:08:25.087 INFO:teuthology.orchestra.run.trial034.stdout:Get:45 https://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-google-auth all 1.5.1-3 [35.7 kB] 2026-02-18T08:08:25.087 INFO:teuthology.orchestra.run.trial034.stdout:Get:46 https://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-requests-oauthlib all 1.3.0+ds-0.1 [18.7 kB] 2026-02-18T08:08:25.088 INFO:teuthology.orchestra.run.trial034.stdout:Get:47 https://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-websocket all 1.2.3-1 [34.7 kB] 2026-02-18T08:08:25.088 INFO:teuthology.orchestra.run.trial034.stdout:Get:48 https://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-kubernetes all 12.0.1-1ubuntu1 [353 kB] 2026-02-18T08:08:25.098 INFO:teuthology.orchestra.run.trial163.stdout:Get:24 https://archive.ubuntu.com/ubuntu jammy-updates/main amd64 liboath0 amd64 2.6.7-3ubuntu0.1 [41.3 kB] 2026-02-18T08:08:25.099 INFO:teuthology.orchestra.run.trial163.stdout:Get:25 https://archive.ubuntu.com/ubuntu jammy/main amd64 python3-jaraco.functools all 3.4.0-2 [9030 B] 2026-02-18T08:08:25.106 INFO:teuthology.orchestra.run.trial163.stdout:Get:26 https://archive.ubuntu.com/ubuntu jammy-updates/main amd64 python3-cheroot all 8.5.2+ds1-1ubuntu3.1 [71.1 kB] 2026-02-18T08:08:25.147 INFO:teuthology.orchestra.run.trial034.stdout:Get:49 https://archive.ubuntu.com/ubuntu jammy/main amd64 libonig5 amd64 6.9.7.1-2build1 [172 kB] 2026-02-18T08:08:25.148 INFO:teuthology.orchestra.run.trial034.stdout:Get:50 https://archive.ubuntu.com/ubuntu jammy-updates/main amd64 libjq1 amd64 1.6-2.1ubuntu3.1 [133 kB] 2026-02-18T08:08:25.149 INFO:teuthology.orchestra.run.trial034.stdout:Get:51 https://archive.ubuntu.com/ubuntu jammy-updates/main amd64 jq amd64 1.6-2.1ubuntu3.1 [52.5 kB] 2026-02-18T08:08:25.187 INFO:teuthology.orchestra.run.trial163.stdout:Get:27 https://archive.ubuntu.com/ubuntu jammy/main amd64 python3-jaraco.classes all 3.2.1-3 [6452 B] 2026-02-18T08:08:25.188 INFO:teuthology.orchestra.run.trial163.stdout:Get:28 https://archive.ubuntu.com/ubuntu jammy/main amd64 python3-jaraco.text all 3.6.0-2 [8716 B] 2026-02-18T08:08:25.188 INFO:teuthology.orchestra.run.trial163.stdout:Get:29 https://archive.ubuntu.com/ubuntu jammy/main amd64 python3-jaraco.collections all 3.4.0-2 [11.4 kB] 2026-02-18T08:08:25.189 INFO:teuthology.orchestra.run.trial163.stdout:Get:30 https://archive.ubuntu.com/ubuntu jammy/main amd64 python3-tempora all 4.1.2-1 [14.8 kB] 2026-02-18T08:08:25.189 INFO:teuthology.orchestra.run.trial163.stdout:Get:31 https://archive.ubuntu.com/ubuntu jammy/main amd64 python3-portend all 3.0.0-1 [7240 B] 2026-02-18T08:08:25.189 INFO:teuthology.orchestra.run.trial163.stdout:Get:32 https://archive.ubuntu.com/ubuntu jammy/main amd64 python3-zc.lockfile all 2.0-1 [8980 B] 2026-02-18T08:08:25.189 INFO:teuthology.orchestra.run.trial163.stdout:Get:33 https://archive.ubuntu.com/ubuntu jammy/main amd64 python3-cherrypy3 all 18.6.1-4 [208 kB] 2026-02-18T08:08:25.194 INFO:teuthology.orchestra.run.trial163.stdout:Get:34 https://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-natsort all 8.0.2-1 [35.3 kB] 2026-02-18T08:08:25.195 INFO:teuthology.orchestra.run.trial163.stdout:Get:35 https://archive.ubuntu.com/ubuntu jammy/universe amd64 libfuse2 amd64 2.9.9-5ubuntu3 [90.3 kB] 2026-02-18T08:08:25.219 INFO:teuthology.orchestra.run.trial034.stdout:Get:52 https://archive.ubuntu.com/ubuntu jammy/main amd64 socat amd64 1.7.4.1-3ubuntu4 [349 kB] 2026-02-18T08:08:25.222 INFO:teuthology.orchestra.run.trial034.stdout:Get:53 https://archive.ubuntu.com/ubuntu jammy/universe amd64 xmlstarlet amd64 1.6.1-2.1 [265 kB] 2026-02-18T08:08:25.224 INFO:teuthology.orchestra.run.trial034.stdout:Get:54 https://archive.ubuntu.com/ubuntu jammy-updates/main amd64 nvme-cli amd64 1.16-3ubuntu0.3 [474 kB] 2026-02-18T08:08:25.228 INFO:teuthology.orchestra.run.trial034.stdout:Get:55 https://archive.ubuntu.com/ubuntu jammy-updates/universe amd64 python-asyncssh-doc all 2.5.0-1ubuntu0.1 [309 kB] 2026-02-18T08:08:25.230 INFO:teuthology.orchestra.run.trial034.stdout:Get:56 https://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-iniconfig all 1.1.1-2 [6024 B] 2026-02-18T08:08:25.230 INFO:teuthology.orchestra.run.trial034.stdout:Get:57 https://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-pluggy all 0.13.0-7.1 [19.0 kB] 2026-02-18T08:08:25.231 INFO:teuthology.orchestra.run.trial034.stdout:Get:58 https://archive.ubuntu.com/ubuntu jammy/main amd64 python3-psutil amd64 5.9.0-1build1 [158 kB] 2026-02-18T08:08:25.280 INFO:teuthology.orchestra.run.trial163.stdout:Get:36 https://archive.ubuntu.com/ubuntu jammy-updates/universe amd64 python3-asyncssh all 2.5.0-1ubuntu0.1 [189 kB] 2026-02-18T08:08:25.283 INFO:teuthology.orchestra.run.trial163.stdout:Get:37 https://archive.ubuntu.com/ubuntu jammy/main amd64 python3-repoze.lru all 0.7-2 [12.1 kB] 2026-02-18T08:08:25.283 INFO:teuthology.orchestra.run.trial163.stdout:Get:38 https://archive.ubuntu.com/ubuntu jammy/main amd64 python3-routes all 2.5.1-1ubuntu1 [89.0 kB] 2026-02-18T08:08:25.284 INFO:teuthology.orchestra.run.trial163.stdout:Get:39 https://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-sklearn-lib amd64 0.23.2-5ubuntu6 [2058 kB] 2026-02-18T08:08:25.291 INFO:teuthology.orchestra.run.trial034.stdout:Get:59 https://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-py all 1.10.0-1 [71.9 kB] 2026-02-18T08:08:25.291 INFO:teuthology.orchestra.run.trial034.stdout:Get:60 https://archive.ubuntu.com/ubuntu jammy-updates/main amd64 python3-pygments all 2.11.2+dfsg-2ubuntu0.1 [750 kB] 2026-02-18T08:08:25.297 INFO:teuthology.orchestra.run.trial034.stdout:Get:61 https://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-toml all 0.10.2-1 [16.5 kB] 2026-02-18T08:08:25.314 INFO:teuthology.orchestra.run.trial163.stdout:Get:40 https://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-joblib all 0.17.0-4ubuntu1 [204 kB] 2026-02-18T08:08:25.316 INFO:teuthology.orchestra.run.trial163.stdout:Get:41 https://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-threadpoolctl all 3.1.0-1 [21.3 kB] 2026-02-18T08:08:25.317 INFO:teuthology.orchestra.run.trial163.stdout:Get:42 https://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-sklearn all 0.23.2-5ubuntu6 [1829 kB] 2026-02-18T08:08:25.362 INFO:teuthology.orchestra.run.trial034.stdout:Get:62 https://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-pytest all 6.2.5-1ubuntu2 [214 kB] 2026-02-18T08:08:25.364 INFO:teuthology.orchestra.run.trial034.stdout:Get:63 https://archive.ubuntu.com/ubuntu jammy/main amd64 python3-simplejson amd64 3.17.6-1build1 [54.7 kB] 2026-02-18T08:08:25.365 INFO:teuthology.orchestra.run.trial034.stdout:Get:64 https://archive.ubuntu.com/ubuntu jammy-updates/main amd64 python3-webob all 1:1.8.6-1.1ubuntu0.1 [86.7 kB] 2026-02-18T08:08:25.365 INFO:teuthology.orchestra.run.trial034.stdout:Get:65 https://archive.ubuntu.com/ubuntu jammy/universe amd64 qttranslations5-l10n all 5.15.3-1 [1983 kB] 2026-02-18T08:08:25.387 INFO:teuthology.orchestra.run.trial163.stdout:Get:43 https://archive.ubuntu.com/ubuntu jammy/main amd64 python3-cachetools all 5.0.0-1 [9722 B] 2026-02-18T08:08:25.387 INFO:teuthology.orchestra.run.trial163.stdout:Get:44 https://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-rsa all 4.8-1 [28.4 kB] 2026-02-18T08:08:25.387 INFO:teuthology.orchestra.run.trial163.stdout:Get:45 https://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-google-auth all 1.5.1-3 [35.7 kB] 2026-02-18T08:08:25.388 INFO:teuthology.orchestra.run.trial163.stdout:Get:46 https://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-requests-oauthlib all 1.3.0+ds-0.1 [18.7 kB] 2026-02-18T08:08:25.388 INFO:teuthology.orchestra.run.trial163.stdout:Get:47 https://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-websocket all 1.2.3-1 [34.7 kB] 2026-02-18T08:08:25.389 INFO:teuthology.orchestra.run.trial163.stdout:Get:48 https://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-kubernetes all 12.0.1-1ubuntu1 [353 kB] 2026-02-18T08:08:25.460 INFO:teuthology.orchestra.run.trial163.stdout:Get:49 https://archive.ubuntu.com/ubuntu jammy/main amd64 libonig5 amd64 6.9.7.1-2build1 [172 kB] 2026-02-18T08:08:25.462 INFO:teuthology.orchestra.run.trial163.stdout:Get:50 https://archive.ubuntu.com/ubuntu jammy-updates/main amd64 libjq1 amd64 1.6-2.1ubuntu3.1 [133 kB] 2026-02-18T08:08:25.463 INFO:teuthology.orchestra.run.trial163.stdout:Get:51 https://archive.ubuntu.com/ubuntu jammy-updates/main amd64 jq amd64 1.6-2.1ubuntu3.1 [52.5 kB] 2026-02-18T08:08:25.548 INFO:teuthology.orchestra.run.trial163.stdout:Get:52 https://archive.ubuntu.com/ubuntu jammy/main amd64 socat amd64 1.7.4.1-3ubuntu4 [349 kB] 2026-02-18T08:08:25.551 INFO:teuthology.orchestra.run.trial163.stdout:Get:53 https://archive.ubuntu.com/ubuntu jammy/universe amd64 xmlstarlet amd64 1.6.1-2.1 [265 kB] 2026-02-18T08:08:25.554 INFO:teuthology.orchestra.run.trial163.stdout:Get:54 https://archive.ubuntu.com/ubuntu jammy-updates/main amd64 nvme-cli amd64 1.16-3ubuntu0.3 [474 kB] 2026-02-18T08:08:25.558 INFO:teuthology.orchestra.run.trial163.stdout:Get:55 https://archive.ubuntu.com/ubuntu jammy-updates/universe amd64 python-asyncssh-doc all 2.5.0-1ubuntu0.1 [309 kB] 2026-02-18T08:08:25.566 INFO:teuthology.orchestra.run.trial163.stdout:Get:56 https://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-iniconfig all 1.1.1-2 [6024 B] 2026-02-18T08:08:25.566 INFO:teuthology.orchestra.run.trial163.stdout:Get:57 https://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-pluggy all 0.13.0-7.1 [19.0 kB] 2026-02-18T08:08:25.566 INFO:teuthology.orchestra.run.trial163.stdout:Get:58 https://archive.ubuntu.com/ubuntu jammy/main amd64 python3-psutil amd64 5.9.0-1build1 [158 kB] 2026-02-18T08:08:25.636 INFO:teuthology.orchestra.run.trial163.stdout:Get:59 https://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-py all 1.10.0-1 [71.9 kB] 2026-02-18T08:08:25.636 INFO:teuthology.orchestra.run.trial163.stdout:Get:60 https://archive.ubuntu.com/ubuntu jammy-updates/main amd64 python3-pygments all 2.11.2+dfsg-2ubuntu0.1 [750 kB] 2026-02-18T08:08:25.644 INFO:teuthology.orchestra.run.trial163.stdout:Get:61 https://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-toml all 0.10.2-1 [16.5 kB] 2026-02-18T08:08:25.727 INFO:teuthology.orchestra.run.trial163.stdout:Get:62 https://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-pytest all 6.2.5-1ubuntu2 [214 kB] 2026-02-18T08:08:25.729 INFO:teuthology.orchestra.run.trial163.stdout:Get:63 https://archive.ubuntu.com/ubuntu jammy/main amd64 python3-simplejson amd64 3.17.6-1build1 [54.7 kB] 2026-02-18T08:08:25.729 INFO:teuthology.orchestra.run.trial163.stdout:Get:64 https://archive.ubuntu.com/ubuntu jammy-updates/main amd64 python3-webob all 1:1.8.6-1.1ubuntu0.1 [86.7 kB] 2026-02-18T08:08:25.730 INFO:teuthology.orchestra.run.trial163.stdout:Get:65 https://archive.ubuntu.com/ubuntu jammy/universe amd64 qttranslations5-l10n all 5.15.3-1 [1983 kB] 2026-02-18T08:08:25.771 INFO:teuthology.orchestra.run.trial034.stdout:Get:66 https://2.chacra.ceph.com/r/ceph/20.2.1_2.17.26-baseline_2/adc1923395f2bcd6ae89b5d331d278364d1848ed/ubuntu/jammy/flavors/default jammy/main amd64 ceph-base amd64 20.2.0-670-gadc19233-1jammy [5416 kB] 2026-02-18T08:08:25.832 INFO:teuthology.orchestra.run.trial163.stdout:Get:66 https://2.chacra.ceph.com/r/ceph/20.2.1_2.17.26-baseline_2/adc1923395f2bcd6ae89b5d331d278364d1848ed/ubuntu/jammy/flavors/default jammy/main amd64 ceph-base amd64 20.2.0-670-gadc19233-1jammy [5416 kB] 2026-02-18T08:08:25.911 INFO:teuthology.orchestra.run.trial034.stdout:Get:67 https://2.chacra.ceph.com/r/ceph/20.2.1_2.17.26-baseline_2/adc1923395f2bcd6ae89b5d331d278364d1848ed/ubuntu/jammy/flavors/default jammy/main amd64 ceph-mgr-modules-core all 20.2.0-670-gadc19233-1jammy [246 kB] 2026-02-18T08:08:25.916 INFO:teuthology.orchestra.run.trial034.stdout:Get:68 https://2.chacra.ceph.com/r/ceph/20.2.1_2.17.26-baseline_2/adc1923395f2bcd6ae89b5d331d278364d1848ed/ubuntu/jammy/flavors/default jammy/main amd64 libsqlite3-mod-ceph amd64 20.2.0-670-gadc19233-1jammy [124 kB] 2026-02-18T08:08:25.920 INFO:teuthology.orchestra.run.trial034.stdout:Get:69 https://2.chacra.ceph.com/r/ceph/20.2.1_2.17.26-baseline_2/adc1923395f2bcd6ae89b5d331d278364d1848ed/ubuntu/jammy/flavors/default jammy/main amd64 ceph-mgr amd64 20.2.0-670-gadc19233-1jammy [907 kB] 2026-02-18T08:08:25.959 INFO:teuthology.orchestra.run.trial034.stdout:Get:70 https://2.chacra.ceph.com/r/ceph/20.2.1_2.17.26-baseline_2/adc1923395f2bcd6ae89b5d331d278364d1848ed/ubuntu/jammy/flavors/default jammy/main amd64 ceph-mon amd64 20.2.0-670-gadc19233-1jammy [6397 kB] 2026-02-18T08:08:26.001 INFO:teuthology.orchestra.run.trial163.stdout:Get:67 https://2.chacra.ceph.com/r/ceph/20.2.1_2.17.26-baseline_2/adc1923395f2bcd6ae89b5d331d278364d1848ed/ubuntu/jammy/flavors/default jammy/main amd64 ceph-mgr-modules-core all 20.2.0-670-gadc19233-1jammy [246 kB] 2026-02-18T08:08:26.004 INFO:teuthology.orchestra.run.trial163.stdout:Get:68 https://2.chacra.ceph.com/r/ceph/20.2.1_2.17.26-baseline_2/adc1923395f2bcd6ae89b5d331d278364d1848ed/ubuntu/jammy/flavors/default jammy/main amd64 libsqlite3-mod-ceph amd64 20.2.0-670-gadc19233-1jammy [124 kB] 2026-02-18T08:08:26.007 INFO:teuthology.orchestra.run.trial163.stdout:Get:69 https://2.chacra.ceph.com/r/ceph/20.2.1_2.17.26-baseline_2/adc1923395f2bcd6ae89b5d331d278364d1848ed/ubuntu/jammy/flavors/default jammy/main amd64 ceph-mgr amd64 20.2.0-670-gadc19233-1jammy [907 kB] 2026-02-18T08:08:26.039 INFO:teuthology.orchestra.run.trial163.stdout:Get:70 https://2.chacra.ceph.com/r/ceph/20.2.1_2.17.26-baseline_2/adc1923395f2bcd6ae89b5d331d278364d1848ed/ubuntu/jammy/flavors/default jammy/main amd64 ceph-mon amd64 20.2.0-670-gadc19233-1jammy [6397 kB] 2026-02-18T08:08:26.192 INFO:teuthology.orchestra.run.trial034.stdout:Get:71 https://2.chacra.ceph.com/r/ceph/20.2.1_2.17.26-baseline_2/adc1923395f2bcd6ae89b5d331d278364d1848ed/ubuntu/jammy/flavors/default jammy/main amd64 ceph-osd amd64 20.2.0-670-gadc19233-1jammy [21.7 MB] 2026-02-18T08:08:26.227 INFO:teuthology.orchestra.run.trial163.stdout:Get:71 https://2.chacra.ceph.com/r/ceph/20.2.1_2.17.26-baseline_2/adc1923395f2bcd6ae89b5d331d278364d1848ed/ubuntu/jammy/flavors/default jammy/main amd64 ceph-osd amd64 20.2.0-670-gadc19233-1jammy [21.7 MB] 2026-02-18T08:08:27.262 INFO:teuthology.orchestra.run.trial034.stdout:Get:72 https://2.chacra.ceph.com/r/ceph/20.2.1_2.17.26-baseline_2/adc1923395f2bcd6ae89b5d331d278364d1848ed/ubuntu/jammy/flavors/default jammy/main amd64 ceph amd64 20.2.0-670-gadc19233-1jammy [14.1 kB] 2026-02-18T08:08:27.263 INFO:teuthology.orchestra.run.trial163.stdout:Get:72 https://2.chacra.ceph.com/r/ceph/20.2.1_2.17.26-baseline_2/adc1923395f2bcd6ae89b5d331d278364d1848ed/ubuntu/jammy/flavors/default jammy/main amd64 ceph amd64 20.2.0-670-gadc19233-1jammy [14.1 kB] 2026-02-18T08:08:27.286 INFO:teuthology.orchestra.run.trial034.stdout:Get:73 https://2.chacra.ceph.com/r/ceph/20.2.1_2.17.26-baseline_2/adc1923395f2bcd6ae89b5d331d278364d1848ed/ubuntu/jammy/flavors/default jammy/main amd64 ceph-fuse amd64 20.2.0-670-gadc19233-1jammy [957 kB] 2026-02-18T08:08:27.287 INFO:teuthology.orchestra.run.trial163.stdout:Get:73 https://2.chacra.ceph.com/r/ceph/20.2.1_2.17.26-baseline_2/adc1923395f2bcd6ae89b5d331d278364d1848ed/ubuntu/jammy/flavors/default jammy/main amd64 ceph-fuse amd64 20.2.0-670-gadc19233-1jammy [957 kB] 2026-02-18T08:08:27.437 INFO:teuthology.orchestra.run.trial034.stdout:Get:74 https://2.chacra.ceph.com/r/ceph/20.2.1_2.17.26-baseline_2/adc1923395f2bcd6ae89b5d331d278364d1848ed/ubuntu/jammy/flavors/default jammy/main amd64 ceph-mds amd64 20.2.0-670-gadc19233-1jammy [2345 kB] 2026-02-18T08:08:27.438 INFO:teuthology.orchestra.run.trial163.stdout:Get:74 https://2.chacra.ceph.com/r/ceph/20.2.1_2.17.26-baseline_2/adc1923395f2bcd6ae89b5d331d278364d1848ed/ubuntu/jammy/flavors/default jammy/main amd64 ceph-mds amd64 20.2.0-670-gadc19233-1jammy [2345 kB] 2026-02-18T08:08:27.535 INFO:teuthology.orchestra.run.trial163.stdout:Get:75 https://2.chacra.ceph.com/r/ceph/20.2.1_2.17.26-baseline_2/adc1923395f2bcd6ae89b5d331d278364d1848ed/ubuntu/jammy/flavors/default jammy/main amd64 cephadm amd64 20.2.0-670-gadc19233-1jammy [1049 kB] 2026-02-18T08:08:27.536 INFO:teuthology.orchestra.run.trial034.stdout:Get:75 https://2.chacra.ceph.com/r/ceph/20.2.1_2.17.26-baseline_2/adc1923395f2bcd6ae89b5d331d278364d1848ed/ubuntu/jammy/flavors/default jammy/main amd64 cephadm amd64 20.2.0-670-gadc19233-1jammy [1049 kB] 2026-02-18T08:08:27.587 INFO:teuthology.orchestra.run.trial163.stdout:Get:76 https://2.chacra.ceph.com/r/ceph/20.2.1_2.17.26-baseline_2/adc1923395f2bcd6ae89b5d331d278364d1848ed/ubuntu/jammy/flavors/default jammy/main amd64 ceph-mgr-cephadm all 20.2.0-670-gadc19233-1jammy [179 kB] 2026-02-18T08:08:27.587 INFO:teuthology.orchestra.run.trial034.stdout:Get:76 https://2.chacra.ceph.com/r/ceph/20.2.1_2.17.26-baseline_2/adc1923395f2bcd6ae89b5d331d278364d1848ed/ubuntu/jammy/flavors/default jammy/main amd64 ceph-mgr-cephadm all 20.2.0-670-gadc19233-1jammy [179 kB] 2026-02-18T08:08:27.762 INFO:teuthology.orchestra.run.trial034.stdout:Get:77 https://2.chacra.ceph.com/r/ceph/20.2.1_2.17.26-baseline_2/adc1923395f2bcd6ae89b5d331d278364d1848ed/ubuntu/jammy/flavors/default jammy/main amd64 ceph-mgr-dashboard all 20.2.0-670-gadc19233-1jammy [45.5 MB] 2026-02-18T08:08:27.762 INFO:teuthology.orchestra.run.trial163.stdout:Get:77 https://2.chacra.ceph.com/r/ceph/20.2.1_2.17.26-baseline_2/adc1923395f2bcd6ae89b5d331d278364d1848ed/ubuntu/jammy/flavors/default jammy/main amd64 ceph-mgr-dashboard all 20.2.0-670-gadc19233-1jammy [45.5 MB] 2026-02-18T08:08:29.323 INFO:teuthology.orchestra.run.trial034.stdout:Get:78 https://2.chacra.ceph.com/r/ceph/20.2.1_2.17.26-baseline_2/adc1923395f2bcd6ae89b5d331d278364d1848ed/ubuntu/jammy/flavors/default jammy/main amd64 ceph-mgr-diskprediction-local all 20.2.0-670-gadc19233-1jammy [8625 kB] 2026-02-18T08:08:29.578 INFO:teuthology.orchestra.run.trial034.stdout:Get:79 https://2.chacra.ceph.com/r/ceph/20.2.1_2.17.26-baseline_2/adc1923395f2bcd6ae89b5d331d278364d1848ed/ubuntu/jammy/flavors/default jammy/main amd64 ceph-mgr-k8sevents all 20.2.0-670-gadc19233-1jammy [14.2 kB] 2026-02-18T08:08:29.578 INFO:teuthology.orchestra.run.trial034.stdout:Get:80 https://2.chacra.ceph.com/r/ceph/20.2.1_2.17.26-baseline_2/adc1923395f2bcd6ae89b5d331d278364d1848ed/ubuntu/jammy/flavors/default jammy/main amd64 ceph-test amd64 20.2.0-670-gadc19233-1jammy [99.5 MB] 2026-02-18T08:08:29.642 INFO:teuthology.orchestra.run.trial163.stdout:Get:78 https://2.chacra.ceph.com/r/ceph/20.2.1_2.17.26-baseline_2/adc1923395f2bcd6ae89b5d331d278364d1848ed/ubuntu/jammy/flavors/default jammy/main amd64 ceph-mgr-diskprediction-local all 20.2.0-670-gadc19233-1jammy [8625 kB] 2026-02-18T08:08:30.069 INFO:teuthology.orchestra.run.trial163.stdout:Get:79 https://2.chacra.ceph.com/r/ceph/20.2.1_2.17.26-baseline_2/adc1923395f2bcd6ae89b5d331d278364d1848ed/ubuntu/jammy/flavors/default jammy/main amd64 ceph-mgr-k8sevents all 20.2.0-670-gadc19233-1jammy [14.2 kB] 2026-02-18T08:08:30.070 INFO:teuthology.orchestra.run.trial163.stdout:Get:80 https://2.chacra.ceph.com/r/ceph/20.2.1_2.17.26-baseline_2/adc1923395f2bcd6ae89b5d331d278364d1848ed/ubuntu/jammy/flavors/default jammy/main amd64 ceph-test amd64 20.2.0-670-gadc19233-1jammy [99.5 MB] 2026-02-18T08:08:33.370 INFO:teuthology.orchestra.run.trial034.stdout:Get:81 https://2.chacra.ceph.com/r/ceph/20.2.1_2.17.26-baseline_2/adc1923395f2bcd6ae89b5d331d278364d1848ed/ubuntu/jammy/flavors/default jammy/main amd64 ceph-volume all 20.2.0-670-gadc19233-1jammy [135 kB] 2026-02-18T08:08:33.371 INFO:teuthology.orchestra.run.trial034.stdout:Get:82 https://2.chacra.ceph.com/r/ceph/20.2.1_2.17.26-baseline_2/adc1923395f2bcd6ae89b5d331d278364d1848ed/ubuntu/jammy/flavors/default jammy/main amd64 libcephfs-daemon amd64 20.2.0-670-gadc19233-1jammy [43.3 kB] 2026-02-18T08:08:33.371 INFO:teuthology.orchestra.run.trial034.stdout:Get:83 https://2.chacra.ceph.com/r/ceph/20.2.1_2.17.26-baseline_2/adc1923395f2bcd6ae89b5d331d278364d1848ed/ubuntu/jammy/flavors/default jammy/main amd64 libcephfs-proxy2 amd64 20.2.0-670-gadc19233-1jammy [30.7 kB] 2026-02-18T08:08:33.372 INFO:teuthology.orchestra.run.trial034.stdout:Get:84 https://2.chacra.ceph.com/r/ceph/20.2.1_2.17.26-baseline_2/adc1923395f2bcd6ae89b5d331d278364d1848ed/ubuntu/jammy/flavors/default jammy/main amd64 libcephfs-dev amd64 20.2.0-670-gadc19233-1jammy [41.5 kB] 2026-02-18T08:08:33.377 INFO:teuthology.orchestra.run.trial034.stdout:Get:85 https://2.chacra.ceph.com/r/ceph/20.2.1_2.17.26-baseline_2/adc1923395f2bcd6ae89b5d331d278364d1848ed/ubuntu/jammy/flavors/default jammy/main amd64 radosgw amd64 20.2.0-670-gadc19233-1jammy [25.1 MB] 2026-02-18T08:08:34.642 INFO:teuthology.orchestra.run.trial034.stdout:Get:86 https://2.chacra.ceph.com/r/ceph/20.2.1_2.17.26-baseline_2/adc1923395f2bcd6ae89b5d331d278364d1848ed/ubuntu/jammy/flavors/default jammy/main amd64 rbd-fuse amd64 20.2.0-670-gadc19233-1jammy [97.5 kB] 2026-02-18T08:08:34.642 INFO:teuthology.orchestra.run.trial034.stdout:Get:87 https://2.chacra.ceph.com/r/ceph/20.2.1_2.17.26-baseline_2/adc1923395f2bcd6ae89b5d331d278364d1848ed/ubuntu/jammy/flavors/default jammy/main amd64 rbd-mirror amd64 20.2.0-670-gadc19233-1jammy [2883 kB] 2026-02-18T08:08:35.093 INFO:teuthology.orchestra.run.trial034.stdout:Fetched 283 MB in 11s (25.7 MB/s) 2026-02-18T08:08:35.109 INFO:teuthology.orchestra.run.trial034.stdout:Selecting previously unselected package liblttng-ust1:amd64. 2026-02-18T08:08:35.263 INFO:teuthology.orchestra.run.trial034.stdout:(Reading database ... (Reading database ... 5% (Reading database ... 10% (Reading database ... 15% (Reading database ... 20% (Reading database ... 25% (Reading database ... 30% (Reading database ... 35% (Reading database ... 40% (Reading database ... 45% (Reading database ... 50% (Reading database ... 55% (Reading database ... 60% (Reading database ... 65% (Reading database ... 70% (Reading database ... 75% (Reading database ... 80% (Reading database ... 85% (Reading database ... 90% (Reading database ... 95% (Reading database ... 100% (Reading database ... 167438 files and directories currently installed.) 2026-02-18T08:08:35.265 INFO:teuthology.orchestra.run.trial034.stdout:Preparing to unpack .../00-liblttng-ust1_2.13.1-1ubuntu1_amd64.deb ... 2026-02-18T08:08:35.266 INFO:teuthology.orchestra.run.trial034.stdout:Unpacking liblttng-ust1:amd64 (2.13.1-1ubuntu1) ... 2026-02-18T08:08:35.286 INFO:teuthology.orchestra.run.trial034.stdout:Selecting previously unselected package libdouble-conversion3:amd64. 2026-02-18T08:08:35.299 INFO:teuthology.orchestra.run.trial034.stdout:Preparing to unpack .../01-libdouble-conversion3_3.1.7-4_amd64.deb ... 2026-02-18T08:08:35.299 INFO:teuthology.orchestra.run.trial034.stdout:Unpacking libdouble-conversion3:amd64 (3.1.7-4) ... 2026-02-18T08:08:35.315 INFO:teuthology.orchestra.run.trial034.stdout:Selecting previously unselected package libpcre2-16-0:amd64. 2026-02-18T08:08:35.327 INFO:teuthology.orchestra.run.trial034.stdout:Preparing to unpack .../02-libpcre2-16-0_10.39-3ubuntu0.1_amd64.deb ... 2026-02-18T08:08:35.327 INFO:teuthology.orchestra.run.trial034.stdout:Unpacking libpcre2-16-0:amd64 (10.39-3ubuntu0.1) ... 2026-02-18T08:08:35.346 INFO:teuthology.orchestra.run.trial034.stdout:Selecting previously unselected package libqt5core5a:amd64. 2026-02-18T08:08:35.354 INFO:teuthology.orchestra.run.trial034.stdout:Preparing to unpack .../03-libqt5core5a_5.15.3+dfsg-2ubuntu0.2_amd64.deb ... 2026-02-18T08:08:35.359 INFO:teuthology.orchestra.run.trial034.stdout:Unpacking libqt5core5a:amd64 (5.15.3+dfsg-2ubuntu0.2) ... 2026-02-18T08:08:35.391 INFO:teuthology.orchestra.run.trial034.stdout:Selecting previously unselected package libqt5dbus5:amd64. 2026-02-18T08:08:35.399 INFO:teuthology.orchestra.run.trial034.stdout:Preparing to unpack .../04-libqt5dbus5_5.15.3+dfsg-2ubuntu0.2_amd64.deb ... 2026-02-18T08:08:35.400 INFO:teuthology.orchestra.run.trial034.stdout:Unpacking libqt5dbus5:amd64 (5.15.3+dfsg-2ubuntu0.2) ... 2026-02-18T08:08:35.416 INFO:teuthology.orchestra.run.trial034.stdout:Selecting previously unselected package libqt5network5:amd64. 2026-02-18T08:08:35.424 INFO:teuthology.orchestra.run.trial034.stdout:Preparing to unpack .../05-libqt5network5_5.15.3+dfsg-2ubuntu0.2_amd64.deb ... 2026-02-18T08:08:35.424 INFO:teuthology.orchestra.run.trial034.stdout:Unpacking libqt5network5:amd64 (5.15.3+dfsg-2ubuntu0.2) ... 2026-02-18T08:08:35.444 INFO:teuthology.orchestra.run.trial034.stdout:Selecting previously unselected package libthrift-0.16.0:amd64. 2026-02-18T08:08:35.452 INFO:teuthology.orchestra.run.trial034.stdout:Preparing to unpack .../06-libthrift-0.16.0_0.16.0-2_amd64.deb ... 2026-02-18T08:08:35.452 INFO:teuthology.orchestra.run.trial034.stdout:Unpacking libthrift-0.16.0:amd64 (0.16.0-2) ... 2026-02-18T08:08:35.468 INFO:teuthology.orchestra.run.trial034.stdout:Selecting previously unselected package librados2. 2026-02-18T08:08:35.476 INFO:teuthology.orchestra.run.trial034.stdout:Preparing to unpack .../07-librados2_20.2.0-670-gadc19233-1jammy_amd64.deb ... 2026-02-18T08:08:35.477 INFO:teuthology.orchestra.run.trial034.stdout:Unpacking librados2 (20.2.0-670-gadc19233-1jammy) ... 2026-02-18T08:08:35.517 INFO:teuthology.orchestra.run.trial034.stdout:Selecting previously unselected package libnbd0. 2026-02-18T08:08:35.525 INFO:teuthology.orchestra.run.trial034.stdout:Preparing to unpack .../08-libnbd0_1.10.5-1_amd64.deb ... 2026-02-18T08:08:35.525 INFO:teuthology.orchestra.run.trial034.stdout:Unpacking libnbd0 (1.10.5-1) ... 2026-02-18T08:08:35.540 INFO:teuthology.orchestra.run.trial034.stdout:Selecting previously unselected package librbd1. 2026-02-18T08:08:35.548 INFO:teuthology.orchestra.run.trial034.stdout:Preparing to unpack .../09-librbd1_20.2.0-670-gadc19233-1jammy_amd64.deb ... 2026-02-18T08:08:35.548 INFO:teuthology.orchestra.run.trial034.stdout:Unpacking librbd1 (20.2.0-670-gadc19233-1jammy) ... 2026-02-18T08:08:35.585 INFO:teuthology.orchestra.run.trial034.stdout:Selecting previously unselected package libcephfs2. 2026-02-18T08:08:35.593 INFO:teuthology.orchestra.run.trial034.stdout:Preparing to unpack .../10-libcephfs2_20.2.0-670-gadc19233-1jammy_amd64.deb ... 2026-02-18T08:08:35.593 INFO:teuthology.orchestra.run.trial034.stdout:Unpacking libcephfs2 (20.2.0-670-gadc19233-1jammy) ... 2026-02-18T08:08:35.613 INFO:teuthology.orchestra.run.trial034.stdout:Selecting previously unselected package python3-rados. 2026-02-18T08:08:35.621 INFO:teuthology.orchestra.run.trial034.stdout:Preparing to unpack .../11-python3-rados_20.2.0-670-gadc19233-1jammy_amd64.deb ... 2026-02-18T08:08:35.621 INFO:teuthology.orchestra.run.trial034.stdout:Unpacking python3-rados (20.2.0-670-gadc19233-1jammy) ... 2026-02-18T08:08:35.638 INFO:teuthology.orchestra.run.trial034.stdout:Selecting previously unselected package python3-ceph-argparse. 2026-02-18T08:08:35.646 INFO:teuthology.orchestra.run.trial034.stdout:Preparing to unpack .../12-python3-ceph-argparse_20.2.0-670-gadc19233-1jammy_all.deb ... 2026-02-18T08:08:35.646 INFO:teuthology.orchestra.run.trial034.stdout:Unpacking python3-ceph-argparse (20.2.0-670-gadc19233-1jammy) ... 2026-02-18T08:08:35.660 INFO:teuthology.orchestra.run.trial034.stdout:Selecting previously unselected package python3-cephfs. 2026-02-18T08:08:35.668 INFO:teuthology.orchestra.run.trial034.stdout:Preparing to unpack .../13-python3-cephfs_20.2.0-670-gadc19233-1jammy_amd64.deb ... 2026-02-18T08:08:35.669 INFO:teuthology.orchestra.run.trial034.stdout:Unpacking python3-cephfs (20.2.0-670-gadc19233-1jammy) ... 2026-02-18T08:08:35.684 INFO:teuthology.orchestra.run.trial034.stdout:Selecting previously unselected package python3-ceph-common. 2026-02-18T08:08:35.692 INFO:teuthology.orchestra.run.trial034.stdout:Preparing to unpack .../14-python3-ceph-common_20.2.0-670-gadc19233-1jammy_all.deb ... 2026-02-18T08:08:35.693 INFO:teuthology.orchestra.run.trial034.stdout:Unpacking python3-ceph-common (20.2.0-670-gadc19233-1jammy) ... 2026-02-18T08:08:35.710 INFO:teuthology.orchestra.run.trial034.stdout:Selecting previously unselected package python3-wcwidth. 2026-02-18T08:08:35.718 INFO:teuthology.orchestra.run.trial034.stdout:Preparing to unpack .../15-python3-wcwidth_0.2.5+dfsg1-1_all.deb ... 2026-02-18T08:08:35.719 INFO:teuthology.orchestra.run.trial034.stdout:Unpacking python3-wcwidth (0.2.5+dfsg1-1) ... 2026-02-18T08:08:35.734 INFO:teuthology.orchestra.run.trial034.stdout:Selecting previously unselected package python3-prettytable. 2026-02-18T08:08:35.742 INFO:teuthology.orchestra.run.trial034.stdout:Preparing to unpack .../16-python3-prettytable_2.5.0-2_all.deb ... 2026-02-18T08:08:35.743 INFO:teuthology.orchestra.run.trial034.stdout:Unpacking python3-prettytable (2.5.0-2) ... 2026-02-18T08:08:35.757 INFO:teuthology.orchestra.run.trial034.stdout:Selecting previously unselected package python3-rbd. 2026-02-18T08:08:35.764 INFO:teuthology.orchestra.run.trial034.stdout:Preparing to unpack .../17-python3-rbd_20.2.0-670-gadc19233-1jammy_amd64.deb ... 2026-02-18T08:08:35.765 INFO:teuthology.orchestra.run.trial034.stdout:Unpacking python3-rbd (20.2.0-670-gadc19233-1jammy) ... 2026-02-18T08:08:35.781 INFO:teuthology.orchestra.run.trial034.stdout:Selecting previously unselected package librdkafka1:amd64. 2026-02-18T08:08:35.790 INFO:teuthology.orchestra.run.trial034.stdout:Preparing to unpack .../18-librdkafka1_1.8.0-1build1_amd64.deb ... 2026-02-18T08:08:35.790 INFO:teuthology.orchestra.run.trial034.stdout:Unpacking librdkafka1:amd64 (1.8.0-1build1) ... 2026-02-18T08:08:35.808 INFO:teuthology.orchestra.run.trial034.stdout:Selecting previously unselected package librgw2. 2026-02-18T08:08:35.815 INFO:teuthology.orchestra.run.trial034.stdout:Preparing to unpack .../19-librgw2_20.2.0-670-gadc19233-1jammy_amd64.deb ... 2026-02-18T08:08:35.816 INFO:teuthology.orchestra.run.trial034.stdout:Unpacking librgw2 (20.2.0-670-gadc19233-1jammy) ... 2026-02-18T08:08:35.910 INFO:teuthology.orchestra.run.trial034.stdout:Selecting previously unselected package python3-rgw. 2026-02-18T08:08:35.919 INFO:teuthology.orchestra.run.trial034.stdout:Preparing to unpack .../20-python3-rgw_20.2.0-670-gadc19233-1jammy_amd64.deb ... 2026-02-18T08:08:35.919 INFO:teuthology.orchestra.run.trial034.stdout:Unpacking python3-rgw (20.2.0-670-gadc19233-1jammy) ... 2026-02-18T08:08:35.934 INFO:teuthology.orchestra.run.trial034.stdout:Selecting previously unselected package liboath0:amd64. 2026-02-18T08:08:35.941 INFO:teuthology.orchestra.run.trial034.stdout:Preparing to unpack .../21-liboath0_2.6.7-3ubuntu0.1_amd64.deb ... 2026-02-18T08:08:35.942 INFO:teuthology.orchestra.run.trial034.stdout:Unpacking liboath0:amd64 (2.6.7-3ubuntu0.1) ... 2026-02-18T08:08:35.956 INFO:teuthology.orchestra.run.trial034.stdout:Selecting previously unselected package libradosstriper1. 2026-02-18T08:08:35.964 INFO:teuthology.orchestra.run.trial034.stdout:Preparing to unpack .../22-libradosstriper1_20.2.0-670-gadc19233-1jammy_amd64.deb ... 2026-02-18T08:08:35.965 INFO:teuthology.orchestra.run.trial034.stdout:Unpacking libradosstriper1 (20.2.0-670-gadc19233-1jammy) ... 2026-02-18T08:08:35.982 INFO:teuthology.orchestra.run.trial034.stdout:Selecting previously unselected package ceph-common. 2026-02-18T08:08:35.990 INFO:teuthology.orchestra.run.trial034.stdout:Preparing to unpack .../23-ceph-common_20.2.0-670-gadc19233-1jammy_amd64.deb ... 2026-02-18T08:08:35.990 INFO:teuthology.orchestra.run.trial034.stdout:Unpacking ceph-common (20.2.0-670-gadc19233-1jammy) ... 2026-02-18T08:08:36.235 INFO:teuthology.orchestra.run.trial034.stdout:Selecting previously unselected package ceph-base. 2026-02-18T08:08:36.244 INFO:teuthology.orchestra.run.trial034.stdout:Preparing to unpack .../24-ceph-base_20.2.0-670-gadc19233-1jammy_amd64.deb ... 2026-02-18T08:08:36.249 INFO:teuthology.orchestra.run.trial034.stdout:Unpacking ceph-base (20.2.0-670-gadc19233-1jammy) ... 2026-02-18T08:08:36.313 INFO:teuthology.orchestra.run.trial034.stdout:Selecting previously unselected package python3-jaraco.functools. 2026-02-18T08:08:36.321 INFO:teuthology.orchestra.run.trial034.stdout:Preparing to unpack .../25-python3-jaraco.functools_3.4.0-2_all.deb ... 2026-02-18T08:08:36.322 INFO:teuthology.orchestra.run.trial034.stdout:Unpacking python3-jaraco.functools (3.4.0-2) ... 2026-02-18T08:08:36.336 INFO:teuthology.orchestra.run.trial034.stdout:Selecting previously unselected package python3-cheroot. 2026-02-18T08:08:36.344 INFO:teuthology.orchestra.run.trial034.stdout:Preparing to unpack .../26-python3-cheroot_8.5.2+ds1-1ubuntu3.1_all.deb ... 2026-02-18T08:08:36.345 INFO:teuthology.orchestra.run.trial034.stdout:Unpacking python3-cheroot (8.5.2+ds1-1ubuntu3.1) ... 2026-02-18T08:08:36.361 INFO:teuthology.orchestra.run.trial034.stdout:Selecting previously unselected package python3-jaraco.classes. 2026-02-18T08:08:36.369 INFO:teuthology.orchestra.run.trial034.stdout:Preparing to unpack .../27-python3-jaraco.classes_3.2.1-3_all.deb ... 2026-02-18T08:08:36.370 INFO:teuthology.orchestra.run.trial034.stdout:Unpacking python3-jaraco.classes (3.2.1-3) ... 2026-02-18T08:08:36.383 INFO:teuthology.orchestra.run.trial034.stdout:Selecting previously unselected package python3-jaraco.text. 2026-02-18T08:08:36.392 INFO:teuthology.orchestra.run.trial034.stdout:Preparing to unpack .../28-python3-jaraco.text_3.6.0-2_all.deb ... 2026-02-18T08:08:36.392 INFO:teuthology.orchestra.run.trial034.stdout:Unpacking python3-jaraco.text (3.6.0-2) ... 2026-02-18T08:08:36.406 INFO:teuthology.orchestra.run.trial034.stdout:Selecting previously unselected package python3-jaraco.collections. 2026-02-18T08:08:36.414 INFO:teuthology.orchestra.run.trial034.stdout:Preparing to unpack .../29-python3-jaraco.collections_3.4.0-2_all.deb ... 2026-02-18T08:08:36.415 INFO:teuthology.orchestra.run.trial034.stdout:Unpacking python3-jaraco.collections (3.4.0-2) ... 2026-02-18T08:08:36.429 INFO:teuthology.orchestra.run.trial034.stdout:Selecting previously unselected package python3-tempora. 2026-02-18T08:08:36.437 INFO:teuthology.orchestra.run.trial034.stdout:Preparing to unpack .../30-python3-tempora_4.1.2-1_all.deb ... 2026-02-18T08:08:36.437 INFO:teuthology.orchestra.run.trial034.stdout:Unpacking python3-tempora (4.1.2-1) ... 2026-02-18T08:08:36.451 INFO:teuthology.orchestra.run.trial034.stdout:Selecting previously unselected package python3-portend. 2026-02-18T08:08:36.459 INFO:teuthology.orchestra.run.trial034.stdout:Preparing to unpack .../31-python3-portend_3.0.0-1_all.deb ... 2026-02-18T08:08:36.460 INFO:teuthology.orchestra.run.trial034.stdout:Unpacking python3-portend (3.0.0-1) ... 2026-02-18T08:08:36.473 INFO:teuthology.orchestra.run.trial034.stdout:Selecting previously unselected package python3-zc.lockfile. 2026-02-18T08:08:36.481 INFO:teuthology.orchestra.run.trial034.stdout:Preparing to unpack .../32-python3-zc.lockfile_2.0-1_all.deb ... 2026-02-18T08:08:36.482 INFO:teuthology.orchestra.run.trial034.stdout:Unpacking python3-zc.lockfile (2.0-1) ... 2026-02-18T08:08:36.496 INFO:teuthology.orchestra.run.trial034.stdout:Selecting previously unselected package python3-cherrypy3. 2026-02-18T08:08:36.504 INFO:teuthology.orchestra.run.trial034.stdout:Preparing to unpack .../33-python3-cherrypy3_18.6.1-4_all.deb ... 2026-02-18T08:08:36.505 INFO:teuthology.orchestra.run.trial034.stdout:Unpacking python3-cherrypy3 (18.6.1-4) ... 2026-02-18T08:08:36.528 INFO:teuthology.orchestra.run.trial034.stdout:Selecting previously unselected package python3-natsort. 2026-02-18T08:08:36.536 INFO:teuthology.orchestra.run.trial034.stdout:Preparing to unpack .../34-python3-natsort_8.0.2-1_all.deb ... 2026-02-18T08:08:36.536 INFO:teuthology.orchestra.run.trial034.stdout:Unpacking python3-natsort (8.0.2-1) ... 2026-02-18T08:08:36.552 INFO:teuthology.orchestra.run.trial034.stdout:Selecting previously unselected package ceph-mgr-modules-core. 2026-02-18T08:08:36.560 INFO:teuthology.orchestra.run.trial034.stdout:Preparing to unpack .../35-ceph-mgr-modules-core_20.2.0-670-gadc19233-1jammy_all.deb ... 2026-02-18T08:08:36.561 INFO:teuthology.orchestra.run.trial034.stdout:Unpacking ceph-mgr-modules-core (20.2.0-670-gadc19233-1jammy) ... 2026-02-18T08:08:36.587 INFO:teuthology.orchestra.run.trial034.stdout:Selecting previously unselected package libsqlite3-mod-ceph. 2026-02-18T08:08:36.595 INFO:teuthology.orchestra.run.trial034.stdout:Preparing to unpack .../36-libsqlite3-mod-ceph_20.2.0-670-gadc19233-1jammy_amd64.deb ... 2026-02-18T08:08:36.596 INFO:teuthology.orchestra.run.trial034.stdout:Unpacking libsqlite3-mod-ceph (20.2.0-670-gadc19233-1jammy) ... 2026-02-18T08:08:36.611 INFO:teuthology.orchestra.run.trial034.stdout:Selecting previously unselected package ceph-mgr. 2026-02-18T08:08:36.619 INFO:teuthology.orchestra.run.trial034.stdout:Preparing to unpack .../37-ceph-mgr_20.2.0-670-gadc19233-1jammy_amd64.deb ... 2026-02-18T08:08:36.619 INFO:teuthology.orchestra.run.trial034.stdout:Unpacking ceph-mgr (20.2.0-670-gadc19233-1jammy) ... 2026-02-18T08:08:36.639 INFO:teuthology.orchestra.run.trial034.stdout:Selecting previously unselected package ceph-mon. 2026-02-18T08:08:36.647 INFO:teuthology.orchestra.run.trial034.stdout:Preparing to unpack .../38-ceph-mon_20.2.0-670-gadc19233-1jammy_amd64.deb ... 2026-02-18T08:08:36.648 INFO:teuthology.orchestra.run.trial034.stdout:Unpacking ceph-mon (20.2.0-670-gadc19233-1jammy) ... 2026-02-18T08:08:36.703 INFO:teuthology.orchestra.run.trial034.stdout:Selecting previously unselected package libfuse2:amd64. 2026-02-18T08:08:36.712 INFO:teuthology.orchestra.run.trial034.stdout:Preparing to unpack .../39-libfuse2_2.9.9-5ubuntu3_amd64.deb ... 2026-02-18T08:08:36.712 INFO:teuthology.orchestra.run.trial034.stdout:Unpacking libfuse2:amd64 (2.9.9-5ubuntu3) ... 2026-02-18T08:08:36.728 INFO:teuthology.orchestra.run.trial034.stdout:Selecting previously unselected package ceph-osd. 2026-02-18T08:08:36.736 INFO:teuthology.orchestra.run.trial034.stdout:Preparing to unpack .../40-ceph-osd_20.2.0-670-gadc19233-1jammy_amd64.deb ... 2026-02-18T08:08:36.737 INFO:teuthology.orchestra.run.trial034.stdout:Unpacking ceph-osd (20.2.0-670-gadc19233-1jammy) ... 2026-02-18T08:08:36.879 INFO:teuthology.orchestra.run.trial034.stdout:Selecting previously unselected package ceph. 2026-02-18T08:08:36.888 INFO:teuthology.orchestra.run.trial034.stdout:Preparing to unpack .../41-ceph_20.2.0-670-gadc19233-1jammy_amd64.deb ... 2026-02-18T08:08:36.888 INFO:teuthology.orchestra.run.trial034.stdout:Unpacking ceph (20.2.0-670-gadc19233-1jammy) ... 2026-02-18T08:08:36.903 INFO:teuthology.orchestra.run.trial034.stdout:Selecting previously unselected package ceph-fuse. 2026-02-18T08:08:36.911 INFO:teuthology.orchestra.run.trial034.stdout:Preparing to unpack .../42-ceph-fuse_20.2.0-670-gadc19233-1jammy_amd64.deb ... 2026-02-18T08:08:36.911 INFO:teuthology.orchestra.run.trial034.stdout:Unpacking ceph-fuse (20.2.0-670-gadc19233-1jammy) ... 2026-02-18T08:08:36.932 INFO:teuthology.orchestra.run.trial034.stdout:Selecting previously unselected package ceph-mds. 2026-02-18T08:08:36.940 INFO:teuthology.orchestra.run.trial034.stdout:Preparing to unpack .../43-ceph-mds_20.2.0-670-gadc19233-1jammy_amd64.deb ... 2026-02-18T08:08:36.941 INFO:teuthology.orchestra.run.trial034.stdout:Unpacking ceph-mds (20.2.0-670-gadc19233-1jammy) ... 2026-02-18T08:08:36.979 INFO:teuthology.orchestra.run.trial034.stdout:Selecting previously unselected package cephadm. 2026-02-18T08:08:36.987 INFO:teuthology.orchestra.run.trial034.stdout:Preparing to unpack .../44-cephadm_20.2.0-670-gadc19233-1jammy_amd64.deb ... 2026-02-18T08:08:36.988 INFO:teuthology.orchestra.run.trial034.stdout:Unpacking cephadm (20.2.0-670-gadc19233-1jammy) ... 2026-02-18T08:08:37.003 INFO:teuthology.orchestra.run.trial034.stdout:Selecting previously unselected package python3-asyncssh. 2026-02-18T08:08:37.011 INFO:teuthology.orchestra.run.trial034.stdout:Preparing to unpack .../45-python3-asyncssh_2.5.0-1ubuntu0.1_all.deb ... 2026-02-18T08:08:37.011 INFO:teuthology.orchestra.run.trial034.stdout:Unpacking python3-asyncssh (2.5.0-1ubuntu0.1) ... 2026-02-18T08:08:37.032 INFO:teuthology.orchestra.run.trial034.stdout:Selecting previously unselected package ceph-mgr-cephadm. 2026-02-18T08:08:37.040 INFO:teuthology.orchestra.run.trial034.stdout:Preparing to unpack .../46-ceph-mgr-cephadm_20.2.0-670-gadc19233-1jammy_all.deb ... 2026-02-18T08:08:37.041 INFO:teuthology.orchestra.run.trial034.stdout:Unpacking ceph-mgr-cephadm (20.2.0-670-gadc19233-1jammy) ... 2026-02-18T08:08:37.061 INFO:teuthology.orchestra.run.trial034.stdout:Selecting previously unselected package python3-repoze.lru. 2026-02-18T08:08:37.069 INFO:teuthology.orchestra.run.trial034.stdout:Preparing to unpack .../47-python3-repoze.lru_0.7-2_all.deb ... 2026-02-18T08:08:37.069 INFO:teuthology.orchestra.run.trial034.stdout:Unpacking python3-repoze.lru (0.7-2) ... 2026-02-18T08:08:37.085 INFO:teuthology.orchestra.run.trial034.stdout:Selecting previously unselected package python3-routes. 2026-02-18T08:08:37.093 INFO:teuthology.orchestra.run.trial034.stdout:Preparing to unpack .../48-python3-routes_2.5.1-1ubuntu1_all.deb ... 2026-02-18T08:08:37.093 INFO:teuthology.orchestra.run.trial034.stdout:Unpacking python3-routes (2.5.1-1ubuntu1) ... 2026-02-18T08:08:37.113 INFO:teuthology.orchestra.run.trial034.stdout:Selecting previously unselected package ceph-mgr-dashboard. 2026-02-18T08:08:37.121 INFO:teuthology.orchestra.run.trial034.stdout:Preparing to unpack .../49-ceph-mgr-dashboard_20.2.0-670-gadc19233-1jammy_all.deb ... 2026-02-18T08:08:37.122 INFO:teuthology.orchestra.run.trial034.stdout:Unpacking ceph-mgr-dashboard (20.2.0-670-gadc19233-1jammy) ... 2026-02-18T08:08:37.552 INFO:teuthology.orchestra.run.trial034.stdout:Selecting previously unselected package python3-sklearn-lib:amd64. 2026-02-18T08:08:37.562 INFO:teuthology.orchestra.run.trial034.stdout:Preparing to unpack .../50-python3-sklearn-lib_0.23.2-5ubuntu6_amd64.deb ... 2026-02-18T08:08:37.562 INFO:teuthology.orchestra.run.trial034.stdout:Unpacking python3-sklearn-lib:amd64 (0.23.2-5ubuntu6) ... 2026-02-18T08:08:37.600 INFO:teuthology.orchestra.run.trial034.stdout:Selecting previously unselected package python3-joblib. 2026-02-18T08:08:37.609 INFO:teuthology.orchestra.run.trial034.stdout:Preparing to unpack .../51-python3-joblib_0.17.0-4ubuntu1_all.deb ... 2026-02-18T08:08:37.609 INFO:teuthology.orchestra.run.trial034.stdout:Unpacking python3-joblib (0.17.0-4ubuntu1) ... 2026-02-18T08:08:37.635 INFO:teuthology.orchestra.run.trial034.stdout:Selecting previously unselected package python3-threadpoolctl. 2026-02-18T08:08:37.644 INFO:teuthology.orchestra.run.trial034.stdout:Preparing to unpack .../52-python3-threadpoolctl_3.1.0-1_all.deb ... 2026-02-18T08:08:37.644 INFO:teuthology.orchestra.run.trial034.stdout:Unpacking python3-threadpoolctl (3.1.0-1) ... 2026-02-18T08:08:37.659 INFO:teuthology.orchestra.run.trial034.stdout:Selecting previously unselected package python3-sklearn. 2026-02-18T08:08:37.667 INFO:teuthology.orchestra.run.trial034.stdout:Preparing to unpack .../53-python3-sklearn_0.23.2-5ubuntu6_all.deb ... 2026-02-18T08:08:37.668 INFO:teuthology.orchestra.run.trial034.stdout:Unpacking python3-sklearn (0.23.2-5ubuntu6) ... 2026-02-18T08:08:37.750 INFO:teuthology.orchestra.run.trial034.stdout:Selecting previously unselected package ceph-mgr-diskprediction-local. 2026-02-18T08:08:37.758 INFO:teuthology.orchestra.run.trial034.stdout:Preparing to unpack .../54-ceph-mgr-diskprediction-local_20.2.0-670-gadc19233-1jammy_all.deb ... 2026-02-18T08:08:37.759 INFO:teuthology.orchestra.run.trial034.stdout:Unpacking ceph-mgr-diskprediction-local (20.2.0-670-gadc19233-1jammy) ... 2026-02-18T08:08:37.924 INFO:teuthology.orchestra.run.trial034.stdout:Selecting previously unselected package python3-cachetools. 2026-02-18T08:08:37.932 INFO:teuthology.orchestra.run.trial034.stdout:Preparing to unpack .../55-python3-cachetools_5.0.0-1_all.deb ... 2026-02-18T08:08:37.933 INFO:teuthology.orchestra.run.trial034.stdout:Unpacking python3-cachetools (5.0.0-1) ... 2026-02-18T08:08:37.948 INFO:teuthology.orchestra.run.trial034.stdout:Selecting previously unselected package python3-rsa. 2026-02-18T08:08:37.956 INFO:teuthology.orchestra.run.trial034.stdout:Preparing to unpack .../56-python3-rsa_4.8-1_all.deb ... 2026-02-18T08:08:37.956 INFO:teuthology.orchestra.run.trial034.stdout:Unpacking python3-rsa (4.8-1) ... 2026-02-18T08:08:37.974 INFO:teuthology.orchestra.run.trial034.stdout:Selecting previously unselected package python3-google-auth. 2026-02-18T08:08:37.982 INFO:teuthology.orchestra.run.trial034.stdout:Preparing to unpack .../57-python3-google-auth_1.5.1-3_all.deb ... 2026-02-18T08:08:37.983 INFO:teuthology.orchestra.run.trial034.stdout:Unpacking python3-google-auth (1.5.1-3) ... 2026-02-18T08:08:37.999 INFO:teuthology.orchestra.run.trial034.stdout:Selecting previously unselected package python3-requests-oauthlib. 2026-02-18T08:08:38.008 INFO:teuthology.orchestra.run.trial034.stdout:Preparing to unpack .../58-python3-requests-oauthlib_1.3.0+ds-0.1_all.deb ... 2026-02-18T08:08:38.009 INFO:teuthology.orchestra.run.trial034.stdout:Unpacking python3-requests-oauthlib (1.3.0+ds-0.1) ... 2026-02-18T08:08:38.024 INFO:teuthology.orchestra.run.trial034.stdout:Selecting previously unselected package python3-websocket. 2026-02-18T08:08:38.032 INFO:teuthology.orchestra.run.trial034.stdout:Preparing to unpack .../59-python3-websocket_1.2.3-1_all.deb ... 2026-02-18T08:08:38.032 INFO:teuthology.orchestra.run.trial034.stdout:Unpacking python3-websocket (1.2.3-1) ... 2026-02-18T08:08:38.050 INFO:teuthology.orchestra.run.trial034.stdout:Selecting previously unselected package python3-kubernetes. 2026-02-18T08:08:38.058 INFO:teuthology.orchestra.run.trial034.stdout:Preparing to unpack .../60-python3-kubernetes_12.0.1-1ubuntu1_all.deb ... 2026-02-18T08:08:38.059 INFO:teuthology.orchestra.run.trial034.stdout:Unpacking python3-kubernetes (12.0.1-1ubuntu1) ... 2026-02-18T08:08:38.152 INFO:teuthology.orchestra.run.trial034.stdout:Selecting previously unselected package ceph-mgr-k8sevents. 2026-02-18T08:08:38.160 INFO:teuthology.orchestra.run.trial034.stdout:Preparing to unpack .../61-ceph-mgr-k8sevents_20.2.0-670-gadc19233-1jammy_all.deb ... 2026-02-18T08:08:38.160 INFO:teuthology.orchestra.run.trial034.stdout:Unpacking ceph-mgr-k8sevents (20.2.0-670-gadc19233-1jammy) ... 2026-02-18T08:08:38.174 INFO:teuthology.orchestra.run.trial034.stdout:Selecting previously unselected package libonig5:amd64. 2026-02-18T08:08:38.182 INFO:teuthology.orchestra.run.trial034.stdout:Preparing to unpack .../62-libonig5_6.9.7.1-2build1_amd64.deb ... 2026-02-18T08:08:38.183 INFO:teuthology.orchestra.run.trial034.stdout:Unpacking libonig5:amd64 (6.9.7.1-2build1) ... 2026-02-18T08:08:38.198 INFO:teuthology.orchestra.run.trial034.stdout:Selecting previously unselected package libjq1:amd64. 2026-02-18T08:08:38.207 INFO:teuthology.orchestra.run.trial034.stdout:Preparing to unpack .../63-libjq1_1.6-2.1ubuntu3.1_amd64.deb ... 2026-02-18T08:08:38.207 INFO:teuthology.orchestra.run.trial034.stdout:Unpacking libjq1:amd64 (1.6-2.1ubuntu3.1) ... 2026-02-18T08:08:38.221 INFO:teuthology.orchestra.run.trial034.stdout:Selecting previously unselected package jq. 2026-02-18T08:08:38.230 INFO:teuthology.orchestra.run.trial034.stdout:Preparing to unpack .../64-jq_1.6-2.1ubuntu3.1_amd64.deb ... 2026-02-18T08:08:38.230 INFO:teuthology.orchestra.run.trial034.stdout:Unpacking jq (1.6-2.1ubuntu3.1) ... 2026-02-18T08:08:38.244 INFO:teuthology.orchestra.run.trial034.stdout:Selecting previously unselected package socat. 2026-02-18T08:08:38.252 INFO:teuthology.orchestra.run.trial034.stdout:Preparing to unpack .../65-socat_1.7.4.1-3ubuntu4_amd64.deb ... 2026-02-18T08:08:38.253 INFO:teuthology.orchestra.run.trial034.stdout:Unpacking socat (1.7.4.1-3ubuntu4) ... 2026-02-18T08:08:38.274 INFO:teuthology.orchestra.run.trial034.stdout:Selecting previously unselected package xmlstarlet. 2026-02-18T08:08:38.282 INFO:teuthology.orchestra.run.trial034.stdout:Preparing to unpack .../66-xmlstarlet_1.6.1-2.1_amd64.deb ... 2026-02-18T08:08:38.282 INFO:teuthology.orchestra.run.trial034.stdout:Unpacking xmlstarlet (1.6.1-2.1) ... 2026-02-18T08:08:38.318 INFO:teuthology.orchestra.run.trial034.stdout:Selecting previously unselected package ceph-test. 2026-02-18T08:08:38.326 INFO:teuthology.orchestra.run.trial034.stdout:Preparing to unpack .../67-ceph-test_20.2.0-670-gadc19233-1jammy_amd64.deb ... 2026-02-18T08:08:38.327 INFO:teuthology.orchestra.run.trial034.stdout:Unpacking ceph-test (20.2.0-670-gadc19233-1jammy) ... 2026-02-18T08:08:38.426 INFO:teuthology.orchestra.run.trial163.stdout:Get:81 https://2.chacra.ceph.com/r/ceph/20.2.1_2.17.26-baseline_2/adc1923395f2bcd6ae89b5d331d278364d1848ed/ubuntu/jammy/flavors/default jammy/main amd64 ceph-volume all 20.2.0-670-gadc19233-1jammy [135 kB] 2026-02-18T08:08:38.427 INFO:teuthology.orchestra.run.trial163.stdout:Get:82 https://2.chacra.ceph.com/r/ceph/20.2.1_2.17.26-baseline_2/adc1923395f2bcd6ae89b5d331d278364d1848ed/ubuntu/jammy/flavors/default jammy/main amd64 libcephfs-daemon amd64 20.2.0-670-gadc19233-1jammy [43.3 kB] 2026-02-18T08:08:38.428 INFO:teuthology.orchestra.run.trial163.stdout:Get:83 https://2.chacra.ceph.com/r/ceph/20.2.1_2.17.26-baseline_2/adc1923395f2bcd6ae89b5d331d278364d1848ed/ubuntu/jammy/flavors/default jammy/main amd64 libcephfs-proxy2 amd64 20.2.0-670-gadc19233-1jammy [30.7 kB] 2026-02-18T08:08:38.428 INFO:teuthology.orchestra.run.trial163.stdout:Get:84 https://2.chacra.ceph.com/r/ceph/20.2.1_2.17.26-baseline_2/adc1923395f2bcd6ae89b5d331d278364d1848ed/ubuntu/jammy/flavors/default jammy/main amd64 libcephfs-dev amd64 20.2.0-670-gadc19233-1jammy [41.5 kB] 2026-02-18T08:08:38.428 INFO:teuthology.orchestra.run.trial163.stdout:Get:85 https://2.chacra.ceph.com/r/ceph/20.2.1_2.17.26-baseline_2/adc1923395f2bcd6ae89b5d331d278364d1848ed/ubuntu/jammy/flavors/default jammy/main amd64 radosgw amd64 20.2.0-670-gadc19233-1jammy [25.1 MB] 2026-02-18T08:08:39.200 INFO:teuthology.orchestra.run.trial034.stdout:Selecting previously unselected package ceph-volume. 2026-02-18T08:08:39.208 INFO:teuthology.orchestra.run.trial034.stdout:Preparing to unpack .../68-ceph-volume_20.2.0-670-gadc19233-1jammy_all.deb ... 2026-02-18T08:08:39.209 INFO:teuthology.orchestra.run.trial034.stdout:Unpacking ceph-volume (20.2.0-670-gadc19233-1jammy) ... 2026-02-18T08:08:39.232 INFO:teuthology.orchestra.run.trial034.stdout:Selecting previously unselected package libcephfs-daemon. 2026-02-18T08:08:39.241 INFO:teuthology.orchestra.run.trial034.stdout:Preparing to unpack .../69-libcephfs-daemon_20.2.0-670-gadc19233-1jammy_amd64.deb ... 2026-02-18T08:08:39.242 INFO:teuthology.orchestra.run.trial034.stdout:Unpacking libcephfs-daemon (20.2.0-670-gadc19233-1jammy) ... 2026-02-18T08:08:39.256 INFO:teuthology.orchestra.run.trial034.stdout:Selecting previously unselected package libcephfs-proxy2. 2026-02-18T08:08:39.265 INFO:teuthology.orchestra.run.trial034.stdout:Preparing to unpack .../70-libcephfs-proxy2_20.2.0-670-gadc19233-1jammy_amd64.deb ... 2026-02-18T08:08:39.265 INFO:teuthology.orchestra.run.trial034.stdout:Unpacking libcephfs-proxy2 (20.2.0-670-gadc19233-1jammy) ... 2026-02-18T08:08:39.270 INFO:teuthology.orchestra.run.trial163.stdout:Get:86 https://2.chacra.ceph.com/r/ceph/20.2.1_2.17.26-baseline_2/adc1923395f2bcd6ae89b5d331d278364d1848ed/ubuntu/jammy/flavors/default jammy/main amd64 rbd-fuse amd64 20.2.0-670-gadc19233-1jammy [97.5 kB] 2026-02-18T08:08:39.271 INFO:teuthology.orchestra.run.trial163.stdout:Get:87 https://2.chacra.ceph.com/r/ceph/20.2.1_2.17.26-baseline_2/adc1923395f2bcd6ae89b5d331d278364d1848ed/ubuntu/jammy/flavors/default jammy/main amd64 rbd-mirror amd64 20.2.0-670-gadc19233-1jammy [2883 kB] 2026-02-18T08:08:39.279 INFO:teuthology.orchestra.run.trial034.stdout:Selecting previously unselected package libcephfs-dev. 2026-02-18T08:08:39.288 INFO:teuthology.orchestra.run.trial034.stdout:Preparing to unpack .../71-libcephfs-dev_20.2.0-670-gadc19233-1jammy_amd64.deb ... 2026-02-18T08:08:39.288 INFO:teuthology.orchestra.run.trial034.stdout:Unpacking libcephfs-dev (20.2.0-670-gadc19233-1jammy) ... 2026-02-18T08:08:39.303 INFO:teuthology.orchestra.run.trial034.stdout:Selecting previously unselected package nvme-cli. 2026-02-18T08:08:39.311 INFO:teuthology.orchestra.run.trial034.stdout:Preparing to unpack .../72-nvme-cli_1.16-3ubuntu0.3_amd64.deb ... 2026-02-18T08:08:39.312 INFO:teuthology.orchestra.run.trial034.stdout:Unpacking nvme-cli (1.16-3ubuntu0.3) ... 2026-02-18T08:08:39.342 INFO:teuthology.orchestra.run.trial034.stdout:Selecting previously unselected package python-asyncssh-doc. 2026-02-18T08:08:39.350 INFO:teuthology.orchestra.run.trial034.stdout:Preparing to unpack .../73-python-asyncssh-doc_2.5.0-1ubuntu0.1_all.deb ... 2026-02-18T08:08:39.351 INFO:teuthology.orchestra.run.trial034.stdout:Unpacking python-asyncssh-doc (2.5.0-1ubuntu0.1) ... 2026-02-18T08:08:39.380 INFO:teuthology.orchestra.run.trial034.stdout:Selecting previously unselected package python3-iniconfig. 2026-02-18T08:08:39.388 INFO:teuthology.orchestra.run.trial034.stdout:Preparing to unpack .../74-python3-iniconfig_1.1.1-2_all.deb ... 2026-02-18T08:08:39.388 INFO:teuthology.orchestra.run.trial034.stdout:Unpacking python3-iniconfig (1.1.1-2) ... 2026-02-18T08:08:39.402 INFO:teuthology.orchestra.run.trial034.stdout:Selecting previously unselected package python3-pluggy. 2026-02-18T08:08:39.411 INFO:teuthology.orchestra.run.trial034.stdout:Preparing to unpack .../75-python3-pluggy_0.13.0-7.1_all.deb ... 2026-02-18T08:08:39.411 INFO:teuthology.orchestra.run.trial034.stdout:Unpacking python3-pluggy (0.13.0-7.1) ... 2026-02-18T08:08:39.426 INFO:teuthology.orchestra.run.trial034.stdout:Selecting previously unselected package python3-psutil. 2026-02-18T08:08:39.434 INFO:teuthology.orchestra.run.trial034.stdout:Preparing to unpack .../76-python3-psutil_5.9.0-1build1_amd64.deb ... 2026-02-18T08:08:39.435 INFO:teuthology.orchestra.run.trial034.stdout:Unpacking python3-psutil (5.9.0-1build1) ... 2026-02-18T08:08:39.453 INFO:teuthology.orchestra.run.trial034.stdout:Selecting previously unselected package python3-py. 2026-02-18T08:08:39.462 INFO:teuthology.orchestra.run.trial034.stdout:Preparing to unpack .../77-python3-py_1.10.0-1_all.deb ... 2026-02-18T08:08:39.462 INFO:teuthology.orchestra.run.trial034.stdout:Unpacking python3-py (1.10.0-1) ... 2026-02-18T08:08:39.481 INFO:teuthology.orchestra.run.trial034.stdout:Selecting previously unselected package python3-pygments. 2026-02-18T08:08:39.489 INFO:teuthology.orchestra.run.trial034.stdout:Preparing to unpack .../78-python3-pygments_2.11.2+dfsg-2ubuntu0.1_all.deb ... 2026-02-18T08:08:39.490 INFO:teuthology.orchestra.run.trial034.stdout:Unpacking python3-pygments (2.11.2+dfsg-2ubuntu0.1) ... 2026-02-18T08:08:39.531 INFO:teuthology.orchestra.run.trial034.stdout:Selecting previously unselected package python3-toml. 2026-02-18T08:08:39.540 INFO:teuthology.orchestra.run.trial034.stdout:Preparing to unpack .../79-python3-toml_0.10.2-1_all.deb ... 2026-02-18T08:08:39.540 INFO:teuthology.orchestra.run.trial034.stdout:Unpacking python3-toml (0.10.2-1) ... 2026-02-18T08:08:39.555 INFO:teuthology.orchestra.run.trial034.stdout:Selecting previously unselected package python3-pytest. 2026-02-18T08:08:39.563 INFO:teuthology.orchestra.run.trial034.stdout:Preparing to unpack .../80-python3-pytest_6.2.5-1ubuntu2_all.deb ... 2026-02-18T08:08:39.563 INFO:teuthology.orchestra.run.trial034.stdout:Unpacking python3-pytest (6.2.5-1ubuntu2) ... 2026-02-18T08:08:39.585 INFO:teuthology.orchestra.run.trial034.stdout:Selecting previously unselected package python3-simplejson. 2026-02-18T08:08:39.593 INFO:teuthology.orchestra.run.trial034.stdout:Preparing to unpack .../81-python3-simplejson_3.17.6-1build1_amd64.deb ... 2026-02-18T08:08:39.593 INFO:teuthology.orchestra.run.trial034.stdout:Unpacking python3-simplejson (3.17.6-1build1) ... 2026-02-18T08:08:39.611 INFO:teuthology.orchestra.run.trial034.stdout:Selecting previously unselected package python3-webob. 2026-02-18T08:08:39.619 INFO:teuthology.orchestra.run.trial034.stdout:Preparing to unpack .../82-python3-webob_1%3a1.8.6-1.1ubuntu0.1_all.deb ... 2026-02-18T08:08:39.620 INFO:teuthology.orchestra.run.trial034.stdout:Unpacking python3-webob (1:1.8.6-1.1ubuntu0.1) ... 2026-02-18T08:08:39.638 INFO:teuthology.orchestra.run.trial163.stdout:Fetched 283 MB in 16s (18.2 MB/s) 2026-02-18T08:08:39.643 INFO:teuthology.orchestra.run.trial034.stdout:Selecting previously unselected package qttranslations5-l10n. 2026-02-18T08:08:39.651 INFO:teuthology.orchestra.run.trial034.stdout:Preparing to unpack .../83-qttranslations5-l10n_5.15.3-1_all.deb ... 2026-02-18T08:08:39.652 INFO:teuthology.orchestra.run.trial034.stdout:Unpacking qttranslations5-l10n (5.15.3-1) ... 2026-02-18T08:08:39.654 INFO:teuthology.orchestra.run.trial163.stdout:Selecting previously unselected package liblttng-ust1:amd64. 2026-02-18T08:08:39.718 INFO:teuthology.orchestra.run.trial034.stdout:Selecting previously unselected package radosgw. 2026-02-18T08:08:39.726 INFO:teuthology.orchestra.run.trial034.stdout:Preparing to unpack .../84-radosgw_20.2.0-670-gadc19233-1jammy_amd64.deb ... 2026-02-18T08:08:39.727 INFO:teuthology.orchestra.run.trial034.stdout:Unpacking radosgw (20.2.0-670-gadc19233-1jammy) ... 2026-02-18T08:08:39.814 INFO:teuthology.orchestra.run.trial163.stdout:(Reading database ... (Reading database ... 5% (Reading database ... 10% (Reading database ... 15% (Reading database ... 20% (Reading database ... 25% (Reading database ... 30% (Reading database ... 35% (Reading database ... 40% (Reading database ... 45% (Reading database ... 50% (Reading database ... 55% (Reading database ... 60% (Reading database ... 65% (Reading database ... 70% (Reading database ... 75% (Reading database ... 80% (Reading database ... 85% (Reading database ... 90% (Reading database ... 95% (Reading database ... 100% (Reading database ... 167438 files and directories currently installed.) 2026-02-18T08:08:39.816 INFO:teuthology.orchestra.run.trial163.stdout:Preparing to unpack .../00-liblttng-ust1_2.13.1-1ubuntu1_amd64.deb ... 2026-02-18T08:08:39.817 INFO:teuthology.orchestra.run.trial163.stdout:Unpacking liblttng-ust1:amd64 (2.13.1-1ubuntu1) ... 2026-02-18T08:08:39.838 INFO:teuthology.orchestra.run.trial163.stdout:Selecting previously unselected package libdouble-conversion3:amd64. 2026-02-18T08:08:39.851 INFO:teuthology.orchestra.run.trial163.stdout:Preparing to unpack .../01-libdouble-conversion3_3.1.7-4_amd64.deb ... 2026-02-18T08:08:39.851 INFO:teuthology.orchestra.run.trial163.stdout:Unpacking libdouble-conversion3:amd64 (3.1.7-4) ... 2026-02-18T08:08:39.869 INFO:teuthology.orchestra.run.trial163.stdout:Selecting previously unselected package libpcre2-16-0:amd64. 2026-02-18T08:08:39.880 INFO:teuthology.orchestra.run.trial163.stdout:Preparing to unpack .../02-libpcre2-16-0_10.39-3ubuntu0.1_amd64.deb ... 2026-02-18T08:08:39.881 INFO:teuthology.orchestra.run.trial163.stdout:Unpacking libpcre2-16-0:amd64 (10.39-3ubuntu0.1) ... 2026-02-18T08:08:39.901 INFO:teuthology.orchestra.run.trial163.stdout:Selecting previously unselected package libqt5core5a:amd64. 2026-02-18T08:08:39.909 INFO:teuthology.orchestra.run.trial163.stdout:Preparing to unpack .../03-libqt5core5a_5.15.3+dfsg-2ubuntu0.2_amd64.deb ... 2026-02-18T08:08:39.915 INFO:teuthology.orchestra.run.trial163.stdout:Unpacking libqt5core5a:amd64 (5.15.3+dfsg-2ubuntu0.2) ... 2026-02-18T08:08:39.934 INFO:teuthology.orchestra.run.trial034.stdout:Selecting previously unselected package rbd-fuse. 2026-02-18T08:08:39.943 INFO:teuthology.orchestra.run.trial034.stdout:Preparing to unpack .../85-rbd-fuse_20.2.0-670-gadc19233-1jammy_amd64.deb ... 2026-02-18T08:08:39.943 INFO:teuthology.orchestra.run.trial034.stdout:Unpacking rbd-fuse (20.2.0-670-gadc19233-1jammy) ... 2026-02-18T08:08:39.950 INFO:teuthology.orchestra.run.trial163.stdout:Selecting previously unselected package libqt5dbus5:amd64. 2026-02-18T08:08:39.958 INFO:teuthology.orchestra.run.trial034.stdout:Selecting previously unselected package rbd-mirror. 2026-02-18T08:08:39.960 INFO:teuthology.orchestra.run.trial163.stdout:Preparing to unpack .../04-libqt5dbus5_5.15.3+dfsg-2ubuntu0.2_amd64.deb ... 2026-02-18T08:08:39.960 INFO:teuthology.orchestra.run.trial163.stdout:Unpacking libqt5dbus5:amd64 (5.15.3+dfsg-2ubuntu0.2) ... 2026-02-18T08:08:39.967 INFO:teuthology.orchestra.run.trial034.stdout:Preparing to unpack .../86-rbd-mirror_20.2.0-670-gadc19233-1jammy_amd64.deb ... 2026-02-18T08:08:39.967 INFO:teuthology.orchestra.run.trial034.stdout:Unpacking rbd-mirror (20.2.0-670-gadc19233-1jammy) ... 2026-02-18T08:08:39.976 INFO:teuthology.orchestra.run.trial163.stdout:Selecting previously unselected package libqt5network5:amd64. 2026-02-18T08:08:39.984 INFO:teuthology.orchestra.run.trial163.stdout:Preparing to unpack .../05-libqt5network5_5.15.3+dfsg-2ubuntu0.2_amd64.deb ... 2026-02-18T08:08:39.985 INFO:teuthology.orchestra.run.trial163.stdout:Unpacking libqt5network5:amd64 (5.15.3+dfsg-2ubuntu0.2) ... 2026-02-18T08:08:40.004 INFO:teuthology.orchestra.run.trial163.stdout:Selecting previously unselected package libthrift-0.16.0:amd64. 2026-02-18T08:08:40.012 INFO:teuthology.orchestra.run.trial163.stdout:Preparing to unpack .../06-libthrift-0.16.0_0.16.0-2_amd64.deb ... 2026-02-18T08:08:40.012 INFO:teuthology.orchestra.run.trial163.stdout:Unpacking libthrift-0.16.0:amd64 (0.16.0-2) ... 2026-02-18T08:08:40.019 INFO:teuthology.orchestra.run.trial034.stdout:Setting up python3-iniconfig (1.1.1-2) ... 2026-02-18T08:08:40.029 INFO:teuthology.orchestra.run.trial163.stdout:Selecting previously unselected package librados2. 2026-02-18T08:08:40.037 INFO:teuthology.orchestra.run.trial163.stdout:Preparing to unpack .../07-librados2_20.2.0-670-gadc19233-1jammy_amd64.deb ... 2026-02-18T08:08:40.038 INFO:teuthology.orchestra.run.trial163.stdout:Unpacking librados2 (20.2.0-670-gadc19233-1jammy) ... 2026-02-18T08:08:40.079 INFO:teuthology.orchestra.run.trial163.stdout:Selecting previously unselected package libnbd0. 2026-02-18T08:08:40.085 INFO:teuthology.orchestra.run.trial034.stdout:Setting up libdouble-conversion3:amd64 (3.1.7-4) ... 2026-02-18T08:08:40.086 INFO:teuthology.orchestra.run.trial034.stdout:Setting up nvme-cli (1.16-3ubuntu0.3) ... 2026-02-18T08:08:40.087 INFO:teuthology.orchestra.run.trial163.stdout:Preparing to unpack .../08-libnbd0_1.10.5-1_amd64.deb ... 2026-02-18T08:08:40.088 INFO:teuthology.orchestra.run.trial163.stdout:Unpacking libnbd0 (1.10.5-1) ... 2026-02-18T08:08:40.102 INFO:teuthology.orchestra.run.trial163.stdout:Selecting previously unselected package librbd1. 2026-02-18T08:08:40.110 INFO:teuthology.orchestra.run.trial163.stdout:Preparing to unpack .../09-librbd1_20.2.0-670-gadc19233-1jammy_amd64.deb ... 2026-02-18T08:08:40.110 INFO:teuthology.orchestra.run.trial163.stdout:Unpacking librbd1 (20.2.0-670-gadc19233-1jammy) ... 2026-02-18T08:08:40.147 INFO:teuthology.orchestra.run.trial163.stdout:Selecting previously unselected package libcephfs2. 2026-02-18T08:08:40.156 INFO:teuthology.orchestra.run.trial034.stdout:Created symlink /etc/systemd/system/default.target.wants/nvmefc-boot-connections.service → /lib/systemd/system/nvmefc-boot-connections.service. 2026-02-18T08:08:40.156 INFO:teuthology.orchestra.run.trial163.stdout:Preparing to unpack .../10-libcephfs2_20.2.0-670-gadc19233-1jammy_amd64.deb ... 2026-02-18T08:08:40.156 INFO:teuthology.orchestra.run.trial163.stdout:Unpacking libcephfs2 (20.2.0-670-gadc19233-1jammy) ... 2026-02-18T08:08:40.176 INFO:teuthology.orchestra.run.trial163.stdout:Selecting previously unselected package python3-rados. 2026-02-18T08:08:40.185 INFO:teuthology.orchestra.run.trial163.stdout:Preparing to unpack .../11-python3-rados_20.2.0-670-gadc19233-1jammy_amd64.deb ... 2026-02-18T08:08:40.185 INFO:teuthology.orchestra.run.trial163.stdout:Unpacking python3-rados (20.2.0-670-gadc19233-1jammy) ... 2026-02-18T08:08:40.202 INFO:teuthology.orchestra.run.trial163.stdout:Selecting previously unselected package python3-ceph-argparse. 2026-02-18T08:08:40.211 INFO:teuthology.orchestra.run.trial163.stdout:Preparing to unpack .../12-python3-ceph-argparse_20.2.0-670-gadc19233-1jammy_all.deb ... 2026-02-18T08:08:40.211 INFO:teuthology.orchestra.run.trial163.stdout:Unpacking python3-ceph-argparse (20.2.0-670-gadc19233-1jammy) ... 2026-02-18T08:08:40.225 INFO:teuthology.orchestra.run.trial163.stdout:Selecting previously unselected package python3-cephfs. 2026-02-18T08:08:40.233 INFO:teuthology.orchestra.run.trial163.stdout:Preparing to unpack .../13-python3-cephfs_20.2.0-670-gadc19233-1jammy_amd64.deb ... 2026-02-18T08:08:40.233 INFO:teuthology.orchestra.run.trial163.stdout:Unpacking python3-cephfs (20.2.0-670-gadc19233-1jammy) ... 2026-02-18T08:08:40.249 INFO:teuthology.orchestra.run.trial163.stdout:Selecting previously unselected package python3-ceph-common. 2026-02-18T08:08:40.257 INFO:teuthology.orchestra.run.trial163.stdout:Preparing to unpack .../14-python3-ceph-common_20.2.0-670-gadc19233-1jammy_all.deb ... 2026-02-18T08:08:40.257 INFO:teuthology.orchestra.run.trial163.stdout:Unpacking python3-ceph-common (20.2.0-670-gadc19233-1jammy) ... 2026-02-18T08:08:40.276 INFO:teuthology.orchestra.run.trial163.stdout:Selecting previously unselected package python3-wcwidth. 2026-02-18T08:08:40.284 INFO:teuthology.orchestra.run.trial163.stdout:Preparing to unpack .../15-python3-wcwidth_0.2.5+dfsg1-1_all.deb ... 2026-02-18T08:08:40.285 INFO:teuthology.orchestra.run.trial163.stdout:Unpacking python3-wcwidth (0.2.5+dfsg1-1) ... 2026-02-18T08:08:40.300 INFO:teuthology.orchestra.run.trial163.stdout:Selecting previously unselected package python3-prettytable. 2026-02-18T08:08:40.308 INFO:teuthology.orchestra.run.trial163.stdout:Preparing to unpack .../16-python3-prettytable_2.5.0-2_all.deb ... 2026-02-18T08:08:40.308 INFO:teuthology.orchestra.run.trial163.stdout:Unpacking python3-prettytable (2.5.0-2) ... 2026-02-18T08:08:40.323 INFO:teuthology.orchestra.run.trial163.stdout:Selecting previously unselected package python3-rbd. 2026-02-18T08:08:40.330 INFO:teuthology.orchestra.run.trial163.stdout:Preparing to unpack .../17-python3-rbd_20.2.0-670-gadc19233-1jammy_amd64.deb ... 2026-02-18T08:08:40.331 INFO:teuthology.orchestra.run.trial163.stdout:Unpacking python3-rbd (20.2.0-670-gadc19233-1jammy) ... 2026-02-18T08:08:40.348 INFO:teuthology.orchestra.run.trial163.stdout:Selecting previously unselected package librdkafka1:amd64. 2026-02-18T08:08:40.356 INFO:teuthology.orchestra.run.trial163.stdout:Preparing to unpack .../18-librdkafka1_1.8.0-1build1_amd64.deb ... 2026-02-18T08:08:40.356 INFO:teuthology.orchestra.run.trial163.stdout:Unpacking librdkafka1:amd64 (1.8.0-1build1) ... 2026-02-18T08:08:40.375 INFO:teuthology.orchestra.run.trial163.stdout:Selecting previously unselected package librgw2. 2026-02-18T08:08:40.383 INFO:teuthology.orchestra.run.trial163.stdout:Preparing to unpack .../19-librgw2_20.2.0-670-gadc19233-1jammy_amd64.deb ... 2026-02-18T08:08:40.383 INFO:teuthology.orchestra.run.trial163.stdout:Unpacking librgw2 (20.2.0-670-gadc19233-1jammy) ... 2026-02-18T08:08:40.424 INFO:teuthology.orchestra.run.trial034.stdout:Created symlink /etc/systemd/system/default.target.wants/nvmf-autoconnect.service → /lib/systemd/system/nvmf-autoconnect.service. 2026-02-18T08:08:40.476 INFO:teuthology.orchestra.run.trial163.stdout:Selecting previously unselected package python3-rgw. 2026-02-18T08:08:40.484 INFO:teuthology.orchestra.run.trial163.stdout:Preparing to unpack .../20-python3-rgw_20.2.0-670-gadc19233-1jammy_amd64.deb ... 2026-02-18T08:08:40.485 INFO:teuthology.orchestra.run.trial163.stdout:Unpacking python3-rgw (20.2.0-670-gadc19233-1jammy) ... 2026-02-18T08:08:40.499 INFO:teuthology.orchestra.run.trial163.stdout:Selecting previously unselected package liboath0:amd64. 2026-02-18T08:08:40.507 INFO:teuthology.orchestra.run.trial163.stdout:Preparing to unpack .../21-liboath0_2.6.7-3ubuntu0.1_amd64.deb ... 2026-02-18T08:08:40.508 INFO:teuthology.orchestra.run.trial163.stdout:Unpacking liboath0:amd64 (2.6.7-3ubuntu0.1) ... 2026-02-18T08:08:40.522 INFO:teuthology.orchestra.run.trial163.stdout:Selecting previously unselected package libradosstriper1. 2026-02-18T08:08:40.530 INFO:teuthology.orchestra.run.trial163.stdout:Preparing to unpack .../22-libradosstriper1_20.2.0-670-gadc19233-1jammy_amd64.deb ... 2026-02-18T08:08:40.531 INFO:teuthology.orchestra.run.trial163.stdout:Unpacking libradosstriper1 (20.2.0-670-gadc19233-1jammy) ... 2026-02-18T08:08:40.547 INFO:teuthology.orchestra.run.trial163.stdout:Selecting previously unselected package ceph-common. 2026-02-18T08:08:40.555 INFO:teuthology.orchestra.run.trial163.stdout:Preparing to unpack .../23-ceph-common_20.2.0-670-gadc19233-1jammy_amd64.deb ... 2026-02-18T08:08:40.555 INFO:teuthology.orchestra.run.trial163.stdout:Unpacking ceph-common (20.2.0-670-gadc19233-1jammy) ... 2026-02-18T08:08:40.795 INFO:teuthology.orchestra.run.trial034.stdout:nvmf-connect.target is a disabled or a static unit, not starting it. 2026-02-18T08:08:40.806 INFO:teuthology.orchestra.run.trial034.stdout:Setting up cephadm (20.2.0-670-gadc19233-1jammy) ... 2026-02-18T08:08:40.811 INFO:teuthology.orchestra.run.trial163.stdout:Selecting previously unselected package ceph-base. 2026-02-18T08:08:40.820 INFO:teuthology.orchestra.run.trial163.stdout:Preparing to unpack .../24-ceph-base_20.2.0-670-gadc19233-1jammy_amd64.deb ... 2026-02-18T08:08:40.825 INFO:teuthology.orchestra.run.trial163.stdout:Unpacking ceph-base (20.2.0-670-gadc19233-1jammy) ... 2026-02-18T08:08:40.850 INFO:teuthology.orchestra.run.trial034.stdout:Adding system user cephadm....done 2026-02-18T08:08:40.859 INFO:teuthology.orchestra.run.trial034.stdout:Setting up python3-jaraco.classes (3.2.1-3) ... 2026-02-18T08:08:40.891 INFO:teuthology.orchestra.run.trial163.stdout:Selecting previously unselected package python3-jaraco.functools. 2026-02-18T08:08:40.899 INFO:teuthology.orchestra.run.trial163.stdout:Preparing to unpack .../25-python3-jaraco.functools_3.4.0-2_all.deb ... 2026-02-18T08:08:40.900 INFO:teuthology.orchestra.run.trial163.stdout:Unpacking python3-jaraco.functools (3.4.0-2) ... 2026-02-18T08:08:40.915 INFO:teuthology.orchestra.run.trial163.stdout:Selecting previously unselected package python3-cheroot. 2026-02-18T08:08:40.923 INFO:teuthology.orchestra.run.trial163.stdout:Preparing to unpack .../26-python3-cheroot_8.5.2+ds1-1ubuntu3.1_all.deb ... 2026-02-18T08:08:40.923 INFO:teuthology.orchestra.run.trial163.stdout:Unpacking python3-cheroot (8.5.2+ds1-1ubuntu3.1) ... 2026-02-18T08:08:40.929 INFO:teuthology.orchestra.run.trial034.stdout:Setting up python-asyncssh-doc (2.5.0-1ubuntu0.1) ... 2026-02-18T08:08:40.930 INFO:teuthology.orchestra.run.trial034.stdout:Setting up python3-jaraco.functools (3.4.0-2) ... 2026-02-18T08:08:40.939 INFO:teuthology.orchestra.run.trial163.stdout:Selecting previously unselected package python3-jaraco.classes. 2026-02-18T08:08:40.948 INFO:teuthology.orchestra.run.trial163.stdout:Preparing to unpack .../27-python3-jaraco.classes_3.2.1-3_all.deb ... 2026-02-18T08:08:40.948 INFO:teuthology.orchestra.run.trial163.stdout:Unpacking python3-jaraco.classes (3.2.1-3) ... 2026-02-18T08:08:40.963 INFO:teuthology.orchestra.run.trial163.stdout:Selecting previously unselected package python3-jaraco.text. 2026-02-18T08:08:40.971 INFO:teuthology.orchestra.run.trial163.stdout:Preparing to unpack .../28-python3-jaraco.text_3.6.0-2_all.deb ... 2026-02-18T08:08:40.971 INFO:teuthology.orchestra.run.trial163.stdout:Unpacking python3-jaraco.text (3.6.0-2) ... 2026-02-18T08:08:40.985 INFO:teuthology.orchestra.run.trial163.stdout:Selecting previously unselected package python3-jaraco.collections. 2026-02-18T08:08:40.993 INFO:teuthology.orchestra.run.trial163.stdout:Preparing to unpack .../29-python3-jaraco.collections_3.4.0-2_all.deb ... 2026-02-18T08:08:40.994 INFO:teuthology.orchestra.run.trial163.stdout:Unpacking python3-jaraco.collections (3.4.0-2) ... 2026-02-18T08:08:41.001 INFO:teuthology.orchestra.run.trial034.stdout:Setting up python3-repoze.lru (0.7-2) ... 2026-02-18T08:08:41.007 INFO:teuthology.orchestra.run.trial163.stdout:Selecting previously unselected package python3-tempora. 2026-02-18T08:08:41.015 INFO:teuthology.orchestra.run.trial163.stdout:Preparing to unpack .../30-python3-tempora_4.1.2-1_all.deb ... 2026-02-18T08:08:41.015 INFO:teuthology.orchestra.run.trial163.stdout:Unpacking python3-tempora (4.1.2-1) ... 2026-02-18T08:08:41.031 INFO:teuthology.orchestra.run.trial163.stdout:Selecting previously unselected package python3-portend. 2026-02-18T08:08:41.039 INFO:teuthology.orchestra.run.trial163.stdout:Preparing to unpack .../31-python3-portend_3.0.0-1_all.deb ... 2026-02-18T08:08:41.039 INFO:teuthology.orchestra.run.trial163.stdout:Unpacking python3-portend (3.0.0-1) ... 2026-02-18T08:08:41.054 INFO:teuthology.orchestra.run.trial163.stdout:Selecting previously unselected package python3-zc.lockfile. 2026-02-18T08:08:41.062 INFO:teuthology.orchestra.run.trial163.stdout:Preparing to unpack .../32-python3-zc.lockfile_2.0-1_all.deb ... 2026-02-18T08:08:41.062 INFO:teuthology.orchestra.run.trial163.stdout:Unpacking python3-zc.lockfile (2.0-1) ... 2026-02-18T08:08:41.076 INFO:teuthology.orchestra.run.trial034.stdout:Setting up liboath0:amd64 (2.6.7-3ubuntu0.1) ... 2026-02-18T08:08:41.077 INFO:teuthology.orchestra.run.trial163.stdout:Selecting previously unselected package python3-cherrypy3. 2026-02-18T08:08:41.078 INFO:teuthology.orchestra.run.trial034.stdout:Setting up python3-py (1.10.0-1) ... 2026-02-18T08:08:41.086 INFO:teuthology.orchestra.run.trial163.stdout:Preparing to unpack .../33-python3-cherrypy3_18.6.1-4_all.deb ... 2026-02-18T08:08:41.086 INFO:teuthology.orchestra.run.trial163.stdout:Unpacking python3-cherrypy3 (18.6.1-4) ... 2026-02-18T08:08:41.109 INFO:teuthology.orchestra.run.trial163.stdout:Selecting previously unselected package python3-natsort. 2026-02-18T08:08:41.117 INFO:teuthology.orchestra.run.trial163.stdout:Preparing to unpack .../34-python3-natsort_8.0.2-1_all.deb ... 2026-02-18T08:08:41.117 INFO:teuthology.orchestra.run.trial163.stdout:Unpacking python3-natsort (8.0.2-1) ... 2026-02-18T08:08:41.134 INFO:teuthology.orchestra.run.trial163.stdout:Selecting previously unselected package ceph-mgr-modules-core. 2026-02-18T08:08:41.143 INFO:teuthology.orchestra.run.trial163.stdout:Preparing to unpack .../35-ceph-mgr-modules-core_20.2.0-670-gadc19233-1jammy_all.deb ... 2026-02-18T08:08:41.143 INFO:teuthology.orchestra.run.trial163.stdout:Unpacking ceph-mgr-modules-core (20.2.0-670-gadc19233-1jammy) ... 2026-02-18T08:08:41.170 INFO:teuthology.orchestra.run.trial163.stdout:Selecting previously unselected package libsqlite3-mod-ceph. 2026-02-18T08:08:41.172 INFO:teuthology.orchestra.run.trial034.stdout:Setting up python3-joblib (0.17.0-4ubuntu1) ... 2026-02-18T08:08:41.178 INFO:teuthology.orchestra.run.trial163.stdout:Preparing to unpack .../36-libsqlite3-mod-ceph_20.2.0-670-gadc19233-1jammy_amd64.deb ... 2026-02-18T08:08:41.178 INFO:teuthology.orchestra.run.trial163.stdout:Unpacking libsqlite3-mod-ceph (20.2.0-670-gadc19233-1jammy) ... 2026-02-18T08:08:41.193 INFO:teuthology.orchestra.run.trial163.stdout:Selecting previously unselected package ceph-mgr. 2026-02-18T08:08:41.201 INFO:teuthology.orchestra.run.trial163.stdout:Preparing to unpack .../37-ceph-mgr_20.2.0-670-gadc19233-1jammy_amd64.deb ... 2026-02-18T08:08:41.201 INFO:teuthology.orchestra.run.trial163.stdout:Unpacking ceph-mgr (20.2.0-670-gadc19233-1jammy) ... 2026-02-18T08:08:41.222 INFO:teuthology.orchestra.run.trial163.stdout:Selecting previously unselected package ceph-mon. 2026-02-18T08:08:41.230 INFO:teuthology.orchestra.run.trial163.stdout:Preparing to unpack .../38-ceph-mon_20.2.0-670-gadc19233-1jammy_amd64.deb ... 2026-02-18T08:08:41.231 INFO:teuthology.orchestra.run.trial163.stdout:Unpacking ceph-mon (20.2.0-670-gadc19233-1jammy) ... 2026-02-18T08:08:41.292 INFO:teuthology.orchestra.run.trial163.stdout:Selecting previously unselected package libfuse2:amd64. 2026-02-18T08:08:41.297 INFO:teuthology.orchestra.run.trial034.stdout:Setting up python3-cachetools (5.0.0-1) ... 2026-02-18T08:08:41.300 INFO:teuthology.orchestra.run.trial163.stdout:Preparing to unpack .../39-libfuse2_2.9.9-5ubuntu3_amd64.deb ... 2026-02-18T08:08:41.300 INFO:teuthology.orchestra.run.trial163.stdout:Unpacking libfuse2:amd64 (2.9.9-5ubuntu3) ... 2026-02-18T08:08:41.316 INFO:teuthology.orchestra.run.trial163.stdout:Selecting previously unselected package ceph-osd. 2026-02-18T08:08:41.324 INFO:teuthology.orchestra.run.trial163.stdout:Preparing to unpack .../40-ceph-osd_20.2.0-670-gadc19233-1jammy_amd64.deb ... 2026-02-18T08:08:41.324 INFO:teuthology.orchestra.run.trial163.stdout:Unpacking ceph-osd (20.2.0-670-gadc19233-1jammy) ... 2026-02-18T08:08:41.372 INFO:teuthology.orchestra.run.trial034.stdout:Setting up python3-threadpoolctl (3.1.0-1) ... 2026-02-18T08:08:41.447 INFO:teuthology.orchestra.run.trial034.stdout:Setting up python3-ceph-argparse (20.2.0-670-gadc19233-1jammy) ... 2026-02-18T08:08:41.472 INFO:teuthology.orchestra.run.trial163.stdout:Selecting previously unselected package ceph. 2026-02-18T08:08:41.481 INFO:teuthology.orchestra.run.trial163.stdout:Preparing to unpack .../41-ceph_20.2.0-670-gadc19233-1jammy_amd64.deb ... 2026-02-18T08:08:41.481 INFO:teuthology.orchestra.run.trial163.stdout:Unpacking ceph (20.2.0-670-gadc19233-1jammy) ... 2026-02-18T08:08:41.495 INFO:teuthology.orchestra.run.trial163.stdout:Selecting previously unselected package ceph-fuse. 2026-02-18T08:08:41.503 INFO:teuthology.orchestra.run.trial163.stdout:Preparing to unpack .../42-ceph-fuse_20.2.0-670-gadc19233-1jammy_amd64.deb ... 2026-02-18T08:08:41.503 INFO:teuthology.orchestra.run.trial163.stdout:Unpacking ceph-fuse (20.2.0-670-gadc19233-1jammy) ... 2026-02-18T08:08:41.515 INFO:teuthology.orchestra.run.trial034.stdout:Setting up python3-sklearn-lib:amd64 (0.23.2-5ubuntu6) ... 2026-02-18T08:08:41.516 INFO:teuthology.orchestra.run.trial034.stdout:Setting up libnbd0 (1.10.5-1) ... 2026-02-18T08:08:41.517 INFO:teuthology.orchestra.run.trial034.stdout:Setting up libfuse2:amd64 (2.9.9-5ubuntu3) ... 2026-02-18T08:08:41.517 INFO:teuthology.orchestra.run.trial034.stdout:Setting up libpcre2-16-0:amd64 (10.39-3ubuntu0.1) ... 2026-02-18T08:08:41.518 INFO:teuthology.orchestra.run.trial034.stdout:Setting up python3-psutil (5.9.0-1build1) ... 2026-02-18T08:08:41.527 INFO:teuthology.orchestra.run.trial163.stdout:Selecting previously unselected package ceph-mds. 2026-02-18T08:08:41.535 INFO:teuthology.orchestra.run.trial163.stdout:Preparing to unpack .../43-ceph-mds_20.2.0-670-gadc19233-1jammy_amd64.deb ... 2026-02-18T08:08:41.536 INFO:teuthology.orchestra.run.trial163.stdout:Unpacking ceph-mds (20.2.0-670-gadc19233-1jammy) ... 2026-02-18T08:08:41.572 INFO:teuthology.orchestra.run.trial163.stdout:Selecting previously unselected package cephadm. 2026-02-18T08:08:41.580 INFO:teuthology.orchestra.run.trial163.stdout:Preparing to unpack .../44-cephadm_20.2.0-670-gadc19233-1jammy_amd64.deb ... 2026-02-18T08:08:41.581 INFO:teuthology.orchestra.run.trial163.stdout:Unpacking cephadm (20.2.0-670-gadc19233-1jammy) ... 2026-02-18T08:08:41.597 INFO:teuthology.orchestra.run.trial163.stdout:Selecting previously unselected package python3-asyncssh. 2026-02-18T08:08:41.605 INFO:teuthology.orchestra.run.trial163.stdout:Preparing to unpack .../45-python3-asyncssh_2.5.0-1ubuntu0.1_all.deb ... 2026-02-18T08:08:41.606 INFO:teuthology.orchestra.run.trial163.stdout:Unpacking python3-asyncssh (2.5.0-1ubuntu0.1) ... 2026-02-18T08:08:41.627 INFO:teuthology.orchestra.run.trial163.stdout:Selecting previously unselected package ceph-mgr-cephadm. 2026-02-18T08:08:41.635 INFO:teuthology.orchestra.run.trial163.stdout:Preparing to unpack .../46-ceph-mgr-cephadm_20.2.0-670-gadc19233-1jammy_all.deb ... 2026-02-18T08:08:41.636 INFO:teuthology.orchestra.run.trial163.stdout:Unpacking ceph-mgr-cephadm (20.2.0-670-gadc19233-1jammy) ... 2026-02-18T08:08:41.639 INFO:teuthology.orchestra.run.trial034.stdout:Setting up python3-natsort (8.0.2-1) ... 2026-02-18T08:08:41.656 INFO:teuthology.orchestra.run.trial163.stdout:Selecting previously unselected package python3-repoze.lru. 2026-02-18T08:08:41.665 INFO:teuthology.orchestra.run.trial163.stdout:Preparing to unpack .../47-python3-repoze.lru_0.7-2_all.deb ... 2026-02-18T08:08:41.665 INFO:teuthology.orchestra.run.trial163.stdout:Unpacking python3-repoze.lru (0.7-2) ... 2026-02-18T08:08:41.679 INFO:teuthology.orchestra.run.trial163.stdout:Selecting previously unselected package python3-routes. 2026-02-18T08:08:41.687 INFO:teuthology.orchestra.run.trial163.stdout:Preparing to unpack .../48-python3-routes_2.5.1-1ubuntu1_all.deb ... 2026-02-18T08:08:41.688 INFO:teuthology.orchestra.run.trial163.stdout:Unpacking python3-routes (2.5.1-1ubuntu1) ... 2026-02-18T08:08:41.707 INFO:teuthology.orchestra.run.trial163.stdout:Selecting previously unselected package ceph-mgr-dashboard. 2026-02-18T08:08:41.715 INFO:teuthology.orchestra.run.trial034.stdout:Setting up libcephfs-proxy2 (20.2.0-670-gadc19233-1jammy) ... 2026-02-18T08:08:41.715 INFO:teuthology.orchestra.run.trial163.stdout:Preparing to unpack .../49-ceph-mgr-dashboard_20.2.0-670-gadc19233-1jammy_all.deb ... 2026-02-18T08:08:41.716 INFO:teuthology.orchestra.run.trial163.stdout:Unpacking ceph-mgr-dashboard (20.2.0-670-gadc19233-1jammy) ... 2026-02-18T08:08:41.716 INFO:teuthology.orchestra.run.trial034.stdout:Setting up python3-routes (2.5.1-1ubuntu1) ... 2026-02-18T08:08:41.795 INFO:teuthology.orchestra.run.trial034.stdout:Setting up python3-simplejson (3.17.6-1build1) ... 2026-02-18T08:08:41.885 INFO:teuthology.orchestra.run.trial034.stdout:Setting up python3-pygments (2.11.2+dfsg-2ubuntu0.1) ... 2026-02-18T08:08:42.138 INFO:teuthology.orchestra.run.trial034.stdout:Setting up qttranslations5-l10n (5.15.3-1) ... 2026-02-18T08:08:42.139 INFO:teuthology.orchestra.run.trial034.stdout:Setting up python3-wcwidth (0.2.5+dfsg1-1) ... 2026-02-18T08:08:42.152 INFO:teuthology.orchestra.run.trial163.stdout:Selecting previously unselected package python3-sklearn-lib:amd64. 2026-02-18T08:08:42.160 INFO:teuthology.orchestra.run.trial163.stdout:Preparing to unpack .../50-python3-sklearn-lib_0.23.2-5ubuntu6_amd64.deb ... 2026-02-18T08:08:42.161 INFO:teuthology.orchestra.run.trial163.stdout:Unpacking python3-sklearn-lib:amd64 (0.23.2-5ubuntu6) ... 2026-02-18T08:08:42.199 INFO:teuthology.orchestra.run.trial163.stdout:Selecting previously unselected package python3-joblib. 2026-02-18T08:08:42.208 INFO:teuthology.orchestra.run.trial163.stdout:Preparing to unpack .../51-python3-joblib_0.17.0-4ubuntu1_all.deb ... 2026-02-18T08:08:42.208 INFO:teuthology.orchestra.run.trial163.stdout:Unpacking python3-joblib (0.17.0-4ubuntu1) ... 2026-02-18T08:08:42.230 INFO:teuthology.orchestra.run.trial034.stdout:Setting up python3-asyncssh (2.5.0-1ubuntu0.1) ... 2026-02-18T08:08:42.235 INFO:teuthology.orchestra.run.trial163.stdout:Selecting previously unselected package python3-threadpoolctl. 2026-02-18T08:08:42.243 INFO:teuthology.orchestra.run.trial163.stdout:Preparing to unpack .../52-python3-threadpoolctl_3.1.0-1_all.deb ... 2026-02-18T08:08:42.243 INFO:teuthology.orchestra.run.trial163.stdout:Unpacking python3-threadpoolctl (3.1.0-1) ... 2026-02-18T08:08:42.259 INFO:teuthology.orchestra.run.trial163.stdout:Selecting previously unselected package python3-sklearn. 2026-02-18T08:08:42.268 INFO:teuthology.orchestra.run.trial163.stdout:Preparing to unpack .../53-python3-sklearn_0.23.2-5ubuntu6_all.deb ... 2026-02-18T08:08:42.268 INFO:teuthology.orchestra.run.trial163.stdout:Unpacking python3-sklearn (0.23.2-5ubuntu6) ... 2026-02-18T08:08:42.352 INFO:teuthology.orchestra.run.trial163.stdout:Selecting previously unselected package ceph-mgr-diskprediction-local. 2026-02-18T08:08:42.354 INFO:teuthology.orchestra.run.trial034.stdout:Setting up python3-cheroot (8.5.2+ds1-1ubuntu3.1) ... 2026-02-18T08:08:42.360 INFO:teuthology.orchestra.run.trial163.stdout:Preparing to unpack .../54-ceph-mgr-diskprediction-local_20.2.0-670-gadc19233-1jammy_all.deb ... 2026-02-18T08:08:42.361 INFO:teuthology.orchestra.run.trial163.stdout:Unpacking ceph-mgr-diskprediction-local (20.2.0-670-gadc19233-1jammy) ... 2026-02-18T08:08:42.434 INFO:teuthology.orchestra.run.trial034.stdout:Setting up python3-jaraco.text (3.6.0-2) ... 2026-02-18T08:08:42.503 INFO:teuthology.orchestra.run.trial034.stdout:Setting up socat (1.7.4.1-3ubuntu4) ... 2026-02-18T08:08:42.504 INFO:teuthology.orchestra.run.trial034.stdout:Setting up python3-ceph-common (20.2.0-670-gadc19233-1jammy) ... 2026-02-18T08:08:42.529 INFO:teuthology.orchestra.run.trial163.stdout:Selecting previously unselected package python3-cachetools. 2026-02-18T08:08:42.537 INFO:teuthology.orchestra.run.trial163.stdout:Preparing to unpack .../55-python3-cachetools_5.0.0-1_all.deb ... 2026-02-18T08:08:42.538 INFO:teuthology.orchestra.run.trial163.stdout:Unpacking python3-cachetools (5.0.0-1) ... 2026-02-18T08:08:42.553 INFO:teuthology.orchestra.run.trial163.stdout:Selecting previously unselected package python3-rsa. 2026-02-18T08:08:42.561 INFO:teuthology.orchestra.run.trial163.stdout:Preparing to unpack .../56-python3-rsa_4.8-1_all.deb ... 2026-02-18T08:08:42.561 INFO:teuthology.orchestra.run.trial163.stdout:Unpacking python3-rsa (4.8-1) ... 2026-02-18T08:08:42.579 INFO:teuthology.orchestra.run.trial163.stdout:Selecting previously unselected package python3-google-auth. 2026-02-18T08:08:42.587 INFO:teuthology.orchestra.run.trial163.stdout:Preparing to unpack .../57-python3-google-auth_1.5.1-3_all.deb ... 2026-02-18T08:08:42.588 INFO:teuthology.orchestra.run.trial163.stdout:Unpacking python3-google-auth (1.5.1-3) ... 2026-02-18T08:08:42.590 INFO:teuthology.orchestra.run.trial034.stdout:Setting up python3-sklearn (0.23.2-5ubuntu6) ... 2026-02-18T08:08:42.604 INFO:teuthology.orchestra.run.trial163.stdout:Selecting previously unselected package python3-requests-oauthlib. 2026-02-18T08:08:42.613 INFO:teuthology.orchestra.run.trial163.stdout:Preparing to unpack .../58-python3-requests-oauthlib_1.3.0+ds-0.1_all.deb ... 2026-02-18T08:08:42.613 INFO:teuthology.orchestra.run.trial163.stdout:Unpacking python3-requests-oauthlib (1.3.0+ds-0.1) ... 2026-02-18T08:08:42.630 INFO:teuthology.orchestra.run.trial163.stdout:Selecting previously unselected package python3-websocket. 2026-02-18T08:08:42.638 INFO:teuthology.orchestra.run.trial163.stdout:Preparing to unpack .../59-python3-websocket_1.2.3-1_all.deb ... 2026-02-18T08:08:42.638 INFO:teuthology.orchestra.run.trial163.stdout:Unpacking python3-websocket (1.2.3-1) ... 2026-02-18T08:08:42.655 INFO:teuthology.orchestra.run.trial163.stdout:Selecting previously unselected package python3-kubernetes. 2026-02-18T08:08:42.664 INFO:teuthology.orchestra.run.trial163.stdout:Preparing to unpack .../60-python3-kubernetes_12.0.1-1ubuntu1_all.deb ... 2026-02-18T08:08:42.664 INFO:teuthology.orchestra.run.trial163.stdout:Unpacking python3-kubernetes (12.0.1-1ubuntu1) ... 2026-02-18T08:08:42.761 INFO:teuthology.orchestra.run.trial163.stdout:Selecting previously unselected package ceph-mgr-k8sevents. 2026-02-18T08:08:42.769 INFO:teuthology.orchestra.run.trial163.stdout:Preparing to unpack .../61-ceph-mgr-k8sevents_20.2.0-670-gadc19233-1jammy_all.deb ... 2026-02-18T08:08:42.770 INFO:teuthology.orchestra.run.trial163.stdout:Unpacking ceph-mgr-k8sevents (20.2.0-670-gadc19233-1jammy) ... 2026-02-18T08:08:42.784 INFO:teuthology.orchestra.run.trial163.stdout:Selecting previously unselected package libonig5:amd64. 2026-02-18T08:08:42.792 INFO:teuthology.orchestra.run.trial163.stdout:Preparing to unpack .../62-libonig5_6.9.7.1-2build1_amd64.deb ... 2026-02-18T08:08:42.793 INFO:teuthology.orchestra.run.trial163.stdout:Unpacking libonig5:amd64 (6.9.7.1-2build1) ... 2026-02-18T08:08:42.809 INFO:teuthology.orchestra.run.trial163.stdout:Selecting previously unselected package libjq1:amd64. 2026-02-18T08:08:42.817 INFO:teuthology.orchestra.run.trial163.stdout:Preparing to unpack .../63-libjq1_1.6-2.1ubuntu3.1_amd64.deb ... 2026-02-18T08:08:42.818 INFO:teuthology.orchestra.run.trial163.stdout:Unpacking libjq1:amd64 (1.6-2.1ubuntu3.1) ... 2026-02-18T08:08:42.832 INFO:teuthology.orchestra.run.trial163.stdout:Selecting previously unselected package jq. 2026-02-18T08:08:42.841 INFO:teuthology.orchestra.run.trial163.stdout:Preparing to unpack .../64-jq_1.6-2.1ubuntu3.1_amd64.deb ... 2026-02-18T08:08:42.841 INFO:teuthology.orchestra.run.trial163.stdout:Unpacking jq (1.6-2.1ubuntu3.1) ... 2026-02-18T08:08:42.856 INFO:teuthology.orchestra.run.trial163.stdout:Selecting previously unselected package socat. 2026-02-18T08:08:42.865 INFO:teuthology.orchestra.run.trial163.stdout:Preparing to unpack .../65-socat_1.7.4.1-3ubuntu4_amd64.deb ... 2026-02-18T08:08:42.865 INFO:teuthology.orchestra.run.trial163.stdout:Unpacking socat (1.7.4.1-3ubuntu4) ... 2026-02-18T08:08:42.885 INFO:teuthology.orchestra.run.trial163.stdout:Selecting previously unselected package xmlstarlet. 2026-02-18T08:08:42.893 INFO:teuthology.orchestra.run.trial163.stdout:Preparing to unpack .../66-xmlstarlet_1.6.1-2.1_amd64.deb ... 2026-02-18T08:08:42.893 INFO:teuthology.orchestra.run.trial163.stdout:Unpacking xmlstarlet (1.6.1-2.1) ... 2026-02-18T08:08:42.931 INFO:teuthology.orchestra.run.trial163.stdout:Selecting previously unselected package ceph-test. 2026-02-18T08:08:42.939 INFO:teuthology.orchestra.run.trial163.stdout:Preparing to unpack .../67-ceph-test_20.2.0-670-gadc19233-1jammy_amd64.deb ... 2026-02-18T08:08:42.939 INFO:teuthology.orchestra.run.trial163.stdout:Unpacking ceph-test (20.2.0-670-gadc19233-1jammy) ... 2026-02-18T08:08:43.097 INFO:teuthology.orchestra.run.trial034.stdout:Setting up libqt5core5a:amd64 (5.15.3+dfsg-2ubuntu0.2) ... 2026-02-18T08:08:43.101 INFO:teuthology.orchestra.run.trial034.stdout:Setting up python3-toml (0.10.2-1) ... 2026-02-18T08:08:43.177 INFO:teuthology.orchestra.run.trial034.stdout:Setting up librdkafka1:amd64 (1.8.0-1build1) ... 2026-02-18T08:08:43.178 INFO:teuthology.orchestra.run.trial034.stdout:Setting up xmlstarlet (1.6.1-2.1) ... 2026-02-18T08:08:43.179 INFO:teuthology.orchestra.run.trial034.stdout:Setting up python3-pluggy (0.13.0-7.1) ... 2026-02-18T08:08:43.256 INFO:teuthology.orchestra.run.trial034.stdout:Setting up python3-zc.lockfile (2.0-1) ... 2026-02-18T08:08:43.329 INFO:teuthology.orchestra.run.trial034.stdout:Setting up libqt5dbus5:amd64 (5.15.3+dfsg-2ubuntu0.2) ... 2026-02-18T08:08:43.330 INFO:teuthology.orchestra.run.trial034.stdout:Setting up python3-rsa (4.8-1) ... 2026-02-18T08:08:43.407 INFO:teuthology.orchestra.run.trial034.stdout:Setting up python3-tempora (4.1.2-1) ... 2026-02-18T08:08:43.480 INFO:teuthology.orchestra.run.trial034.stdout:Setting up python3-prettytable (2.5.0-2) ... 2026-02-18T08:08:43.558 INFO:teuthology.orchestra.run.trial034.stdout:Setting up liblttng-ust1:amd64 (2.13.1-1ubuntu1) ... 2026-02-18T08:08:43.559 INFO:teuthology.orchestra.run.trial034.stdout:Setting up python3-websocket (1.2.3-1) ... 2026-02-18T08:08:43.640 INFO:teuthology.orchestra.run.trial034.stdout:Setting up libonig5:amd64 (6.9.7.1-2build1) ... 2026-02-18T08:08:43.641 INFO:teuthology.orchestra.run.trial034.stdout:Setting up python3-requests-oauthlib (1.3.0+ds-0.1) ... 2026-02-18T08:08:43.717 INFO:teuthology.orchestra.run.trial034.stdout:Setting up python3-webob (1:1.8.6-1.1ubuntu0.1) ... 2026-02-18T08:08:43.800 INFO:teuthology.orchestra.run.trial034.stdout:Setting up python3-jaraco.collections (3.4.0-2) ... 2026-02-18T08:08:43.874 INFO:teuthology.orchestra.run.trial034.stdout:Setting up libjq1:amd64 (1.6-2.1ubuntu3.1) ... 2026-02-18T08:08:43.875 INFO:teuthology.orchestra.run.trial034.stdout:Setting up python3-pytest (6.2.5-1ubuntu2) ... 2026-02-18T08:08:43.877 INFO:teuthology.orchestra.run.trial163.stdout:Selecting previously unselected package ceph-volume. 2026-02-18T08:08:43.886 INFO:teuthology.orchestra.run.trial163.stdout:Preparing to unpack .../68-ceph-volume_20.2.0-670-gadc19233-1jammy_all.deb ... 2026-02-18T08:08:43.886 INFO:teuthology.orchestra.run.trial163.stdout:Unpacking ceph-volume (20.2.0-670-gadc19233-1jammy) ... 2026-02-18T08:08:43.909 INFO:teuthology.orchestra.run.trial163.stdout:Selecting previously unselected package libcephfs-daemon. 2026-02-18T08:08:43.917 INFO:teuthology.orchestra.run.trial163.stdout:Preparing to unpack .../69-libcephfs-daemon_20.2.0-670-gadc19233-1jammy_amd64.deb ... 2026-02-18T08:08:43.918 INFO:teuthology.orchestra.run.trial163.stdout:Unpacking libcephfs-daemon (20.2.0-670-gadc19233-1jammy) ... 2026-02-18T08:08:43.932 INFO:teuthology.orchestra.run.trial163.stdout:Selecting previously unselected package libcephfs-proxy2. 2026-02-18T08:08:43.941 INFO:teuthology.orchestra.run.trial163.stdout:Preparing to unpack .../70-libcephfs-proxy2_20.2.0-670-gadc19233-1jammy_amd64.deb ... 2026-02-18T08:08:43.941 INFO:teuthology.orchestra.run.trial163.stdout:Unpacking libcephfs-proxy2 (20.2.0-670-gadc19233-1jammy) ... 2026-02-18T08:08:43.955 INFO:teuthology.orchestra.run.trial163.stdout:Selecting previously unselected package libcephfs-dev. 2026-02-18T08:08:43.963 INFO:teuthology.orchestra.run.trial163.stdout:Preparing to unpack .../71-libcephfs-dev_20.2.0-670-gadc19233-1jammy_amd64.deb ... 2026-02-18T08:08:43.963 INFO:teuthology.orchestra.run.trial163.stdout:Unpacking libcephfs-dev (20.2.0-670-gadc19233-1jammy) ... 2026-02-18T08:08:43.979 INFO:teuthology.orchestra.run.trial163.stdout:Selecting previously unselected package nvme-cli. 2026-02-18T08:08:43.988 INFO:teuthology.orchestra.run.trial163.stdout:Preparing to unpack .../72-nvme-cli_1.16-3ubuntu0.3_amd64.deb ... 2026-02-18T08:08:43.988 INFO:teuthology.orchestra.run.trial163.stdout:Unpacking nvme-cli (1.16-3ubuntu0.3) ... 2026-02-18T08:08:44.008 INFO:teuthology.orchestra.run.trial034.stdout:Setting up python3-portend (3.0.0-1) ... 2026-02-18T08:08:44.020 INFO:teuthology.orchestra.run.trial163.stdout:Selecting previously unselected package python-asyncssh-doc. 2026-02-18T08:08:44.029 INFO:teuthology.orchestra.run.trial163.stdout:Preparing to unpack .../73-python-asyncssh-doc_2.5.0-1ubuntu0.1_all.deb ... 2026-02-18T08:08:44.029 INFO:teuthology.orchestra.run.trial163.stdout:Unpacking python-asyncssh-doc (2.5.0-1ubuntu0.1) ... 2026-02-18T08:08:44.059 INFO:teuthology.orchestra.run.trial163.stdout:Selecting previously unselected package python3-iniconfig. 2026-02-18T08:08:44.068 INFO:teuthology.orchestra.run.trial163.stdout:Preparing to unpack .../74-python3-iniconfig_1.1.1-2_all.deb ... 2026-02-18T08:08:44.068 INFO:teuthology.orchestra.run.trial163.stdout:Unpacking python3-iniconfig (1.1.1-2) ... 2026-02-18T08:08:44.081 INFO:teuthology.orchestra.run.trial034.stdout:Setting up libqt5network5:amd64 (5.15.3+dfsg-2ubuntu0.2) ... 2026-02-18T08:08:44.082 INFO:teuthology.orchestra.run.trial034.stdout:Setting up python3-google-auth (1.5.1-3) ... 2026-02-18T08:08:44.083 INFO:teuthology.orchestra.run.trial163.stdout:Selecting previously unselected package python3-pluggy. 2026-02-18T08:08:44.091 INFO:teuthology.orchestra.run.trial163.stdout:Preparing to unpack .../75-python3-pluggy_0.13.0-7.1_all.deb ... 2026-02-18T08:08:44.091 INFO:teuthology.orchestra.run.trial163.stdout:Unpacking python3-pluggy (0.13.0-7.1) ... 2026-02-18T08:08:44.107 INFO:teuthology.orchestra.run.trial163.stdout:Selecting previously unselected package python3-psutil. 2026-02-18T08:08:44.115 INFO:teuthology.orchestra.run.trial163.stdout:Preparing to unpack .../76-python3-psutil_5.9.0-1build1_amd64.deb ... 2026-02-18T08:08:44.116 INFO:teuthology.orchestra.run.trial163.stdout:Unpacking python3-psutil (5.9.0-1build1) ... 2026-02-18T08:08:44.134 INFO:teuthology.orchestra.run.trial163.stdout:Selecting previously unselected package python3-py. 2026-02-18T08:08:44.142 INFO:teuthology.orchestra.run.trial163.stdout:Preparing to unpack .../77-python3-py_1.10.0-1_all.deb ... 2026-02-18T08:08:44.142 INFO:teuthology.orchestra.run.trial163.stdout:Unpacking python3-py (1.10.0-1) ... 2026-02-18T08:08:44.162 INFO:teuthology.orchestra.run.trial163.stdout:Selecting previously unselected package python3-pygments. 2026-02-18T08:08:44.163 INFO:teuthology.orchestra.run.trial034.stdout:Setting up jq (1.6-2.1ubuntu3.1) ... 2026-02-18T08:08:44.163 INFO:teuthology.orchestra.run.trial034.stdout:Setting up python3-cherrypy3 (18.6.1-4) ... 2026-02-18T08:08:44.171 INFO:teuthology.orchestra.run.trial163.stdout:Preparing to unpack .../78-python3-pygments_2.11.2+dfsg-2ubuntu0.1_all.deb ... 2026-02-18T08:08:44.171 INFO:teuthology.orchestra.run.trial163.stdout:Unpacking python3-pygments (2.11.2+dfsg-2ubuntu0.1) ... 2026-02-18T08:08:44.214 INFO:teuthology.orchestra.run.trial163.stdout:Selecting previously unselected package python3-toml. 2026-02-18T08:08:44.222 INFO:teuthology.orchestra.run.trial163.stdout:Preparing to unpack .../79-python3-toml_0.10.2-1_all.deb ... 2026-02-18T08:08:44.223 INFO:teuthology.orchestra.run.trial163.stdout:Unpacking python3-toml (0.10.2-1) ... 2026-02-18T08:08:44.238 INFO:teuthology.orchestra.run.trial163.stdout:Selecting previously unselected package python3-pytest. 2026-02-18T08:08:44.246 INFO:teuthology.orchestra.run.trial163.stdout:Preparing to unpack .../80-python3-pytest_6.2.5-1ubuntu2_all.deb ... 2026-02-18T08:08:44.247 INFO:teuthology.orchestra.run.trial163.stdout:Unpacking python3-pytest (6.2.5-1ubuntu2) ... 2026-02-18T08:08:44.269 INFO:teuthology.orchestra.run.trial163.stdout:Selecting previously unselected package python3-simplejson. 2026-02-18T08:08:44.277 INFO:teuthology.orchestra.run.trial163.stdout:Preparing to unpack .../81-python3-simplejson_3.17.6-1build1_amd64.deb ... 2026-02-18T08:08:44.278 INFO:teuthology.orchestra.run.trial163.stdout:Unpacking python3-simplejson (3.17.6-1build1) ... 2026-02-18T08:08:44.292 INFO:teuthology.orchestra.run.trial034.stdout:Setting up libthrift-0.16.0:amd64 (0.16.0-2) ... 2026-02-18T08:08:44.293 INFO:teuthology.orchestra.run.trial034.stdout:Setting up librados2 (20.2.0-670-gadc19233-1jammy) ... 2026-02-18T08:08:44.294 INFO:teuthology.orchestra.run.trial034.stdout:Setting up librgw2 (20.2.0-670-gadc19233-1jammy) ... 2026-02-18T08:08:44.295 INFO:teuthology.orchestra.run.trial034.stdout:Setting up libsqlite3-mod-ceph (20.2.0-670-gadc19233-1jammy) ... 2026-02-18T08:08:44.296 INFO:teuthology.orchestra.run.trial163.stdout:Selecting previously unselected package python3-webob. 2026-02-18T08:08:44.296 INFO:teuthology.orchestra.run.trial034.stdout:Setting up python3-kubernetes (12.0.1-1ubuntu1) ... 2026-02-18T08:08:44.304 INFO:teuthology.orchestra.run.trial163.stdout:Preparing to unpack .../82-python3-webob_1%3a1.8.6-1.1ubuntu0.1_all.deb ... 2026-02-18T08:08:44.304 INFO:teuthology.orchestra.run.trial163.stdout:Unpacking python3-webob (1:1.8.6-1.1ubuntu0.1) ... 2026-02-18T08:08:44.328 INFO:teuthology.orchestra.run.trial163.stdout:Selecting previously unselected package qttranslations5-l10n. 2026-02-18T08:08:44.336 INFO:teuthology.orchestra.run.trial163.stdout:Preparing to unpack .../83-qttranslations5-l10n_5.15.3-1_all.deb ... 2026-02-18T08:08:44.337 INFO:teuthology.orchestra.run.trial163.stdout:Unpacking qttranslations5-l10n (5.15.3-1) ... 2026-02-18T08:08:44.404 INFO:teuthology.orchestra.run.trial163.stdout:Selecting previously unselected package radosgw. 2026-02-18T08:08:44.413 INFO:teuthology.orchestra.run.trial163.stdout:Preparing to unpack .../84-radosgw_20.2.0-670-gadc19233-1jammy_amd64.deb ... 2026-02-18T08:08:44.414 INFO:teuthology.orchestra.run.trial163.stdout:Unpacking radosgw (20.2.0-670-gadc19233-1jammy) ... 2026-02-18T08:08:44.640 INFO:teuthology.orchestra.run.trial163.stdout:Selecting previously unselected package rbd-fuse. 2026-02-18T08:08:44.649 INFO:teuthology.orchestra.run.trial163.stdout:Preparing to unpack .../85-rbd-fuse_20.2.0-670-gadc19233-1jammy_amd64.deb ... 2026-02-18T08:08:44.649 INFO:teuthology.orchestra.run.trial163.stdout:Unpacking rbd-fuse (20.2.0-670-gadc19233-1jammy) ... 2026-02-18T08:08:44.664 INFO:teuthology.orchestra.run.trial163.stdout:Selecting previously unselected package rbd-mirror. 2026-02-18T08:08:44.673 INFO:teuthology.orchestra.run.trial163.stdout:Preparing to unpack .../86-rbd-mirror_20.2.0-670-gadc19233-1jammy_amd64.deb ... 2026-02-18T08:08:44.673 INFO:teuthology.orchestra.run.trial163.stdout:Unpacking rbd-mirror (20.2.0-670-gadc19233-1jammy) ... 2026-02-18T08:08:44.728 INFO:teuthology.orchestra.run.trial163.stdout:Setting up python3-iniconfig (1.1.1-2) ... 2026-02-18T08:08:44.795 INFO:teuthology.orchestra.run.trial163.stdout:Setting up libdouble-conversion3:amd64 (3.1.7-4) ... 2026-02-18T08:08:44.795 INFO:teuthology.orchestra.run.trial163.stdout:Setting up nvme-cli (1.16-3ubuntu0.3) ... 2026-02-18T08:08:44.824 INFO:teuthology.orchestra.run.trial034.stdout:Setting up libcephfs2 (20.2.0-670-gadc19233-1jammy) ... 2026-02-18T08:08:44.825 INFO:teuthology.orchestra.run.trial034.stdout:Setting up libradosstriper1 (20.2.0-670-gadc19233-1jammy) ... 2026-02-18T08:08:44.826 INFO:teuthology.orchestra.run.trial034.stdout:Setting up librbd1 (20.2.0-670-gadc19233-1jammy) ... 2026-02-18T08:08:44.827 INFO:teuthology.orchestra.run.trial034.stdout:Setting up ceph-mgr-modules-core (20.2.0-670-gadc19233-1jammy) ... 2026-02-18T08:08:44.828 INFO:teuthology.orchestra.run.trial034.stdout:Setting up ceph-fuse (20.2.0-670-gadc19233-1jammy) ... 2026-02-18T08:08:44.855 INFO:teuthology.orchestra.run.trial163.stdout:Created symlink /etc/systemd/system/default.target.wants/nvmefc-boot-connections.service → /lib/systemd/system/nvmefc-boot-connections.service. 2026-02-18T08:08:44.900 INFO:teuthology.orchestra.run.trial034.stdout:Created symlink /etc/systemd/system/remote-fs.target.wants/ceph-fuse.target → /lib/systemd/system/ceph-fuse.target. 2026-02-18T08:08:44.900 INFO:teuthology.orchestra.run.trial034.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-fuse.target → /lib/systemd/system/ceph-fuse.target. 2026-02-18T08:08:45.114 INFO:teuthology.orchestra.run.trial163.stdout:Created symlink /etc/systemd/system/default.target.wants/nvmf-autoconnect.service → /lib/systemd/system/nvmf-autoconnect.service. 2026-02-18T08:08:45.275 INFO:teuthology.orchestra.run.trial034.stdout:Setting up libcephfs-dev (20.2.0-670-gadc19233-1jammy) ... 2026-02-18T08:08:45.276 INFO:teuthology.orchestra.run.trial034.stdout:Setting up python3-rados (20.2.0-670-gadc19233-1jammy) ... 2026-02-18T08:08:45.277 INFO:teuthology.orchestra.run.trial034.stdout:Setting up libcephfs-daemon (20.2.0-670-gadc19233-1jammy) ... 2026-02-18T08:08:45.278 INFO:teuthology.orchestra.run.trial034.stdout:Setting up python3-rbd (20.2.0-670-gadc19233-1jammy) ... 2026-02-18T08:08:45.279 INFO:teuthology.orchestra.run.trial034.stdout:Setting up rbd-fuse (20.2.0-670-gadc19233-1jammy) ... 2026-02-18T08:08:45.280 INFO:teuthology.orchestra.run.trial034.stdout:Setting up python3-rgw (20.2.0-670-gadc19233-1jammy) ... 2026-02-18T08:08:45.281 INFO:teuthology.orchestra.run.trial034.stdout:Setting up python3-cephfs (20.2.0-670-gadc19233-1jammy) ... 2026-02-18T08:08:45.281 INFO:teuthology.orchestra.run.trial034.stdout:Setting up ceph-common (20.2.0-670-gadc19233-1jammy) ... 2026-02-18T08:08:45.309 INFO:teuthology.orchestra.run.trial034.stdout:Adding group ceph....done 2026-02-18T08:08:45.343 INFO:teuthology.orchestra.run.trial034.stdout:Adding system user ceph....done 2026-02-18T08:08:45.353 INFO:teuthology.orchestra.run.trial034.stdout:Setting system user ceph properties....done 2026-02-18T08:08:45.359 INFO:teuthology.orchestra.run.trial034.stdout:chown: cannot access '/var/log/ceph/*.log*': No such file or directory 2026-02-18T08:08:45.447 INFO:teuthology.orchestra.run.trial034.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph.target → /lib/systemd/system/ceph.target. 2026-02-18T08:08:45.497 INFO:teuthology.orchestra.run.trial163.stdout:nvmf-connect.target is a disabled or a static unit, not starting it. 2026-02-18T08:08:45.508 INFO:teuthology.orchestra.run.trial163.stdout:Setting up cephadm (20.2.0-670-gadc19233-1jammy) ... 2026-02-18T08:08:45.552 INFO:teuthology.orchestra.run.trial163.stdout:Adding system user cephadm....done 2026-02-18T08:08:45.562 INFO:teuthology.orchestra.run.trial163.stdout:Setting up python3-jaraco.classes (3.2.1-3) ... 2026-02-18T08:08:45.635 INFO:teuthology.orchestra.run.trial163.stdout:Setting up python-asyncssh-doc (2.5.0-1ubuntu0.1) ... 2026-02-18T08:08:45.635 INFO:teuthology.orchestra.run.trial163.stdout:Setting up python3-jaraco.functools (3.4.0-2) ... 2026-02-18T08:08:45.707 INFO:teuthology.orchestra.run.trial163.stdout:Setting up python3-repoze.lru (0.7-2) ... 2026-02-18T08:08:45.723 INFO:teuthology.orchestra.run.trial034.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/rbdmap.service → /lib/systemd/system/rbdmap.service. 2026-02-18T08:08:45.780 INFO:teuthology.orchestra.run.trial163.stdout:Setting up liboath0:amd64 (2.6.7-3ubuntu0.1) ... 2026-02-18T08:08:45.781 INFO:teuthology.orchestra.run.trial163.stdout:Setting up python3-py (1.10.0-1) ... 2026-02-18T08:08:45.878 INFO:teuthology.orchestra.run.trial163.stdout:Setting up python3-joblib (0.17.0-4ubuntu1) ... 2026-02-18T08:08:46.002 INFO:teuthology.orchestra.run.trial163.stdout:Setting up python3-cachetools (5.0.0-1) ... 2026-02-18T08:08:46.077 INFO:teuthology.orchestra.run.trial163.stdout:Setting up python3-threadpoolctl (3.1.0-1) ... 2026-02-18T08:08:46.078 INFO:teuthology.orchestra.run.trial034.stdout:Setting up ceph-test (20.2.0-670-gadc19233-1jammy) ... 2026-02-18T08:08:46.079 INFO:teuthology.orchestra.run.trial034.stdout:Setting up radosgw (20.2.0-670-gadc19233-1jammy) ... 2026-02-18T08:08:46.152 INFO:teuthology.orchestra.run.trial163.stdout:Setting up python3-ceph-argparse (20.2.0-670-gadc19233-1jammy) ... 2026-02-18T08:08:46.216 INFO:teuthology.orchestra.run.trial163.stdout:Setting up python3-sklearn-lib:amd64 (0.23.2-5ubuntu6) ... 2026-02-18T08:08:46.217 INFO:teuthology.orchestra.run.trial163.stdout:Setting up libnbd0 (1.10.5-1) ... 2026-02-18T08:08:46.218 INFO:teuthology.orchestra.run.trial163.stdout:Setting up libfuse2:amd64 (2.9.9-5ubuntu3) ... 2026-02-18T08:08:46.219 INFO:teuthology.orchestra.run.trial163.stdout:Setting up libpcre2-16-0:amd64 (10.39-3ubuntu0.1) ... 2026-02-18T08:08:46.220 INFO:teuthology.orchestra.run.trial163.stdout:Setting up python3-psutil (5.9.0-1build1) ... 2026-02-18T08:08:46.331 INFO:teuthology.orchestra.run.trial034.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-radosgw.target → /lib/systemd/system/ceph-radosgw.target. 2026-02-18T08:08:46.331 INFO:teuthology.orchestra.run.trial034.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-radosgw.target → /lib/systemd/system/ceph-radosgw.target. 2026-02-18T08:08:46.341 INFO:teuthology.orchestra.run.trial163.stdout:Setting up python3-natsort (8.0.2-1) ... 2026-02-18T08:08:46.420 INFO:teuthology.orchestra.run.trial163.stdout:Setting up libcephfs-proxy2 (20.2.0-670-gadc19233-1jammy) ... 2026-02-18T08:08:46.421 INFO:teuthology.orchestra.run.trial163.stdout:Setting up python3-routes (2.5.1-1ubuntu1) ... 2026-02-18T08:08:46.500 INFO:teuthology.orchestra.run.trial163.stdout:Setting up python3-simplejson (3.17.6-1build1) ... 2026-02-18T08:08:46.587 INFO:teuthology.orchestra.run.trial163.stdout:Setting up python3-pygments (2.11.2+dfsg-2ubuntu0.1) ... 2026-02-18T08:08:46.667 INFO:teuthology.orchestra.run.trial034.stdout:Setting up ceph-base (20.2.0-670-gadc19233-1jammy) ... 2026-02-18T08:08:46.783 INFO:teuthology.orchestra.run.trial034.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-crash.service → /lib/systemd/system/ceph-crash.service. 2026-02-18T08:08:46.840 INFO:teuthology.orchestra.run.trial163.stdout:Setting up qttranslations5-l10n (5.15.3-1) ... 2026-02-18T08:08:46.841 INFO:teuthology.orchestra.run.trial163.stdout:Setting up python3-wcwidth (0.2.5+dfsg1-1) ... 2026-02-18T08:08:46.937 INFO:teuthology.orchestra.run.trial163.stdout:Setting up python3-asyncssh (2.5.0-1ubuntu0.1) ... 2026-02-18T08:08:47.063 INFO:teuthology.orchestra.run.trial163.stdout:Setting up python3-cheroot (8.5.2+ds1-1ubuntu3.1) ... 2026-02-18T08:08:47.143 INFO:teuthology.orchestra.run.trial163.stdout:Setting up python3-jaraco.text (3.6.0-2) ... 2026-02-18T08:08:47.144 INFO:teuthology.orchestra.run.trial034.stdout:Setting up ceph-mds (20.2.0-670-gadc19233-1jammy) ... 2026-02-18T08:08:47.203 INFO:teuthology.orchestra.run.trial034.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mds.target → /lib/systemd/system/ceph-mds.target. 2026-02-18T08:08:47.203 INFO:teuthology.orchestra.run.trial034.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mds.target → /lib/systemd/system/ceph-mds.target. 2026-02-18T08:08:47.218 INFO:teuthology.orchestra.run.trial163.stdout:Setting up socat (1.7.4.1-3ubuntu4) ... 2026-02-18T08:08:47.219 INFO:teuthology.orchestra.run.trial163.stdout:Setting up python3-ceph-common (20.2.0-670-gadc19233-1jammy) ... 2026-02-18T08:08:47.307 INFO:teuthology.orchestra.run.trial163.stdout:Setting up python3-sklearn (0.23.2-5ubuntu6) ... 2026-02-18T08:08:47.554 INFO:teuthology.orchestra.run.trial034.stdout:Setting up rbd-mirror (20.2.0-670-gadc19233-1jammy) ... 2026-02-18T08:08:47.628 INFO:teuthology.orchestra.run.trial034.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-rbd-mirror.target → /lib/systemd/system/ceph-rbd-mirror.target. 2026-02-18T08:08:47.628 INFO:teuthology.orchestra.run.trial034.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-rbd-mirror.target → /lib/systemd/system/ceph-rbd-mirror.target. 2026-02-18T08:08:47.811 INFO:teuthology.orchestra.run.trial163.stdout:Setting up libqt5core5a:amd64 (5.15.3+dfsg-2ubuntu0.2) ... 2026-02-18T08:08:47.816 INFO:teuthology.orchestra.run.trial163.stdout:Setting up python3-toml (0.10.2-1) ... 2026-02-18T08:08:47.895 INFO:teuthology.orchestra.run.trial163.stdout:Setting up librdkafka1:amd64 (1.8.0-1build1) ... 2026-02-18T08:08:47.896 INFO:teuthology.orchestra.run.trial163.stdout:Setting up xmlstarlet (1.6.1-2.1) ... 2026-02-18T08:08:47.897 INFO:teuthology.orchestra.run.trial163.stdout:Setting up python3-pluggy (0.13.0-7.1) ... 2026-02-18T08:08:47.966 INFO:teuthology.orchestra.run.trial034.stdout:Setting up ceph-mgr (20.2.0-670-gadc19233-1jammy) ... 2026-02-18T08:08:47.973 INFO:teuthology.orchestra.run.trial163.stdout:Setting up python3-zc.lockfile (2.0-1) ... 2026-02-18T08:08:48.043 INFO:teuthology.orchestra.run.trial034.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mgr.target → /lib/systemd/system/ceph-mgr.target. 2026-02-18T08:08:48.043 INFO:teuthology.orchestra.run.trial034.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mgr.target → /lib/systemd/system/ceph-mgr.target. 2026-02-18T08:08:48.046 INFO:teuthology.orchestra.run.trial163.stdout:Setting up libqt5dbus5:amd64 (5.15.3+dfsg-2ubuntu0.2) ... 2026-02-18T08:08:48.047 INFO:teuthology.orchestra.run.trial163.stdout:Setting up python3-rsa (4.8-1) ... 2026-02-18T08:08:48.128 INFO:teuthology.orchestra.run.trial163.stdout:Setting up python3-tempora (4.1.2-1) ... 2026-02-18T08:08:48.201 INFO:teuthology.orchestra.run.trial163.stdout:Setting up python3-prettytable (2.5.0-2) ... 2026-02-18T08:08:48.279 INFO:teuthology.orchestra.run.trial163.stdout:Setting up liblttng-ust1:amd64 (2.13.1-1ubuntu1) ... 2026-02-18T08:08:48.280 INFO:teuthology.orchestra.run.trial163.stdout:Setting up python3-websocket (1.2.3-1) ... 2026-02-18T08:08:48.361 INFO:teuthology.orchestra.run.trial163.stdout:Setting up libonig5:amd64 (6.9.7.1-2build1) ... 2026-02-18T08:08:48.362 INFO:teuthology.orchestra.run.trial163.stdout:Setting up python3-requests-oauthlib (1.3.0+ds-0.1) ... 2026-02-18T08:08:48.386 INFO:teuthology.orchestra.run.trial034.stdout:Setting up ceph-osd (20.2.0-670-gadc19233-1jammy) ... 2026-02-18T08:08:48.438 INFO:teuthology.orchestra.run.trial163.stdout:Setting up python3-webob (1:1.8.6-1.1ubuntu0.1) ... 2026-02-18T08:08:48.468 INFO:teuthology.orchestra.run.trial034.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-osd.target → /lib/systemd/system/ceph-osd.target. 2026-02-18T08:08:48.468 INFO:teuthology.orchestra.run.trial034.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-osd.target → /lib/systemd/system/ceph-osd.target. 2026-02-18T08:08:48.523 INFO:teuthology.orchestra.run.trial163.stdout:Setting up python3-jaraco.collections (3.4.0-2) ... 2026-02-18T08:08:48.596 INFO:teuthology.orchestra.run.trial163.stdout:Setting up libjq1:amd64 (1.6-2.1ubuntu3.1) ... 2026-02-18T08:08:48.597 INFO:teuthology.orchestra.run.trial163.stdout:Setting up python3-pytest (6.2.5-1ubuntu2) ... 2026-02-18T08:08:48.733 INFO:teuthology.orchestra.run.trial163.stdout:Setting up python3-portend (3.0.0-1) ... 2026-02-18T08:08:48.806 INFO:teuthology.orchestra.run.trial163.stdout:Setting up libqt5network5:amd64 (5.15.3+dfsg-2ubuntu0.2) ... 2026-02-18T08:08:48.807 INFO:teuthology.orchestra.run.trial163.stdout:Setting up python3-google-auth (1.5.1-3) ... 2026-02-18T08:08:48.823 INFO:teuthology.orchestra.run.trial034.stdout:Setting up ceph-mgr-k8sevents (20.2.0-670-gadc19233-1jammy) ... 2026-02-18T08:08:48.824 INFO:teuthology.orchestra.run.trial034.stdout:Setting up ceph-mgr-diskprediction-local (20.2.0-670-gadc19233-1jammy) ... 2026-02-18T08:08:48.834 INFO:teuthology.orchestra.run.trial034.stdout:Setting up ceph-mon (20.2.0-670-gadc19233-1jammy) ... 2026-02-18T08:08:48.890 INFO:teuthology.orchestra.run.trial163.stdout:Setting up jq (1.6-2.1ubuntu3.1) ... 2026-02-18T08:08:48.891 INFO:teuthology.orchestra.run.trial163.stdout:Setting up python3-cherrypy3 (18.6.1-4) ... 2026-02-18T08:08:48.905 INFO:teuthology.orchestra.run.trial034.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mon.target → /lib/systemd/system/ceph-mon.target. 2026-02-18T08:08:48.905 INFO:teuthology.orchestra.run.trial034.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mon.target → /lib/systemd/system/ceph-mon.target. 2026-02-18T08:08:49.025 INFO:teuthology.orchestra.run.trial163.stdout:Setting up libthrift-0.16.0:amd64 (0.16.0-2) ... 2026-02-18T08:08:49.027 INFO:teuthology.orchestra.run.trial163.stdout:Setting up librados2 (20.2.0-670-gadc19233-1jammy) ... 2026-02-18T08:08:49.027 INFO:teuthology.orchestra.run.trial163.stdout:Setting up librgw2 (20.2.0-670-gadc19233-1jammy) ... 2026-02-18T08:08:49.028 INFO:teuthology.orchestra.run.trial163.stdout:Setting up libsqlite3-mod-ceph (20.2.0-670-gadc19233-1jammy) ... 2026-02-18T08:08:49.029 INFO:teuthology.orchestra.run.trial163.stdout:Setting up python3-kubernetes (12.0.1-1ubuntu1) ... 2026-02-18T08:08:49.270 INFO:teuthology.orchestra.run.trial034.stdout:Setting up ceph-mgr-cephadm (20.2.0-670-gadc19233-1jammy) ... 2026-02-18T08:08:49.281 INFO:teuthology.orchestra.run.trial034.stdout:Setting up ceph (20.2.0-670-gadc19233-1jammy) ... 2026-02-18T08:08:49.282 INFO:teuthology.orchestra.run.trial034.stdout:Setting up ceph-mgr-dashboard (20.2.0-670-gadc19233-1jammy) ... 2026-02-18T08:08:49.292 INFO:teuthology.orchestra.run.trial034.stdout:Setting up ceph-volume (20.2.0-670-gadc19233-1jammy) ... 2026-02-18T08:08:49.401 INFO:teuthology.orchestra.run.trial034.stdout:Processing triggers for man-db (2.10.2-1) ... 2026-02-18T08:08:49.485 INFO:teuthology.orchestra.run.trial034.stdout:Processing triggers for libc-bin (2.35-0ubuntu3.11) ... 2026-02-18T08:08:49.560 INFO:teuthology.orchestra.run.trial163.stdout:Setting up libcephfs2 (20.2.0-670-gadc19233-1jammy) ... 2026-02-18T08:08:49.561 INFO:teuthology.orchestra.run.trial163.stdout:Setting up libradosstriper1 (20.2.0-670-gadc19233-1jammy) ... 2026-02-18T08:08:49.562 INFO:teuthology.orchestra.run.trial163.stdout:Setting up librbd1 (20.2.0-670-gadc19233-1jammy) ... 2026-02-18T08:08:49.563 INFO:teuthology.orchestra.run.trial163.stdout:Setting up ceph-mgr-modules-core (20.2.0-670-gadc19233-1jammy) ... 2026-02-18T08:08:49.563 INFO:teuthology.orchestra.run.trial163.stdout:Setting up ceph-fuse (20.2.0-670-gadc19233-1jammy) ... 2026-02-18T08:08:49.634 INFO:teuthology.orchestra.run.trial163.stdout:Created symlink /etc/systemd/system/remote-fs.target.wants/ceph-fuse.target → /lib/systemd/system/ceph-fuse.target. 2026-02-18T08:08:49.634 INFO:teuthology.orchestra.run.trial163.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-fuse.target → /lib/systemd/system/ceph-fuse.target. 2026-02-18T08:08:50.025 INFO:teuthology.orchestra.run.trial163.stdout:Setting up libcephfs-dev (20.2.0-670-gadc19233-1jammy) ... 2026-02-18T08:08:50.025 INFO:teuthology.orchestra.run.trial163.stdout:Setting up python3-rados (20.2.0-670-gadc19233-1jammy) ... 2026-02-18T08:08:50.026 INFO:teuthology.orchestra.run.trial163.stdout:Setting up libcephfs-daemon (20.2.0-670-gadc19233-1jammy) ... 2026-02-18T08:08:50.027 INFO:teuthology.orchestra.run.trial163.stdout:Setting up python3-rbd (20.2.0-670-gadc19233-1jammy) ... 2026-02-18T08:08:50.028 INFO:teuthology.orchestra.run.trial163.stdout:Setting up rbd-fuse (20.2.0-670-gadc19233-1jammy) ... 2026-02-18T08:08:50.029 INFO:teuthology.orchestra.run.trial163.stdout:Setting up python3-rgw (20.2.0-670-gadc19233-1jammy) ... 2026-02-18T08:08:50.030 INFO:teuthology.orchestra.run.trial163.stdout:Setting up python3-cephfs (20.2.0-670-gadc19233-1jammy) ... 2026-02-18T08:08:50.030 INFO:teuthology.orchestra.run.trial163.stdout:Setting up ceph-common (20.2.0-670-gadc19233-1jammy) ... 2026-02-18T08:08:50.059 INFO:teuthology.orchestra.run.trial163.stdout:Adding group ceph....done 2026-02-18T08:08:50.093 INFO:teuthology.orchestra.run.trial163.stdout:Adding system user ceph....done 2026-02-18T08:08:50.103 INFO:teuthology.orchestra.run.trial163.stdout:Setting system user ceph properties....done 2026-02-18T08:08:50.110 INFO:teuthology.orchestra.run.trial163.stdout:chown: cannot access '/var/log/ceph/*.log*': No such file or directory 2026-02-18T08:08:50.140 INFO:teuthology.orchestra.run.trial034.stdout: 2026-02-18T08:08:50.140 INFO:teuthology.orchestra.run.trial034.stdout:Running kernel seems to be up-to-date. 2026-02-18T08:08:50.140 INFO:teuthology.orchestra.run.trial034.stdout: 2026-02-18T08:08:50.140 INFO:teuthology.orchestra.run.trial034.stdout:The processor microcode seems to be up-to-date. 2026-02-18T08:08:50.140 INFO:teuthology.orchestra.run.trial034.stdout: 2026-02-18T08:08:50.140 INFO:teuthology.orchestra.run.trial034.stdout:No services need to be restarted. 2026-02-18T08:08:50.141 INFO:teuthology.orchestra.run.trial034.stdout: 2026-02-18T08:08:50.141 INFO:teuthology.orchestra.run.trial034.stdout:No containers need to be restarted. 2026-02-18T08:08:50.141 INFO:teuthology.orchestra.run.trial034.stdout: 2026-02-18T08:08:50.141 INFO:teuthology.orchestra.run.trial034.stdout:No user sessions are running outdated binaries. 2026-02-18T08:08:50.141 INFO:teuthology.orchestra.run.trial034.stdout: 2026-02-18T08:08:50.141 INFO:teuthology.orchestra.run.trial034.stdout:No VM guests are running outdated hypervisor (qemu) binaries on this host. 2026-02-18T08:08:50.197 INFO:teuthology.orchestra.run.trial163.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph.target → /lib/systemd/system/ceph.target. 2026-02-18T08:08:50.454 INFO:teuthology.orchestra.run.trial163.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/rbdmap.service → /lib/systemd/system/rbdmap.service. 2026-02-18T08:08:50.801 INFO:teuthology.orchestra.run.trial034.stderr:W: --force-yes is deprecated, use one of the options starting with --allow instead. 2026-02-18T08:08:50.808 DEBUG:teuthology.parallel:result is None 2026-02-18T08:08:50.859 INFO:teuthology.orchestra.run.trial163.stdout:Setting up ceph-test (20.2.0-670-gadc19233-1jammy) ... 2026-02-18T08:08:50.860 INFO:teuthology.orchestra.run.trial163.stdout:Setting up radosgw (20.2.0-670-gadc19233-1jammy) ... 2026-02-18T08:08:51.156 INFO:teuthology.orchestra.run.trial163.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-radosgw.target → /lib/systemd/system/ceph-radosgw.target. 2026-02-18T08:08:51.157 INFO:teuthology.orchestra.run.trial163.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-radosgw.target → /lib/systemd/system/ceph-radosgw.target. 2026-02-18T08:08:51.521 INFO:teuthology.orchestra.run.trial163.stdout:Setting up ceph-base (20.2.0-670-gadc19233-1jammy) ... 2026-02-18T08:08:51.641 INFO:teuthology.orchestra.run.trial163.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-crash.service → /lib/systemd/system/ceph-crash.service. 2026-02-18T08:08:52.025 INFO:teuthology.orchestra.run.trial163.stdout:Setting up ceph-mds (20.2.0-670-gadc19233-1jammy) ... 2026-02-18T08:08:52.083 INFO:teuthology.orchestra.run.trial163.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mds.target → /lib/systemd/system/ceph-mds.target. 2026-02-18T08:08:52.083 INFO:teuthology.orchestra.run.trial163.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mds.target → /lib/systemd/system/ceph-mds.target. 2026-02-18T08:08:52.468 INFO:teuthology.orchestra.run.trial163.stdout:Setting up rbd-mirror (20.2.0-670-gadc19233-1jammy) ... 2026-02-18T08:08:52.538 INFO:teuthology.orchestra.run.trial163.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-rbd-mirror.target → /lib/systemd/system/ceph-rbd-mirror.target. 2026-02-18T08:08:52.538 INFO:teuthology.orchestra.run.trial163.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-rbd-mirror.target → /lib/systemd/system/ceph-rbd-mirror.target. 2026-02-18T08:08:52.931 INFO:teuthology.orchestra.run.trial163.stdout:Setting up ceph-mgr (20.2.0-670-gadc19233-1jammy) ... 2026-02-18T08:08:53.006 INFO:teuthology.orchestra.run.trial163.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mgr.target → /lib/systemd/system/ceph-mgr.target. 2026-02-18T08:08:53.006 INFO:teuthology.orchestra.run.trial163.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mgr.target → /lib/systemd/system/ceph-mgr.target. 2026-02-18T08:08:53.395 INFO:teuthology.orchestra.run.trial163.stdout:Setting up ceph-osd (20.2.0-670-gadc19233-1jammy) ... 2026-02-18T08:08:53.488 INFO:teuthology.orchestra.run.trial163.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-osd.target → /lib/systemd/system/ceph-osd.target. 2026-02-18T08:08:53.488 INFO:teuthology.orchestra.run.trial163.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-osd.target → /lib/systemd/system/ceph-osd.target. 2026-02-18T08:08:53.893 INFO:teuthology.orchestra.run.trial163.stdout:Setting up ceph-mgr-k8sevents (20.2.0-670-gadc19233-1jammy) ... 2026-02-18T08:08:53.895 INFO:teuthology.orchestra.run.trial163.stdout:Setting up ceph-mgr-diskprediction-local (20.2.0-670-gadc19233-1jammy) ... 2026-02-18T08:08:53.904 INFO:teuthology.orchestra.run.trial163.stdout:Setting up ceph-mon (20.2.0-670-gadc19233-1jammy) ... 2026-02-18T08:08:53.974 INFO:teuthology.orchestra.run.trial163.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mon.target → /lib/systemd/system/ceph-mon.target. 2026-02-18T08:08:53.974 INFO:teuthology.orchestra.run.trial163.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mon.target → /lib/systemd/system/ceph-mon.target. 2026-02-18T08:08:54.368 INFO:teuthology.orchestra.run.trial163.stdout:Setting up ceph-mgr-cephadm (20.2.0-670-gadc19233-1jammy) ... 2026-02-18T08:08:54.380 INFO:teuthology.orchestra.run.trial163.stdout:Setting up ceph (20.2.0-670-gadc19233-1jammy) ... 2026-02-18T08:08:54.381 INFO:teuthology.orchestra.run.trial163.stdout:Setting up ceph-mgr-dashboard (20.2.0-670-gadc19233-1jammy) ... 2026-02-18T08:08:54.391 INFO:teuthology.orchestra.run.trial163.stdout:Setting up ceph-volume (20.2.0-670-gadc19233-1jammy) ... 2026-02-18T08:08:54.499 INFO:teuthology.orchestra.run.trial163.stdout:Processing triggers for man-db (2.10.2-1) ... 2026-02-18T08:08:54.584 INFO:teuthology.orchestra.run.trial163.stdout:Processing triggers for libc-bin (2.35-0ubuntu3.11) ... 2026-02-18T08:08:55.249 INFO:teuthology.orchestra.run.trial163.stdout: 2026-02-18T08:08:55.249 INFO:teuthology.orchestra.run.trial163.stdout:Running kernel seems to be up-to-date. 2026-02-18T08:08:55.249 INFO:teuthology.orchestra.run.trial163.stdout: 2026-02-18T08:08:55.249 INFO:teuthology.orchestra.run.trial163.stdout:The processor microcode seems to be up-to-date. 2026-02-18T08:08:55.249 INFO:teuthology.orchestra.run.trial163.stdout: 2026-02-18T08:08:55.250 INFO:teuthology.orchestra.run.trial163.stdout:No services need to be restarted. 2026-02-18T08:08:55.250 INFO:teuthology.orchestra.run.trial163.stdout: 2026-02-18T08:08:55.250 INFO:teuthology.orchestra.run.trial163.stdout:No containers need to be restarted. 2026-02-18T08:08:55.250 INFO:teuthology.orchestra.run.trial163.stdout: 2026-02-18T08:08:55.250 INFO:teuthology.orchestra.run.trial163.stdout:No user sessions are running outdated binaries. 2026-02-18T08:08:55.250 INFO:teuthology.orchestra.run.trial163.stdout: 2026-02-18T08:08:55.250 INFO:teuthology.orchestra.run.trial163.stdout:No VM guests are running outdated hypervisor (qemu) binaries on this host. 2026-02-18T08:08:55.907 INFO:teuthology.orchestra.run.trial163.stderr:W: --force-yes is deprecated, use one of the options starting with --allow instead. 2026-02-18T08:08:55.914 DEBUG:teuthology.parallel:result is None 2026-02-18T08:08:55.915 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=ubuntu%2F22.04%2Fx86_64&sha1=adc1923395f2bcd6ae89b5d331d278364d1848ed 2026-02-18T08:08:56.079 DEBUG:teuthology.orchestra.run.trial034:> dpkg-query -W -f '${Version}' ceph 2026-02-18T08:08:56.090 INFO:teuthology.orchestra.run.trial034.stdout:20.2.0-670-gadc19233-1jammy 2026-02-18T08:08:56.090 INFO:teuthology.packaging:The installed version of ceph is 20.2.0-670-gadc19233-1jammy 2026-02-18T08:08:56.091 INFO:teuthology.task.install:The correct ceph version 20.2.0-670-gadc19233-1jammy is installed. 2026-02-18T08:08:56.093 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=ubuntu%2F22.04%2Fx86_64&sha1=adc1923395f2bcd6ae89b5d331d278364d1848ed 2026-02-18T08:08:56.238 DEBUG:teuthology.orchestra.run.trial163:> dpkg-query -W -f '${Version}' ceph 2026-02-18T08:08:56.249 INFO:teuthology.orchestra.run.trial163.stdout:20.2.0-670-gadc19233-1jammy 2026-02-18T08:08:56.249 INFO:teuthology.packaging:The installed version of ceph is 20.2.0-670-gadc19233-1jammy 2026-02-18T08:08:56.250 INFO:teuthology.task.install:The correct ceph version 20.2.0-670-gadc19233-1jammy is installed. 2026-02-18T08:08:56.251 INFO:teuthology.task.install.util:Shipping valgrind.supp... 2026-02-18T08:08:56.252 DEBUG:teuthology.orchestra.run.trial034:> set -ex 2026-02-18T08:08:56.252 DEBUG:teuthology.orchestra.run.trial034:> sudo dd of=/home/ubuntu/cephtest/valgrind.supp 2026-02-18T08:08:56.262 DEBUG:teuthology.orchestra.run.trial163:> set -ex 2026-02-18T08:08:56.263 DEBUG:teuthology.orchestra.run.trial163:> sudo dd of=/home/ubuntu/cephtest/valgrind.supp 2026-02-18T08:08:56.302 INFO:teuthology.task.install.util:Shipping 'daemon-helper'... 2026-02-18T08:08:56.302 DEBUG:teuthology.orchestra.run.trial034:> set -ex 2026-02-18T08:08:56.302 DEBUG:teuthology.orchestra.run.trial034:> sudo dd of=/usr/bin/daemon-helper 2026-02-18T08:08:56.313 DEBUG:teuthology.orchestra.run.trial034:> sudo chmod a=rx -- /usr/bin/daemon-helper 2026-02-18T08:08:56.364 DEBUG:teuthology.orchestra.run.trial163:> set -ex 2026-02-18T08:08:56.364 DEBUG:teuthology.orchestra.run.trial163:> sudo dd of=/usr/bin/daemon-helper 2026-02-18T08:08:56.375 DEBUG:teuthology.orchestra.run.trial163:> sudo chmod a=rx -- /usr/bin/daemon-helper 2026-02-18T08:08:56.426 INFO:teuthology.task.install.util:Shipping 'adjust-ulimits'... 2026-02-18T08:08:56.426 DEBUG:teuthology.orchestra.run.trial034:> set -ex 2026-02-18T08:08:56.427 DEBUG:teuthology.orchestra.run.trial034:> sudo dd of=/usr/bin/adjust-ulimits 2026-02-18T08:08:56.437 DEBUG:teuthology.orchestra.run.trial034:> sudo chmod a=rx -- /usr/bin/adjust-ulimits 2026-02-18T08:08:56.487 DEBUG:teuthology.orchestra.run.trial163:> set -ex 2026-02-18T08:08:56.487 DEBUG:teuthology.orchestra.run.trial163:> sudo dd of=/usr/bin/adjust-ulimits 2026-02-18T08:08:56.498 DEBUG:teuthology.orchestra.run.trial163:> sudo chmod a=rx -- /usr/bin/adjust-ulimits 2026-02-18T08:08:56.553 INFO:teuthology.task.install.util:Shipping 'stdin-killer'... 2026-02-18T08:08:56.553 DEBUG:teuthology.orchestra.run.trial034:> set -ex 2026-02-18T08:08:56.553 DEBUG:teuthology.orchestra.run.trial034:> sudo dd of=/usr/bin/stdin-killer 2026-02-18T08:08:56.564 DEBUG:teuthology.orchestra.run.trial034:> sudo chmod a=rx -- /usr/bin/stdin-killer 2026-02-18T08:08:56.612 DEBUG:teuthology.orchestra.run.trial163:> set -ex 2026-02-18T08:08:56.612 DEBUG:teuthology.orchestra.run.trial163:> sudo dd of=/usr/bin/stdin-killer 2026-02-18T08:08:56.621 DEBUG:teuthology.orchestra.run.trial163:> sudo chmod a=rx -- /usr/bin/stdin-killer 2026-02-18T08:08:56.669 INFO:teuthology.run_tasks:Running task ceph... 2026-02-18T08:08:56.783 INFO:tasks.ceph:Making ceph log dir writeable by non-root... 2026-02-18T08:08:56.783 DEBUG:teuthology.orchestra.run.trial034:> sudo chmod 777 /var/log/ceph 2026-02-18T08:08:56.784 DEBUG:teuthology.orchestra.run.trial163:> sudo chmod 777 /var/log/ceph 2026-02-18T08:08:56.792 INFO:tasks.ceph:Disabling ceph logrotate... 2026-02-18T08:08:56.793 DEBUG:teuthology.orchestra.run.trial034:> sudo rm -f -- /etc/logrotate.d/ceph 2026-02-18T08:08:56.833 DEBUG:teuthology.orchestra.run.trial163:> sudo rm -f -- /etc/logrotate.d/ceph 2026-02-18T08:08:56.840 INFO:tasks.ceph:Creating extra log directories... 2026-02-18T08:08:56.840 DEBUG:teuthology.orchestra.run.trial034:> sudo install -d -m0777 -- /var/log/ceph/valgrind /var/log/ceph/profiling-logger 2026-02-18T08:08:56.881 DEBUG:teuthology.orchestra.run.trial163:> sudo install -d -m0777 -- /var/log/ceph/valgrind /var/log/ceph/profiling-logger 2026-02-18T08:08:56.911 INFO:tasks.ceph:Creating ceph cluster cluster1... 2026-02-18T08:08:56.911 INFO:tasks.ceph:config {'conf': {'client': {'debug rbd': 20, 'debug rbd_mirror': 15, 'log to stderr': False, 'rbd default features': 125, 'rbd mirror image policy type': 'simple'}, 'client.mirror.0': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.1': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.2': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.3': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'global': {'mon client directed command retry': 5, 'mon warn on pool no app': False, 'ms inject socket failures': 5000}, 'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}, 'osd': {'bluefs allocator': 'hybrid', 'bluestore allocator': 'hybrid', 'bluestore block size': 96636764160, 'bluestore fsck on mount': True, 'debug bluefs': '1/20', 'debug bluestore': '1/20', 'debug ms': 1, 'debug osd': 20, 'debug rocksdb': '4/10', 'mon osd backfillfull_ratio': 0.85, 'mon osd full ratio': 0.9, 'mon osd nearfull ratio': 0.8, 'osd failsafe full ratio': 0.95, 'osd objectstore': 'bluestore'}}, 'fs': 'xfs', 'mkfs_options': None, 'mount_options': None, 'skip_mgr_daemons': False, 'log_ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)', '\\(OSD_SLOW_PING_TIME'], 'cpu_profile': set(), 'cluster': 'cluster1', 'mon_bind_msgr2': True, 'mon_bind_addrvec': True} 2026-02-18T08:08:56.911 INFO:tasks.ceph:ctx.config {'archive_path': '/home/teuthworker/mnt/teuthology/yuriw-2026-02-17_21:26:40-rbd-20.2.1_2.17.26-baseline_2-distro-default-trial/54948', 'branch': '20.2.1_2.17.26-baseline_2', 'description': 'rbd/mirror-thrash/{base/install clients/mirror cluster/{2-node} conf/{disable-pool-app} msgr-failures/few objectstore/bluestore-hybrid policy/simple rbd-mirror/four-per-cluster supported-random-distro$/{ubuntu_latest} workloads/rbd-mirror-journal-stress-workunit}', 'email': 'yweinste@redhat.com', 'first_in_suite': False, 'flavor': 'default', 'job_id': '54948', 'kernel': {'branch': 'distro', 'kdb': True, 'sha1': 'distro'}, 'ktype': 'distro', 'last_in_suite': False, 'machine_type': 'trial', 'meta': [{'desc': 'run two ceph clusters and install rbd-mirror'}, {'desc': 'configure the permissions for client.mirror'}, {'desc': '2 ceph clusters with 1 mon and 3 osds each'}, {'desc': 'run four rbd-mirror daemons per cluster'}, {'desc': 'run the rbd_mirror_stress.sh workunit to test the rbd-mirror daemon'}], 'name': 'yuriw-2026-02-17_21:26:40-rbd-20.2.1_2.17.26-baseline_2-distro-default-trial', 'no_nested_subset': False, 'os_type': 'ubuntu', 'os_version': '22.04', 'overrides': {'admin_socket': {'branch': '20.2.1_2.17.26-baseline_2'}, 'ceph': {'conf': {'client': {'debug rbd': 20, 'debug rbd_mirror': 15, 'log to stderr': False, 'rbd default features': 125, 'rbd mirror image policy type': 'simple'}, 'client.mirror.0': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.1': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.2': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.3': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'global': {'mon client directed command retry': 5, 'mon warn on pool no app': False, 'ms inject socket failures': 5000}, 'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}, 'osd': {'bluefs allocator': 'hybrid', 'bluestore allocator': 'hybrid', 'bluestore block size': 96636764160, 'bluestore fsck on mount': True, 'debug bluefs': '1/20', 'debug bluestore': '1/20', 'debug ms': 1, 'debug osd': 20, 'debug rocksdb': '4/10', 'mon osd backfillfull_ratio': 0.85, 'mon osd full ratio': 0.9, 'mon osd nearfull ratio': 0.8, 'osd failsafe full ratio': 0.95, 'osd objectstore': 'bluestore'}}, 'flavor': 'default', 'fs': 'xfs', 'log-ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)', '\\(OSD_SLOW_PING_TIME'], 'sha1': 'adc1923395f2bcd6ae89b5d331d278364d1848ed'}, 'ceph-deploy': {'bluestore': True, 'conf': {'client': {'log file': '/var/log/ceph/ceph-$name.$pid.log'}, 'mon': {}, 'osd': {'bluestore block size': 96636764160, 'bluestore fsck on mount': True, 'debug bluefs': '1/20', 'debug bluestore': '1/20', 'debug rocksdb': '4/10', 'mon osd backfillfull_ratio': 0.85, 'mon osd full ratio': 0.9, 'mon osd nearfull ratio': 0.8, 'osd failsafe full ratio': 0.95, 'osd objectstore': 'bluestore'}}, 'fs': 'xfs'}, 'install': {'ceph': {'flavor': 'default', 'sha1': 'adc1923395f2bcd6ae89b5d331d278364d1848ed'}}, 'thrashosds': {'bdev_inject_crash': 2, 'bdev_inject_crash_probability': 0.5}, 'workunit': {'branch': '20.2.1_2.17.26-baseline_2', 'sha1': 'adc1923395f2bcd6ae89b5d331d278364d1848ed'}}, 'owner': 'scheduled_yuriw@soko04.front.sepia.ceph.com', 'priority': 69, 'repo': 'https://github.com/ceph/ceph-ci.git', 'roles': [['cluster1.mon.a', 'cluster1.mgr.x', 'cluster2.mgr.x', 'cluster1.osd.0', 'cluster1.osd.1', 'cluster1.osd.2', 'cluster1.client.0', 'cluster2.client.0'], ['cluster2.mon.a', 'cluster2.osd.0', 'cluster2.osd.1', 'cluster2.osd.2', 'cluster1.client.mirror', 'cluster1.client.mirror.0', 'cluster1.client.mirror.1', 'cluster1.client.mirror.2', 'cluster1.client.mirror.3', 'cluster1.client.mirror.4', 'cluster1.client.mirror.5', 'cluster1.client.mirror.6', 'cluster2.client.mirror', 'cluster2.client.mirror.0', 'cluster2.client.mirror.1', 'cluster2.client.mirror.2', 'cluster2.client.mirror.3', 'cluster2.client.mirror.4', 'cluster2.client.mirror.5', 'cluster2.client.mirror.6']], 'seed': 7878, 'sha1': 'adc1923395f2bcd6ae89b5d331d278364d1848ed', 'sleep_before_teardown': 0, 'subset': '111/120000', 'suite': 'rbd', 'suite_branch': '20.2.1_2.17.26-baseline_2', 'suite_path': '/home/teuthworker/src/github.com_ceph_ceph-c_adc1923395f2bcd6ae89b5d331d278364d1848ed/qa', 'suite_relpath': 'qa', 'suite_repo': 'https://github.com/ceph/ceph-ci.git', 'suite_sha1': 'adc1923395f2bcd6ae89b5d331d278364d1848ed', 'targets': {'trial034.front.sepia.ceph.com': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBF0BoVKPHOg0QKqwwjlPElLLlbn535mlxpDa0ESvNojoG4w+irzEhqHqNKGlXw1d/sbAP/gj/AqXlTteMMmvQPo=', 'trial163.front.sepia.ceph.com': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBLTZW5+I/rQuc56SPLMku4v11X/kldvg5rnpQDqR+Im5nfwRzHmq3wJFTgZ2+NqmphkPILaOoEHK845hjsXPPN8='}, 'tasks': [{'internal.check_packages': None}, {'internal.buildpackages_prep': None}, {'internal.save_config': None}, {'internal.check_lock': None}, {'internal.add_remotes': None}, {'console_log': None}, {'internal.connect': None}, {'internal.push_inventory': None}, {'internal.serialize_remote_roles': None}, {'internal.check_conflict': None}, {'internal.check_ceph_data': None}, {'internal.vm_setup': None}, {'kernel': {'branch': 'distro', 'kdb': True, 'sha1': 'distro'}}, {'internal.base': None}, {'internal.archive_upload': None}, {'internal.archive': None}, {'internal.coredump': None}, {'internal.sudo': None}, {'internal.syslog': None}, {'internal.timer': None}, {'pcp': None}, {'selinux': None}, {'ansible.cephlab': None}, {'clock': None}, {'install': {'extra_packages': ['rbd-mirror'], 'flavor': 'default', 'sha1': 'adc1923395f2bcd6ae89b5d331d278364d1848ed'}}, {'ceph': {'cluster': 'cluster1', 'conf': {'client': {'debug rbd': 20, 'debug rbd_mirror': 15, 'log to stderr': False, 'rbd default features': 125, 'rbd mirror image policy type': 'simple'}, 'client.mirror.0': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.1': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.2': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.3': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'global': {'mon client directed command retry': 5, 'mon warn on pool no app': False, 'ms inject socket failures': 5000}, 'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}, 'osd': {'bluefs allocator': 'hybrid', 'bluestore allocator': 'hybrid', 'bluestore block size': 96636764160, 'bluestore fsck on mount': True, 'debug bluefs': '1/20', 'debug bluestore': '1/20', 'debug ms': 1, 'debug osd': 20, 'debug rocksdb': '4/10', 'mon osd backfillfull_ratio': 0.85, 'mon osd full ratio': 0.9, 'mon osd nearfull ratio': 0.8, 'osd failsafe full ratio': 0.95, 'osd objectstore': 'bluestore'}}, 'flavor': 'default', 'fs': 'xfs', 'log-ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)', '\\(OSD_SLOW_PING_TIME'], 'sha1': 'adc1923395f2bcd6ae89b5d331d278364d1848ed'}}, {'ceph': {'cluster': 'cluster2'}}, {'exec': {'cluster1.client.mirror.0': ["sudo ceph --cluster cluster1 auth caps client.mirror mon 'profile rbd-mirror-peer' osd 'profile rbd'", "sudo ceph --cluster cluster1 auth caps client.mirror.0 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster1 auth caps client.mirror.1 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster1 auth caps client.mirror.2 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster1 auth caps client.mirror.3 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster2 auth caps client.mirror mon 'profile rbd-mirror-peer' osd 'profile rbd'", "sudo ceph --cluster cluster2 auth caps client.mirror.0 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster2 auth caps client.mirror.1 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster2 auth caps client.mirror.2 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster2 auth caps client.mirror.3 mon 'profile rbd-mirror' osd 'profile rbd'"]}}, {'rbd-mirror': {'client': 'cluster1.client.mirror.0', 'thrash': True}}, {'rbd-mirror': {'client': 'cluster1.client.mirror.1', 'thrash': True}}, {'rbd-mirror': {'client': 'cluster1.client.mirror.2', 'thrash': True}}, {'rbd-mirror': {'client': 'cluster1.client.mirror.3', 'thrash': True}}, {'rbd-mirror': {'client': 'cluster2.client.mirror.0', 'thrash': True}}, {'rbd-mirror': {'client': 'cluster2.client.mirror.1', 'thrash': True}}, {'rbd-mirror': {'client': 'cluster2.client.mirror.2', 'thrash': True}}, {'rbd-mirror': {'client': 'cluster2.client.mirror.3', 'thrash': True}}, {'rbd-mirror-thrash': {'cluster': 'cluster1'}}, {'rbd-mirror-thrash': {'cluster': 'cluster2'}}, {'workunit': {'clients': {'cluster1.client.mirror': ['rbd/rbd_mirror_stress.sh']}, 'env': {'CEPH_ARGS': '', 'MIRROR_POOL_MODE': 'pool', 'RBD_MIRROR_INSTANCES': '4', 'RBD_MIRROR_MODE': 'journal', 'RBD_MIRROR_USE_EXISTING_CLUSTER': '1', 'RBD_MIRROR_USE_RBD_MIRROR': '1'}, 'timeout': '6h'}}], 'teuthology': {'fragments_dropped': [], 'meta': {}, 'postmerge': []}, 'teuthology_branch': 'main', 'teuthology_sha1': 'fa17720d0088c3ac28e473468bfc79eeaff5cd38', 'timestamp': '2026-02-17_21:26:40', 'tube': 'trial', 'user': 'yuriw', 'verbose': True, 'worker_log': '/home/teuthworker/mnt/teuthology/worker_logs/dispatcher.trial.3682'} 2026-02-18T08:08:56.913 DEBUG:teuthology.orchestra.run.trial034:> install -d -m0755 -- /home/ubuntu/cephtest/cluster1.data 2026-02-18T08:08:56.930 DEBUG:teuthology.orchestra.run.trial163:> install -d -m0755 -- /home/ubuntu/cephtest/cluster1.data 2026-02-18T08:08:56.941 DEBUG:teuthology.orchestra.run.trial034:> sudo install -d -m0777 -- /var/run/ceph 2026-02-18T08:08:56.978 DEBUG:teuthology.orchestra.run.trial163:> sudo install -d -m0777 -- /var/run/ceph 2026-02-18T08:08:56.989 DEBUG:teuthology.orchestra.run.trial034:> set -ex 2026-02-18T08:08:56.989 DEBUG:teuthology.orchestra.run.trial034:> dd if=/scratch_devs of=/dev/stdout 2026-02-18T08:08:57.033 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2026-02-18T08:08:57.033 DEBUG:teuthology.orchestra.run.trial034:> stat /dev/vg_nvme/lv_1 2026-02-18T08:08:57.080 INFO:teuthology.orchestra.run.trial034.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2026-02-18T08:08:57.080 INFO:teuthology.orchestra.run.trial034.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2026-02-18T08:08:57.080 INFO:teuthology.orchestra.run.trial034.stdout:Device: 5h/5d Inode: 896 Links: 1 2026-02-18T08:08:57.080 INFO:teuthology.orchestra.run.trial034.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2026-02-18T08:08:57.081 INFO:teuthology.orchestra.run.trial034.stdout:Access: 2026-02-18 08:08:00.932941822 +0000 2026-02-18T08:08:57.081 INFO:teuthology.orchestra.run.trial034.stdout:Modify: 2026-02-18 08:08:00.768946006 +0000 2026-02-18T08:08:57.081 INFO:teuthology.orchestra.run.trial034.stdout:Change: 2026-02-18 08:08:00.768946006 +0000 2026-02-18T08:08:57.081 INFO:teuthology.orchestra.run.trial034.stdout: Birth: - 2026-02-18T08:08:57.081 DEBUG:teuthology.orchestra.run.trial034:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2026-02-18T08:08:57.131 INFO:teuthology.orchestra.run.trial034.stderr:1+0 records in 2026-02-18T08:08:57.131 INFO:teuthology.orchestra.run.trial034.stderr:1+0 records out 2026-02-18T08:08:57.132 INFO:teuthology.orchestra.run.trial034.stderr:512 bytes copied, 0.000196595 s, 2.6 MB/s 2026-02-18T08:08:57.133 DEBUG:teuthology.orchestra.run.trial034:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2026-02-18T08:08:57.182 DEBUG:teuthology.orchestra.run.trial034:> stat /dev/vg_nvme/lv_2 2026-02-18T08:08:57.229 INFO:teuthology.orchestra.run.trial034.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2026-02-18T08:08:57.229 INFO:teuthology.orchestra.run.trial034.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2026-02-18T08:08:57.229 INFO:teuthology.orchestra.run.trial034.stdout:Device: 5h/5d Inode: 931 Links: 1 2026-02-18T08:08:57.229 INFO:teuthology.orchestra.run.trial034.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2026-02-18T08:08:57.229 INFO:teuthology.orchestra.run.trial034.stdout:Access: 2026-02-18 08:08:01.104937431 +0000 2026-02-18T08:08:57.229 INFO:teuthology.orchestra.run.trial034.stdout:Modify: 2026-02-18 08:08:01.100937533 +0000 2026-02-18T08:08:57.229 INFO:teuthology.orchestra.run.trial034.stdout:Change: 2026-02-18 08:08:01.100937533 +0000 2026-02-18T08:08:57.229 INFO:teuthology.orchestra.run.trial034.stdout: Birth: - 2026-02-18T08:08:57.230 DEBUG:teuthology.orchestra.run.trial034:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2026-02-18T08:08:57.279 INFO:teuthology.orchestra.run.trial034.stderr:1+0 records in 2026-02-18T08:08:57.280 INFO:teuthology.orchestra.run.trial034.stderr:1+0 records out 2026-02-18T08:08:57.280 INFO:teuthology.orchestra.run.trial034.stderr:512 bytes copied, 0.0001777 s, 2.9 MB/s 2026-02-18T08:08:57.281 DEBUG:teuthology.orchestra.run.trial034:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2026-02-18T08:08:57.329 DEBUG:teuthology.orchestra.run.trial034:> stat /dev/vg_nvme/lv_3 2026-02-18T08:08:57.376 INFO:teuthology.orchestra.run.trial034.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2026-02-18T08:08:57.376 INFO:teuthology.orchestra.run.trial034.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2026-02-18T08:08:57.376 INFO:teuthology.orchestra.run.trial034.stdout:Device: 5h/5d Inode: 955 Links: 1 2026-02-18T08:08:57.376 INFO:teuthology.orchestra.run.trial034.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2026-02-18T08:08:57.376 INFO:teuthology.orchestra.run.trial034.stdout:Access: 2026-02-18 08:08:01.460928345 +0000 2026-02-18T08:08:57.376 INFO:teuthology.orchestra.run.trial034.stdout:Modify: 2026-02-18 08:08:01.456928447 +0000 2026-02-18T08:08:57.376 INFO:teuthology.orchestra.run.trial034.stdout:Change: 2026-02-18 08:08:01.456928447 +0000 2026-02-18T08:08:57.376 INFO:teuthology.orchestra.run.trial034.stdout: Birth: - 2026-02-18T08:08:57.376 DEBUG:teuthology.orchestra.run.trial034:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2026-02-18T08:08:57.424 INFO:teuthology.orchestra.run.trial034.stderr:1+0 records in 2026-02-18T08:08:57.424 INFO:teuthology.orchestra.run.trial034.stderr:1+0 records out 2026-02-18T08:08:57.424 INFO:teuthology.orchestra.run.trial034.stderr:512 bytes copied, 0.000191896 s, 2.7 MB/s 2026-02-18T08:08:57.426 DEBUG:teuthology.orchestra.run.trial034:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2026-02-18T08:08:57.473 DEBUG:teuthology.orchestra.run.trial034:> stat /dev/vg_nvme/lv_4 2026-02-18T08:08:57.521 INFO:teuthology.orchestra.run.trial034.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2026-02-18T08:08:57.522 INFO:teuthology.orchestra.run.trial034.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2026-02-18T08:08:57.522 INFO:teuthology.orchestra.run.trial034.stdout:Device: 5h/5d Inode: 968 Links: 1 2026-02-18T08:08:57.522 INFO:teuthology.orchestra.run.trial034.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2026-02-18T08:08:57.522 INFO:teuthology.orchestra.run.trial034.stdout:Access: 2026-02-18 08:08:01.800919667 +0000 2026-02-18T08:08:57.522 INFO:teuthology.orchestra.run.trial034.stdout:Modify: 2026-02-18 08:08:01.796919770 +0000 2026-02-18T08:08:57.522 INFO:teuthology.orchestra.run.trial034.stdout:Change: 2026-02-18 08:08:01.796919770 +0000 2026-02-18T08:08:57.522 INFO:teuthology.orchestra.run.trial034.stdout: Birth: - 2026-02-18T08:08:57.522 DEBUG:teuthology.orchestra.run.trial034:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2026-02-18T08:08:57.577 INFO:teuthology.orchestra.run.trial034.stderr:1+0 records in 2026-02-18T08:08:57.577 INFO:teuthology.orchestra.run.trial034.stderr:1+0 records out 2026-02-18T08:08:57.577 INFO:teuthology.orchestra.run.trial034.stderr:512 bytes copied, 0.000165076 s, 3.1 MB/s 2026-02-18T08:08:57.577 DEBUG:teuthology.orchestra.run.trial034:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2026-02-18T08:08:57.625 INFO:tasks.ceph:osd dev map: {'cluster1.osd.0': '/dev/vg_nvme/lv_1', 'cluster1.osd.1': '/dev/vg_nvme/lv_2', 'cluster1.osd.2': '/dev/vg_nvme/lv_3'} 2026-02-18T08:08:57.625 INFO:tasks.ceph:remote_to_roles_to_devs: {Remote(name='ubuntu@trial034.front.sepia.ceph.com'): {'cluster1.osd.0': '/dev/vg_nvme/lv_1', 'cluster1.osd.1': '/dev/vg_nvme/lv_2', 'cluster1.osd.2': '/dev/vg_nvme/lv_3'}} 2026-02-18T08:08:57.625 INFO:tasks.ceph:Generating config... 2026-02-18T08:08:57.626 INFO:tasks.ceph:[client] debug rbd = 20 2026-02-18T08:08:57.626 INFO:tasks.ceph:[client] debug rbd_mirror = 15 2026-02-18T08:08:57.626 INFO:tasks.ceph:[client] log to stderr = False 2026-02-18T08:08:57.626 INFO:tasks.ceph:[client] rbd default features = 125 2026-02-18T08:08:57.626 INFO:tasks.ceph:[client] rbd mirror image policy type = simple 2026-02-18T08:08:57.626 INFO:tasks.ceph:[client.mirror.0] admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-18T08:08:57.626 INFO:tasks.ceph:[client.mirror.0] pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-18T08:08:57.626 INFO:tasks.ceph:[client.mirror.1] admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-18T08:08:57.626 INFO:tasks.ceph:[client.mirror.1] pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-18T08:08:57.626 INFO:tasks.ceph:[client.mirror.2] admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-18T08:08:57.626 INFO:tasks.ceph:[client.mirror.2] pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-18T08:08:57.626 INFO:tasks.ceph:[client.mirror.3] admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-18T08:08:57.626 INFO:tasks.ceph:[client.mirror.3] pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-18T08:08:57.626 INFO:tasks.ceph:[global] mon client directed command retry = 5 2026-02-18T08:08:57.626 INFO:tasks.ceph:[global] mon warn on pool no app = False 2026-02-18T08:08:57.626 INFO:tasks.ceph:[global] ms inject socket failures = 5000 2026-02-18T08:08:57.626 INFO:tasks.ceph:[mgr] debug mgr = 20 2026-02-18T08:08:57.626 INFO:tasks.ceph:[mgr] debug ms = 1 2026-02-18T08:08:57.626 INFO:tasks.ceph:[mon] debug mon = 20 2026-02-18T08:08:57.626 INFO:tasks.ceph:[mon] debug ms = 1 2026-02-18T08:08:57.626 INFO:tasks.ceph:[mon] debug paxos = 20 2026-02-18T08:08:57.626 INFO:tasks.ceph:[osd] bluefs allocator = hybrid 2026-02-18T08:08:57.626 INFO:tasks.ceph:[osd] bluestore allocator = hybrid 2026-02-18T08:08:57.626 INFO:tasks.ceph:[osd] bluestore block size = 96636764160 2026-02-18T08:08:57.626 INFO:tasks.ceph:[osd] bluestore fsck on mount = True 2026-02-18T08:08:57.626 INFO:tasks.ceph:[osd] debug bluefs = 1/20 2026-02-18T08:08:57.626 INFO:tasks.ceph:[osd] debug bluestore = 1/20 2026-02-18T08:08:57.626 INFO:tasks.ceph:[osd] debug ms = 1 2026-02-18T08:08:57.626 INFO:tasks.ceph:[osd] debug osd = 20 2026-02-18T08:08:57.626 INFO:tasks.ceph:[osd] debug rocksdb = 4/10 2026-02-18T08:08:57.626 INFO:tasks.ceph:[osd] mon osd backfillfull_ratio = 0.85 2026-02-18T08:08:57.626 INFO:tasks.ceph:[osd] mon osd full ratio = 0.9 2026-02-18T08:08:57.626 INFO:tasks.ceph:[osd] mon osd nearfull ratio = 0.8 2026-02-18T08:08:57.626 INFO:tasks.ceph:[osd] osd failsafe full ratio = 0.95 2026-02-18T08:08:57.626 INFO:tasks.ceph:[osd] osd objectstore = bluestore 2026-02-18T08:08:57.626 INFO:tasks.ceph:Setting up cluster1.mon.a... 2026-02-18T08:08:57.627 DEBUG:teuthology.orchestra.run.trial034:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring /etc/ceph/cluster1.keyring 2026-02-18T08:08:57.684 INFO:teuthology.orchestra.run.trial034.stdout:creating /etc/ceph/cluster1.keyring 2026-02-18T08:08:57.686 DEBUG:teuthology.orchestra.run.trial034:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --gen-key --name=mon. /etc/ceph/cluster1.keyring 2026-02-18T08:08:57.751 DEBUG:teuthology.orchestra.run.trial034:> sudo chmod 0644 /etc/ceph/cluster1.keyring 2026-02-18T08:08:57.799 DEBUG:tasks.ceph:Ceph mon addresses: [('cluster1.mon.a', '10.20.193.34')] 2026-02-18T08:08:57.799 DEBUG:tasks.ceph:writing out conf {'global': {'chdir': '', 'pid file': '/var/run/ceph/$cluster-$name.pid', 'auth supported': 'cephx', 'filestore xattr use omap': 'true', 'mon clock drift allowed': '1.000', 'osd crush chooseleaf type': '0', 'auth debug': 'true', 'ms die on old message': 'true', 'ms die on bug': 'true', 'mon max pg per osd': '10000', 'mon pg warn max object skew': '0', 'osd_pool_default_pg_autoscale_mode': 'off', 'osd pool default size': '2', 'mon osd allow primary affinity': 'true', 'mon osd allow pg remap': 'true', 'mon warn on legacy crush tunables': 'false', 'mon warn on crush straw calc version zero': 'false', 'mon warn on no sortbitwise': 'false', 'mon warn on osd down out interval zero': 'false', 'mon warn on too few osds': 'false', 'mon_warn_on_pool_pg_num_not_power_of_two': 'false', 'mon_warn_on_pool_no_redundancy': 'false', 'mon_allow_pool_size_one': 'true', 'osd pool default erasure code profile': 'plugin=isa technique=reed_sol_van k=2 m=1 crush-failure-domain=osd', 'osd default data pool replay window': '5', 'mon allow pool delete': 'true', 'mon cluster log file level': 'debug', 'debug asserts on shutdown': 'true', 'mon health detail to clog': 'false', 'mon host': '10.20.193.34', 'mon client directed command retry': 5, 'mon warn on pool no app': False, 'ms inject socket failures': 5000}, 'osd': {'osd journal size': '100', 'osd scrub load threshold': '5.0', 'osd scrub max interval': '600', 'osd mclock profile': 'high_recovery_ops', 'osd mclock skip benchmark': 'true', 'osd recover clone overlap': 'true', 'osd recovery max chunk': '1048576', 'osd debug shutdown': 'true', 'osd debug op order': 'true', 'osd debug verify stray on activate': 'true', 'osd debug trim objects': 'true', 'osd open classes on start': 'true', 'osd debug pg log writeout': 'true', 'osd deep scrub update digest min age': '30', 'osd map max advance': '10', 'journal zero on create': 'true', 'filestore ondisk finisher threads': '3', 'filestore apply finisher threads': '3', 'bdev debug aio': 'true', 'osd debug misdirected ops': 'true', 'bluefs allocator': 'hybrid', 'bluestore allocator': 'hybrid', 'bluestore block size': 96636764160, 'bluestore fsck on mount': True, 'debug bluefs': '1/20', 'debug bluestore': '1/20', 'debug ms': 1, 'debug osd': 20, 'debug rocksdb': '4/10', 'mon osd backfillfull_ratio': 0.85, 'mon osd full ratio': 0.9, 'mon osd nearfull ratio': 0.8, 'osd failsafe full ratio': 0.95, 'osd objectstore': 'bluestore'}, 'mgr': {'debug ms': 1, 'debug mgr': 20, 'debug mon': '20', 'debug auth': '20', 'mon reweight min pgs per osd': '4', 'mon reweight min bytes per osd': '10', 'mgr/telemetry/nag': 'false'}, 'mon': {'debug ms': 1, 'debug mon': 20, 'debug paxos': 20, 'debug auth': '20', 'mon data avail warn': '5', 'mon mgr mkfs grace': '240', 'mon reweight min pgs per osd': '4', 'mon osd reporter subtree level': 'osd', 'mon osd prime pg temp': 'true', 'mon reweight min bytes per osd': '10', 'auth mon ticket ttl': '660', 'auth service ticket ttl': '240', 'mon_warn_on_insecure_global_id_reclaim': 'false', 'mon_warn_on_insecure_global_id_reclaim_allowed': 'false', 'mon_down_mkfs_grace': '2m', 'mon_warn_on_filestore_osds': 'false'}, 'client': {'rgw cache enabled': 'true', 'rgw enable ops log': 'true', 'rgw enable usage log': 'true', 'log file': '/var/log/ceph/$cluster-$name.$pid.log', 'admin socket': '/var/run/ceph/$cluster-$name.$pid.asok', 'debug rbd': 20, 'debug rbd_mirror': 15, 'log to stderr': False, 'rbd default features': 125, 'rbd mirror image policy type': 'simple'}, 'mon.a': {}, 'client.mirror.0': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.1': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.2': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.3': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}} 2026-02-18T08:08:57.800 DEBUG:teuthology.orchestra.run.trial034:> set -ex 2026-02-18T08:08:57.800 DEBUG:teuthology.orchestra.run.trial034:> dd of=/home/ubuntu/cephtest/ceph.tmp.conf 2026-02-18T08:08:57.843 DEBUG:teuthology.orchestra.run.trial034:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage monmaptool -c /home/ubuntu/cephtest/ceph.tmp.conf --create --clobber --enable-all-features --add a 10.20.193.34 --print /home/ubuntu/cephtest/cluster1.monmap 2026-02-18T08:08:57.900 INFO:teuthology.orchestra.run.trial034.stdout:monmaptool: monmap file /home/ubuntu/cephtest/cluster1.monmap 2026-02-18T08:08:57.900 INFO:teuthology.orchestra.run.trial034.stdout:monmaptool: generated fsid 538b5201-9b84-4a73-9eef-5646b3360b9d 2026-02-18T08:08:57.900 INFO:teuthology.orchestra.run.trial034.stdout:setting min_mon_release = tentacle 2026-02-18T08:08:57.900 INFO:teuthology.orchestra.run.trial034.stdout:epoch 0 2026-02-18T08:08:57.900 INFO:teuthology.orchestra.run.trial034.stdout:fsid 538b5201-9b84-4a73-9eef-5646b3360b9d 2026-02-18T08:08:57.900 INFO:teuthology.orchestra.run.trial034.stdout:last_changed 2026-02-18T08:08:57.900166+0000 2026-02-18T08:08:57.900 INFO:teuthology.orchestra.run.trial034.stdout:created 2026-02-18T08:08:57.900166+0000 2026-02-18T08:08:57.901 INFO:teuthology.orchestra.run.trial034.stdout:min_mon_release 20 (tentacle) 2026-02-18T08:08:57.901 INFO:teuthology.orchestra.run.trial034.stdout:election_strategy: 1 2026-02-18T08:08:57.901 INFO:teuthology.orchestra.run.trial034.stdout:0: [v2:10.20.193.34:3300/0,v1:10.20.193.34:6789/0] mon.a 2026-02-18T08:08:57.901 INFO:teuthology.orchestra.run.trial034.stdout:monmaptool: writing epoch 0 to /home/ubuntu/cephtest/cluster1.monmap (1 monitors) 2026-02-18T08:08:57.902 DEBUG:teuthology.orchestra.run.trial034:> rm -- /home/ubuntu/cephtest/ceph.tmp.conf 2026-02-18T08:08:57.947 INFO:tasks.ceph:Writing /etc/ceph/cluster1.conf for FSID 538b5201-9b84-4a73-9eef-5646b3360b9d... 2026-02-18T08:08:57.947 DEBUG:teuthology.orchestra.run.trial034:> sudo mkdir -p /etc/ceph && sudo chmod 0755 /etc/ceph && sudo tee /etc/ceph/cluster1.conf && sudo chmod 0644 /etc/ceph/cluster1.conf > /dev/null 2026-02-18T08:08:57.990 DEBUG:teuthology.orchestra.run.trial163:> sudo mkdir -p /etc/ceph && sudo chmod 0755 /etc/ceph && sudo tee /etc/ceph/cluster1.conf && sudo chmod 0644 /etc/ceph/cluster1.conf > /dev/null 2026-02-18T08:08:58.007 INFO:teuthology.orchestra.run.trial034.stdout:[global] 2026-02-18T08:08:58.007 INFO:teuthology.orchestra.run.trial034.stdout: chdir = "" 2026-02-18T08:08:58.007 INFO:teuthology.orchestra.run.trial034.stdout: pid file = /var/run/ceph/$cluster-$name.pid 2026-02-18T08:08:58.008 INFO:teuthology.orchestra.run.trial034.stdout: auth supported = cephx 2026-02-18T08:08:58.008 INFO:teuthology.orchestra.run.trial034.stdout: 2026-02-18T08:08:58.008 INFO:teuthology.orchestra.run.trial034.stdout: filestore xattr use omap = true 2026-02-18T08:08:58.008 INFO:teuthology.orchestra.run.trial034.stdout: 2026-02-18T08:08:58.008 INFO:teuthology.orchestra.run.trial034.stdout: mon clock drift allowed = 1.000 2026-02-18T08:08:58.008 INFO:teuthology.orchestra.run.trial034.stdout: 2026-02-18T08:08:58.008 INFO:teuthology.orchestra.run.trial034.stdout: osd crush chooseleaf type = 0 2026-02-18T08:08:58.008 INFO:teuthology.orchestra.run.trial034.stdout: auth debug = true 2026-02-18T08:08:58.008 INFO:teuthology.orchestra.run.trial034.stdout: 2026-02-18T08:08:58.008 INFO:teuthology.orchestra.run.trial034.stdout: ms die on old message = true 2026-02-18T08:08:58.008 INFO:teuthology.orchestra.run.trial034.stdout: ms die on bug = true 2026-02-18T08:08:58.009 INFO:teuthology.orchestra.run.trial034.stdout: 2026-02-18T08:08:58.009 INFO:teuthology.orchestra.run.trial034.stdout: mon max pg per osd = 10000 # >= luminous 2026-02-18T08:08:58.009 INFO:teuthology.orchestra.run.trial034.stdout: mon pg warn max object skew = 0 2026-02-18T08:08:58.009 INFO:teuthology.orchestra.run.trial034.stdout: 2026-02-18T08:08:58.009 INFO:teuthology.orchestra.run.trial034.stdout: # disable pg_autoscaler by default for new pools 2026-02-18T08:08:58.009 INFO:teuthology.orchestra.run.trial034.stdout: osd_pool_default_pg_autoscale_mode = off 2026-02-18T08:08:58.009 INFO:teuthology.orchestra.run.trial034.stdout: 2026-02-18T08:08:58.009 INFO:teuthology.orchestra.run.trial034.stdout: osd pool default size = 2 2026-02-18T08:08:58.010 INFO:teuthology.orchestra.run.trial034.stdout: 2026-02-18T08:08:58.010 INFO:teuthology.orchestra.run.trial034.stdout: mon osd allow primary affinity = true 2026-02-18T08:08:58.010 INFO:teuthology.orchestra.run.trial034.stdout: mon osd allow pg remap = true 2026-02-18T08:08:58.010 INFO:teuthology.orchestra.run.trial034.stdout: mon warn on legacy crush tunables = false 2026-02-18T08:08:58.010 INFO:teuthology.orchestra.run.trial034.stdout: mon warn on crush straw calc version zero = false 2026-02-18T08:08:58.010 INFO:teuthology.orchestra.run.trial034.stdout: mon warn on no sortbitwise = false 2026-02-18T08:08:58.010 INFO:teuthology.orchestra.run.trial034.stdout: mon warn on osd down out interval zero = false 2026-02-18T08:08:58.010 INFO:teuthology.orchestra.run.trial034.stdout: mon warn on too few osds = false 2026-02-18T08:08:58.010 INFO:teuthology.orchestra.run.trial034.stdout: mon_warn_on_pool_pg_num_not_power_of_two = false 2026-02-18T08:08:58.010 INFO:teuthology.orchestra.run.trial034.stdout: mon_warn_on_pool_no_redundancy = false 2026-02-18T08:08:58.010 INFO:teuthology.orchestra.run.trial034.stdout: mon_allow_pool_size_one = true 2026-02-18T08:08:58.010 INFO:teuthology.orchestra.run.trial034.stdout: 2026-02-18T08:08:58.011 INFO:teuthology.orchestra.run.trial034.stdout: osd pool default erasure code profile = plugin=isa technique=reed_sol_van k=2 m=1 crush-failure-domain=osd 2026-02-18T08:08:58.011 INFO:teuthology.orchestra.run.trial034.stdout: 2026-02-18T08:08:58.011 INFO:teuthology.orchestra.run.trial034.stdout: osd default data pool replay window = 5 2026-02-18T08:08:58.011 INFO:teuthology.orchestra.run.trial034.stdout: 2026-02-18T08:08:58.011 INFO:teuthology.orchestra.run.trial034.stdout: mon allow pool delete = true 2026-02-18T08:08:58.011 INFO:teuthology.orchestra.run.trial034.stdout: 2026-02-18T08:08:58.011 INFO:teuthology.orchestra.run.trial034.stdout: mon cluster log file level = debug 2026-02-18T08:08:58.011 INFO:teuthology.orchestra.run.trial034.stdout: debug asserts on shutdown = true 2026-02-18T08:08:58.011 INFO:teuthology.orchestra.run.trial034.stdout: mon health detail to clog = false 2026-02-18T08:08:58.011 INFO:teuthology.orchestra.run.trial034.stdout: mon host = 10.20.193.34 2026-02-18T08:08:58.011 INFO:teuthology.orchestra.run.trial034.stdout: mon client directed command retry = 5 2026-02-18T08:08:58.011 INFO:teuthology.orchestra.run.trial034.stdout: mon warn on pool no app = False 2026-02-18T08:08:58.012 INFO:teuthology.orchestra.run.trial034.stdout: ms inject socket failures = 5000 2026-02-18T08:08:58.012 INFO:teuthology.orchestra.run.trial034.stdout: fsid = 538b5201-9b84-4a73-9eef-5646b3360b9d 2026-02-18T08:08:58.012 INFO:teuthology.orchestra.run.trial034.stdout: 2026-02-18T08:08:58.012 INFO:teuthology.orchestra.run.trial034.stdout:[osd] 2026-02-18T08:08:58.012 INFO:teuthology.orchestra.run.trial034.stdout: osd journal size = 100 2026-02-18T08:08:58.012 INFO:teuthology.orchestra.run.trial034.stdout: 2026-02-18T08:08:58.012 INFO:teuthology.orchestra.run.trial034.stdout: osd scrub load threshold = 5.0 2026-02-18T08:08:58.012 INFO:teuthology.orchestra.run.trial034.stdout: osd scrub max interval = 600 2026-02-18T08:08:58.012 INFO:teuthology.orchestra.run.trial034.stdout: osd mclock profile = high_recovery_ops 2026-02-18T08:08:58.012 INFO:teuthology.orchestra.run.trial034.stdout: osd mclock skip benchmark = true 2026-02-18T08:08:58.012 INFO:teuthology.orchestra.run.trial034.stdout: 2026-02-18T08:08:58.012 INFO:teuthology.orchestra.run.trial034.stdout: osd recover clone overlap = true 2026-02-18T08:08:58.012 INFO:teuthology.orchestra.run.trial034.stdout: osd recovery max chunk = 1048576 2026-02-18T08:08:58.013 INFO:teuthology.orchestra.run.trial034.stdout: 2026-02-18T08:08:58.013 INFO:teuthology.orchestra.run.trial034.stdout: osd debug shutdown = true 2026-02-18T08:08:58.013 INFO:teuthology.orchestra.run.trial034.stdout: osd debug op order = true 2026-02-18T08:08:58.013 INFO:teuthology.orchestra.run.trial034.stdout: osd debug verify stray on activate = true 2026-02-18T08:08:58.013 INFO:teuthology.orchestra.run.trial034.stdout: osd debug trim objects = true 2026-02-18T08:08:58.013 INFO:teuthology.orchestra.run.trial034.stdout: 2026-02-18T08:08:58.013 INFO:teuthology.orchestra.run.trial034.stdout: osd open classes on start = true 2026-02-18T08:08:58.013 INFO:teuthology.orchestra.run.trial034.stdout: osd debug pg log writeout = true 2026-02-18T08:08:58.013 INFO:teuthology.orchestra.run.trial034.stdout: 2026-02-18T08:08:58.013 INFO:teuthology.orchestra.run.trial034.stdout: osd deep scrub update digest min age = 30 2026-02-18T08:08:58.013 INFO:teuthology.orchestra.run.trial034.stdout: 2026-02-18T08:08:58.013 INFO:teuthology.orchestra.run.trial034.stdout: osd map max advance = 10 2026-02-18T08:08:58.014 INFO:teuthology.orchestra.run.trial034.stdout: 2026-02-18T08:08:58.014 INFO:teuthology.orchestra.run.trial034.stdout: journal zero on create = true 2026-02-18T08:08:58.014 INFO:teuthology.orchestra.run.trial034.stdout: 2026-02-18T08:08:58.014 INFO:teuthology.orchestra.run.trial034.stdout: filestore ondisk finisher threads = 3 2026-02-18T08:08:58.014 INFO:teuthology.orchestra.run.trial034.stdout: filestore apply finisher threads = 3 2026-02-18T08:08:58.014 INFO:teuthology.orchestra.run.trial034.stdout: 2026-02-18T08:08:58.014 INFO:teuthology.orchestra.run.trial034.stdout: bdev debug aio = true 2026-02-18T08:08:58.014 INFO:teuthology.orchestra.run.trial034.stdout: osd debug misdirected ops = true 2026-02-18T08:08:58.014 INFO:teuthology.orchestra.run.trial034.stdout: bluefs allocator = hybrid 2026-02-18T08:08:58.014 INFO:teuthology.orchestra.run.trial034.stdout: bluestore allocator = hybrid 2026-02-18T08:08:58.014 INFO:teuthology.orchestra.run.trial034.stdout: bluestore block size = 96636764160 2026-02-18T08:08:58.014 INFO:teuthology.orchestra.run.trial034.stdout: bluestore fsck on mount = True 2026-02-18T08:08:58.014 INFO:teuthology.orchestra.run.trial034.stdout: debug bluefs = 1/20 2026-02-18T08:08:58.015 INFO:teuthology.orchestra.run.trial034.stdout: debug bluestore = 1/20 2026-02-18T08:08:58.015 INFO:teuthology.orchestra.run.trial034.stdout: debug ms = 1 2026-02-18T08:08:58.015 INFO:teuthology.orchestra.run.trial034.stdout: debug osd = 20 2026-02-18T08:08:58.015 INFO:teuthology.orchestra.run.trial034.stdout: debug rocksdb = 4/10 2026-02-18T08:08:58.015 INFO:teuthology.orchestra.run.trial034.stdout: mon osd backfillfull_ratio = 0.85 2026-02-18T08:08:58.015 INFO:teuthology.orchestra.run.trial034.stdout: mon osd full ratio = 0.9 2026-02-18T08:08:58.015 INFO:teuthology.orchestra.run.trial034.stdout: mon osd nearfull ratio = 0.8 2026-02-18T08:08:58.015 INFO:teuthology.orchestra.run.trial034.stdout: osd failsafe full ratio = 0.95 2026-02-18T08:08:58.015 INFO:teuthology.orchestra.run.trial034.stdout: osd objectstore = bluestore 2026-02-18T08:08:58.015 INFO:teuthology.orchestra.run.trial034.stdout: 2026-02-18T08:08:58.015 INFO:teuthology.orchestra.run.trial034.stdout:[mgr] 2026-02-18T08:08:58.015 INFO:teuthology.orchestra.run.trial034.stdout: debug ms = 1 2026-02-18T08:08:58.015 INFO:teuthology.orchestra.run.trial034.stdout: debug mgr = 20 2026-02-18T08:08:58.016 INFO:teuthology.orchestra.run.trial034.stdout: debug mon = 20 2026-02-18T08:08:58.016 INFO:teuthology.orchestra.run.trial034.stdout: debug auth = 20 2026-02-18T08:08:58.016 INFO:teuthology.orchestra.run.trial034.stdout: mon reweight min pgs per osd = 4 2026-02-18T08:08:58.016 INFO:teuthology.orchestra.run.trial034.stdout: mon reweight min bytes per osd = 10 2026-02-18T08:08:58.016 INFO:teuthology.orchestra.run.trial034.stdout: mgr/telemetry/nag = false 2026-02-18T08:08:58.016 INFO:teuthology.orchestra.run.trial034.stdout: 2026-02-18T08:08:58.016 INFO:teuthology.orchestra.run.trial034.stdout:[mon] 2026-02-18T08:08:58.016 INFO:teuthology.orchestra.run.trial034.stdout: debug ms = 1 2026-02-18T08:08:58.016 INFO:teuthology.orchestra.run.trial034.stdout: debug mon = 20 2026-02-18T08:08:58.016 INFO:teuthology.orchestra.run.trial034.stdout: debug paxos = 20 2026-02-18T08:08:58.016 INFO:teuthology.orchestra.run.trial034.stdout: debug auth = 20 2026-02-18T08:08:58.016 INFO:teuthology.orchestra.run.trial034.stdout: mon data avail warn = 5 2026-02-18T08:08:58.016 INFO:teuthology.orchestra.run.trial034.stdout: mon mgr mkfs grace = 240 2026-02-18T08:08:58.017 INFO:teuthology.orchestra.run.trial034.stdout: mon reweight min pgs per osd = 4 2026-02-18T08:08:58.017 INFO:teuthology.orchestra.run.trial034.stdout: mon osd reporter subtree level = osd 2026-02-18T08:08:58.017 INFO:teuthology.orchestra.run.trial034.stdout: mon osd prime pg temp = true 2026-02-18T08:08:58.017 INFO:teuthology.orchestra.run.trial034.stdout: mon reweight min bytes per osd = 10 2026-02-18T08:08:58.017 INFO:teuthology.orchestra.run.trial034.stdout: 2026-02-18T08:08:58.017 INFO:teuthology.orchestra.run.trial034.stdout: # rotate auth tickets quickly to exercise renewal paths 2026-02-18T08:08:58.017 INFO:teuthology.orchestra.run.trial034.stdout: auth mon ticket ttl = 660 # 11m 2026-02-18T08:08:58.017 INFO:teuthology.orchestra.run.trial034.stdout: auth service ticket ttl = 240 # 4m 2026-02-18T08:08:58.017 INFO:teuthology.orchestra.run.trial034.stdout: 2026-02-18T08:08:58.017 INFO:teuthology.orchestra.run.trial034.stdout: # don't complain about insecure global_id in the test suite 2026-02-18T08:08:58.017 INFO:teuthology.orchestra.run.trial034.stdout: mon_warn_on_insecure_global_id_reclaim = false 2026-02-18T08:08:58.017 INFO:teuthology.orchestra.run.trial034.stdout: mon_warn_on_insecure_global_id_reclaim_allowed = false 2026-02-18T08:08:58.018 INFO:teuthology.orchestra.run.trial034.stdout: 2026-02-18T08:08:58.018 INFO:teuthology.orchestra.run.trial034.stdout: # 1m isn't quite enough 2026-02-18T08:08:58.018 INFO:teuthology.orchestra.run.trial034.stdout: mon_down_mkfs_grace = 2m 2026-02-18T08:08:58.018 INFO:teuthology.orchestra.run.trial034.stdout: 2026-02-18T08:08:58.018 INFO:teuthology.orchestra.run.trial034.stdout: mon_warn_on_filestore_osds = false 2026-02-18T08:08:58.018 INFO:teuthology.orchestra.run.trial034.stdout: 2026-02-18T08:08:58.018 INFO:teuthology.orchestra.run.trial034.stdout:[client] 2026-02-18T08:08:58.018 INFO:teuthology.orchestra.run.trial034.stdout: rgw cache enabled = true 2026-02-18T08:08:58.018 INFO:teuthology.orchestra.run.trial034.stdout: rgw enable ops log = true 2026-02-18T08:08:58.018 INFO:teuthology.orchestra.run.trial034.stdout: rgw enable usage log = true 2026-02-18T08:08:58.018 INFO:teuthology.orchestra.run.trial034.stdout: log file = /var/log/ceph/$cluster-$name.$pid.log 2026-02-18T08:08:58.018 INFO:teuthology.orchestra.run.trial034.stdout: admin socket = /var/run/ceph/$cluster-$name.$pid.asok 2026-02-18T08:08:58.018 INFO:teuthology.orchestra.run.trial034.stdout: debug rbd = 20 2026-02-18T08:08:58.019 INFO:teuthology.orchestra.run.trial034.stdout: debug rbd_mirror = 15 2026-02-18T08:08:58.019 INFO:teuthology.orchestra.run.trial034.stdout: log to stderr = False 2026-02-18T08:08:58.019 INFO:teuthology.orchestra.run.trial034.stdout: rbd default features = 125 2026-02-18T08:08:58.019 INFO:teuthology.orchestra.run.trial034.stdout: rbd mirror image policy type = simple 2026-02-18T08:08:58.019 INFO:teuthology.orchestra.run.trial034.stdout:[mon.a] 2026-02-18T08:08:58.019 INFO:teuthology.orchestra.run.trial034.stdout:[client.mirror.0] 2026-02-18T08:08:58.019 INFO:teuthology.orchestra.run.trial034.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-18T08:08:58.019 INFO:teuthology.orchestra.run.trial034.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-18T08:08:58.019 INFO:teuthology.orchestra.run.trial034.stdout:[client.mirror.1] 2026-02-18T08:08:58.019 INFO:teuthology.orchestra.run.trial034.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-18T08:08:58.019 INFO:teuthology.orchestra.run.trial034.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-18T08:08:58.020 INFO:teuthology.orchestra.run.trial034.stdout:[client.mirror.2] 2026-02-18T08:08:58.020 INFO:teuthology.orchestra.run.trial034.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-18T08:08:58.020 INFO:teuthology.orchestra.run.trial034.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-18T08:08:58.020 INFO:teuthology.orchestra.run.trial034.stdout:[client.mirror.3] 2026-02-18T08:08:58.020 INFO:teuthology.orchestra.run.trial034.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-18T08:08:58.020 INFO:teuthology.orchestra.run.trial034.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-18T08:08:58.022 INFO:teuthology.orchestra.run.trial163.stdout:[global] 2026-02-18T08:08:58.022 INFO:teuthology.orchestra.run.trial163.stdout: chdir = "" 2026-02-18T08:08:58.022 INFO:teuthology.orchestra.run.trial163.stdout: pid file = /var/run/ceph/$cluster-$name.pid 2026-02-18T08:08:58.022 INFO:teuthology.orchestra.run.trial163.stdout: auth supported = cephx 2026-02-18T08:08:58.022 INFO:teuthology.orchestra.run.trial163.stdout: 2026-02-18T08:08:58.023 INFO:teuthology.orchestra.run.trial163.stdout: filestore xattr use omap = true 2026-02-18T08:08:58.023 INFO:teuthology.orchestra.run.trial163.stdout: 2026-02-18T08:08:58.023 INFO:teuthology.orchestra.run.trial163.stdout: mon clock drift allowed = 1.000 2026-02-18T08:08:58.023 INFO:teuthology.orchestra.run.trial163.stdout: 2026-02-18T08:08:58.023 INFO:teuthology.orchestra.run.trial163.stdout: osd crush chooseleaf type = 0 2026-02-18T08:08:58.023 INFO:teuthology.orchestra.run.trial163.stdout: auth debug = true 2026-02-18T08:08:58.023 INFO:teuthology.orchestra.run.trial163.stdout: 2026-02-18T08:08:58.023 INFO:teuthology.orchestra.run.trial163.stdout: ms die on old message = true 2026-02-18T08:08:58.023 INFO:teuthology.orchestra.run.trial163.stdout: ms die on bug = true 2026-02-18T08:08:58.023 INFO:teuthology.orchestra.run.trial163.stdout: 2026-02-18T08:08:58.023 INFO:teuthology.orchestra.run.trial163.stdout: mon max pg per osd = 10000 # >= luminous 2026-02-18T08:08:58.023 INFO:teuthology.orchestra.run.trial163.stdout: mon pg warn max object skew = 0 2026-02-18T08:08:58.024 INFO:teuthology.orchestra.run.trial163.stdout: 2026-02-18T08:08:58.024 INFO:teuthology.orchestra.run.trial163.stdout: # disable pg_autoscaler by default for new pools 2026-02-18T08:08:58.024 INFO:teuthology.orchestra.run.trial163.stdout: osd_pool_default_pg_autoscale_mode = off 2026-02-18T08:08:58.024 INFO:teuthology.orchestra.run.trial163.stdout: 2026-02-18T08:08:58.024 INFO:teuthology.orchestra.run.trial163.stdout: osd pool default size = 2 2026-02-18T08:08:58.024 INFO:teuthology.orchestra.run.trial163.stdout: 2026-02-18T08:08:58.024 INFO:teuthology.orchestra.run.trial163.stdout: mon osd allow primary affinity = true 2026-02-18T08:08:58.024 INFO:teuthology.orchestra.run.trial163.stdout: mon osd allow pg remap = true 2026-02-18T08:08:58.024 INFO:teuthology.orchestra.run.trial163.stdout: mon warn on legacy crush tunables = false 2026-02-18T08:08:58.024 INFO:teuthology.orchestra.run.trial163.stdout: mon warn on crush straw calc version zero = false 2026-02-18T08:08:58.024 INFO:teuthology.orchestra.run.trial163.stdout: mon warn on no sortbitwise = false 2026-02-18T08:08:58.024 INFO:teuthology.orchestra.run.trial163.stdout: mon warn on osd down out interval zero = false 2026-02-18T08:08:58.024 INFO:teuthology.orchestra.run.trial163.stdout: mon warn on too few osds = false 2026-02-18T08:08:58.025 INFO:teuthology.orchestra.run.trial163.stdout: mon_warn_on_pool_pg_num_not_power_of_two = false 2026-02-18T08:08:58.025 INFO:teuthology.orchestra.run.trial163.stdout: mon_warn_on_pool_no_redundancy = false 2026-02-18T08:08:58.025 INFO:teuthology.orchestra.run.trial163.stdout: mon_allow_pool_size_one = true 2026-02-18T08:08:58.025 INFO:teuthology.orchestra.run.trial163.stdout: 2026-02-18T08:08:58.025 INFO:teuthology.orchestra.run.trial163.stdout: osd pool default erasure code profile = plugin=isa technique=reed_sol_van k=2 m=1 crush-failure-domain=osd 2026-02-18T08:08:58.025 INFO:teuthology.orchestra.run.trial163.stdout: 2026-02-18T08:08:58.025 INFO:teuthology.orchestra.run.trial163.stdout: osd default data pool replay window = 5 2026-02-18T08:08:58.025 INFO:teuthology.orchestra.run.trial163.stdout: 2026-02-18T08:08:58.025 INFO:teuthology.orchestra.run.trial163.stdout: mon allow pool delete = true 2026-02-18T08:08:58.025 INFO:teuthology.orchestra.run.trial163.stdout: 2026-02-18T08:08:58.025 INFO:teuthology.orchestra.run.trial163.stdout: mon cluster log file level = debug 2026-02-18T08:08:58.025 INFO:teuthology.orchestra.run.trial163.stdout: debug asserts on shutdown = true 2026-02-18T08:08:58.026 INFO:teuthology.orchestra.run.trial163.stdout: mon health detail to clog = false 2026-02-18T08:08:58.026 INFO:teuthology.orchestra.run.trial163.stdout: mon host = 10.20.193.34 2026-02-18T08:08:58.026 INFO:teuthology.orchestra.run.trial163.stdout: mon client directed command retry = 5 2026-02-18T08:08:58.026 INFO:teuthology.orchestra.run.trial163.stdout: mon warn on pool no app = False 2026-02-18T08:08:58.026 INFO:teuthology.orchestra.run.trial163.stdout: ms inject socket failures = 5000 2026-02-18T08:08:58.026 INFO:teuthology.orchestra.run.trial163.stdout: fsid = 538b5201-9b84-4a73-9eef-5646b3360b9d 2026-02-18T08:08:58.026 INFO:teuthology.orchestra.run.trial163.stdout: 2026-02-18T08:08:58.026 INFO:teuthology.orchestra.run.trial163.stdout:[osd] 2026-02-18T08:08:58.026 INFO:teuthology.orchestra.run.trial163.stdout: osd journal size = 100 2026-02-18T08:08:58.026 INFO:teuthology.orchestra.run.trial163.stdout: 2026-02-18T08:08:58.026 INFO:teuthology.orchestra.run.trial163.stdout: osd scrub load threshold = 5.0 2026-02-18T08:08:58.026 INFO:teuthology.orchestra.run.trial163.stdout: osd scrub max interval = 600 2026-02-18T08:08:58.026 INFO:teuthology.orchestra.run.trial163.stdout: osd mclock profile = high_recovery_ops 2026-02-18T08:08:58.027 INFO:teuthology.orchestra.run.trial163.stdout: osd mclock skip benchmark = true 2026-02-18T08:08:58.027 INFO:teuthology.orchestra.run.trial163.stdout: 2026-02-18T08:08:58.027 INFO:teuthology.orchestra.run.trial163.stdout: osd recover clone overlap = true 2026-02-18T08:08:58.027 INFO:teuthology.orchestra.run.trial163.stdout: osd recovery max chunk = 1048576 2026-02-18T08:08:58.027 INFO:teuthology.orchestra.run.trial163.stdout: 2026-02-18T08:08:58.027 INFO:teuthology.orchestra.run.trial163.stdout: osd debug shutdown = true 2026-02-18T08:08:58.027 INFO:teuthology.orchestra.run.trial163.stdout: osd debug op order = true 2026-02-18T08:08:58.027 INFO:teuthology.orchestra.run.trial163.stdout: osd debug verify stray on activate = true 2026-02-18T08:08:58.027 INFO:teuthology.orchestra.run.trial163.stdout: osd debug trim objects = true 2026-02-18T08:08:58.027 INFO:teuthology.orchestra.run.trial163.stdout: 2026-02-18T08:08:58.027 INFO:teuthology.orchestra.run.trial163.stdout: osd open classes on start = true 2026-02-18T08:08:58.027 INFO:teuthology.orchestra.run.trial163.stdout: osd debug pg log writeout = true 2026-02-18T08:08:58.027 INFO:teuthology.orchestra.run.trial163.stdout: 2026-02-18T08:08:58.028 INFO:teuthology.orchestra.run.trial163.stdout: osd deep scrub update digest min age = 30 2026-02-18T08:08:58.028 INFO:teuthology.orchestra.run.trial163.stdout: 2026-02-18T08:08:58.028 INFO:teuthology.orchestra.run.trial163.stdout: osd map max advance = 10 2026-02-18T08:08:58.028 INFO:teuthology.orchestra.run.trial163.stdout: 2026-02-18T08:08:58.028 INFO:teuthology.orchestra.run.trial163.stdout: journal zero on create = true 2026-02-18T08:08:58.028 INFO:teuthology.orchestra.run.trial163.stdout: 2026-02-18T08:08:58.028 INFO:teuthology.orchestra.run.trial163.stdout: filestore ondisk finisher threads = 3 2026-02-18T08:08:58.028 INFO:teuthology.orchestra.run.trial163.stdout: filestore apply finisher threads = 3 2026-02-18T08:08:58.028 INFO:teuthology.orchestra.run.trial163.stdout: 2026-02-18T08:08:58.028 INFO:teuthology.orchestra.run.trial163.stdout: bdev debug aio = true 2026-02-18T08:08:58.028 INFO:teuthology.orchestra.run.trial163.stdout: osd debug misdirected ops = true 2026-02-18T08:08:58.028 INFO:teuthology.orchestra.run.trial163.stdout: bluefs allocator = hybrid 2026-02-18T08:08:58.029 INFO:teuthology.orchestra.run.trial163.stdout: bluestore allocator = hybrid 2026-02-18T08:08:58.029 INFO:teuthology.orchestra.run.trial163.stdout: bluestore block size = 96636764160 2026-02-18T08:08:58.029 INFO:teuthology.orchestra.run.trial163.stdout: bluestore fsck on mount = True 2026-02-18T08:08:58.029 INFO:teuthology.orchestra.run.trial163.stdout: debug bluefs = 1/20 2026-02-18T08:08:58.029 INFO:teuthology.orchestra.run.trial163.stdout: debug bluestore = 1/20 2026-02-18T08:08:58.029 INFO:teuthology.orchestra.run.trial163.stdout: debug ms = 1 2026-02-18T08:08:58.029 INFO:teuthology.orchestra.run.trial163.stdout: debug osd = 20 2026-02-18T08:08:58.029 INFO:teuthology.orchestra.run.trial163.stdout: debug rocksdb = 4/10 2026-02-18T08:08:58.029 INFO:teuthology.orchestra.run.trial163.stdout: mon osd backfillfull_ratio = 0.85 2026-02-18T08:08:58.029 INFO:teuthology.orchestra.run.trial163.stdout: mon osd full ratio = 0.9 2026-02-18T08:08:58.029 INFO:teuthology.orchestra.run.trial163.stdout: mon osd nearfull ratio = 0.8 2026-02-18T08:08:58.029 INFO:teuthology.orchestra.run.trial163.stdout: osd failsafe full ratio = 0.95 2026-02-18T08:08:58.029 INFO:teuthology.orchestra.run.trial163.stdout: osd objectstore = bluestore 2026-02-18T08:08:58.030 INFO:teuthology.orchestra.run.trial163.stdout: 2026-02-18T08:08:58.030 INFO:teuthology.orchestra.run.trial163.stdout:[mgr] 2026-02-18T08:08:58.030 INFO:teuthology.orchestra.run.trial163.stdout: debug ms = 1 2026-02-18T08:08:58.030 INFO:teuthology.orchestra.run.trial163.stdout: debug mgr = 20 2026-02-18T08:08:58.030 INFO:teuthology.orchestra.run.trial163.stdout: debug mon = 20 2026-02-18T08:08:58.030 INFO:teuthology.orchestra.run.trial163.stdout: debug auth = 20 2026-02-18T08:08:58.030 INFO:teuthology.orchestra.run.trial163.stdout: mon reweight min pgs per osd = 4 2026-02-18T08:08:58.030 INFO:teuthology.orchestra.run.trial163.stdout: mon reweight min bytes per osd = 10 2026-02-18T08:08:58.031 INFO:teuthology.orchestra.run.trial163.stdout: mgr/telemetry/nag = false 2026-02-18T08:08:58.031 INFO:teuthology.orchestra.run.trial163.stdout: 2026-02-18T08:08:58.031 INFO:teuthology.orchestra.run.trial163.stdout:[mon] 2026-02-18T08:08:58.031 INFO:teuthology.orchestra.run.trial163.stdout: debug ms = 1 2026-02-18T08:08:58.031 INFO:teuthology.orchestra.run.trial163.stdout: debug mon = 20 2026-02-18T08:08:58.031 INFO:teuthology.orchestra.run.trial163.stdout: debug paxos = 20 2026-02-18T08:08:58.031 INFO:teuthology.orchestra.run.trial163.stdout: debug auth = 20 2026-02-18T08:08:58.031 INFO:teuthology.orchestra.run.trial163.stdout: mon data avail warn = 5 2026-02-18T08:08:58.031 INFO:teuthology.orchestra.run.trial163.stdout: mon mgr mkfs grace = 240 2026-02-18T08:08:58.031 INFO:teuthology.orchestra.run.trial163.stdout: mon reweight min pgs per osd = 4 2026-02-18T08:08:58.031 INFO:teuthology.orchestra.run.trial163.stdout: mon osd reporter subtree level = osd 2026-02-18T08:08:58.032 INFO:teuthology.orchestra.run.trial163.stdout: mon osd prime pg temp = true 2026-02-18T08:08:58.032 INFO:teuthology.orchestra.run.trial163.stdout: mon reweight min bytes per osd = 10 2026-02-18T08:08:58.032 INFO:teuthology.orchestra.run.trial163.stdout: 2026-02-18T08:08:58.032 INFO:teuthology.orchestra.run.trial163.stdout: # rotate auth tickets quickly to exercise renewal paths 2026-02-18T08:08:58.032 INFO:teuthology.orchestra.run.trial163.stdout: auth mon ticket ttl = 660 # 11m 2026-02-18T08:08:58.032 INFO:teuthology.orchestra.run.trial163.stdout: auth service ticket ttl = 240 # 4m 2026-02-18T08:08:58.032 INFO:teuthology.orchestra.run.trial163.stdout: 2026-02-18T08:08:58.032 INFO:teuthology.orchestra.run.trial163.stdout: # don't complain about insecure global_id in the test suite 2026-02-18T08:08:58.032 INFO:teuthology.orchestra.run.trial163.stdout: mon_warn_on_insecure_global_id_reclaim = false 2026-02-18T08:08:58.032 INFO:teuthology.orchestra.run.trial163.stdout: mon_warn_on_insecure_global_id_reclaim_allowed = false 2026-02-18T08:08:58.032 INFO:teuthology.orchestra.run.trial163.stdout: 2026-02-18T08:08:58.033 INFO:teuthology.orchestra.run.trial163.stdout: # 1m isn't quite enough 2026-02-18T08:08:58.033 INFO:teuthology.orchestra.run.trial163.stdout: mon_down_mkfs_grace = 2m 2026-02-18T08:08:58.033 INFO:teuthology.orchestra.run.trial163.stdout: 2026-02-18T08:08:58.033 INFO:teuthology.orchestra.run.trial163.stdout: mon_warn_on_filestore_osds = false 2026-02-18T08:08:58.033 INFO:teuthology.orchestra.run.trial163.stdout: 2026-02-18T08:08:58.033 INFO:teuthology.orchestra.run.trial163.stdout:[client] 2026-02-18T08:08:58.033 INFO:teuthology.orchestra.run.trial163.stdout: rgw cache enabled = true 2026-02-18T08:08:58.033 INFO:teuthology.orchestra.run.trial163.stdout: rgw enable ops log = true 2026-02-18T08:08:58.033 INFO:teuthology.orchestra.run.trial163.stdout: rgw enable usage log = true 2026-02-18T08:08:58.033 INFO:teuthology.orchestra.run.trial163.stdout: log file = /var/log/ceph/$cluster-$name.$pid.log 2026-02-18T08:08:58.033 INFO:teuthology.orchestra.run.trial163.stdout: admin socket = /var/run/ceph/$cluster-$name.$pid.asok 2026-02-18T08:08:58.033 INFO:teuthology.orchestra.run.trial163.stdout: debug rbd = 20 2026-02-18T08:08:58.033 INFO:teuthology.orchestra.run.trial163.stdout: debug rbd_mirror = 15 2026-02-18T08:08:58.034 INFO:teuthology.orchestra.run.trial163.stdout: log to stderr = False 2026-02-18T08:08:58.034 INFO:teuthology.orchestra.run.trial163.stdout: rbd default features = 125 2026-02-18T08:08:58.034 INFO:teuthology.orchestra.run.trial163.stdout: rbd mirror image policy type = simple 2026-02-18T08:08:58.034 INFO:teuthology.orchestra.run.trial163.stdout:[mon.a] 2026-02-18T08:08:58.034 INFO:teuthology.orchestra.run.trial163.stdout:[client.mirror.0] 2026-02-18T08:08:58.034 INFO:teuthology.orchestra.run.trial163.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-18T08:08:58.034 INFO:teuthology.orchestra.run.trial163.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-18T08:08:58.034 INFO:teuthology.orchestra.run.trial163.stdout:[client.mirror.1] 2026-02-18T08:08:58.034 INFO:teuthology.orchestra.run.trial163.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-18T08:08:58.034 INFO:teuthology.orchestra.run.trial163.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-18T08:08:58.034 INFO:teuthology.orchestra.run.trial163.stdout:[client.mirror.2] 2026-02-18T08:08:58.034 INFO:teuthology.orchestra.run.trial163.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-18T08:08:58.034 INFO:teuthology.orchestra.run.trial163.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-18T08:08:58.035 INFO:teuthology.orchestra.run.trial163.stdout:[client.mirror.3] 2026-02-18T08:08:58.035 INFO:teuthology.orchestra.run.trial163.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-18T08:08:58.035 INFO:teuthology.orchestra.run.trial163.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-18T08:08:58.035 INFO:tasks.ceph:Creating admin key on cluster1.mon.a... 2026-02-18T08:08:58.035 DEBUG:teuthology.orchestra.run.trial034:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --gen-key --name=client.admin --cap mon 'allow *' --cap osd 'allow *' --cap mds 'allow *' --cap mgr 'allow *' /etc/ceph/cluster1.keyring 2026-02-18T08:08:58.088 INFO:tasks.ceph:Copying monmap to all nodes... 2026-02-18T08:08:58.088 DEBUG:teuthology.orchestra.run.trial034:> set -ex 2026-02-18T08:08:58.088 DEBUG:teuthology.orchestra.run.trial034:> dd if=/etc/ceph/cluster1.keyring of=/dev/stdout 2026-02-18T08:08:58.136 DEBUG:teuthology.orchestra.run.trial034:> set -ex 2026-02-18T08:08:58.137 DEBUG:teuthology.orchestra.run.trial034:> dd if=/home/ubuntu/cephtest/cluster1.monmap of=/dev/stdout 2026-02-18T08:08:58.184 INFO:tasks.ceph:Sending monmap to node ubuntu@trial034.front.sepia.ceph.com 2026-02-18T08:08:58.184 DEBUG:teuthology.orchestra.run.trial034:> set -ex 2026-02-18T08:08:58.185 DEBUG:teuthology.orchestra.run.trial034:> sudo dd of=/etc/ceph/cluster1.keyring 2026-02-18T08:08:58.185 DEBUG:teuthology.orchestra.run.trial034:> sudo chmod 0644 /etc/ceph/cluster1.keyring 2026-02-18T08:08:58.243 DEBUG:teuthology.orchestra.run.trial034:> set -ex 2026-02-18T08:08:58.243 DEBUG:teuthology.orchestra.run.trial034:> dd of=/home/ubuntu/cephtest/cluster1.monmap 2026-02-18T08:08:58.288 INFO:tasks.ceph:Sending monmap to node ubuntu@trial163.front.sepia.ceph.com 2026-02-18T08:08:58.288 DEBUG:teuthology.orchestra.run.trial163:> set -ex 2026-02-18T08:08:58.289 DEBUG:teuthology.orchestra.run.trial163:> sudo dd of=/etc/ceph/cluster1.keyring 2026-02-18T08:08:58.289 DEBUG:teuthology.orchestra.run.trial163:> sudo chmod 0644 /etc/ceph/cluster1.keyring 2026-02-18T08:08:58.303 DEBUG:teuthology.orchestra.run.trial163:> set -ex 2026-02-18T08:08:58.303 DEBUG:teuthology.orchestra.run.trial163:> dd of=/home/ubuntu/cephtest/cluster1.monmap 2026-02-18T08:08:58.353 INFO:tasks.ceph:Setting up mon nodes... 2026-02-18T08:08:58.354 INFO:tasks.ceph:Setting up mgr nodes... 2026-02-18T08:08:58.354 DEBUG:teuthology.orchestra.run.trial034:> sudo mkdir -p /var/lib/ceph/mgr/cluster1-x && sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=mgr.x /var/lib/ceph/mgr/cluster1-x/keyring 2026-02-18T08:08:58.377 INFO:teuthology.orchestra.run.trial034.stdout:creating /var/lib/ceph/mgr/cluster1-x/keyring 2026-02-18T08:08:58.379 INFO:tasks.ceph:Setting up mds nodes... 2026-02-18T08:08:58.380 INFO:tasks.ceph_client:Setting up client nodes... 2026-02-18T08:08:58.380 DEBUG:teuthology.orchestra.run.trial034:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.0 /etc/ceph/cluster1.client.0.keyring && sudo chmod 0644 /etc/ceph/cluster1.client.0.keyring 2026-02-18T08:08:58.437 INFO:teuthology.orchestra.run.trial034.stdout:creating /etc/ceph/cluster1.client.0.keyring 2026-02-18T08:08:58.444 DEBUG:teuthology.orchestra.run.trial163:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.mirror /etc/ceph/cluster1.client.mirror.keyring && sudo chmod 0644 /etc/ceph/cluster1.client.mirror.keyring 2026-02-18T08:08:58.463 INFO:teuthology.orchestra.run.trial163.stdout:creating /etc/ceph/cluster1.client.mirror.keyring 2026-02-18T08:08:58.473 DEBUG:teuthology.orchestra.run.trial163:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.mirror.0 /etc/ceph/cluster1.client.mirror.0.keyring && sudo chmod 0644 /etc/ceph/cluster1.client.mirror.0.keyring 2026-02-18T08:08:58.530 INFO:teuthology.orchestra.run.trial163.stdout:creating /etc/ceph/cluster1.client.mirror.0.keyring 2026-02-18T08:08:58.538 DEBUG:teuthology.orchestra.run.trial163:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.mirror.1 /etc/ceph/cluster1.client.mirror.1.keyring && sudo chmod 0644 /etc/ceph/cluster1.client.mirror.1.keyring 2026-02-18T08:08:58.602 INFO:teuthology.orchestra.run.trial163.stdout:creating /etc/ceph/cluster1.client.mirror.1.keyring 2026-02-18T08:08:58.609 DEBUG:teuthology.orchestra.run.trial163:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.mirror.2 /etc/ceph/cluster1.client.mirror.2.keyring && sudo chmod 0644 /etc/ceph/cluster1.client.mirror.2.keyring 2026-02-18T08:08:58.670 INFO:teuthology.orchestra.run.trial163.stdout:creating /etc/ceph/cluster1.client.mirror.2.keyring 2026-02-18T08:08:58.677 DEBUG:teuthology.orchestra.run.trial163:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.mirror.3 /etc/ceph/cluster1.client.mirror.3.keyring && sudo chmod 0644 /etc/ceph/cluster1.client.mirror.3.keyring 2026-02-18T08:08:58.734 INFO:teuthology.orchestra.run.trial163.stdout:creating /etc/ceph/cluster1.client.mirror.3.keyring 2026-02-18T08:08:58.742 DEBUG:teuthology.orchestra.run.trial163:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.mirror.4 /etc/ceph/cluster1.client.mirror.4.keyring && sudo chmod 0644 /etc/ceph/cluster1.client.mirror.4.keyring 2026-02-18T08:08:58.801 INFO:teuthology.orchestra.run.trial163.stdout:creating /etc/ceph/cluster1.client.mirror.4.keyring 2026-02-18T08:08:58.809 DEBUG:teuthology.orchestra.run.trial163:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.mirror.5 /etc/ceph/cluster1.client.mirror.5.keyring && sudo chmod 0644 /etc/ceph/cluster1.client.mirror.5.keyring 2026-02-18T08:08:58.865 INFO:teuthology.orchestra.run.trial163.stdout:creating /etc/ceph/cluster1.client.mirror.5.keyring 2026-02-18T08:08:58.873 DEBUG:teuthology.orchestra.run.trial163:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.mirror.6 /etc/ceph/cluster1.client.mirror.6.keyring && sudo chmod 0644 /etc/ceph/cluster1.client.mirror.6.keyring 2026-02-18T08:08:58.934 INFO:teuthology.orchestra.run.trial163.stdout:creating /etc/ceph/cluster1.client.mirror.6.keyring 2026-02-18T08:08:58.940 INFO:tasks.ceph:Running mkfs on osd nodes... 2026-02-18T08:08:58.940 INFO:tasks.ceph:ctx.disk_config.remote_to_roles_to_dev: {Remote(name='ubuntu@trial034.front.sepia.ceph.com'): {'cluster1.osd.0': '/dev/vg_nvme/lv_1', 'cluster1.osd.1': '/dev/vg_nvme/lv_2', 'cluster1.osd.2': '/dev/vg_nvme/lv_3'}} 2026-02-18T08:08:58.941 DEBUG:teuthology.orchestra.run.trial034:> sudo mkdir -p /var/lib/ceph/osd/cluster1-0 2026-02-18T08:08:58.950 INFO:tasks.ceph:roles_to_devs: {'cluster1.osd.0': '/dev/vg_nvme/lv_1', 'cluster1.osd.1': '/dev/vg_nvme/lv_2', 'cluster1.osd.2': '/dev/vg_nvme/lv_3'} 2026-02-18T08:08:58.950 INFO:tasks.ceph:role: cluster1.osd.0 2026-02-18T08:08:58.950 INFO:tasks.ceph:['mkfs.xfs', '-f', '-i', 'size=2048'] on /dev/vg_nvme/lv_1 on ubuntu@trial034.front.sepia.ceph.com 2026-02-18T08:08:58.950 DEBUG:teuthology.orchestra.run.trial034:> yes | sudo mkfs.xfs -f -i size=2048 /dev/vg_nvme/lv_1 2026-02-18T08:08:59.001 INFO:teuthology.orchestra.run.trial034.stdout:meta-data=/dev/vg_nvme/lv_1 isize=2048 agcount=4, agsize=45875200 blks 2026-02-18T08:08:59.002 INFO:teuthology.orchestra.run.trial034.stdout: = sectsz=4096 attr=2, projid32bit=1 2026-02-18T08:08:59.002 INFO:teuthology.orchestra.run.trial034.stdout: = crc=1 finobt=1, sparse=1, rmapbt=0 2026-02-18T08:08:59.002 INFO:teuthology.orchestra.run.trial034.stdout: = reflink=1 bigtime=0 inobtcount=0 2026-02-18T08:08:59.002 INFO:teuthology.orchestra.run.trial034.stdout:data = bsize=4096 blocks=183500800, imaxpct=25 2026-02-18T08:08:59.002 INFO:teuthology.orchestra.run.trial034.stdout: = sunit=0 swidth=0 blks 2026-02-18T08:08:59.002 INFO:teuthology.orchestra.run.trial034.stdout:naming =version 2 bsize=4096 ascii-ci=0, ftype=1 2026-02-18T08:08:59.002 INFO:teuthology.orchestra.run.trial034.stdout:log =internal log bsize=4096 blocks=89600, version=2 2026-02-18T08:08:59.002 INFO:teuthology.orchestra.run.trial034.stdout: = sectsz=4096 sunit=1 blks, lazy-count=1 2026-02-18T08:08:59.002 INFO:teuthology.orchestra.run.trial034.stdout:realtime =none extsz=4096 blocks=0, rtextents=0 2026-02-18T08:08:59.578 INFO:teuthology.orchestra.run.trial034.stdout:Discarding blocks...Done. 2026-02-18T08:08:59.580 INFO:tasks.ceph:mount /dev/vg_nvme/lv_1 on ubuntu@trial034.front.sepia.ceph.com -o noatime 2026-02-18T08:08:59.580 DEBUG:teuthology.orchestra.run.trial034:> sudo mount -t xfs -o noatime /dev/vg_nvme/lv_1 /var/lib/ceph/osd/cluster1-0 2026-02-18T08:08:59.636 DEBUG:teuthology.orchestra.run.trial034:> sudo /sbin/restorecon /var/lib/ceph/osd/cluster1-0 2026-02-18T08:08:59.686 INFO:teuthology.orchestra.run.trial034.stderr:sudo: /sbin/restorecon: command not found 2026-02-18T08:08:59.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2026-02-18T08:08:59.687 DEBUG:teuthology.orchestra.run.trial034:> sudo mkdir -p /var/lib/ceph/osd/cluster1-1 2026-02-18T08:08:59.736 INFO:tasks.ceph:roles_to_devs: {'cluster1.osd.0': '/dev/vg_nvme/lv_1', 'cluster1.osd.1': '/dev/vg_nvme/lv_2', 'cluster1.osd.2': '/dev/vg_nvme/lv_3'} 2026-02-18T08:08:59.736 INFO:tasks.ceph:role: cluster1.osd.1 2026-02-18T08:08:59.736 INFO:tasks.ceph:['mkfs.xfs', '-f', '-i', 'size=2048'] on /dev/vg_nvme/lv_2 on ubuntu@trial034.front.sepia.ceph.com 2026-02-18T08:08:59.736 DEBUG:teuthology.orchestra.run.trial034:> yes | sudo mkfs.xfs -f -i size=2048 /dev/vg_nvme/lv_2 2026-02-18T08:08:59.786 INFO:teuthology.orchestra.run.trial034.stdout:meta-data=/dev/vg_nvme/lv_2 isize=2048 agcount=4, agsize=45875200 blks 2026-02-18T08:08:59.786 INFO:teuthology.orchestra.run.trial034.stdout: = sectsz=4096 attr=2, projid32bit=1 2026-02-18T08:08:59.786 INFO:teuthology.orchestra.run.trial034.stdout: = crc=1 finobt=1, sparse=1, rmapbt=0 2026-02-18T08:08:59.786 INFO:teuthology.orchestra.run.trial034.stdout: = reflink=1 bigtime=0 inobtcount=0 2026-02-18T08:08:59.786 INFO:teuthology.orchestra.run.trial034.stdout:data = bsize=4096 blocks=183500800, imaxpct=25 2026-02-18T08:08:59.786 INFO:teuthology.orchestra.run.trial034.stdout: = sunit=0 swidth=0 blks 2026-02-18T08:08:59.786 INFO:teuthology.orchestra.run.trial034.stdout:naming =version 2 bsize=4096 ascii-ci=0, ftype=1 2026-02-18T08:08:59.786 INFO:teuthology.orchestra.run.trial034.stdout:log =internal log bsize=4096 blocks=89600, version=2 2026-02-18T08:08:59.786 INFO:teuthology.orchestra.run.trial034.stdout: = sectsz=4096 sunit=1 blks, lazy-count=1 2026-02-18T08:08:59.786 INFO:teuthology.orchestra.run.trial034.stdout:realtime =none extsz=4096 blocks=0, rtextents=0 2026-02-18T08:09:00.354 INFO:teuthology.orchestra.run.trial034.stdout:Discarding blocks...Done. 2026-02-18T08:09:00.355 INFO:tasks.ceph:mount /dev/vg_nvme/lv_2 on ubuntu@trial034.front.sepia.ceph.com -o noatime 2026-02-18T08:09:00.355 DEBUG:teuthology.orchestra.run.trial034:> sudo mount -t xfs -o noatime /dev/vg_nvme/lv_2 /var/lib/ceph/osd/cluster1-1 2026-02-18T08:09:00.410 DEBUG:teuthology.orchestra.run.trial034:> sudo /sbin/restorecon /var/lib/ceph/osd/cluster1-1 2026-02-18T08:09:00.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2026-02-18T08:09:00.458 INFO:teuthology.orchestra.run.trial034.stderr:sudo: /sbin/restorecon: command not found 2026-02-18T08:09:00.459 DEBUG:teuthology.orchestra.run.trial034:> sudo mkdir -p /var/lib/ceph/osd/cluster1-2 2026-02-18T08:09:00.508 INFO:tasks.ceph:roles_to_devs: {'cluster1.osd.0': '/dev/vg_nvme/lv_1', 'cluster1.osd.1': '/dev/vg_nvme/lv_2', 'cluster1.osd.2': '/dev/vg_nvme/lv_3'} 2026-02-18T08:09:00.508 INFO:tasks.ceph:role: cluster1.osd.2 2026-02-18T08:09:00.509 INFO:tasks.ceph:['mkfs.xfs', '-f', '-i', 'size=2048'] on /dev/vg_nvme/lv_3 on ubuntu@trial034.front.sepia.ceph.com 2026-02-18T08:09:00.509 DEBUG:teuthology.orchestra.run.trial034:> yes | sudo mkfs.xfs -f -i size=2048 /dev/vg_nvme/lv_3 2026-02-18T08:09:00.562 INFO:teuthology.orchestra.run.trial034.stdout:meta-data=/dev/vg_nvme/lv_3 isize=2048 agcount=4, agsize=45875200 blks 2026-02-18T08:09:00.562 INFO:teuthology.orchestra.run.trial034.stdout: = sectsz=4096 attr=2, projid32bit=1 2026-02-18T08:09:00.562 INFO:teuthology.orchestra.run.trial034.stdout: = crc=1 finobt=1, sparse=1, rmapbt=0 2026-02-18T08:09:00.562 INFO:teuthology.orchestra.run.trial034.stdout: = reflink=1 bigtime=0 inobtcount=0 2026-02-18T08:09:00.562 INFO:teuthology.orchestra.run.trial034.stdout:data = bsize=4096 blocks=183500800, imaxpct=25 2026-02-18T08:09:00.562 INFO:teuthology.orchestra.run.trial034.stdout: = sunit=0 swidth=0 blks 2026-02-18T08:09:00.562 INFO:teuthology.orchestra.run.trial034.stdout:naming =version 2 bsize=4096 ascii-ci=0, ftype=1 2026-02-18T08:09:00.562 INFO:teuthology.orchestra.run.trial034.stdout:log =internal log bsize=4096 blocks=89600, version=2 2026-02-18T08:09:00.563 INFO:teuthology.orchestra.run.trial034.stdout: = sectsz=4096 sunit=1 blks, lazy-count=1 2026-02-18T08:09:00.563 INFO:teuthology.orchestra.run.trial034.stdout:realtime =none extsz=4096 blocks=0, rtextents=0 2026-02-18T08:09:01.133 INFO:teuthology.orchestra.run.trial034.stdout:Discarding blocks...Done. 2026-02-18T08:09:01.135 INFO:tasks.ceph:mount /dev/vg_nvme/lv_3 on ubuntu@trial034.front.sepia.ceph.com -o noatime 2026-02-18T08:09:01.135 DEBUG:teuthology.orchestra.run.trial034:> sudo mount -t xfs -o noatime /dev/vg_nvme/lv_3 /var/lib/ceph/osd/cluster1-2 2026-02-18T08:09:01.189 DEBUG:teuthology.orchestra.run.trial034:> sudo /sbin/restorecon /var/lib/ceph/osd/cluster1-2 2026-02-18T08:09:01.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2026-02-18T08:09:01.239 INFO:teuthology.orchestra.run.trial034.stderr:sudo: /sbin/restorecon: command not found 2026-02-18T08:09:01.239 DEBUG:teuthology.orchestra.run.trial034:> sudo MALLOC_CHECK_=3 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-osd --no-mon-config --cluster cluster1 --mkfs --mkkey -i 0 --monmap /home/ubuntu/cephtest/cluster1.monmap 2026-02-18T08:09:01.302 INFO:teuthology.orchestra.run.trial034.stderr:2026-02-18T08:09:01.299+0000 7f5d2ee9ba40 -1 auth: error reading file: /var/lib/ceph/osd/cluster1-0/keyring: can't open /var/lib/ceph/osd/cluster1-0/keyring: (2) No such file or directory 2026-02-18T08:09:01.302 INFO:teuthology.orchestra.run.trial034.stderr:2026-02-18T08:09:01.299+0000 7f5d2ee9ba40 -1 created new key in keyring /var/lib/ceph/osd/cluster1-0/keyring 2026-02-18T08:09:01.303 INFO:teuthology.orchestra.run.trial034.stderr:2026-02-18T08:09:01.299+0000 7f5d2ee9ba40 -1 bdev(0x560deb16d800 /var/lib/ceph/osd/cluster1-0/block) open stat got: (1) Operation not permitted 2026-02-18T08:09:01.303 INFO:teuthology.orchestra.run.trial034.stderr:2026-02-18T08:09:01.299+0000 7f5d2ee9ba40 -1 bluestore(/var/lib/ceph/osd/cluster1-0) _read_fsid unparsable uuid 2026-02-18T08:09:02.206 DEBUG:teuthology.orchestra.run.trial034:> sudo chown -R ceph:ceph /var/lib/ceph/osd/cluster1-0 2026-02-18T08:09:02.257 DEBUG:teuthology.orchestra.run.trial034:> sudo MALLOC_CHECK_=3 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-osd --no-mon-config --cluster cluster1 --mkfs --mkkey -i 1 --monmap /home/ubuntu/cephtest/cluster1.monmap 2026-02-18T08:09:02.321 INFO:teuthology.orchestra.run.trial034.stderr:2026-02-18T08:09:02.319+0000 7f99c2205a40 -1 auth: error reading file: /var/lib/ceph/osd/cluster1-1/keyring: can't open /var/lib/ceph/osd/cluster1-1/keyring: (2) No such file or directory 2026-02-18T08:09:02.321 INFO:teuthology.orchestra.run.trial034.stderr:2026-02-18T08:09:02.319+0000 7f99c2205a40 -1 created new key in keyring /var/lib/ceph/osd/cluster1-1/keyring 2026-02-18T08:09:02.321 INFO:teuthology.orchestra.run.trial034.stderr:2026-02-18T08:09:02.319+0000 7f99c2205a40 -1 bdev(0x5628a67b5800 /var/lib/ceph/osd/cluster1-1/block) open stat got: (1) Operation not permitted 2026-02-18T08:09:02.321 INFO:teuthology.orchestra.run.trial034.stderr:2026-02-18T08:09:02.319+0000 7f99c2205a40 -1 bluestore(/var/lib/ceph/osd/cluster1-1) _read_fsid unparsable uuid 2026-02-18T08:09:03.181 DEBUG:teuthology.orchestra.run.trial034:> sudo chown -R ceph:ceph /var/lib/ceph/osd/cluster1-1 2026-02-18T08:09:03.233 DEBUG:teuthology.orchestra.run.trial034:> sudo MALLOC_CHECK_=3 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-osd --no-mon-config --cluster cluster1 --mkfs --mkkey -i 2 --monmap /home/ubuntu/cephtest/cluster1.monmap 2026-02-18T08:09:03.297 INFO:teuthology.orchestra.run.trial034.stderr:2026-02-18T08:09:03.295+0000 7f31a105da40 -1 auth: error reading file: /var/lib/ceph/osd/cluster1-2/keyring: can't open /var/lib/ceph/osd/cluster1-2/keyring: (2) No such file or directory 2026-02-18T08:09:03.297 INFO:teuthology.orchestra.run.trial034.stderr:2026-02-18T08:09:03.295+0000 7f31a105da40 -1 created new key in keyring /var/lib/ceph/osd/cluster1-2/keyring 2026-02-18T08:09:03.297 INFO:teuthology.orchestra.run.trial034.stderr:2026-02-18T08:09:03.295+0000 7f31a105da40 -1 bdev(0x562659ef1800 /var/lib/ceph/osd/cluster1-2/block) open stat got: (1) Operation not permitted 2026-02-18T08:09:03.298 INFO:teuthology.orchestra.run.trial034.stderr:2026-02-18T08:09:03.295+0000 7f31a105da40 -1 bluestore(/var/lib/ceph/osd/cluster1-2) _read_fsid unparsable uuid 2026-02-18T08:09:04.236 DEBUG:teuthology.orchestra.run.trial034:> sudo chown -R ceph:ceph /var/lib/ceph/osd/cluster1-2 2026-02-18T08:09:04.292 INFO:tasks.ceph:Reading keys from all nodes... 2026-02-18T08:09:04.292 DEBUG:teuthology.orchestra.run.trial034:> set -ex 2026-02-18T08:09:04.292 DEBUG:teuthology.orchestra.run.trial034:> sudo dd if=/var/lib/ceph/mgr/cluster1-x/keyring of=/dev/stdout 2026-02-18T08:09:04.344 DEBUG:teuthology.orchestra.run.trial034:> set -ex 2026-02-18T08:09:04.344 DEBUG:teuthology.orchestra.run.trial034:> sudo dd if=/var/lib/ceph/osd/cluster1-0/keyring of=/dev/stdout 2026-02-18T08:09:04.395 DEBUG:teuthology.orchestra.run.trial034:> set -ex 2026-02-18T08:09:04.395 DEBUG:teuthology.orchestra.run.trial034:> sudo dd if=/var/lib/ceph/osd/cluster1-1/keyring of=/dev/stdout 2026-02-18T08:09:04.443 DEBUG:teuthology.orchestra.run.trial034:> set -ex 2026-02-18T08:09:04.443 DEBUG:teuthology.orchestra.run.trial034:> sudo dd if=/var/lib/ceph/osd/cluster1-2/keyring of=/dev/stdout 2026-02-18T08:09:04.491 DEBUG:teuthology.orchestra.run.trial034:> set -ex 2026-02-18T08:09:04.491 DEBUG:teuthology.orchestra.run.trial034:> dd if=/etc/ceph/cluster1.client.0.keyring of=/dev/stdout 2026-02-18T08:09:04.535 DEBUG:teuthology.orchestra.run.trial163:> set -ex 2026-02-18T08:09:04.535 DEBUG:teuthology.orchestra.run.trial163:> dd if=/etc/ceph/cluster1.client.mirror.keyring of=/dev/stdout 2026-02-18T08:09:04.539 DEBUG:teuthology.orchestra.run.trial163:> set -ex 2026-02-18T08:09:04.539 DEBUG:teuthology.orchestra.run.trial163:> dd if=/etc/ceph/cluster1.client.mirror.0.keyring of=/dev/stdout 2026-02-18T08:09:04.585 DEBUG:teuthology.orchestra.run.trial163:> set -ex 2026-02-18T08:09:04.585 DEBUG:teuthology.orchestra.run.trial163:> dd if=/etc/ceph/cluster1.client.mirror.1.keyring of=/dev/stdout 2026-02-18T08:09:04.637 DEBUG:teuthology.orchestra.run.trial163:> set -ex 2026-02-18T08:09:04.637 DEBUG:teuthology.orchestra.run.trial163:> dd if=/etc/ceph/cluster1.client.mirror.2.keyring of=/dev/stdout 2026-02-18T08:09:04.684 DEBUG:teuthology.orchestra.run.trial163:> set -ex 2026-02-18T08:09:04.684 DEBUG:teuthology.orchestra.run.trial163:> dd if=/etc/ceph/cluster1.client.mirror.3.keyring of=/dev/stdout 2026-02-18T08:09:04.728 DEBUG:teuthology.orchestra.run.trial163:> set -ex 2026-02-18T08:09:04.728 DEBUG:teuthology.orchestra.run.trial163:> dd if=/etc/ceph/cluster1.client.mirror.4.keyring of=/dev/stdout 2026-02-18T08:09:04.773 DEBUG:teuthology.orchestra.run.trial163:> set -ex 2026-02-18T08:09:04.773 DEBUG:teuthology.orchestra.run.trial163:> dd if=/etc/ceph/cluster1.client.mirror.5.keyring of=/dev/stdout 2026-02-18T08:09:04.816 DEBUG:teuthology.orchestra.run.trial163:> set -ex 2026-02-18T08:09:04.816 DEBUG:teuthology.orchestra.run.trial163:> dd if=/etc/ceph/cluster1.client.mirror.6.keyring of=/dev/stdout 2026-02-18T08:09:04.860 INFO:tasks.ceph:Adding keys to all mons... 2026-02-18T08:09:04.860 DEBUG:teuthology.orchestra.run.trial034:> sudo tee -a /etc/ceph/cluster1.keyring 2026-02-18T08:09:04.868 INFO:teuthology.orchestra.run.trial034.stdout:[mgr.x] 2026-02-18T08:09:04.868 INFO:teuthology.orchestra.run.trial034.stdout: key = AQCac5VpljZ2FhAA9K1kCopGKOq1tv9c26pxZg== 2026-02-18T08:09:04.869 INFO:teuthology.orchestra.run.trial034.stdout:[osd.0] 2026-02-18T08:09:04.869 INFO:teuthology.orchestra.run.trial034.stdout: key = AQCdc5Vp4Q/2ERAAvfLxFwbm6Vu07hjt7eCBkg== 2026-02-18T08:09:04.869 INFO:teuthology.orchestra.run.trial034.stdout:[osd.1] 2026-02-18T08:09:04.869 INFO:teuthology.orchestra.run.trial034.stdout: key = AQCec5Vp2IUaExAAR3pzOqXZ8Fk9/7H57yNW6Q== 2026-02-18T08:09:04.869 INFO:teuthology.orchestra.run.trial034.stdout:[osd.2] 2026-02-18T08:09:04.869 INFO:teuthology.orchestra.run.trial034.stdout: key = AQCfc5VpIT6pERAAY/1n/q2yc+Rcw4inNdAX4A== 2026-02-18T08:09:04.869 INFO:teuthology.orchestra.run.trial034.stdout:[client.0] 2026-02-18T08:09:04.869 INFO:teuthology.orchestra.run.trial034.stdout: key = AQCac5Vp4hoFGhAAq0jjOEzRUmbDAyerrTKeFA== 2026-02-18T08:09:04.869 INFO:teuthology.orchestra.run.trial034.stdout:[client.mirror] 2026-02-18T08:09:04.869 INFO:teuthology.orchestra.run.trial034.stdout: key = AQCac5VptF2JGxAA8fwGPHy3Vimem1mhfATLqQ== 2026-02-18T08:09:04.869 INFO:teuthology.orchestra.run.trial034.stdout:[client.mirror.0] 2026-02-18T08:09:04.870 INFO:teuthology.orchestra.run.trial034.stdout: key = AQCac5VpY4WIHxAAb8lF8oj1LHV4QwUL5CjlEA== 2026-02-18T08:09:04.870 INFO:teuthology.orchestra.run.trial034.stdout:[client.mirror.1] 2026-02-18T08:09:04.870 INFO:teuthology.orchestra.run.trial034.stdout: key = AQCac5VpXW/OIxAA/gkf9pv7/1PZCRjYBasG5g== 2026-02-18T08:09:04.870 INFO:teuthology.orchestra.run.trial034.stdout:[client.mirror.2] 2026-02-18T08:09:04.870 INFO:teuthology.orchestra.run.trial034.stdout: key = AQCac5VpxRvaJxAAqziyv6zUs3W9g1C4i6ZfSQ== 2026-02-18T08:09:04.870 INFO:teuthology.orchestra.run.trial034.stdout:[client.mirror.3] 2026-02-18T08:09:04.870 INFO:teuthology.orchestra.run.trial034.stdout: key = AQCac5Vp+RCqKxAAtMZG+CvONyOSXdhiObn7JA== 2026-02-18T08:09:04.870 INFO:teuthology.orchestra.run.trial034.stdout:[client.mirror.4] 2026-02-18T08:09:04.870 INFO:teuthology.orchestra.run.trial034.stdout: key = AQCac5Vp6A2qLxAAWE5Z15U2bjEUO/2SgaKudg== 2026-02-18T08:09:04.870 INFO:teuthology.orchestra.run.trial034.stdout:[client.mirror.5] 2026-02-18T08:09:04.870 INFO:teuthology.orchestra.run.trial034.stdout: key = AQCac5Vppo98MxAAQAl6bdR0xTMbe8cXjZjKCw== 2026-02-18T08:09:04.870 INFO:teuthology.orchestra.run.trial034.stdout:[client.mirror.6] 2026-02-18T08:09:04.871 INFO:teuthology.orchestra.run.trial034.stdout: key = AQCac5VpBguQNxAARnK6QzxaQVJIVSKxb+YRKA== 2026-02-18T08:09:04.872 DEBUG:teuthology.orchestra.run.trial034:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster1.keyring --name=mgr.x --cap mon 'allow profile mgr' --cap osd 'allow *' --cap mds 'allow *' 2026-02-18T08:09:04.934 DEBUG:teuthology.orchestra.run.trial034:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster1.keyring --name=osd.0 --cap mon 'allow profile osd' --cap mgr 'allow profile osd' --cap osd 'allow *' 2026-02-18T08:09:04.997 DEBUG:teuthology.orchestra.run.trial034:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster1.keyring --name=osd.1 --cap mon 'allow profile osd' --cap mgr 'allow profile osd' --cap osd 'allow *' 2026-02-18T08:09:05.061 DEBUG:teuthology.orchestra.run.trial034:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster1.keyring --name=osd.2 --cap mon 'allow profile osd' --cap mgr 'allow profile osd' --cap osd 'allow *' 2026-02-18T08:09:05.125 DEBUG:teuthology.orchestra.run.trial034:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster1.keyring --name=client.0 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2026-02-18T08:09:05.190 DEBUG:teuthology.orchestra.run.trial034:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster1.keyring --name=client.mirror --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2026-02-18T08:09:05.254 DEBUG:teuthology.orchestra.run.trial034:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster1.keyring --name=client.mirror.0 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2026-02-18T08:09:05.322 DEBUG:teuthology.orchestra.run.trial034:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster1.keyring --name=client.mirror.1 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2026-02-18T08:09:05.386 DEBUG:teuthology.orchestra.run.trial034:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster1.keyring --name=client.mirror.2 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2026-02-18T08:09:05.450 DEBUG:teuthology.orchestra.run.trial034:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster1.keyring --name=client.mirror.3 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2026-02-18T08:09:05.514 DEBUG:teuthology.orchestra.run.trial034:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster1.keyring --name=client.mirror.4 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2026-02-18T08:09:05.578 DEBUG:teuthology.orchestra.run.trial034:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster1.keyring --name=client.mirror.5 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2026-02-18T08:09:05.641 DEBUG:teuthology.orchestra.run.trial034:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster1.keyring --name=client.mirror.6 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2026-02-18T08:09:05.706 INFO:tasks.ceph:Running mkfs on mon nodes... 2026-02-18T08:09:05.706 DEBUG:teuthology.orchestra.run.trial034:> sudo mkdir -p /var/lib/ceph/mon/cluster1-a 2026-02-18T08:09:05.756 DEBUG:teuthology.orchestra.run.trial034:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-mon --cluster cluster1 --mkfs -i a --monmap /home/ubuntu/cephtest/cluster1.monmap --keyring /etc/ceph/cluster1.keyring 2026-02-18T08:09:05.822 DEBUG:teuthology.orchestra.run.trial034:> sudo chown -R ceph:ceph /var/lib/ceph/mon/cluster1-a 2026-02-18T08:09:05.872 DEBUG:teuthology.orchestra.run.trial034:> rm -- /home/ubuntu/cephtest/cluster1.monmap 2026-02-18T08:09:05.920 INFO:tasks.ceph:Starting mon daemons in cluster cluster1... 2026-02-18T08:09:05.920 INFO:tasks.ceph.cluster1.mon.a:Restarting daemon 2026-02-18T08:09:05.920 DEBUG:teuthology.orchestra.run.trial034:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-mon -f --cluster cluster1 -i a 2026-02-18T08:09:05.961 INFO:tasks.ceph.cluster1.mon.a:Started 2026-02-18T08:09:05.962 INFO:tasks.ceph:Starting mgr daemons in cluster cluster1... 2026-02-18T08:09:05.962 INFO:tasks.ceph.cluster1.mgr.x:Restarting daemon 2026-02-18T08:09:05.962 DEBUG:teuthology.orchestra.run.trial034:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-mgr -f --cluster cluster1 -i x 2026-02-18T08:09:05.964 INFO:tasks.ceph.cluster1.mgr.x:Started 2026-02-18T08:09:05.964 DEBUG:tasks.ceph:set 0 configs 2026-02-18T08:09:05.965 DEBUG:teuthology.orchestra.run.trial034:> sudo ceph --cluster cluster1 config dump 2026-02-18T08:09:06.091 INFO:teuthology.orchestra.run.trial034.stdout:WHO MASK LEVEL OPTION VALUE RO 2026-02-18T08:09:06.103 INFO:tasks.ceph:Setting crush tunables to default 2026-02-18T08:09:06.103 DEBUG:teuthology.orchestra.run.trial034:> sudo ceph --cluster cluster1 osd crush tunables default 2026-02-18T08:09:06.199 INFO:teuthology.orchestra.run.trial034.stderr:adjusted tunables profile to default 2026-02-18T08:09:06.210 INFO:tasks.ceph:check_enable_crimson: False 2026-02-18T08:09:06.210 INFO:tasks.ceph:Starting osd daemons in cluster cluster1... 2026-02-18T08:09:06.211 DEBUG:teuthology.orchestra.run.trial034:> set -ex 2026-02-18T08:09:06.211 DEBUG:teuthology.orchestra.run.trial034:> sudo dd if=/var/lib/ceph/osd/cluster1-0/fsid of=/dev/stdout 2026-02-18T08:09:06.222 DEBUG:teuthology.orchestra.run.trial034:> set -ex 2026-02-18T08:09:06.222 DEBUG:teuthology.orchestra.run.trial034:> sudo dd if=/var/lib/ceph/osd/cluster1-1/fsid of=/dev/stdout 2026-02-18T08:09:06.271 DEBUG:teuthology.orchestra.run.trial034:> set -ex 2026-02-18T08:09:06.272 DEBUG:teuthology.orchestra.run.trial034:> sudo dd if=/var/lib/ceph/osd/cluster1-2/fsid of=/dev/stdout 2026-02-18T08:09:06.319 DEBUG:teuthology.orchestra.run.trial034:> sudo ceph --cluster cluster1 osd new 6ca6cdc0-6879-42a3-a4d6-eee71da49395 0 2026-02-18T08:09:06.458 INFO:teuthology.orchestra.run.trial034.stdout:0 2026-02-18T08:09:06.469 DEBUG:teuthology.orchestra.run.trial034:> sudo ceph --cluster cluster1 osd new eb801757-b69b-4914-a101-65b314371e7c 1 2026-02-18T08:09:06.566 INFO:teuthology.orchestra.run.trial034.stdout:1 2026-02-18T08:09:06.575 DEBUG:teuthology.orchestra.run.trial034:> sudo ceph --cluster cluster1 osd new 8cff614b-3827-4a60-8c9f-a83bed4c4d45 2 2026-02-18T08:09:06.673 INFO:teuthology.orchestra.run.trial034.stdout:2 2026-02-18T08:09:06.683 INFO:tasks.ceph.cluster1.osd.0:Restarting daemon 2026-02-18T08:09:06.683 DEBUG:teuthology.orchestra.run.trial034:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-osd -f --cluster cluster1 -i 0 2026-02-18T08:09:06.685 INFO:tasks.ceph.cluster1.osd.0:Started 2026-02-18T08:09:06.685 INFO:tasks.ceph.cluster1.osd.1:Restarting daemon 2026-02-18T08:09:06.686 DEBUG:teuthology.orchestra.run.trial034:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-osd -f --cluster cluster1 -i 1 2026-02-18T08:09:06.687 INFO:tasks.ceph.cluster1.osd.1:Started 2026-02-18T08:09:06.688 INFO:tasks.ceph.cluster1.osd.2:Restarting daemon 2026-02-18T08:09:06.688 DEBUG:teuthology.orchestra.run.trial034:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-osd -f --cluster cluster1 -i 2 2026-02-18T08:09:06.690 INFO:tasks.ceph.cluster1.osd.2:Started 2026-02-18T08:09:06.690 DEBUG:teuthology.orchestra.run.trial034:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 osd dump --format=json 2026-02-18T08:09:06.789 INFO:teuthology.orchestra.run.trial034.stdout: 2026-02-18T08:09:06.789 INFO:teuthology.orchestra.run.trial034.stdout:{"epoch":5,"fsid":"538b5201-9b84-4a73-9eef-5646b3360b9d","created":"2026-02-18T08:09:06.049413+0000","modified":"2026-02-18T08:09:06.672265+0000","last_up_change":"0.000000","last_in_change":"2026-02-18T08:09:06.672265+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":2,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":0,"max_osd":3,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"tentacle","allow_crimson":false,"pools":[],"osds":[{"osd":0,"uuid":"6ca6cdc0-6879-42a3-a4d6-eee71da49395","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":1,"uuid":"eb801757-b69b-4914-a101-65b314371e7c","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":2,"uuid":"8cff614b-3827-4a60-8c9f-a83bed4c4d45","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"isa","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2026-02-18T08:09:06.800 INFO:tasks.ceph.ceph_manager.cluster1:[] 2026-02-18T08:09:06.800 INFO:tasks.ceph:Waiting for OSDs to come up 2026-02-18T08:09:07.102 DEBUG:teuthology.orchestra.run.trial034:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph --cluster cluster1 osd dump --format=json 2026-02-18T08:09:07.241 INFO:tasks.ceph.cluster1.osd.1.trial034.stderr:2026-02-18T08:09:07.238+0000 7f20e83caa40 -1 Falling back to public interface 2026-02-18T08:09:07.261 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:09:07.258+0000 7fb00dac0a40 -1 Falling back to public interface 2026-02-18T08:09:07.269 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:09:07.266+0000 7fb28f957a40 -1 Falling back to public interface 2026-02-18T08:09:07.360 INFO:teuthology.misc.health.trial034.stderr:2026-02-18T08:09:07.358+0000 7fef55a43640 0 --2- 10.20.193.34:0/4281747710 >> v2:10.20.193.34:3300/0 conn(0x7fef5014bbd0 0x7fef500a8c40 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request get_initial_auth_request returned -2 2026-02-18T08:09:07.395 INFO:teuthology.misc.health.trial034.stdout: 2026-02-18T08:09:07.395 INFO:teuthology.misc.health.trial034.stdout:{"epoch":5,"fsid":"538b5201-9b84-4a73-9eef-5646b3360b9d","created":"2026-02-18T08:09:06.049413+0000","modified":"2026-02-18T08:09:06.672265+0000","last_up_change":"0.000000","last_in_change":"2026-02-18T08:09:06.672265+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":2,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":0,"max_osd":3,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"tentacle","allow_crimson":false,"pools":[],"osds":[{"osd":0,"uuid":"6ca6cdc0-6879-42a3-a4d6-eee71da49395","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":1,"uuid":"eb801757-b69b-4914-a101-65b314371e7c","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":2,"uuid":"8cff614b-3827-4a60-8c9f-a83bed4c4d45","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"isa","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2026-02-18T08:09:07.403 DEBUG:teuthology.misc:0 of 3 OSDs are up 2026-02-18T08:09:07.542 INFO:tasks.ceph.cluster1.mgr.x.trial034.stderr:ERROR:root:Module 'jmespath' is not installed. 2026-02-18T08:09:07.596 INFO:tasks.ceph.cluster1.mgr.x.trial034.stderr:ERROR:dashboard.controllers.nvmeof:Failed to import NVMeoFClient and related components: cannot import name 'NVMeoFClient' from 'dashboard.services.nvmeof_client' (/usr/share/ceph/mgr/dashboard/services/nvmeof_client.py) 2026-02-18T08:09:07.602 INFO:tasks.ceph.cluster1.mgr.x.trial034.stderr:ERROR:sso:Module 'jmespath' is not installed. 2026-02-18T08:09:07.649 INFO:tasks.ceph.cluster1.mgr.x.trial034.stderr:ERROR:root:Module 'xmltodict' is not installed. 2026-02-18T08:09:08.081 INFO:tasks.ceph.cluster1.mgr.x.trial034.stderr:/usr/lib/python3/dist-packages/scipy/__init__.py:67: UserWarning: NumPy was imported from a Python sub-interpreter but NumPy does not properly support sub-interpreters. This will likely work for most users but might cause hard to track down issues or subtle bugs. A common user of the rare sub-interpreter feature is wsgi which also allows single-interpreter mode. 2026-02-18T08:09:08.081 INFO:tasks.ceph.cluster1.mgr.x.trial034.stderr:Improvements in the case of bugs are welcome, but is not on the NumPy roadmap, and full support may require significant effort to achieve. 2026-02-18T08:09:08.081 INFO:tasks.ceph.cluster1.mgr.x.trial034.stderr: from numpy import show_config as show_numpy_config 2026-02-18T08:09:08.661 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:09:08.658+0000 7fb00dac0a40 -1 osd.0 0 log_to_monitors true 2026-02-18T08:09:08.683 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:09:08.678+0000 7fb28f957a40 -1 osd.2 0 log_to_monitors true 2026-02-18T08:09:08.716 INFO:tasks.ceph.cluster1.osd.1.trial034.stderr:2026-02-18T08:09:08.710+0000 7f20e83caa40 -1 osd.1 0 log_to_monitors true 2026-02-18T08:09:09.153 INFO:tasks.ceph.cluster1.mgr.x.trial034.stderr:2026-02-18T08:09:09.150+0000 7f7ad3cb9640 -1 mgr.server handle_report got status from non-daemon mon.a 2026-02-18T08:09:10.213 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:09:10.210+0000 7fb009a69640 -1 osd.0 0 waiting for initial osdmap 2026-02-18T08:09:10.213 INFO:tasks.ceph.cluster1.osd.1.trial034.stderr:2026-02-18T08:09:10.210+0000 7f20e4373640 -1 osd.1 0 waiting for initial osdmap 2026-02-18T08:09:10.213 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:09:10.210+0000 7fb28b900640 -1 osd.2 0 waiting for initial osdmap 2026-02-18T08:09:10.215 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:09:10.210+0000 7fb001070640 -1 osd.0 7 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2026-02-18T08:09:10.216 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:09:10.210+0000 7fb282f07640 -1 osd.2 7 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2026-02-18T08:09:10.216 INFO:tasks.ceph.cluster1.osd.1.trial034.stderr:2026-02-18T08:09:10.210+0000 7f20db97a640 -1 osd.1 7 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2026-02-18T08:09:13.706 DEBUG:teuthology.orchestra.run.trial034:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph --cluster cluster1 osd dump --format=json 2026-02-18T08:09:13.855 INFO:teuthology.misc.health.trial034.stdout: 2026-02-18T08:09:13.855 INFO:teuthology.misc.health.trial034.stdout:{"epoch":10,"fsid":"538b5201-9b84-4a73-9eef-5646b3360b9d","created":"2026-02-18T08:09:06.049413+0000","modified":"2026-02-18T08:09:13.192788+0000","last_up_change":"2026-02-18T08:09:11.186536+0000","last_in_change":"2026-02-18T08:09:06.672265+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":4,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":3,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"tentacle","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2026-02-18T08:09:12.184009+0000","flags":32769,"flags_names":"hashpspool,creating","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"10","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"nonprimary_shards":"{}","options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":2.9900000095367432,"score_stable":2.9900000095367432,"optimal_score":0.67000001668930054,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"6ca6cdc0-6879-42a3-a4d6-eee71da49395","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.34:6808","nonce":3130378423},{"type":"v1","addr":"10.20.193.34:6809","nonce":3130378423}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.34:6810","nonce":3130378423},{"type":"v1","addr":"10.20.193.34:6811","nonce":3130378423}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.34:6814","nonce":3130378423},{"type":"v1","addr":"10.20.193.34:6815","nonce":3130378423}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.34:6812","nonce":3130378423},{"type":"v1","addr":"10.20.193.34:6813","nonce":3130378423}]},"public_addr":"10.20.193.34:6809/3130378423","cluster_addr":"10.20.193.34:6811/3130378423","heartbeat_back_addr":"10.20.193.34:6815/3130378423","heartbeat_front_addr":"10.20.193.34:6813/3130378423","state":["exists","up"]},{"osd":1,"uuid":"eb801757-b69b-4914-a101-65b314371e7c","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":9,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.34:6800","nonce":2597388232},{"type":"v1","addr":"10.20.193.34:6801","nonce":2597388232}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.34:6802","nonce":2597388232},{"type":"v1","addr":"10.20.193.34:6803","nonce":2597388232}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.34:6806","nonce":2597388232},{"type":"v1","addr":"10.20.193.34:6807","nonce":2597388232}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.34:6804","nonce":2597388232},{"type":"v1","addr":"10.20.193.34:6805","nonce":2597388232}]},"public_addr":"10.20.193.34:6801/2597388232","cluster_addr":"10.20.193.34:6803/2597388232","heartbeat_back_addr":"10.20.193.34:6807/2597388232","heartbeat_front_addr":"10.20.193.34:6805/2597388232","state":["exists","up"]},{"osd":2,"uuid":"8cff614b-3827-4a60-8c9f-a83bed4c4d45","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.34:6816","nonce":1006015704},{"type":"v1","addr":"10.20.193.34:6817","nonce":1006015704}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.34:6818","nonce":1006015704},{"type":"v1","addr":"10.20.193.34:6819","nonce":1006015704}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.34:6822","nonce":1006015704},{"type":"v1","addr":"10.20.193.34:6823","nonce":1006015704}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.34:6820","nonce":1006015704},{"type":"v1","addr":"10.20.193.34:6821","nonce":1006015704}]},"public_addr":"10.20.193.34:6817/1006015704","cluster_addr":"10.20.193.34:6819/1006015704","heartbeat_back_addr":"10.20.193.34:6823/1006015704","heartbeat_front_addr":"10.20.193.34:6821/1006015704","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4544132024016699391,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4544132024016699391,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4544132024016699391,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"isa","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2026-02-18T08:09:13.864 DEBUG:teuthology.misc:3 of 3 OSDs are up 2026-02-18T08:09:13.864 INFO:tasks.ceph:Creating RBD pool 2026-02-18T08:09:13.865 DEBUG:teuthology.orchestra.run.trial034:> sudo ceph --cluster cluster1 osd pool create rbd 8 2026-02-18T08:09:14.194 INFO:teuthology.orchestra.run.trial034.stderr:pool 'rbd' created 2026-02-18T08:09:14.207 DEBUG:teuthology.orchestra.run.trial034:> rbd --cluster cluster1 pool init rbd 2026-02-18T08:09:17.212 INFO:tasks.ceph:Starting mds daemons in cluster cluster1... 2026-02-18T08:09:17.213 DEBUG:teuthology.orchestra.run.trial034:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 config log 1 --format=json 2026-02-18T08:09:17.213 INFO:tasks.daemonwatchdog.daemon_watchdog:watchdog starting 2026-02-18T08:09:17.373 INFO:teuthology.orchestra.run.trial034.stdout: 2026-02-18T08:09:17.383 INFO:teuthology.orchestra.run.trial034.stdout:[{"version":1,"timestamp":"0.000000","name":"","changes":[]}] 2026-02-18T08:09:17.383 INFO:tasks.ceph_manager:config epoch is 1 2026-02-18T08:09:17.384 INFO:tasks.ceph:Waiting until cluster1 daemons up and pgs clean... 2026-02-18T08:09:17.384 INFO:tasks.ceph.ceph_manager.cluster1:waiting for mgr available 2026-02-18T08:09:17.384 DEBUG:teuthology.orchestra.run.trial034:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 mgr dump --format=json 2026-02-18T08:09:17.557 INFO:teuthology.orchestra.run.trial034.stdout: 2026-02-18T08:09:17.566 INFO:teuthology.orchestra.run.trial034.stdout:{"epoch":5,"flags":0,"active_gid":4104,"active_name":"x","active_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.34:6824","nonce":2746112233},{"type":"v1","addr":"10.20.193.34:6825","nonce":2746112233}]},"active_addr":"10.20.193.34:6825/2746112233","active_change":"2026-02-18T08:09:08.144784+0000","active_mgr_features":4544132024016699391,"available":true,"standbys":[],"modules":["iostat","nfs"],"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to, use commas to separate multiple","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2359","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","read","upmap","upmap-read"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"update_pg_upmap_activity":{"name":"update_pg_upmap_activity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Updates pg_upmap activity stats to be used in `balancer status detail`","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cephadm_log_destination":{"name":"cephadm_log_destination","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":["file","file,syslog","syslog"],"desc":"Destination for cephadm command's persistent logging","long_desc":"","tags":[],"see_also":[]},"certificate_automated_rotation_enabled":{"name":"certificate_automated_rotation_enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"This flag controls whether cephadm automatically rotates certificates upon expiration.","long_desc":"","tags":[],"see_also":[]},"certificate_check_debug_mode":{"name":"certificate_check_debug_mode","type":"bool","level":"dev","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"FOR TESTING ONLY: This flag forces the certificate check instead of waiting for certificate_check_period.","long_desc":"","tags":[],"see_also":[]},"certificate_check_period":{"name":"certificate_check_period","type":"int","level":"advanced","flags":0,"default_value":"1","min":"0","max":"30","enum_allowed":[],"desc":"Specifies how often (in days) the certificate should be checked for validity.","long_desc":"","tags":[],"see_also":[]},"certificate_duration_days":{"name":"certificate_duration_days","type":"int","level":"advanced","flags":0,"default_value":"1095","min":"90","max":"3650","enum_allowed":[],"desc":"Specifies the duration of self certificates generated and signed by cephadm root CA","long_desc":"","tags":[],"see_also":[]},"certificate_renewal_threshold_days":{"name":"certificate_renewal_threshold_days","type":"int","level":"advanced","flags":0,"default_value":"30","min":"10","max":"90","enum_allowed":[],"desc":"Specifies the lead time in days to initiate certificate renewal before expiration.","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.28.1","min":"","max":"","enum_allowed":[],"desc":"Alertmanager container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_elasticsearch":{"name":"container_image_elasticsearch","type":"str","level":"advanced","flags":0,"default_value":"quay.io/omrizeneva/elasticsearch:6.8.23","min":"","max":"","enum_allowed":[],"desc":"Elasticsearch container image","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/grafana:12.3.1","min":"","max":"","enum_allowed":[],"desc":"Grafana container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"Haproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_agent":{"name":"container_image_jaeger_agent","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-agent:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger agent container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_collector":{"name":"container_image_jaeger_collector","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-collector:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger collector container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_query":{"name":"container_image_jaeger_query","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-query:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger query container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.2.4","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:3.0.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_nginx":{"name":"container_image_nginx","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/nginx:sclorg-nginx-126","min":"","max":"","enum_allowed":[],"desc":"Nginx container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.9.1","min":"","max":"","enum_allowed":[],"desc":"Node exporter container image","long_desc":"","tags":[],"see_also":[]},"container_image_nvmeof":{"name":"container_image_nvmeof","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/nvmeof:1.5","min":"","max":"","enum_allowed":[],"desc":"Nvmeof container image","long_desc":"","tags":[],"see_also":[]},"container_image_oauth2_proxy":{"name":"container_image_oauth2_proxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/oauth2-proxy/oauth2-proxy:v7.6.0","min":"","max":"","enum_allowed":[],"desc":"Oauth2 proxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v3.6.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:3.0.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_samba":{"name":"container_image_samba","type":"str","level":"advanced","flags":0,"default_value":"quay.io/samba.org/samba-server:ceph20-centos-amd64","min":"","max":"","enum_allowed":[],"desc":"Samba container image","long_desc":"","tags":[],"see_also":[]},"container_image_samba_metrics":{"name":"container_image_samba_metrics","type":"str","level":"advanced","flags":0,"default_value":"quay.io/samba.org/samba-metrics:ceph20-centos-amd64","min":"","max":"","enum_allowed":[],"desc":"Samba metrics container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"Snmp gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"int","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"quay.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"grafana_dashboards_path":{"name":"grafana_dashboards_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/grafana/dashboards/ceph-dashboard/","min":"","max":"","enum_allowed":[],"desc":"location of dashboards to include in grafana deployments","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"hw_monitoring":{"name":"hw_monitoring","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Deploy hw monitoring daemon on every host.","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"oob_default_addr":{"name":"oob_default_addr","type":"str","level":"advanced","flags":0,"default_value":"169.254.1.1","min":"","max":"","enum_allowed":[],"desc":"Default address for RedFish API (oob management).","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"secure_monitoring_stack":{"name":"secure_monitoring_stack","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable TLS security for all the monitoring stack daemons","long_desc":"","tags":[],"see_also":[]},"service_discovery_port":{"name":"service_discovery_port","type":"int","level":"advanced","flags":0,"default_value":"8765","min":"","max":"","enum_allowed":[],"desc":"cephadm service discovery port","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"ssh_keepalive_count_max":{"name":"ssh_keepalive_count_max","type":"int","level":"advanced","flags":0,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"How many times ssh connections can fail liveness checks before the host is marked offline","long_desc":"","tags":[],"see_also":[]},"ssh_keepalive_interval":{"name":"ssh_keepalive_interval","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"How often ssh connections are checked for liveness","long_desc":"","tags":[],"see_also":[]},"stray_daemon_check_interval":{"name":"stray_daemon_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"how frequently cephadm should check for the presence of stray daemons","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"MANAGED_BY_CLUSTERS":{"name":"MANAGED_BY_CLUSTERS","type":"str","level":"advanced","flags":0,"default_value":"[]","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"MULTICLUSTER_CONFIG":{"name":"MULTICLUSTER_CONFIG","type":"str","level":"advanced","flags":0,"default_value":"{}","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROM_ALERT_CREDENTIAL_CACHE_TTL":{"name":"PROM_ALERT_CREDENTIAL_CACHE_TTL","type":"int","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_HOSTNAME_PER_DAEMON":{"name":"RGW_HOSTNAME_PER_DAEMON","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"UNSAFE_TLS_v1_2":{"name":"UNSAFE_TLS_v1_2","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crypto_caller":{"name":"crypto_caller","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sso_oauth2":{"name":"sso_oauth2","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"fail_fs":{"name":"fail_fs","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Fail filesystem for rapid multi-rank mds upgrade","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"exclude_perf_counters":{"name":"exclude_perf_counters","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Do not include perf-counters in the metrics output","long_desc":"Gathering perf-counters from a single Prometheus exporter can degrade ceph-mgr performance, especially in large clusters. Instead, Ceph-exporter daemons are now used by default for perf-counter gathering. This should only be disabled when no ceph-exporters are deployed.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"secondary_zone_period_retry_limit":{"name":"secondary_zone_period_retry_limit","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"RGW module period update retry limit for secondary site","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard_description":{"name":"leaderboard_description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"pause_cloning":{"name":"pause_cloning","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Pause asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"pause_purging":{"name":"pause_purging","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Pause asynchronous subvolume purge threads","long_desc":"","tags":[],"see_also":[]},"periodic_async_work":{"name":"periodic_async_work","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Periodically check for async work","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_no_wait":{"name":"snapshot_clone_no_wait","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Reject subvolume clone request when cloner threads are busy","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}],"services":{},"always_on_modules":{"octopus":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"pacific":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"quincy":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"reef":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"squid":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"tentacle":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"]},"force_disabled_modules":{},"last_failure_osd_epoch":0,"active_clients":[{"name":"devicehealth","addrvec":[{"type":"v2","addr":"10.20.193.34:0","nonce":969662005}]},{"name":"libcephsqlite","addrvec":[{"type":"v2","addr":"10.20.193.34:0","nonce":2902585643}]},{"name":"rbd_support","addrvec":[{"type":"v2","addr":"10.20.193.34:0","nonce":2928122038}]},{"name":"volumes","addrvec":[{"type":"v2","addr":"10.20.193.34:0","nonce":2597524855}]}]} 2026-02-18T08:09:17.570 INFO:tasks.ceph.ceph_manager.cluster1:mgr available! 2026-02-18T08:09:17.570 INFO:tasks.ceph.ceph_manager.cluster1:waiting for all up 2026-02-18T08:09:17.570 DEBUG:teuthology.orchestra.run.trial034:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 osd dump --format=json 2026-02-18T08:09:17.727 INFO:teuthology.orchestra.run.trial034.stdout: 2026-02-18T08:09:17.728 INFO:teuthology.orchestra.run.trial034.stdout:{"epoch":14,"fsid":"538b5201-9b84-4a73-9eef-5646b3360b9d","created":"2026-02-18T08:09:06.049413+0000","modified":"2026-02-18T08:09:17.202008+0000","last_up_change":"2026-02-18T08:09:11.186536+0000","last_in_change":"2026-02-18T08:09:06.672265+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":4,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":2,"max_osd":3,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"tentacle","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2026-02-18T08:09:12.184009+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"11","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"nonprimary_shards":"{}","options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":2.9900000095367432,"score_stable":2.9900000095367432,"optimal_score":0.67000001668930054,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2026-02-18T08:09:14.017646+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"14","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":2,"snap_epoch":14,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"nonprimary_shards":"{}","options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_type":"Fair distribution","score_acting":1.8799999952316284,"score_stable":1.8799999952316284,"optimal_score":1,"raw_score_acting":1.8799999952316284,"raw_score_stable":1.8799999952316284,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"6ca6cdc0-6879-42a3-a4d6-eee71da49395","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":11,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.34:6808","nonce":3130378423},{"type":"v1","addr":"10.20.193.34:6809","nonce":3130378423}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.34:6810","nonce":3130378423},{"type":"v1","addr":"10.20.193.34:6811","nonce":3130378423}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.34:6814","nonce":3130378423},{"type":"v1","addr":"10.20.193.34:6815","nonce":3130378423}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.34:6812","nonce":3130378423},{"type":"v1","addr":"10.20.193.34:6813","nonce":3130378423}]},"public_addr":"10.20.193.34:6809/3130378423","cluster_addr":"10.20.193.34:6811/3130378423","heartbeat_back_addr":"10.20.193.34:6815/3130378423","heartbeat_front_addr":"10.20.193.34:6813/3130378423","state":["exists","up"]},{"osd":1,"uuid":"eb801757-b69b-4914-a101-65b314371e7c","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":11,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.34:6800","nonce":2597388232},{"type":"v1","addr":"10.20.193.34:6801","nonce":2597388232}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.34:6802","nonce":2597388232},{"type":"v1","addr":"10.20.193.34:6803","nonce":2597388232}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.34:6806","nonce":2597388232},{"type":"v1","addr":"10.20.193.34:6807","nonce":2597388232}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.34:6804","nonce":2597388232},{"type":"v1","addr":"10.20.193.34:6805","nonce":2597388232}]},"public_addr":"10.20.193.34:6801/2597388232","cluster_addr":"10.20.193.34:6803/2597388232","heartbeat_back_addr":"10.20.193.34:6807/2597388232","heartbeat_front_addr":"10.20.193.34:6805/2597388232","state":["exists","up"]},{"osd":2,"uuid":"8cff614b-3827-4a60-8c9f-a83bed4c4d45","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":11,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.34:6816","nonce":1006015704},{"type":"v1","addr":"10.20.193.34:6817","nonce":1006015704}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.34:6818","nonce":1006015704},{"type":"v1","addr":"10.20.193.34:6819","nonce":1006015704}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.34:6822","nonce":1006015704},{"type":"v1","addr":"10.20.193.34:6823","nonce":1006015704}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.34:6820","nonce":1006015704},{"type":"v1","addr":"10.20.193.34:6821","nonce":1006015704}]},"public_addr":"10.20.193.34:6817/1006015704","cluster_addr":"10.20.193.34:6819/1006015704","heartbeat_back_addr":"10.20.193.34:6823/1006015704","heartbeat_front_addr":"10.20.193.34:6821/1006015704","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4544132024016699391,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4544132024016699391,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4544132024016699391,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"isa","technique":"reed_sol_van"}},"removed_snaps_queue":[{"pool":2,"snaps":[{"begin":2,"length":1}]}],"new_removed_snaps":[{"pool":2,"snaps":[{"begin":2,"length":1}]}],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2026-02-18T08:09:17.738 INFO:tasks.ceph.ceph_manager.cluster1:all up! 2026-02-18T08:09:17.738 DEBUG:teuthology.orchestra.run.trial034:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 osd dump --format=json 2026-02-18T08:09:17.899 INFO:teuthology.orchestra.run.trial034.stdout: 2026-02-18T08:09:17.899 INFO:teuthology.orchestra.run.trial034.stdout:{"epoch":14,"fsid":"538b5201-9b84-4a73-9eef-5646b3360b9d","created":"2026-02-18T08:09:06.049413+0000","modified":"2026-02-18T08:09:17.202008+0000","last_up_change":"2026-02-18T08:09:11.186536+0000","last_in_change":"2026-02-18T08:09:06.672265+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":4,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":2,"max_osd":3,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"tentacle","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2026-02-18T08:09:12.184009+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"11","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"nonprimary_shards":"{}","options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":2.9900000095367432,"score_stable":2.9900000095367432,"optimal_score":0.67000001668930054,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2026-02-18T08:09:14.017646+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"14","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":2,"snap_epoch":14,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"nonprimary_shards":"{}","options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_type":"Fair distribution","score_acting":1.8799999952316284,"score_stable":1.8799999952316284,"optimal_score":1,"raw_score_acting":1.8799999952316284,"raw_score_stable":1.8799999952316284,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"6ca6cdc0-6879-42a3-a4d6-eee71da49395","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":11,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.34:6808","nonce":3130378423},{"type":"v1","addr":"10.20.193.34:6809","nonce":3130378423}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.34:6810","nonce":3130378423},{"type":"v1","addr":"10.20.193.34:6811","nonce":3130378423}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.34:6814","nonce":3130378423},{"type":"v1","addr":"10.20.193.34:6815","nonce":3130378423}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.34:6812","nonce":3130378423},{"type":"v1","addr":"10.20.193.34:6813","nonce":3130378423}]},"public_addr":"10.20.193.34:6809/3130378423","cluster_addr":"10.20.193.34:6811/3130378423","heartbeat_back_addr":"10.20.193.34:6815/3130378423","heartbeat_front_addr":"10.20.193.34:6813/3130378423","state":["exists","up"]},{"osd":1,"uuid":"eb801757-b69b-4914-a101-65b314371e7c","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":11,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.34:6800","nonce":2597388232},{"type":"v1","addr":"10.20.193.34:6801","nonce":2597388232}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.34:6802","nonce":2597388232},{"type":"v1","addr":"10.20.193.34:6803","nonce":2597388232}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.34:6806","nonce":2597388232},{"type":"v1","addr":"10.20.193.34:6807","nonce":2597388232}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.34:6804","nonce":2597388232},{"type":"v1","addr":"10.20.193.34:6805","nonce":2597388232}]},"public_addr":"10.20.193.34:6801/2597388232","cluster_addr":"10.20.193.34:6803/2597388232","heartbeat_back_addr":"10.20.193.34:6807/2597388232","heartbeat_front_addr":"10.20.193.34:6805/2597388232","state":["exists","up"]},{"osd":2,"uuid":"8cff614b-3827-4a60-8c9f-a83bed4c4d45","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":11,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.34:6816","nonce":1006015704},{"type":"v1","addr":"10.20.193.34:6817","nonce":1006015704}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.34:6818","nonce":1006015704},{"type":"v1","addr":"10.20.193.34:6819","nonce":1006015704}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.34:6822","nonce":1006015704},{"type":"v1","addr":"10.20.193.34:6823","nonce":1006015704}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.34:6820","nonce":1006015704},{"type":"v1","addr":"10.20.193.34:6821","nonce":1006015704}]},"public_addr":"10.20.193.34:6817/1006015704","cluster_addr":"10.20.193.34:6819/1006015704","heartbeat_back_addr":"10.20.193.34:6823/1006015704","heartbeat_front_addr":"10.20.193.34:6821/1006015704","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4544132024016699391,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4544132024016699391,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4544132024016699391,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"isa","technique":"reed_sol_van"}},"removed_snaps_queue":[{"pool":2,"snaps":[{"begin":2,"length":1}]}],"new_removed_snaps":[{"pool":2,"snaps":[{"begin":2,"length":1}]}],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2026-02-18T08:09:17.912 DEBUG:teuthology.orchestra.run.trial034:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 tell osd.0 flush_pg_stats 2026-02-18T08:09:17.913 DEBUG:teuthology.orchestra.run.trial034:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 tell osd.1 flush_pg_stats 2026-02-18T08:09:17.913 DEBUG:teuthology.orchestra.run.trial034:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 tell osd.2 flush_pg_stats 2026-02-18T08:09:17.998 INFO:teuthology.orchestra.run.trial034.stdout:34359738373 2026-02-18T08:09:17.999 DEBUG:teuthology.orchestra.run.trial034:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 osd last-stat-seq osd.2 2026-02-18T08:09:18.000 INFO:teuthology.orchestra.run.trial034.stdout:34359738371 2026-02-18T08:09:18.000 DEBUG:teuthology.orchestra.run.trial034:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 osd last-stat-seq osd.0 2026-02-18T08:09:18.002 INFO:teuthology.orchestra.run.trial034.stdout:34359738372 2026-02-18T08:09:18.003 DEBUG:teuthology.orchestra.run.trial034:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 osd last-stat-seq osd.1 2026-02-18T08:09:18.204 INFO:teuthology.orchestra.run.trial034.stdout:34359738371 2026-02-18T08:09:18.216 INFO:tasks.ceph.ceph_manager.cluster1:need seq 34359738373 got 34359738371 for osd.2 2026-02-18T08:09:18.225 INFO:teuthology.orchestra.run.trial034.stdout:34359738371 2026-02-18T08:09:18.241 INFO:tasks.ceph.ceph_manager.cluster1:need seq 34359738371 got 34359738371 for osd.0 2026-02-18T08:09:18.241 DEBUG:teuthology.parallel:result is None 2026-02-18T08:09:18.286 INFO:teuthology.orchestra.run.trial034.stdout:34359738372 2026-02-18T08:09:18.298 INFO:tasks.ceph.ceph_manager.cluster1:need seq 34359738372 got 34359738372 for osd.1 2026-02-18T08:09:18.299 DEBUG:teuthology.parallel:result is None 2026-02-18T08:09:19.218 DEBUG:teuthology.orchestra.run.trial034:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 osd last-stat-seq osd.2 2026-02-18T08:09:19.372 INFO:teuthology.orchestra.run.trial034.stdout:34359738373 2026-02-18T08:09:19.385 INFO:tasks.ceph.ceph_manager.cluster1:need seq 34359738373 got 34359738373 for osd.2 2026-02-18T08:09:19.385 DEBUG:teuthology.parallel:result is None 2026-02-18T08:09:19.385 INFO:tasks.ceph.ceph_manager.cluster1:waiting for clean 2026-02-18T08:09:19.385 DEBUG:teuthology.orchestra.run.trial034:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 pg dump --format=json 2026-02-18T08:09:19.546 INFO:teuthology.orchestra.run.trial034.stdout: 2026-02-18T08:09:19.546 INFO:teuthology.orchestra.run.trial034.stderr:dumped all 2026-02-18T08:09:19.556 INFO:teuthology.orchestra.run.trial034.stdout:{"pg_ready":true,"pg_map":{"version":15,"stamp":"2026-02-18T08:09:18.149261+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590387,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":103,"num_write_kb":1630,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":75,"ondisk_log_size":75,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":18,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":82400,"kb_used_data":1520,"kb_used_omap":19,"kb_used_meta":80812,"kb_avail":283033120,"statfs":{"total":289910292480,"available":289825914880,"internally_reserved":0,"allocated":1556480,"data_stored":1283458,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":19825,"internal_metadata":82752143},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"0.946493"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":20,"reported_epoch":14,"state":"active+clean","last_fresh":"2026-02-18T08:09:17.206713+0000","last_change":"2026-02-18T08:09:17.206844+0000","last_active":"2026-02-18T08:09:17.206713+0000","last_peered":"2026-02-18T08:09:17.206713+0000","last_clean":"2026-02-18T08:09:17.206713+0000","last_became_active":"2026-02-18T08:09:15.204488+0000","last_became_peered":"2026-02-18T08:09:15.204488+0000","last_unstale":"2026-02-18T08:09:17.206713+0000","last_undegraded":"2026-02-18T08:09:17.206713+0000","last_fullsized":"2026-02-18T08:09:17.206713+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-18T08:09:14.193205+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-18T08:09:14.193205+0000","last_clean_scrub_stamp":"2026-02-18T08:09:14.193205+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-19T08:17:13.769843+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00094066400000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.6","version":"0'0","reported_seq":20,"reported_epoch":14,"state":"active+clean","last_fresh":"2026-02-18T08:09:17.206537+0000","last_change":"2026-02-18T08:09:17.206676+0000","last_active":"2026-02-18T08:09:17.206537+0000","last_peered":"2026-02-18T08:09:17.206537+0000","last_clean":"2026-02-18T08:09:17.206537+0000","last_became_active":"2026-02-18T08:09:15.204451+0000","last_became_peered":"2026-02-18T08:09:15.204451+0000","last_unstale":"2026-02-18T08:09:17.206537+0000","last_undegraded":"2026-02-18T08:09:17.206537+0000","last_fullsized":"2026-02-18T08:09:17.206537+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-18T08:09:14.193205+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-18T08:09:14.193205+0000","last_clean_scrub_stamp":"2026-02-18T08:09:14.193205+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-19T10:15:15.160787+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00075490900000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.5","version":"0'0","reported_seq":20,"reported_epoch":14,"state":"active+clean","last_fresh":"2026-02-18T08:09:17.206352+0000","last_change":"2026-02-18T08:09:17.206497+0000","last_active":"2026-02-18T08:09:17.206352+0000","last_peered":"2026-02-18T08:09:17.206352+0000","last_clean":"2026-02-18T08:09:17.206352+0000","last_became_active":"2026-02-18T08:09:15.204532+0000","last_became_peered":"2026-02-18T08:09:15.204532+0000","last_unstale":"2026-02-18T08:09:17.206352+0000","last_undegraded":"2026-02-18T08:09:17.206352+0000","last_fullsized":"2026-02-18T08:09:17.206352+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-18T08:09:14.193205+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-18T08:09:14.193205+0000","last_clean_scrub_stamp":"2026-02-18T08:09:14.193205+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-19T17:38:17.949604+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00059731699999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.4","version":"0'0","reported_seq":20,"reported_epoch":14,"state":"active+clean","last_fresh":"2026-02-18T08:09:17.206190+0000","last_change":"2026-02-18T08:09:17.206457+0000","last_active":"2026-02-18T08:09:17.206190+0000","last_peered":"2026-02-18T08:09:17.206190+0000","last_clean":"2026-02-18T08:09:17.206190+0000","last_became_active":"2026-02-18T08:09:15.204567+0000","last_became_peered":"2026-02-18T08:09:15.204567+0000","last_unstale":"2026-02-18T08:09:17.206190+0000","last_undegraded":"2026-02-18T08:09:17.206190+0000","last_fullsized":"2026-02-18T08:09:17.206190+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-18T08:09:14.193205+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-18T08:09:14.193205+0000","last_clean_scrub_stamp":"2026-02-18T08:09:14.193205+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-19T08:39:51.433130+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00062999799999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.2","version":"14'2","reported_seq":22,"reported_epoch":14,"state":"active+clean","last_fresh":"2026-02-18T08:09:17.204700+0000","last_change":"2026-02-18T08:09:17.204700+0000","last_active":"2026-02-18T08:09:17.204700+0000","last_peered":"2026-02-18T08:09:17.204700+0000","last_clean":"2026-02-18T08:09:17.204700+0000","last_became_active":"2026-02-18T08:09:15.204550+0000","last_became_peered":"2026-02-18T08:09:15.204550+0000","last_unstale":"2026-02-18T08:09:17.204700+0000","last_undegraded":"2026-02-18T08:09:17.204700+0000","last_fullsized":"2026-02-18T08:09:17.204700+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-18T08:09:14.193205+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-18T08:09:14.193205+0000","last_clean_scrub_stamp":"2026-02-18T08:09:14.193205+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-19T14:33:42.672738+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00095267599999999996,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.1","version":"0'0","reported_seq":20,"reported_epoch":14,"state":"active+clean","last_fresh":"2026-02-18T08:09:17.602891+0000","last_change":"2026-02-18T08:09:17.603054+0000","last_active":"2026-02-18T08:09:17.602891+0000","last_peered":"2026-02-18T08:09:17.602891+0000","last_clean":"2026-02-18T08:09:17.602891+0000","last_became_active":"2026-02-18T08:09:15.204585+0000","last_became_peered":"2026-02-18T08:09:15.204585+0000","last_unstale":"2026-02-18T08:09:17.602891+0000","last_undegraded":"2026-02-18T08:09:17.602891+0000","last_fullsized":"2026-02-18T08:09:17.602891+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-18T08:09:14.193205+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-18T08:09:14.193205+0000","last_clean_scrub_stamp":"2026-02-18T08:09:14.193205+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-19T19:11:28.258653+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00029873299999999997,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.0","version":"0'0","reported_seq":20,"reported_epoch":14,"state":"active+clean","last_fresh":"2026-02-18T08:09:17.602915+0000","last_change":"2026-02-18T08:09:17.603088+0000","last_active":"2026-02-18T08:09:17.602915+0000","last_peered":"2026-02-18T08:09:17.602915+0000","last_clean":"2026-02-18T08:09:17.602915+0000","last_became_active":"2026-02-18T08:09:15.204604+0000","last_became_peered":"2026-02-18T08:09:15.204604+0000","last_unstale":"2026-02-18T08:09:17.602915+0000","last_undegraded":"2026-02-18T08:09:17.602915+0000","last_fullsized":"2026-02-18T08:09:17.602915+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-18T08:09:14.193205+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-18T08:09:14.193205+0000","last_clean_scrub_stamp":"2026-02-18T08:09:14.193205+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-19T14:02:23.790633+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00033665399999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.3","version":"12'1","reported_seq":21,"reported_epoch":14,"state":"active+clean","last_fresh":"2026-02-18T08:09:17.206503+0000","last_change":"2026-02-18T08:09:17.206665+0000","last_active":"2026-02-18T08:09:17.206503+0000","last_peered":"2026-02-18T08:09:17.206503+0000","last_clean":"2026-02-18T08:09:17.206503+0000","last_became_active":"2026-02-18T08:09:15.204115+0000","last_became_peered":"2026-02-18T08:09:15.204115+0000","last_unstale":"2026-02-18T08:09:17.206503+0000","last_undegraded":"2026-02-18T08:09:17.206503+0000","last_fullsized":"2026-02-18T08:09:17.206503+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-18T08:09:14.193205+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-18T08:09:14.193205+0000","last_clean_scrub_stamp":"2026-02-18T08:09:14.193205+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-19T15:05:10.251315+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00077986500000000003,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"1.0","version":"10'72","reported_seq":109,"reported_epoch":14,"state":"active+clean","last_fresh":"2026-02-18T08:09:17.205586+0000","last_change":"2026-02-18T08:09:13.198212+0000","last_active":"2026-02-18T08:09:17.205586+0000","last_peered":"2026-02-18T08:09:17.205586+0000","last_clean":"2026-02-18T08:09:17.205586+0000","last_became_active":"2026-02-18T08:09:13.196393+0000","last_became_peered":"2026-02-18T08:09:13.196393+0000","last_unstale":"2026-02-18T08:09:17.205586+0000","last_undegraded":"2026-02-18T08:09:17.205586+0000","last_fullsized":"2026-02-18T08:09:17.205586+0000","mapping_epoch":9,"log_start":"0'0","ondisk_log_start":"0'0","created":9,"last_epoch_clean":10,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-18T08:09:12.188963+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-18T08:09:12.188963+0000","last_clean_scrub_stamp":"2026-02-18T08:09:12.188963+0000","objects_scrubbed":0,"log_size":72,"log_dups_size":0,"ondisk_log_size":72,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-19T11:44:22.114942+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":38,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":16,"acting":16,"num_store_stats":3},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1187840,"data_stored":1180736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":72,"ondisk_log_size":72,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":2,"up_from":8,"seq":34359738373,"num_pgs":3,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27072,"kb_used_data":112,"kb_used_omap":6,"kb_used_meta":26937,"kb_avail":94344768,"statfs":{"total":96636764160,"available":96609042432,"internally_reserved":0,"allocated":114688,"data_stored":31852,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6825,"internal_metadata":27583831},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":8,"seq":34359738372,"num_pgs":9,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27664,"kb_used_data":704,"kb_used_omap":7,"kb_used_meta":26936,"kb_avail":94344176,"statfs":{"total":96636764160,"available":96608436224,"internally_reserved":0,"allocated":720896,"data_stored":625803,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8125,"internal_metadata":27582531},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":8,"seq":34359738371,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27664,"kb_used_data":704,"kb_used_omap":4,"kb_used_meta":26939,"kb_avail":94344176,"statfs":{"total":96636764160,"available":96608436224,"internally_reserved":0,"allocated":720896,"data_stored":625803,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4875,"internal_metadata":27585781},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2026-02-18T08:09:19.558 DEBUG:teuthology.orchestra.run.trial034:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 pg dump --format=json 2026-02-18T08:09:19.713 INFO:teuthology.orchestra.run.trial034.stdout: 2026-02-18T08:09:19.714 INFO:teuthology.orchestra.run.trial034.stderr:dumped all 2026-02-18T08:09:19.725 INFO:teuthology.orchestra.run.trial034.stdout:{"pg_ready":true,"pg_map":{"version":15,"stamp":"2026-02-18T08:09:18.149261+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590387,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":103,"num_write_kb":1630,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":75,"ondisk_log_size":75,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":18,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":82400,"kb_used_data":1520,"kb_used_omap":19,"kb_used_meta":80812,"kb_avail":283033120,"statfs":{"total":289910292480,"available":289825914880,"internally_reserved":0,"allocated":1556480,"data_stored":1283458,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":19825,"internal_metadata":82752143},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"0.946493"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":20,"reported_epoch":14,"state":"active+clean","last_fresh":"2026-02-18T08:09:17.206713+0000","last_change":"2026-02-18T08:09:17.206844+0000","last_active":"2026-02-18T08:09:17.206713+0000","last_peered":"2026-02-18T08:09:17.206713+0000","last_clean":"2026-02-18T08:09:17.206713+0000","last_became_active":"2026-02-18T08:09:15.204488+0000","last_became_peered":"2026-02-18T08:09:15.204488+0000","last_unstale":"2026-02-18T08:09:17.206713+0000","last_undegraded":"2026-02-18T08:09:17.206713+0000","last_fullsized":"2026-02-18T08:09:17.206713+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-18T08:09:14.193205+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-18T08:09:14.193205+0000","last_clean_scrub_stamp":"2026-02-18T08:09:14.193205+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-19T08:17:13.769843+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00094066400000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.6","version":"0'0","reported_seq":20,"reported_epoch":14,"state":"active+clean","last_fresh":"2026-02-18T08:09:17.206537+0000","last_change":"2026-02-18T08:09:17.206676+0000","last_active":"2026-02-18T08:09:17.206537+0000","last_peered":"2026-02-18T08:09:17.206537+0000","last_clean":"2026-02-18T08:09:17.206537+0000","last_became_active":"2026-02-18T08:09:15.204451+0000","last_became_peered":"2026-02-18T08:09:15.204451+0000","last_unstale":"2026-02-18T08:09:17.206537+0000","last_undegraded":"2026-02-18T08:09:17.206537+0000","last_fullsized":"2026-02-18T08:09:17.206537+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-18T08:09:14.193205+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-18T08:09:14.193205+0000","last_clean_scrub_stamp":"2026-02-18T08:09:14.193205+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-19T10:15:15.160787+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00075490900000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.5","version":"0'0","reported_seq":20,"reported_epoch":14,"state":"active+clean","last_fresh":"2026-02-18T08:09:17.206352+0000","last_change":"2026-02-18T08:09:17.206497+0000","last_active":"2026-02-18T08:09:17.206352+0000","last_peered":"2026-02-18T08:09:17.206352+0000","last_clean":"2026-02-18T08:09:17.206352+0000","last_became_active":"2026-02-18T08:09:15.204532+0000","last_became_peered":"2026-02-18T08:09:15.204532+0000","last_unstale":"2026-02-18T08:09:17.206352+0000","last_undegraded":"2026-02-18T08:09:17.206352+0000","last_fullsized":"2026-02-18T08:09:17.206352+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-18T08:09:14.193205+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-18T08:09:14.193205+0000","last_clean_scrub_stamp":"2026-02-18T08:09:14.193205+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-19T17:38:17.949604+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00059731699999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.4","version":"0'0","reported_seq":20,"reported_epoch":14,"state":"active+clean","last_fresh":"2026-02-18T08:09:17.206190+0000","last_change":"2026-02-18T08:09:17.206457+0000","last_active":"2026-02-18T08:09:17.206190+0000","last_peered":"2026-02-18T08:09:17.206190+0000","last_clean":"2026-02-18T08:09:17.206190+0000","last_became_active":"2026-02-18T08:09:15.204567+0000","last_became_peered":"2026-02-18T08:09:15.204567+0000","last_unstale":"2026-02-18T08:09:17.206190+0000","last_undegraded":"2026-02-18T08:09:17.206190+0000","last_fullsized":"2026-02-18T08:09:17.206190+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-18T08:09:14.193205+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-18T08:09:14.193205+0000","last_clean_scrub_stamp":"2026-02-18T08:09:14.193205+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-19T08:39:51.433130+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00062999799999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.2","version":"14'2","reported_seq":22,"reported_epoch":14,"state":"active+clean","last_fresh":"2026-02-18T08:09:17.204700+0000","last_change":"2026-02-18T08:09:17.204700+0000","last_active":"2026-02-18T08:09:17.204700+0000","last_peered":"2026-02-18T08:09:17.204700+0000","last_clean":"2026-02-18T08:09:17.204700+0000","last_became_active":"2026-02-18T08:09:15.204550+0000","last_became_peered":"2026-02-18T08:09:15.204550+0000","last_unstale":"2026-02-18T08:09:17.204700+0000","last_undegraded":"2026-02-18T08:09:17.204700+0000","last_fullsized":"2026-02-18T08:09:17.204700+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-18T08:09:14.193205+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-18T08:09:14.193205+0000","last_clean_scrub_stamp":"2026-02-18T08:09:14.193205+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-19T14:33:42.672738+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00095267599999999996,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.1","version":"0'0","reported_seq":20,"reported_epoch":14,"state":"active+clean","last_fresh":"2026-02-18T08:09:17.602891+0000","last_change":"2026-02-18T08:09:17.603054+0000","last_active":"2026-02-18T08:09:17.602891+0000","last_peered":"2026-02-18T08:09:17.602891+0000","last_clean":"2026-02-18T08:09:17.602891+0000","last_became_active":"2026-02-18T08:09:15.204585+0000","last_became_peered":"2026-02-18T08:09:15.204585+0000","last_unstale":"2026-02-18T08:09:17.602891+0000","last_undegraded":"2026-02-18T08:09:17.602891+0000","last_fullsized":"2026-02-18T08:09:17.602891+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-18T08:09:14.193205+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-18T08:09:14.193205+0000","last_clean_scrub_stamp":"2026-02-18T08:09:14.193205+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-19T19:11:28.258653+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00029873299999999997,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.0","version":"0'0","reported_seq":20,"reported_epoch":14,"state":"active+clean","last_fresh":"2026-02-18T08:09:17.602915+0000","last_change":"2026-02-18T08:09:17.603088+0000","last_active":"2026-02-18T08:09:17.602915+0000","last_peered":"2026-02-18T08:09:17.602915+0000","last_clean":"2026-02-18T08:09:17.602915+0000","last_became_active":"2026-02-18T08:09:15.204604+0000","last_became_peered":"2026-02-18T08:09:15.204604+0000","last_unstale":"2026-02-18T08:09:17.602915+0000","last_undegraded":"2026-02-18T08:09:17.602915+0000","last_fullsized":"2026-02-18T08:09:17.602915+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-18T08:09:14.193205+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-18T08:09:14.193205+0000","last_clean_scrub_stamp":"2026-02-18T08:09:14.193205+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-19T14:02:23.790633+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00033665399999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.3","version":"12'1","reported_seq":21,"reported_epoch":14,"state":"active+clean","last_fresh":"2026-02-18T08:09:17.206503+0000","last_change":"2026-02-18T08:09:17.206665+0000","last_active":"2026-02-18T08:09:17.206503+0000","last_peered":"2026-02-18T08:09:17.206503+0000","last_clean":"2026-02-18T08:09:17.206503+0000","last_became_active":"2026-02-18T08:09:15.204115+0000","last_became_peered":"2026-02-18T08:09:15.204115+0000","last_unstale":"2026-02-18T08:09:17.206503+0000","last_undegraded":"2026-02-18T08:09:17.206503+0000","last_fullsized":"2026-02-18T08:09:17.206503+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-18T08:09:14.193205+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-18T08:09:14.193205+0000","last_clean_scrub_stamp":"2026-02-18T08:09:14.193205+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-19T15:05:10.251315+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00077986500000000003,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"1.0","version":"10'72","reported_seq":109,"reported_epoch":14,"state":"active+clean","last_fresh":"2026-02-18T08:09:17.205586+0000","last_change":"2026-02-18T08:09:13.198212+0000","last_active":"2026-02-18T08:09:17.205586+0000","last_peered":"2026-02-18T08:09:17.205586+0000","last_clean":"2026-02-18T08:09:17.205586+0000","last_became_active":"2026-02-18T08:09:13.196393+0000","last_became_peered":"2026-02-18T08:09:13.196393+0000","last_unstale":"2026-02-18T08:09:17.205586+0000","last_undegraded":"2026-02-18T08:09:17.205586+0000","last_fullsized":"2026-02-18T08:09:17.205586+0000","mapping_epoch":9,"log_start":"0'0","ondisk_log_start":"0'0","created":9,"last_epoch_clean":10,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-18T08:09:12.188963+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-18T08:09:12.188963+0000","last_clean_scrub_stamp":"2026-02-18T08:09:12.188963+0000","objects_scrubbed":0,"log_size":72,"log_dups_size":0,"ondisk_log_size":72,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-19T11:44:22.114942+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":38,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":16,"acting":16,"num_store_stats":3},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1187840,"data_stored":1180736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":72,"ondisk_log_size":72,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":2,"up_from":8,"seq":34359738373,"num_pgs":3,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27072,"kb_used_data":112,"kb_used_omap":6,"kb_used_meta":26937,"kb_avail":94344768,"statfs":{"total":96636764160,"available":96609042432,"internally_reserved":0,"allocated":114688,"data_stored":31852,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6825,"internal_metadata":27583831},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":8,"seq":34359738372,"num_pgs":9,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27664,"kb_used_data":704,"kb_used_omap":7,"kb_used_meta":26936,"kb_avail":94344176,"statfs":{"total":96636764160,"available":96608436224,"internally_reserved":0,"allocated":720896,"data_stored":625803,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8125,"internal_metadata":27582531},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":8,"seq":34359738371,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27664,"kb_used_data":704,"kb_used_omap":4,"kb_used_meta":26939,"kb_avail":94344176,"statfs":{"total":96636764160,"available":96608436224,"internally_reserved":0,"allocated":720896,"data_stored":625803,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4875,"internal_metadata":27585781},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2026-02-18T08:09:19.726 INFO:tasks.ceph.ceph_manager.cluster1:clean! 2026-02-18T08:09:19.727 INFO:tasks.ceph:Waiting until ceph cluster cluster1 is healthy... 2026-02-18T08:09:19.727 INFO:tasks.ceph.ceph_manager.cluster1:wait_until_healthy 2026-02-18T08:09:19.727 DEBUG:teuthology.orchestra.run.trial034:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 health --format=json 2026-02-18T08:09:19.899 INFO:teuthology.orchestra.run.trial034.stdout: 2026-02-18T08:09:19.899 INFO:teuthology.orchestra.run.trial034.stdout:{"status":"HEALTH_OK","checks":{},"mutes":[]} 2026-02-18T08:09:19.911 INFO:tasks.ceph.ceph_manager.cluster1:wait_until_healthy done 2026-02-18T08:09:19.912 INFO:teuthology.run_tasks:Running task ceph... 2026-02-18T08:09:19.926 INFO:tasks.ceph:Creating ceph cluster cluster2... 2026-02-18T08:09:19.926 INFO:tasks.ceph:config {'conf': {'client': {'debug rbd': 20, 'debug rbd_mirror': 15, 'log to stderr': False, 'rbd default features': 125, 'rbd mirror image policy type': 'simple'}, 'client.mirror.0': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.1': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.2': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.3': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'global': {'mon client directed command retry': 5, 'mon warn on pool no app': False, 'ms inject socket failures': 5000}, 'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}, 'osd': {'bluefs allocator': 'hybrid', 'bluestore allocator': 'hybrid', 'bluestore block size': 96636764160, 'bluestore fsck on mount': True, 'debug bluefs': '1/20', 'debug bluestore': '1/20', 'debug ms': 1, 'debug osd': 20, 'debug rocksdb': '4/10', 'mon osd backfillfull_ratio': 0.85, 'mon osd full ratio': 0.9, 'mon osd nearfull ratio': 0.8, 'osd failsafe full ratio': 0.95, 'osd objectstore': 'bluestore'}}, 'fs': 'xfs', 'mkfs_options': None, 'mount_options': None, 'skip_mgr_daemons': False, 'log_ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)', '\\(OSD_SLOW_PING_TIME'], 'cpu_profile': set(), 'cluster': 'cluster2', 'mon_bind_msgr2': True, 'mon_bind_addrvec': True} 2026-02-18T08:09:19.927 INFO:tasks.ceph:ctx.config {'archive_path': '/home/teuthworker/mnt/teuthology/yuriw-2026-02-17_21:26:40-rbd-20.2.1_2.17.26-baseline_2-distro-default-trial/54948', 'branch': '20.2.1_2.17.26-baseline_2', 'description': 'rbd/mirror-thrash/{base/install clients/mirror cluster/{2-node} conf/{disable-pool-app} msgr-failures/few objectstore/bluestore-hybrid policy/simple rbd-mirror/four-per-cluster supported-random-distro$/{ubuntu_latest} workloads/rbd-mirror-journal-stress-workunit}', 'email': 'yweinste@redhat.com', 'first_in_suite': False, 'flavor': 'default', 'job_id': '54948', 'kernel': {'branch': 'distro', 'kdb': True, 'sha1': 'distro'}, 'ktype': 'distro', 'last_in_suite': False, 'machine_type': 'trial', 'meta': [{'desc': 'run two ceph clusters and install rbd-mirror'}, {'desc': 'configure the permissions for client.mirror'}, {'desc': '2 ceph clusters with 1 mon and 3 osds each'}, {'desc': 'run four rbd-mirror daemons per cluster'}, {'desc': 'run the rbd_mirror_stress.sh workunit to test the rbd-mirror daemon'}], 'name': 'yuriw-2026-02-17_21:26:40-rbd-20.2.1_2.17.26-baseline_2-distro-default-trial', 'no_nested_subset': False, 'os_type': 'ubuntu', 'os_version': '22.04', 'overrides': {'admin_socket': {'branch': '20.2.1_2.17.26-baseline_2'}, 'ceph': {'conf': {'client': {'debug rbd': 20, 'debug rbd_mirror': 15, 'log to stderr': False, 'rbd default features': 125, 'rbd mirror image policy type': 'simple'}, 'client.mirror.0': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.1': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.2': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.3': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'global': {'mon client directed command retry': 5, 'mon warn on pool no app': False, 'ms inject socket failures': 5000}, 'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}, 'osd': {'bluefs allocator': 'hybrid', 'bluestore allocator': 'hybrid', 'bluestore block size': 96636764160, 'bluestore fsck on mount': True, 'debug bluefs': '1/20', 'debug bluestore': '1/20', 'debug ms': 1, 'debug osd': 20, 'debug rocksdb': '4/10', 'mon osd backfillfull_ratio': 0.85, 'mon osd full ratio': 0.9, 'mon osd nearfull ratio': 0.8, 'osd failsafe full ratio': 0.95, 'osd objectstore': 'bluestore'}}, 'flavor': 'default', 'fs': 'xfs', 'log-ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)', '\\(OSD_SLOW_PING_TIME'], 'sha1': 'adc1923395f2bcd6ae89b5d331d278364d1848ed'}, 'ceph-deploy': {'bluestore': True, 'conf': {'client': {'log file': '/var/log/ceph/ceph-$name.$pid.log'}, 'mon': {}, 'osd': {'bluestore block size': 96636764160, 'bluestore fsck on mount': True, 'debug bluefs': '1/20', 'debug bluestore': '1/20', 'debug rocksdb': '4/10', 'mon osd backfillfull_ratio': 0.85, 'mon osd full ratio': 0.9, 'mon osd nearfull ratio': 0.8, 'osd failsafe full ratio': 0.95, 'osd objectstore': 'bluestore'}}, 'fs': 'xfs'}, 'install': {'ceph': {'flavor': 'default', 'sha1': 'adc1923395f2bcd6ae89b5d331d278364d1848ed'}}, 'thrashosds': {'bdev_inject_crash': 2, 'bdev_inject_crash_probability': 0.5}, 'workunit': {'branch': '20.2.1_2.17.26-baseline_2', 'sha1': 'adc1923395f2bcd6ae89b5d331d278364d1848ed'}}, 'owner': 'scheduled_yuriw@soko04.front.sepia.ceph.com', 'priority': 69, 'repo': 'https://github.com/ceph/ceph-ci.git', 'roles': [['cluster1.mon.a', 'cluster1.mgr.x', 'cluster2.mgr.x', 'cluster1.osd.0', 'cluster1.osd.1', 'cluster1.osd.2', 'cluster1.client.0', 'cluster2.client.0'], ['cluster2.mon.a', 'cluster2.osd.0', 'cluster2.osd.1', 'cluster2.osd.2', 'cluster1.client.mirror', 'cluster1.client.mirror.0', 'cluster1.client.mirror.1', 'cluster1.client.mirror.2', 'cluster1.client.mirror.3', 'cluster1.client.mirror.4', 'cluster1.client.mirror.5', 'cluster1.client.mirror.6', 'cluster2.client.mirror', 'cluster2.client.mirror.0', 'cluster2.client.mirror.1', 'cluster2.client.mirror.2', 'cluster2.client.mirror.3', 'cluster2.client.mirror.4', 'cluster2.client.mirror.5', 'cluster2.client.mirror.6']], 'seed': 7878, 'sha1': 'adc1923395f2bcd6ae89b5d331d278364d1848ed', 'sleep_before_teardown': 0, 'subset': '111/120000', 'suite': 'rbd', 'suite_branch': '20.2.1_2.17.26-baseline_2', 'suite_path': '/home/teuthworker/src/github.com_ceph_ceph-c_adc1923395f2bcd6ae89b5d331d278364d1848ed/qa', 'suite_relpath': 'qa', 'suite_repo': 'https://github.com/ceph/ceph-ci.git', 'suite_sha1': 'adc1923395f2bcd6ae89b5d331d278364d1848ed', 'targets': {'trial034.front.sepia.ceph.com': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBF0BoVKPHOg0QKqwwjlPElLLlbn535mlxpDa0ESvNojoG4w+irzEhqHqNKGlXw1d/sbAP/gj/AqXlTteMMmvQPo=', 'trial163.front.sepia.ceph.com': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBLTZW5+I/rQuc56SPLMku4v11X/kldvg5rnpQDqR+Im5nfwRzHmq3wJFTgZ2+NqmphkPILaOoEHK845hjsXPPN8='}, 'tasks': [{'internal.check_packages': None}, {'internal.buildpackages_prep': None}, {'internal.save_config': None}, {'internal.check_lock': None}, {'internal.add_remotes': None}, {'console_log': None}, {'internal.connect': None}, {'internal.push_inventory': None}, {'internal.serialize_remote_roles': None}, {'internal.check_conflict': None}, {'internal.check_ceph_data': None}, {'internal.vm_setup': None}, {'kernel': {'branch': 'distro', 'kdb': True, 'sha1': 'distro'}}, {'internal.base': None}, {'internal.archive_upload': None}, {'internal.archive': None}, {'internal.coredump': None}, {'internal.sudo': None}, {'internal.syslog': None}, {'internal.timer': None}, {'pcp': None}, {'selinux': None}, {'ansible.cephlab': None}, {'clock': None}, {'install': {'extra_packages': ['rbd-mirror'], 'flavor': 'default', 'sha1': 'adc1923395f2bcd6ae89b5d331d278364d1848ed'}}, {'ceph': {'cluster': 'cluster1', 'conf': {'client': {'debug rbd': 20, 'debug rbd_mirror': 15, 'log to stderr': False, 'rbd default features': 125, 'rbd mirror image policy type': 'simple'}, 'client.mirror.0': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.1': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.2': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.3': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'global': {'mon client directed command retry': 5, 'mon warn on pool no app': False, 'ms inject socket failures': 5000}, 'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}, 'osd': {'bluefs allocator': 'hybrid', 'bluestore allocator': 'hybrid', 'bluestore block size': 96636764160, 'bluestore fsck on mount': True, 'debug bluefs': '1/20', 'debug bluestore': '1/20', 'debug ms': 1, 'debug osd': 20, 'debug rocksdb': '4/10', 'mon osd backfillfull_ratio': 0.85, 'mon osd full ratio': 0.9, 'mon osd nearfull ratio': 0.8, 'osd failsafe full ratio': 0.95, 'osd objectstore': 'bluestore'}}, 'flavor': 'default', 'fs': 'xfs', 'log-ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)', '\\(OSD_SLOW_PING_TIME'], 'sha1': 'adc1923395f2bcd6ae89b5d331d278364d1848ed'}}, {'ceph': {'cluster': 'cluster2', 'conf': {'client': {'debug rbd': 20, 'debug rbd_mirror': 15, 'log to stderr': False, 'rbd default features': 125, 'rbd mirror image policy type': 'simple'}, 'client.mirror.0': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.1': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.2': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.3': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'global': {'mon client directed command retry': 5, 'mon warn on pool no app': False, 'ms inject socket failures': 5000}, 'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}, 'osd': {'bluefs allocator': 'hybrid', 'bluestore allocator': 'hybrid', 'bluestore block size': 96636764160, 'bluestore fsck on mount': True, 'debug bluefs': '1/20', 'debug bluestore': '1/20', 'debug ms': 1, 'debug osd': 20, 'debug rocksdb': '4/10', 'mon osd backfillfull_ratio': 0.85, 'mon osd full ratio': 0.9, 'mon osd nearfull ratio': 0.8, 'osd failsafe full ratio': 0.95, 'osd objectstore': 'bluestore'}}, 'flavor': 'default', 'fs': 'xfs', 'log-ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)', '\\(OSD_SLOW_PING_TIME'], 'sha1': 'adc1923395f2bcd6ae89b5d331d278364d1848ed'}}, {'exec': {'cluster1.client.mirror.0': ["sudo ceph --cluster cluster1 auth caps client.mirror mon 'profile rbd-mirror-peer' osd 'profile rbd'", "sudo ceph --cluster cluster1 auth caps client.mirror.0 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster1 auth caps client.mirror.1 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster1 auth caps client.mirror.2 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster1 auth caps client.mirror.3 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster2 auth caps client.mirror mon 'profile rbd-mirror-peer' osd 'profile rbd'", "sudo ceph --cluster cluster2 auth caps client.mirror.0 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster2 auth caps client.mirror.1 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster2 auth caps client.mirror.2 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster2 auth caps client.mirror.3 mon 'profile rbd-mirror' osd 'profile rbd'"]}}, {'rbd-mirror': {'client': 'cluster1.client.mirror.0', 'thrash': True}}, {'rbd-mirror': {'client': 'cluster1.client.mirror.1', 'thrash': True}}, {'rbd-mirror': {'client': 'cluster1.client.mirror.2', 'thrash': True}}, {'rbd-mirror': {'client': 'cluster1.client.mirror.3', 'thrash': True}}, {'rbd-mirror': {'client': 'cluster2.client.mirror.0', 'thrash': True}}, {'rbd-mirror': {'client': 'cluster2.client.mirror.1', 'thrash': True}}, {'rbd-mirror': {'client': 'cluster2.client.mirror.2', 'thrash': True}}, {'rbd-mirror': {'client': 'cluster2.client.mirror.3', 'thrash': True}}, {'rbd-mirror-thrash': {'cluster': 'cluster1'}}, {'rbd-mirror-thrash': {'cluster': 'cluster2'}}, {'workunit': {'clients': {'cluster1.client.mirror': ['rbd/rbd_mirror_stress.sh']}, 'env': {'CEPH_ARGS': '', 'MIRROR_POOL_MODE': 'pool', 'RBD_MIRROR_INSTANCES': '4', 'RBD_MIRROR_MODE': 'journal', 'RBD_MIRROR_USE_EXISTING_CLUSTER': '1', 'RBD_MIRROR_USE_RBD_MIRROR': '1'}, 'timeout': '6h'}}], 'teuthology': {'fragments_dropped': [], 'meta': {}, 'postmerge': []}, 'teuthology_branch': 'main', 'teuthology_sha1': 'fa17720d0088c3ac28e473468bfc79eeaff5cd38', 'timestamp': '2026-02-17_21:26:40', 'tube': 'trial', 'user': 'yuriw', 'verbose': True, 'worker_log': '/home/teuthworker/mnt/teuthology/worker_logs/dispatcher.trial.3682'} 2026-02-18T08:09:19.928 DEBUG:teuthology.orchestra.run.trial034:> install -d -m0755 -- /home/ubuntu/cephtest/cluster2.data 2026-02-18T08:09:19.930 DEBUG:teuthology.orchestra.run.trial163:> install -d -m0755 -- /home/ubuntu/cephtest/cluster2.data 2026-02-18T08:09:19.935 DEBUG:teuthology.orchestra.run.trial034:> sudo install -d -m0777 -- /var/run/ceph 2026-02-18T08:09:19.978 DEBUG:teuthology.orchestra.run.trial163:> sudo install -d -m0777 -- /var/run/ceph 2026-02-18T08:09:19.986 DEBUG:teuthology.orchestra.run.trial163:> set -ex 2026-02-18T08:09:19.987 DEBUG:teuthology.orchestra.run.trial163:> dd if=/scratch_devs of=/dev/stdout 2026-02-18T08:09:20.034 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2026-02-18T08:09:20.034 DEBUG:teuthology.orchestra.run.trial163:> stat /dev/vg_nvme/lv_1 2026-02-18T08:09:20.081 INFO:teuthology.orchestra.run.trial163.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2026-02-18T08:09:20.081 INFO:teuthology.orchestra.run.trial163.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2026-02-18T08:09:20.082 INFO:teuthology.orchestra.run.trial163.stdout:Device: 5h/5d Inode: 902 Links: 1 2026-02-18T08:09:20.082 INFO:teuthology.orchestra.run.trial163.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2026-02-18T08:09:20.082 INFO:teuthology.orchestra.run.trial163.stdout:Access: 2026-02-18 08:08:00.772883573 +0000 2026-02-18T08:09:20.082 INFO:teuthology.orchestra.run.trial163.stdout:Modify: 2026-02-18 08:08:00.608887760 +0000 2026-02-18T08:09:20.082 INFO:teuthology.orchestra.run.trial163.stdout:Change: 2026-02-18 08:08:00.608887760 +0000 2026-02-18T08:09:20.082 INFO:teuthology.orchestra.run.trial163.stdout: Birth: - 2026-02-18T08:09:20.082 DEBUG:teuthology.orchestra.run.trial163:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2026-02-18T08:09:20.128 INFO:teuthology.orchestra.run.trial163.stderr:1+0 records in 2026-02-18T08:09:20.129 INFO:teuthology.orchestra.run.trial163.stderr:1+0 records out 2026-02-18T08:09:20.129 INFO:teuthology.orchestra.run.trial163.stderr:512 bytes copied, 0.000192917 s, 2.7 MB/s 2026-02-18T08:09:20.130 DEBUG:teuthology.orchestra.run.trial163:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2026-02-18T08:09:20.179 DEBUG:teuthology.orchestra.run.trial163:> stat /dev/vg_nvme/lv_2 2026-02-18T08:09:20.225 INFO:teuthology.orchestra.run.trial163.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2026-02-18T08:09:20.225 INFO:teuthology.orchestra.run.trial163.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2026-02-18T08:09:20.225 INFO:teuthology.orchestra.run.trial163.stdout:Device: 5h/5d Inode: 937 Links: 1 2026-02-18T08:09:20.226 INFO:teuthology.orchestra.run.trial163.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2026-02-18T08:09:20.226 INFO:teuthology.orchestra.run.trial163.stdout:Access: 2026-02-18 08:08:00.956878877 +0000 2026-02-18T08:09:20.226 INFO:teuthology.orchestra.run.trial163.stdout:Modify: 2026-02-18 08:08:00.952878979 +0000 2026-02-18T08:09:20.226 INFO:teuthology.orchestra.run.trial163.stdout:Change: 2026-02-18 08:08:00.952878979 +0000 2026-02-18T08:09:20.226 INFO:teuthology.orchestra.run.trial163.stdout: Birth: - 2026-02-18T08:09:20.226 DEBUG:teuthology.orchestra.run.trial163:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2026-02-18T08:09:20.276 INFO:teuthology.orchestra.run.trial163.stderr:1+0 records in 2026-02-18T08:09:20.277 INFO:teuthology.orchestra.run.trial163.stderr:1+0 records out 2026-02-18T08:09:20.277 INFO:teuthology.orchestra.run.trial163.stderr:512 bytes copied, 0.000175084 s, 2.9 MB/s 2026-02-18T08:09:20.278 DEBUG:teuthology.orchestra.run.trial163:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2026-02-18T08:09:20.327 DEBUG:teuthology.orchestra.run.trial163:> stat /dev/vg_nvme/lv_3 2026-02-18T08:09:20.373 INFO:teuthology.orchestra.run.trial163.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2026-02-18T08:09:20.373 INFO:teuthology.orchestra.run.trial163.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2026-02-18T08:09:20.374 INFO:teuthology.orchestra.run.trial163.stdout:Device: 5h/5d Inode: 951 Links: 1 2026-02-18T08:09:20.374 INFO:teuthology.orchestra.run.trial163.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2026-02-18T08:09:20.374 INFO:teuthology.orchestra.run.trial163.stdout:Access: 2026-02-18 08:08:01.312869790 +0000 2026-02-18T08:09:20.374 INFO:teuthology.orchestra.run.trial163.stdout:Modify: 2026-02-18 08:08:01.308869893 +0000 2026-02-18T08:09:20.374 INFO:teuthology.orchestra.run.trial163.stdout:Change: 2026-02-18 08:08:01.308869893 +0000 2026-02-18T08:09:20.374 INFO:teuthology.orchestra.run.trial163.stdout: Birth: - 2026-02-18T08:09:20.374 DEBUG:teuthology.orchestra.run.trial163:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2026-02-18T08:09:20.420 INFO:teuthology.orchestra.run.trial163.stderr:1+0 records in 2026-02-18T08:09:20.420 INFO:teuthology.orchestra.run.trial163.stderr:1+0 records out 2026-02-18T08:09:20.420 INFO:teuthology.orchestra.run.trial163.stderr:512 bytes copied, 0.000173621 s, 2.9 MB/s 2026-02-18T08:09:20.421 DEBUG:teuthology.orchestra.run.trial163:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2026-02-18T08:09:20.466 DEBUG:teuthology.orchestra.run.trial163:> stat /dev/vg_nvme/lv_4 2026-02-18T08:09:20.513 INFO:teuthology.orchestra.run.trial163.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2026-02-18T08:09:20.513 INFO:teuthology.orchestra.run.trial163.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2026-02-18T08:09:20.513 INFO:teuthology.orchestra.run.trial163.stdout:Device: 5h/5d Inode: 974 Links: 1 2026-02-18T08:09:20.513 INFO:teuthology.orchestra.run.trial163.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2026-02-18T08:09:20.513 INFO:teuthology.orchestra.run.trial163.stdout:Access: 2026-02-18 08:08:01.852856006 +0000 2026-02-18T08:09:20.513 INFO:teuthology.orchestra.run.trial163.stdout:Modify: 2026-02-18 08:08:01.664860804 +0000 2026-02-18T08:09:20.513 INFO:teuthology.orchestra.run.trial163.stdout:Change: 2026-02-18 08:08:01.664860804 +0000 2026-02-18T08:09:20.513 INFO:teuthology.orchestra.run.trial163.stdout: Birth: - 2026-02-18T08:09:20.514 DEBUG:teuthology.orchestra.run.trial163:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2026-02-18T08:09:20.560 INFO:teuthology.orchestra.run.trial163.stderr:1+0 records in 2026-02-18T08:09:20.560 INFO:teuthology.orchestra.run.trial163.stderr:1+0 records out 2026-02-18T08:09:20.560 INFO:teuthology.orchestra.run.trial163.stderr:512 bytes copied, 0.000162801 s, 3.1 MB/s 2026-02-18T08:09:20.561 DEBUG:teuthology.orchestra.run.trial163:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2026-02-18T08:09:20.606 INFO:tasks.ceph:osd dev map: {'cluster2.osd.0': '/dev/vg_nvme/lv_1', 'cluster2.osd.1': '/dev/vg_nvme/lv_2', 'cluster2.osd.2': '/dev/vg_nvme/lv_3'} 2026-02-18T08:09:20.606 INFO:tasks.ceph:remote_to_roles_to_devs: {Remote(name='ubuntu@trial163.front.sepia.ceph.com'): {'cluster2.osd.0': '/dev/vg_nvme/lv_1', 'cluster2.osd.1': '/dev/vg_nvme/lv_2', 'cluster2.osd.2': '/dev/vg_nvme/lv_3'}} 2026-02-18T08:09:20.606 INFO:tasks.ceph:Generating config... 2026-02-18T08:09:20.607 INFO:tasks.ceph:[client] debug rbd = 20 2026-02-18T08:09:20.607 INFO:tasks.ceph:[client] debug rbd_mirror = 15 2026-02-18T08:09:20.607 INFO:tasks.ceph:[client] log to stderr = False 2026-02-18T08:09:20.608 INFO:tasks.ceph:[client] rbd default features = 125 2026-02-18T08:09:20.608 INFO:tasks.ceph:[client] rbd mirror image policy type = simple 2026-02-18T08:09:20.608 INFO:tasks.ceph:[client.mirror.0] admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-18T08:09:20.608 INFO:tasks.ceph:[client.mirror.0] pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-18T08:09:20.608 INFO:tasks.ceph:[client.mirror.1] admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-18T08:09:20.608 INFO:tasks.ceph:[client.mirror.1] pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-18T08:09:20.608 INFO:tasks.ceph:[client.mirror.2] admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-18T08:09:20.608 INFO:tasks.ceph:[client.mirror.2] pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-18T08:09:20.608 INFO:tasks.ceph:[client.mirror.3] admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-18T08:09:20.608 INFO:tasks.ceph:[client.mirror.3] pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-18T08:09:20.608 INFO:tasks.ceph:[global] mon client directed command retry = 5 2026-02-18T08:09:20.608 INFO:tasks.ceph:[global] mon warn on pool no app = False 2026-02-18T08:09:20.608 INFO:tasks.ceph:[global] ms inject socket failures = 5000 2026-02-18T08:09:20.608 INFO:tasks.ceph:[mgr] debug mgr = 20 2026-02-18T08:09:20.608 INFO:tasks.ceph:[mgr] debug ms = 1 2026-02-18T08:09:20.608 INFO:tasks.ceph:[mon] debug mon = 20 2026-02-18T08:09:20.608 INFO:tasks.ceph:[mon] debug ms = 1 2026-02-18T08:09:20.608 INFO:tasks.ceph:[mon] debug paxos = 20 2026-02-18T08:09:20.608 INFO:tasks.ceph:[osd] bluefs allocator = hybrid 2026-02-18T08:09:20.608 INFO:tasks.ceph:[osd] bluestore allocator = hybrid 2026-02-18T08:09:20.609 INFO:tasks.ceph:[osd] bluestore block size = 96636764160 2026-02-18T08:09:20.609 INFO:tasks.ceph:[osd] bluestore fsck on mount = True 2026-02-18T08:09:20.609 INFO:tasks.ceph:[osd] debug bluefs = 1/20 2026-02-18T08:09:20.609 INFO:tasks.ceph:[osd] debug bluestore = 1/20 2026-02-18T08:09:20.609 INFO:tasks.ceph:[osd] debug ms = 1 2026-02-18T08:09:20.609 INFO:tasks.ceph:[osd] debug osd = 20 2026-02-18T08:09:20.609 INFO:tasks.ceph:[osd] debug rocksdb = 4/10 2026-02-18T08:09:20.609 INFO:tasks.ceph:[osd] mon osd backfillfull_ratio = 0.85 2026-02-18T08:09:20.609 INFO:tasks.ceph:[osd] mon osd full ratio = 0.9 2026-02-18T08:09:20.609 INFO:tasks.ceph:[osd] mon osd nearfull ratio = 0.8 2026-02-18T08:09:20.609 INFO:tasks.ceph:[osd] osd failsafe full ratio = 0.95 2026-02-18T08:09:20.609 INFO:tasks.ceph:[osd] osd objectstore = bluestore 2026-02-18T08:09:20.609 INFO:tasks.ceph:Setting up cluster2.mon.a... 2026-02-18T08:09:20.609 DEBUG:teuthology.orchestra.run.trial163:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring /etc/ceph/cluster2.keyring 2026-02-18T08:09:20.665 INFO:teuthology.orchestra.run.trial163.stdout:creating /etc/ceph/cluster2.keyring 2026-02-18T08:09:20.668 DEBUG:teuthology.orchestra.run.trial163:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --gen-key --name=mon. /etc/ceph/cluster2.keyring 2026-02-18T08:09:20.728 DEBUG:teuthology.orchestra.run.trial163:> sudo chmod 0644 /etc/ceph/cluster2.keyring 2026-02-18T08:09:20.777 DEBUG:tasks.ceph:Ceph mon addresses: [('cluster2.mon.a', '10.20.193.163')] 2026-02-18T08:09:20.777 DEBUG:tasks.ceph:writing out conf {'global': {'chdir': '', 'pid file': '/var/run/ceph/$cluster-$name.pid', 'auth supported': 'cephx', 'filestore xattr use omap': 'true', 'mon clock drift allowed': '1.000', 'osd crush chooseleaf type': '0', 'auth debug': 'true', 'ms die on old message': 'true', 'ms die on bug': 'true', 'mon max pg per osd': '10000', 'mon pg warn max object skew': '0', 'osd_pool_default_pg_autoscale_mode': 'off', 'osd pool default size': '2', 'mon osd allow primary affinity': 'true', 'mon osd allow pg remap': 'true', 'mon warn on legacy crush tunables': 'false', 'mon warn on crush straw calc version zero': 'false', 'mon warn on no sortbitwise': 'false', 'mon warn on osd down out interval zero': 'false', 'mon warn on too few osds': 'false', 'mon_warn_on_pool_pg_num_not_power_of_two': 'false', 'mon_warn_on_pool_no_redundancy': 'false', 'mon_allow_pool_size_one': 'true', 'osd pool default erasure code profile': 'plugin=isa technique=reed_sol_van k=2 m=1 crush-failure-domain=osd', 'osd default data pool replay window': '5', 'mon allow pool delete': 'true', 'mon cluster log file level': 'debug', 'debug asserts on shutdown': 'true', 'mon health detail to clog': 'false', 'mon host': '10.20.193.163', 'mon client directed command retry': 5, 'mon warn on pool no app': False, 'ms inject socket failures': 5000}, 'osd': {'osd journal size': '100', 'osd scrub load threshold': '5.0', 'osd scrub max interval': '600', 'osd mclock profile': 'high_recovery_ops', 'osd mclock skip benchmark': 'true', 'osd recover clone overlap': 'true', 'osd recovery max chunk': '1048576', 'osd debug shutdown': 'true', 'osd debug op order': 'true', 'osd debug verify stray on activate': 'true', 'osd debug trim objects': 'true', 'osd open classes on start': 'true', 'osd debug pg log writeout': 'true', 'osd deep scrub update digest min age': '30', 'osd map max advance': '10', 'journal zero on create': 'true', 'filestore ondisk finisher threads': '3', 'filestore apply finisher threads': '3', 'bdev debug aio': 'true', 'osd debug misdirected ops': 'true', 'bluefs allocator': 'hybrid', 'bluestore allocator': 'hybrid', 'bluestore block size': 96636764160, 'bluestore fsck on mount': True, 'debug bluefs': '1/20', 'debug bluestore': '1/20', 'debug ms': 1, 'debug osd': 20, 'debug rocksdb': '4/10', 'mon osd backfillfull_ratio': 0.85, 'mon osd full ratio': 0.9, 'mon osd nearfull ratio': 0.8, 'osd failsafe full ratio': 0.95, 'osd objectstore': 'bluestore'}, 'mgr': {'debug ms': 1, 'debug mgr': 20, 'debug mon': '20', 'debug auth': '20', 'mon reweight min pgs per osd': '4', 'mon reweight min bytes per osd': '10', 'mgr/telemetry/nag': 'false'}, 'mon': {'debug ms': 1, 'debug mon': 20, 'debug paxos': 20, 'debug auth': '20', 'mon data avail warn': '5', 'mon mgr mkfs grace': '240', 'mon reweight min pgs per osd': '4', 'mon osd reporter subtree level': 'osd', 'mon osd prime pg temp': 'true', 'mon reweight min bytes per osd': '10', 'auth mon ticket ttl': '660', 'auth service ticket ttl': '240', 'mon_warn_on_insecure_global_id_reclaim': 'false', 'mon_warn_on_insecure_global_id_reclaim_allowed': 'false', 'mon_down_mkfs_grace': '2m', 'mon_warn_on_filestore_osds': 'false'}, 'client': {'rgw cache enabled': 'true', 'rgw enable ops log': 'true', 'rgw enable usage log': 'true', 'log file': '/var/log/ceph/$cluster-$name.$pid.log', 'admin socket': '/var/run/ceph/$cluster-$name.$pid.asok', 'debug rbd': 20, 'debug rbd_mirror': 15, 'log to stderr': False, 'rbd default features': 125, 'rbd mirror image policy type': 'simple'}, 'mon.a': {}, 'client.mirror.0': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.1': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.2': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.3': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}} 2026-02-18T08:09:20.778 DEBUG:teuthology.orchestra.run.trial163:> set -ex 2026-02-18T08:09:20.778 DEBUG:teuthology.orchestra.run.trial163:> dd of=/home/ubuntu/cephtest/ceph.tmp.conf 2026-02-18T08:09:20.821 DEBUG:teuthology.orchestra.run.trial163:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage monmaptool -c /home/ubuntu/cephtest/ceph.tmp.conf --create --clobber --enable-all-features --add a 10.20.193.163 --print /home/ubuntu/cephtest/cluster2.monmap 2026-02-18T08:09:20.878 INFO:teuthology.orchestra.run.trial163.stdout:monmaptool: monmap file /home/ubuntu/cephtest/cluster2.monmap 2026-02-18T08:09:20.878 INFO:teuthology.orchestra.run.trial163.stdout:monmaptool: generated fsid 892b71e6-8019-418b-97e7-5a7b63a9d866 2026-02-18T08:09:20.878 INFO:teuthology.orchestra.run.trial163.stdout:setting min_mon_release = tentacle 2026-02-18T08:09:20.878 INFO:teuthology.orchestra.run.trial163.stdout:epoch 0 2026-02-18T08:09:20.879 INFO:teuthology.orchestra.run.trial163.stdout:fsid 892b71e6-8019-418b-97e7-5a7b63a9d866 2026-02-18T08:09:20.879 INFO:teuthology.orchestra.run.trial163.stdout:last_changed 2026-02-18T08:09:20.876625+0000 2026-02-18T08:09:20.879 INFO:teuthology.orchestra.run.trial163.stdout:created 2026-02-18T08:09:20.876625+0000 2026-02-18T08:09:20.879 INFO:teuthology.orchestra.run.trial163.stdout:min_mon_release 20 (tentacle) 2026-02-18T08:09:20.879 INFO:teuthology.orchestra.run.trial163.stdout:election_strategy: 1 2026-02-18T08:09:20.879 INFO:teuthology.orchestra.run.trial163.stdout:0: [v2:10.20.193.163:3300/0,v1:10.20.193.163:6789/0] mon.a 2026-02-18T08:09:20.879 INFO:teuthology.orchestra.run.trial163.stdout:monmaptool: writing epoch 0 to /home/ubuntu/cephtest/cluster2.monmap (1 monitors) 2026-02-18T08:09:20.880 DEBUG:teuthology.orchestra.run.trial163:> rm -- /home/ubuntu/cephtest/ceph.tmp.conf 2026-02-18T08:09:20.925 INFO:tasks.ceph:Writing /etc/ceph/cluster2.conf for FSID 892b71e6-8019-418b-97e7-5a7b63a9d866... 2026-02-18T08:09:20.926 DEBUG:teuthology.orchestra.run.trial034:> sudo mkdir -p /etc/ceph && sudo chmod 0755 /etc/ceph && sudo tee /etc/ceph/cluster2.conf && sudo chmod 0644 /etc/ceph/cluster2.conf > /dev/null 2026-02-18T08:09:20.928 DEBUG:teuthology.orchestra.run.trial163:> sudo mkdir -p /etc/ceph && sudo chmod 0755 /etc/ceph && sudo tee /etc/ceph/cluster2.conf && sudo chmod 0644 /etc/ceph/cluster2.conf > /dev/null 2026-02-18T08:09:20.971 INFO:teuthology.orchestra.run.trial034.stdout:[global] 2026-02-18T08:09:20.971 INFO:teuthology.orchestra.run.trial034.stdout: chdir = "" 2026-02-18T08:09:20.971 INFO:teuthology.orchestra.run.trial034.stdout: pid file = /var/run/ceph/$cluster-$name.pid 2026-02-18T08:09:20.971 INFO:teuthology.orchestra.run.trial034.stdout: auth supported = cephx 2026-02-18T08:09:20.971 INFO:teuthology.orchestra.run.trial034.stdout: 2026-02-18T08:09:20.971 INFO:teuthology.orchestra.run.trial034.stdout: filestore xattr use omap = true 2026-02-18T08:09:20.971 INFO:teuthology.orchestra.run.trial034.stdout: 2026-02-18T08:09:20.971 INFO:teuthology.orchestra.run.trial034.stdout: mon clock drift allowed = 1.000 2026-02-18T08:09:20.971 INFO:teuthology.orchestra.run.trial034.stdout: 2026-02-18T08:09:20.971 INFO:teuthology.orchestra.run.trial034.stdout: osd crush chooseleaf type = 0 2026-02-18T08:09:20.971 INFO:teuthology.orchestra.run.trial034.stdout: auth debug = true 2026-02-18T08:09:20.972 INFO:teuthology.orchestra.run.trial034.stdout: 2026-02-18T08:09:20.972 INFO:teuthology.orchestra.run.trial034.stdout: ms die on old message = true 2026-02-18T08:09:20.972 INFO:teuthology.orchestra.run.trial034.stdout: ms die on bug = true 2026-02-18T08:09:20.972 INFO:teuthology.orchestra.run.trial034.stdout: 2026-02-18T08:09:20.972 INFO:teuthology.orchestra.run.trial034.stdout: mon max pg per osd = 10000 # >= luminous 2026-02-18T08:09:20.972 INFO:teuthology.orchestra.run.trial034.stdout: mon pg warn max object skew = 0 2026-02-18T08:09:20.972 INFO:teuthology.orchestra.run.trial034.stdout: 2026-02-18T08:09:20.972 INFO:teuthology.orchestra.run.trial034.stdout: # disable pg_autoscaler by default for new pools 2026-02-18T08:09:20.972 INFO:teuthology.orchestra.run.trial034.stdout: osd_pool_default_pg_autoscale_mode = off 2026-02-18T08:09:20.972 INFO:teuthology.orchestra.run.trial034.stdout: 2026-02-18T08:09:20.972 INFO:teuthology.orchestra.run.trial034.stdout: osd pool default size = 2 2026-02-18T08:09:20.972 INFO:teuthology.orchestra.run.trial034.stdout: 2026-02-18T08:09:20.972 INFO:teuthology.orchestra.run.trial034.stdout: mon osd allow primary affinity = true 2026-02-18T08:09:20.972 INFO:teuthology.orchestra.run.trial034.stdout: mon osd allow pg remap = true 2026-02-18T08:09:20.972 INFO:teuthology.orchestra.run.trial034.stdout: mon warn on legacy crush tunables = false 2026-02-18T08:09:20.972 INFO:teuthology.orchestra.run.trial034.stdout: mon warn on crush straw calc version zero = false 2026-02-18T08:09:20.973 INFO:teuthology.orchestra.run.trial034.stdout: mon warn on no sortbitwise = false 2026-02-18T08:09:20.973 INFO:teuthology.orchestra.run.trial034.stdout: mon warn on osd down out interval zero = false 2026-02-18T08:09:20.973 INFO:teuthology.orchestra.run.trial034.stdout: mon warn on too few osds = false 2026-02-18T08:09:20.973 INFO:teuthology.orchestra.run.trial034.stdout: mon_warn_on_pool_pg_num_not_power_of_two = false 2026-02-18T08:09:20.973 INFO:teuthology.orchestra.run.trial034.stdout: mon_warn_on_pool_no_redundancy = false 2026-02-18T08:09:20.973 INFO:teuthology.orchestra.run.trial034.stdout: mon_allow_pool_size_one = true 2026-02-18T08:09:20.973 INFO:teuthology.orchestra.run.trial034.stdout: 2026-02-18T08:09:20.973 INFO:teuthology.orchestra.run.trial034.stdout: osd pool default erasure code profile = plugin=isa technique=reed_sol_van k=2 m=1 crush-failure-domain=osd 2026-02-18T08:09:20.973 INFO:teuthology.orchestra.run.trial034.stdout: 2026-02-18T08:09:20.973 INFO:teuthology.orchestra.run.trial034.stdout: osd default data pool replay window = 5 2026-02-18T08:09:20.973 INFO:teuthology.orchestra.run.trial034.stdout: 2026-02-18T08:09:20.973 INFO:teuthology.orchestra.run.trial034.stdout: mon allow pool delete = true 2026-02-18T08:09:20.973 INFO:teuthology.orchestra.run.trial034.stdout: 2026-02-18T08:09:20.973 INFO:teuthology.orchestra.run.trial034.stdout: mon cluster log file level = debug 2026-02-18T08:09:20.973 INFO:teuthology.orchestra.run.trial034.stdout: debug asserts on shutdown = true 2026-02-18T08:09:20.973 INFO:teuthology.orchestra.run.trial034.stdout: mon health detail to clog = false 2026-02-18T08:09:20.973 INFO:teuthology.orchestra.run.trial034.stdout: mon host = 10.20.193.163 2026-02-18T08:09:20.973 INFO:teuthology.orchestra.run.trial034.stdout: mon client directed command retry = 5 2026-02-18T08:09:20.974 INFO:teuthology.orchestra.run.trial034.stdout: mon warn on pool no app = False 2026-02-18T08:09:20.974 INFO:teuthology.orchestra.run.trial034.stdout: ms inject socket failures = 5000 2026-02-18T08:09:20.974 INFO:teuthology.orchestra.run.trial034.stdout: fsid = 892b71e6-8019-418b-97e7-5a7b63a9d866 2026-02-18T08:09:20.974 INFO:teuthology.orchestra.run.trial034.stdout: 2026-02-18T08:09:20.974 INFO:teuthology.orchestra.run.trial034.stdout:[osd] 2026-02-18T08:09:20.974 INFO:teuthology.orchestra.run.trial034.stdout: osd journal size = 100 2026-02-18T08:09:20.974 INFO:teuthology.orchestra.run.trial034.stdout: 2026-02-18T08:09:20.974 INFO:teuthology.orchestra.run.trial034.stdout: osd scrub load threshold = 5.0 2026-02-18T08:09:20.974 INFO:teuthology.orchestra.run.trial034.stdout: osd scrub max interval = 600 2026-02-18T08:09:20.974 INFO:teuthology.orchestra.run.trial034.stdout: osd mclock profile = high_recovery_ops 2026-02-18T08:09:20.974 INFO:teuthology.orchestra.run.trial034.stdout: osd mclock skip benchmark = true 2026-02-18T08:09:20.974 INFO:teuthology.orchestra.run.trial034.stdout: 2026-02-18T08:09:20.974 INFO:teuthology.orchestra.run.trial034.stdout: osd recover clone overlap = true 2026-02-18T08:09:20.974 INFO:teuthology.orchestra.run.trial034.stdout: osd recovery max chunk = 1048576 2026-02-18T08:09:20.974 INFO:teuthology.orchestra.run.trial034.stdout: 2026-02-18T08:09:20.974 INFO:teuthology.orchestra.run.trial034.stdout: osd debug shutdown = true 2026-02-18T08:09:20.974 INFO:teuthology.orchestra.run.trial034.stdout: osd debug op order = true 2026-02-18T08:09:20.974 INFO:teuthology.orchestra.run.trial034.stdout: osd debug verify stray on activate = true 2026-02-18T08:09:20.975 INFO:teuthology.orchestra.run.trial034.stdout: osd debug trim objects = true 2026-02-18T08:09:20.975 INFO:teuthology.orchestra.run.trial034.stdout: 2026-02-18T08:09:20.975 INFO:teuthology.orchestra.run.trial034.stdout: osd open classes on start = true 2026-02-18T08:09:20.975 INFO:teuthology.orchestra.run.trial034.stdout: osd debug pg log writeout = true 2026-02-18T08:09:20.975 INFO:teuthology.orchestra.run.trial034.stdout: 2026-02-18T08:09:20.975 INFO:teuthology.orchestra.run.trial034.stdout: osd deep scrub update digest min age = 30 2026-02-18T08:09:20.975 INFO:teuthology.orchestra.run.trial034.stdout: 2026-02-18T08:09:20.975 INFO:teuthology.orchestra.run.trial034.stdout: osd map max advance = 10 2026-02-18T08:09:20.975 INFO:teuthology.orchestra.run.trial034.stdout: 2026-02-18T08:09:20.975 INFO:teuthology.orchestra.run.trial034.stdout: journal zero on create = true 2026-02-18T08:09:20.975 INFO:teuthology.orchestra.run.trial034.stdout: 2026-02-18T08:09:20.975 INFO:teuthology.orchestra.run.trial034.stdout: filestore ondisk finisher threads = 3 2026-02-18T08:09:20.975 INFO:teuthology.orchestra.run.trial034.stdout: filestore apply finisher threads = 3 2026-02-18T08:09:20.975 INFO:teuthology.orchestra.run.trial034.stdout: 2026-02-18T08:09:20.975 INFO:teuthology.orchestra.run.trial034.stdout: bdev debug aio = true 2026-02-18T08:09:20.975 INFO:teuthology.orchestra.run.trial034.stdout: osd debug misdirected ops = true 2026-02-18T08:09:20.975 INFO:teuthology.orchestra.run.trial034.stdout: bluefs allocator = hybrid 2026-02-18T08:09:20.975 INFO:teuthology.orchestra.run.trial034.stdout: bluestore allocator = hybrid 2026-02-18T08:09:20.975 INFO:teuthology.orchestra.run.trial034.stdout: bluestore block size = 96636764160 2026-02-18T08:09:20.975 INFO:teuthology.orchestra.run.trial034.stdout: bluestore fsck on mount = True 2026-02-18T08:09:20.975 INFO:teuthology.orchestra.run.trial034.stdout: debug bluefs = 1/20 2026-02-18T08:09:20.975 INFO:teuthology.orchestra.run.trial034.stdout: debug bluestore = 1/20 2026-02-18T08:09:20.976 INFO:teuthology.orchestra.run.trial034.stdout: debug ms = 1 2026-02-18T08:09:20.976 INFO:teuthology.orchestra.run.trial034.stdout: debug osd = 20 2026-02-18T08:09:20.976 INFO:teuthology.orchestra.run.trial034.stdout: debug rocksdb = 4/10 2026-02-18T08:09:20.976 INFO:teuthology.orchestra.run.trial034.stdout: mon osd backfillfull_ratio = 0.85 2026-02-18T08:09:20.976 INFO:teuthology.orchestra.run.trial034.stdout: mon osd full ratio = 0.9 2026-02-18T08:09:20.976 INFO:teuthology.orchestra.run.trial034.stdout: mon osd nearfull ratio = 0.8 2026-02-18T08:09:20.976 INFO:teuthology.orchestra.run.trial034.stdout: osd failsafe full ratio = 0.95 2026-02-18T08:09:20.976 INFO:teuthology.orchestra.run.trial034.stdout: osd objectstore = bluestore 2026-02-18T08:09:20.976 INFO:teuthology.orchestra.run.trial034.stdout: 2026-02-18T08:09:20.976 INFO:teuthology.orchestra.run.trial034.stdout:[mgr] 2026-02-18T08:09:20.976 INFO:teuthology.orchestra.run.trial034.stdout: debug ms = 1 2026-02-18T08:09:20.976 INFO:teuthology.orchestra.run.trial034.stdout: debug mgr = 20 2026-02-18T08:09:20.976 INFO:teuthology.orchestra.run.trial034.stdout: debug mon = 20 2026-02-18T08:09:20.976 INFO:teuthology.orchestra.run.trial034.stdout: debug auth = 20 2026-02-18T08:09:20.976 INFO:teuthology.orchestra.run.trial034.stdout: mon reweight min pgs per osd = 4 2026-02-18T08:09:20.976 INFO:teuthology.orchestra.run.trial034.stdout: mon reweight min bytes per osd = 10 2026-02-18T08:09:20.976 INFO:teuthology.orchestra.run.trial034.stdout: mgr/telemetry/nag = false 2026-02-18T08:09:20.976 INFO:teuthology.orchestra.run.trial034.stdout: 2026-02-18T08:09:20.976 INFO:teuthology.orchestra.run.trial034.stdout:[mon] 2026-02-18T08:09:20.976 INFO:teuthology.orchestra.run.trial034.stdout: debug ms = 1 2026-02-18T08:09:20.976 INFO:teuthology.orchestra.run.trial034.stdout: debug mon = 20 2026-02-18T08:09:20.976 INFO:teuthology.orchestra.run.trial034.stdout: debug paxos = 20 2026-02-18T08:09:20.976 INFO:teuthology.orchestra.run.trial034.stdout: debug auth = 20 2026-02-18T08:09:20.977 INFO:teuthology.orchestra.run.trial034.stdout: mon data avail warn = 5 2026-02-18T08:09:20.977 INFO:teuthology.orchestra.run.trial034.stdout: mon mgr mkfs grace = 240 2026-02-18T08:09:20.977 INFO:teuthology.orchestra.run.trial034.stdout: mon reweight min pgs per osd = 4 2026-02-18T08:09:20.977 INFO:teuthology.orchestra.run.trial034.stdout: mon osd reporter subtree level = osd 2026-02-18T08:09:20.977 INFO:teuthology.orchestra.run.trial034.stdout: mon osd prime pg temp = true 2026-02-18T08:09:20.977 INFO:teuthology.orchestra.run.trial034.stdout: mon reweight min bytes per osd = 10 2026-02-18T08:09:20.977 INFO:teuthology.orchestra.run.trial034.stdout: 2026-02-18T08:09:20.977 INFO:teuthology.orchestra.run.trial034.stdout: # rotate auth tickets quickly to exercise renewal paths 2026-02-18T08:09:20.977 INFO:teuthology.orchestra.run.trial034.stdout: auth mon ticket ttl = 660 # 11m 2026-02-18T08:09:20.977 INFO:teuthology.orchestra.run.trial034.stdout: auth service ticket ttl = 240 # 4m 2026-02-18T08:09:20.977 INFO:teuthology.orchestra.run.trial034.stdout: 2026-02-18T08:09:20.977 INFO:teuthology.orchestra.run.trial034.stdout: # don't complain about insecure global_id in the test suite 2026-02-18T08:09:20.977 INFO:teuthology.orchestra.run.trial034.stdout: mon_warn_on_insecure_global_id_reclaim = false 2026-02-18T08:09:20.977 INFO:teuthology.orchestra.run.trial034.stdout: mon_warn_on_insecure_global_id_reclaim_allowed = false 2026-02-18T08:09:20.977 INFO:teuthology.orchestra.run.trial034.stdout: 2026-02-18T08:09:20.977 INFO:teuthology.orchestra.run.trial034.stdout: # 1m isn't quite enough 2026-02-18T08:09:20.977 INFO:teuthology.orchestra.run.trial034.stdout: mon_down_mkfs_grace = 2m 2026-02-18T08:09:20.977 INFO:teuthology.orchestra.run.trial034.stdout: 2026-02-18T08:09:20.977 INFO:teuthology.orchestra.run.trial034.stdout: mon_warn_on_filestore_osds = false 2026-02-18T08:09:20.977 INFO:teuthology.orchestra.run.trial034.stdout: 2026-02-18T08:09:20.977 INFO:teuthology.orchestra.run.trial034.stdout:[client] 2026-02-18T08:09:20.977 INFO:teuthology.orchestra.run.trial034.stdout: rgw cache enabled = true 2026-02-18T08:09:20.978 INFO:teuthology.orchestra.run.trial034.stdout: rgw enable ops log = true 2026-02-18T08:09:20.978 INFO:teuthology.orchestra.run.trial034.stdout: rgw enable usage log = true 2026-02-18T08:09:20.978 INFO:teuthology.orchestra.run.trial034.stdout: log file = /var/log/ceph/$cluster-$name.$pid.log 2026-02-18T08:09:20.978 INFO:teuthology.orchestra.run.trial034.stdout: admin socket = /var/run/ceph/$cluster-$name.$pid.asok 2026-02-18T08:09:20.978 INFO:teuthology.orchestra.run.trial034.stdout: debug rbd = 20 2026-02-18T08:09:20.978 INFO:teuthology.orchestra.run.trial034.stdout: debug rbd_mirror = 15 2026-02-18T08:09:20.978 INFO:teuthology.orchestra.run.trial034.stdout: log to stderr = False 2026-02-18T08:09:20.978 INFO:teuthology.orchestra.run.trial034.stdout: rbd default features = 125 2026-02-18T08:09:20.978 INFO:teuthology.orchestra.run.trial034.stdout: rbd mirror image policy type = simple 2026-02-18T08:09:20.978 INFO:teuthology.orchestra.run.trial034.stdout:[mon.a] 2026-02-18T08:09:20.978 INFO:teuthology.orchestra.run.trial034.stdout:[client.mirror.0] 2026-02-18T08:09:20.978 INFO:teuthology.orchestra.run.trial034.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-18T08:09:20.978 INFO:teuthology.orchestra.run.trial034.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-18T08:09:20.978 INFO:teuthology.orchestra.run.trial034.stdout:[client.mirror.1] 2026-02-18T08:09:20.978 INFO:teuthology.orchestra.run.trial034.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-18T08:09:20.978 INFO:teuthology.orchestra.run.trial034.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-18T08:09:20.978 INFO:teuthology.orchestra.run.trial034.stdout:[client.mirror.2] 2026-02-18T08:09:20.978 INFO:teuthology.orchestra.run.trial034.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-18T08:09:20.978 INFO:teuthology.orchestra.run.trial034.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-18T08:09:20.978 INFO:teuthology.orchestra.run.trial034.stdout:[client.mirror.3] 2026-02-18T08:09:20.978 INFO:teuthology.orchestra.run.trial034.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-18T08:09:20.978 INFO:teuthology.orchestra.run.trial034.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-18T08:09:20.988 INFO:teuthology.orchestra.run.trial163.stdout:[global] 2026-02-18T08:09:20.989 INFO:teuthology.orchestra.run.trial163.stdout: chdir = "" 2026-02-18T08:09:20.989 INFO:teuthology.orchestra.run.trial163.stdout: pid file = /var/run/ceph/$cluster-$name.pid 2026-02-18T08:09:20.989 INFO:teuthology.orchestra.run.trial163.stdout: auth supported = cephx 2026-02-18T08:09:20.989 INFO:teuthology.orchestra.run.trial163.stdout: 2026-02-18T08:09:20.989 INFO:teuthology.orchestra.run.trial163.stdout: filestore xattr use omap = true 2026-02-18T08:09:20.989 INFO:teuthology.orchestra.run.trial163.stdout: 2026-02-18T08:09:20.989 INFO:teuthology.orchestra.run.trial163.stdout: mon clock drift allowed = 1.000 2026-02-18T08:09:20.989 INFO:teuthology.orchestra.run.trial163.stdout: 2026-02-18T08:09:20.989 INFO:teuthology.orchestra.run.trial163.stdout: osd crush chooseleaf type = 0 2026-02-18T08:09:20.989 INFO:teuthology.orchestra.run.trial163.stdout: auth debug = true 2026-02-18T08:09:20.989 INFO:teuthology.orchestra.run.trial163.stdout: 2026-02-18T08:09:20.989 INFO:teuthology.orchestra.run.trial163.stdout: ms die on old message = true 2026-02-18T08:09:20.989 INFO:teuthology.orchestra.run.trial163.stdout: ms die on bug = true 2026-02-18T08:09:20.989 INFO:teuthology.orchestra.run.trial163.stdout: 2026-02-18T08:09:20.989 INFO:teuthology.orchestra.run.trial163.stdout: mon max pg per osd = 10000 # >= luminous 2026-02-18T08:09:20.989 INFO:teuthology.orchestra.run.trial163.stdout: mon pg warn max object skew = 0 2026-02-18T08:09:20.989 INFO:teuthology.orchestra.run.trial163.stdout: 2026-02-18T08:09:20.989 INFO:teuthology.orchestra.run.trial163.stdout: # disable pg_autoscaler by default for new pools 2026-02-18T08:09:20.989 INFO:teuthology.orchestra.run.trial163.stdout: osd_pool_default_pg_autoscale_mode = off 2026-02-18T08:09:20.989 INFO:teuthology.orchestra.run.trial163.stdout: 2026-02-18T08:09:20.989 INFO:teuthology.orchestra.run.trial163.stdout: osd pool default size = 2 2026-02-18T08:09:20.990 INFO:teuthology.orchestra.run.trial163.stdout: 2026-02-18T08:09:20.990 INFO:teuthology.orchestra.run.trial163.stdout: mon osd allow primary affinity = true 2026-02-18T08:09:20.990 INFO:teuthology.orchestra.run.trial163.stdout: mon osd allow pg remap = true 2026-02-18T08:09:20.990 INFO:teuthology.orchestra.run.trial163.stdout: mon warn on legacy crush tunables = false 2026-02-18T08:09:20.990 INFO:teuthology.orchestra.run.trial163.stdout: mon warn on crush straw calc version zero = false 2026-02-18T08:09:20.990 INFO:teuthology.orchestra.run.trial163.stdout: mon warn on no sortbitwise = false 2026-02-18T08:09:20.990 INFO:teuthology.orchestra.run.trial163.stdout: mon warn on osd down out interval zero = false 2026-02-18T08:09:20.990 INFO:teuthology.orchestra.run.trial163.stdout: mon warn on too few osds = false 2026-02-18T08:09:20.990 INFO:teuthology.orchestra.run.trial163.stdout: mon_warn_on_pool_pg_num_not_power_of_two = false 2026-02-18T08:09:20.990 INFO:teuthology.orchestra.run.trial163.stdout: mon_warn_on_pool_no_redundancy = false 2026-02-18T08:09:20.990 INFO:teuthology.orchestra.run.trial163.stdout: mon_allow_pool_size_one = true 2026-02-18T08:09:20.990 INFO:teuthology.orchestra.run.trial163.stdout: 2026-02-18T08:09:20.990 INFO:teuthology.orchestra.run.trial163.stdout: osd pool default erasure code profile = plugin=isa technique=reed_sol_van k=2 m=1 crush-failure-domain=osd 2026-02-18T08:09:20.990 INFO:teuthology.orchestra.run.trial163.stdout: 2026-02-18T08:09:20.990 INFO:teuthology.orchestra.run.trial163.stdout: osd default data pool replay window = 5 2026-02-18T08:09:20.990 INFO:teuthology.orchestra.run.trial163.stdout: 2026-02-18T08:09:20.990 INFO:teuthology.orchestra.run.trial163.stdout: mon allow pool delete = true 2026-02-18T08:09:20.990 INFO:teuthology.orchestra.run.trial163.stdout: 2026-02-18T08:09:20.990 INFO:teuthology.orchestra.run.trial163.stdout: mon cluster log file level = debug 2026-02-18T08:09:20.990 INFO:teuthology.orchestra.run.trial163.stdout: debug asserts on shutdown = true 2026-02-18T08:09:20.990 INFO:teuthology.orchestra.run.trial163.stdout: mon health detail to clog = false 2026-02-18T08:09:20.990 INFO:teuthology.orchestra.run.trial163.stdout: mon host = 10.20.193.163 2026-02-18T08:09:20.991 INFO:teuthology.orchestra.run.trial163.stdout: mon client directed command retry = 5 2026-02-18T08:09:20.991 INFO:teuthology.orchestra.run.trial163.stdout: mon warn on pool no app = False 2026-02-18T08:09:20.991 INFO:teuthology.orchestra.run.trial163.stdout: ms inject socket failures = 5000 2026-02-18T08:09:20.991 INFO:teuthology.orchestra.run.trial163.stdout: fsid = 892b71e6-8019-418b-97e7-5a7b63a9d866 2026-02-18T08:09:20.991 INFO:teuthology.orchestra.run.trial163.stdout: 2026-02-18T08:09:20.991 INFO:teuthology.orchestra.run.trial163.stdout:[osd] 2026-02-18T08:09:20.991 INFO:teuthology.orchestra.run.trial163.stdout: osd journal size = 100 2026-02-18T08:09:20.991 INFO:teuthology.orchestra.run.trial163.stdout: 2026-02-18T08:09:20.991 INFO:teuthology.orchestra.run.trial163.stdout: osd scrub load threshold = 5.0 2026-02-18T08:09:20.991 INFO:teuthology.orchestra.run.trial163.stdout: osd scrub max interval = 600 2026-02-18T08:09:20.991 INFO:teuthology.orchestra.run.trial163.stdout: osd mclock profile = high_recovery_ops 2026-02-18T08:09:20.991 INFO:teuthology.orchestra.run.trial163.stdout: osd mclock skip benchmark = true 2026-02-18T08:09:20.991 INFO:teuthology.orchestra.run.trial163.stdout: 2026-02-18T08:09:20.991 INFO:teuthology.orchestra.run.trial163.stdout: osd recover clone overlap = true 2026-02-18T08:09:20.991 INFO:teuthology.orchestra.run.trial163.stdout: osd recovery max chunk = 1048576 2026-02-18T08:09:20.991 INFO:teuthology.orchestra.run.trial163.stdout: 2026-02-18T08:09:20.991 INFO:teuthology.orchestra.run.trial163.stdout: osd debug shutdown = true 2026-02-18T08:09:20.991 INFO:teuthology.orchestra.run.trial163.stdout: osd debug op order = true 2026-02-18T08:09:20.991 INFO:teuthology.orchestra.run.trial163.stdout: osd debug verify stray on activate = true 2026-02-18T08:09:20.991 INFO:teuthology.orchestra.run.trial163.stdout: osd debug trim objects = true 2026-02-18T08:09:20.991 INFO:teuthology.orchestra.run.trial163.stdout: 2026-02-18T08:09:20.991 INFO:teuthology.orchestra.run.trial163.stdout: osd open classes on start = true 2026-02-18T08:09:20.992 INFO:teuthology.orchestra.run.trial163.stdout: osd debug pg log writeout = true 2026-02-18T08:09:20.992 INFO:teuthology.orchestra.run.trial163.stdout: 2026-02-18T08:09:20.992 INFO:teuthology.orchestra.run.trial163.stdout: osd deep scrub update digest min age = 30 2026-02-18T08:09:20.992 INFO:teuthology.orchestra.run.trial163.stdout: 2026-02-18T08:09:20.992 INFO:teuthology.orchestra.run.trial163.stdout: osd map max advance = 10 2026-02-18T08:09:20.992 INFO:teuthology.orchestra.run.trial163.stdout: 2026-02-18T08:09:20.992 INFO:teuthology.orchestra.run.trial163.stdout: journal zero on create = true 2026-02-18T08:09:20.992 INFO:teuthology.orchestra.run.trial163.stdout: 2026-02-18T08:09:20.992 INFO:teuthology.orchestra.run.trial163.stdout: filestore ondisk finisher threads = 3 2026-02-18T08:09:20.992 INFO:teuthology.orchestra.run.trial163.stdout: filestore apply finisher threads = 3 2026-02-18T08:09:20.992 INFO:teuthology.orchestra.run.trial163.stdout: 2026-02-18T08:09:20.992 INFO:teuthology.orchestra.run.trial163.stdout: bdev debug aio = true 2026-02-18T08:09:20.992 INFO:teuthology.orchestra.run.trial163.stdout: osd debug misdirected ops = true 2026-02-18T08:09:20.992 INFO:teuthology.orchestra.run.trial163.stdout: bluefs allocator = hybrid 2026-02-18T08:09:20.992 INFO:teuthology.orchestra.run.trial163.stdout: bluestore allocator = hybrid 2026-02-18T08:09:20.992 INFO:teuthology.orchestra.run.trial163.stdout: bluestore block size = 96636764160 2026-02-18T08:09:20.992 INFO:teuthology.orchestra.run.trial163.stdout: bluestore fsck on mount = True 2026-02-18T08:09:20.992 INFO:teuthology.orchestra.run.trial163.stdout: debug bluefs = 1/20 2026-02-18T08:09:20.992 INFO:teuthology.orchestra.run.trial163.stdout: debug bluestore = 1/20 2026-02-18T08:09:20.992 INFO:teuthology.orchestra.run.trial163.stdout: debug ms = 1 2026-02-18T08:09:20.992 INFO:teuthology.orchestra.run.trial163.stdout: debug osd = 20 2026-02-18T08:09:20.992 INFO:teuthology.orchestra.run.trial163.stdout: debug rocksdb = 4/10 2026-02-18T08:09:20.993 INFO:teuthology.orchestra.run.trial163.stdout: mon osd backfillfull_ratio = 0.85 2026-02-18T08:09:20.993 INFO:teuthology.orchestra.run.trial163.stdout: mon osd full ratio = 0.9 2026-02-18T08:09:20.993 INFO:teuthology.orchestra.run.trial163.stdout: mon osd nearfull ratio = 0.8 2026-02-18T08:09:20.993 INFO:teuthology.orchestra.run.trial163.stdout: osd failsafe full ratio = 0.95 2026-02-18T08:09:20.993 INFO:teuthology.orchestra.run.trial163.stdout: osd objectstore = bluestore 2026-02-18T08:09:20.993 INFO:teuthology.orchestra.run.trial163.stdout: 2026-02-18T08:09:20.993 INFO:teuthology.orchestra.run.trial163.stdout:[mgr] 2026-02-18T08:09:20.993 INFO:teuthology.orchestra.run.trial163.stdout: debug ms = 1 2026-02-18T08:09:20.993 INFO:teuthology.orchestra.run.trial163.stdout: debug mgr = 20 2026-02-18T08:09:20.993 INFO:teuthology.orchestra.run.trial163.stdout: debug mon = 20 2026-02-18T08:09:20.993 INFO:teuthology.orchestra.run.trial163.stdout: debug auth = 20 2026-02-18T08:09:20.993 INFO:teuthology.orchestra.run.trial163.stdout: mon reweight min pgs per osd = 4 2026-02-18T08:09:20.993 INFO:teuthology.orchestra.run.trial163.stdout: mon reweight min bytes per osd = 10 2026-02-18T08:09:20.993 INFO:teuthology.orchestra.run.trial163.stdout: mgr/telemetry/nag = false 2026-02-18T08:09:20.993 INFO:teuthology.orchestra.run.trial163.stdout: 2026-02-18T08:09:20.993 INFO:teuthology.orchestra.run.trial163.stdout:[mon] 2026-02-18T08:09:20.994 INFO:teuthology.orchestra.run.trial163.stdout: debug ms = 1 2026-02-18T08:09:20.994 INFO:teuthology.orchestra.run.trial163.stdout: debug mon = 20 2026-02-18T08:09:20.994 INFO:teuthology.orchestra.run.trial163.stdout: debug paxos = 20 2026-02-18T08:09:20.994 INFO:teuthology.orchestra.run.trial163.stdout: debug auth = 20 2026-02-18T08:09:20.994 INFO:teuthology.orchestra.run.trial163.stdout: mon data avail warn = 5 2026-02-18T08:09:20.994 INFO:teuthology.orchestra.run.trial163.stdout: mon mgr mkfs grace = 240 2026-02-18T08:09:20.994 INFO:teuthology.orchestra.run.trial163.stdout: mon reweight min pgs per osd = 4 2026-02-18T08:09:20.994 INFO:teuthology.orchestra.run.trial163.stdout: mon osd reporter subtree level = osd 2026-02-18T08:09:20.994 INFO:teuthology.orchestra.run.trial163.stdout: mon osd prime pg temp = true 2026-02-18T08:09:20.994 INFO:teuthology.orchestra.run.trial163.stdout: mon reweight min bytes per osd = 10 2026-02-18T08:09:20.994 INFO:teuthology.orchestra.run.trial163.stdout: 2026-02-18T08:09:20.994 INFO:teuthology.orchestra.run.trial163.stdout: # rotate auth tickets quickly to exercise renewal paths 2026-02-18T08:09:20.994 INFO:teuthology.orchestra.run.trial163.stdout: auth mon ticket ttl = 660 # 11m 2026-02-18T08:09:20.994 INFO:teuthology.orchestra.run.trial163.stdout: auth service ticket ttl = 240 # 4m 2026-02-18T08:09:20.994 INFO:teuthology.orchestra.run.trial163.stdout: 2026-02-18T08:09:20.994 INFO:teuthology.orchestra.run.trial163.stdout: # don't complain about insecure global_id in the test suite 2026-02-18T08:09:20.995 INFO:teuthology.orchestra.run.trial163.stdout: mon_warn_on_insecure_global_id_reclaim = false 2026-02-18T08:09:20.995 INFO:teuthology.orchestra.run.trial163.stdout: mon_warn_on_insecure_global_id_reclaim_allowed = false 2026-02-18T08:09:20.995 INFO:teuthology.orchestra.run.trial163.stdout: 2026-02-18T08:09:20.995 INFO:teuthology.orchestra.run.trial163.stdout: # 1m isn't quite enough 2026-02-18T08:09:20.995 INFO:teuthology.orchestra.run.trial163.stdout: mon_down_mkfs_grace = 2m 2026-02-18T08:09:20.995 INFO:teuthology.orchestra.run.trial163.stdout: 2026-02-18T08:09:20.995 INFO:teuthology.orchestra.run.trial163.stdout: mon_warn_on_filestore_osds = false 2026-02-18T08:09:20.995 INFO:teuthology.orchestra.run.trial163.stdout: 2026-02-18T08:09:20.995 INFO:teuthology.orchestra.run.trial163.stdout:[client] 2026-02-18T08:09:20.995 INFO:teuthology.orchestra.run.trial163.stdout: rgw cache enabled = true 2026-02-18T08:09:20.995 INFO:teuthology.orchestra.run.trial163.stdout: rgw enable ops log = true 2026-02-18T08:09:20.995 INFO:teuthology.orchestra.run.trial163.stdout: rgw enable usage log = true 2026-02-18T08:09:20.995 INFO:teuthology.orchestra.run.trial163.stdout: log file = /var/log/ceph/$cluster-$name.$pid.log 2026-02-18T08:09:20.995 INFO:teuthology.orchestra.run.trial163.stdout: admin socket = /var/run/ceph/$cluster-$name.$pid.asok 2026-02-18T08:09:20.995 INFO:teuthology.orchestra.run.trial163.stdout: debug rbd = 20 2026-02-18T08:09:20.996 INFO:teuthology.orchestra.run.trial163.stdout: debug rbd_mirror = 15 2026-02-18T08:09:20.996 INFO:teuthology.orchestra.run.trial163.stdout: log to stderr = False 2026-02-18T08:09:20.996 INFO:teuthology.orchestra.run.trial163.stdout: rbd default features = 125 2026-02-18T08:09:20.996 INFO:teuthology.orchestra.run.trial163.stdout: rbd mirror image policy type = simple 2026-02-18T08:09:20.996 INFO:teuthology.orchestra.run.trial163.stdout:[mon.a] 2026-02-18T08:09:20.996 INFO:teuthology.orchestra.run.trial163.stdout:[client.mirror.0] 2026-02-18T08:09:20.996 INFO:teuthology.orchestra.run.trial163.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-18T08:09:20.996 INFO:teuthology.orchestra.run.trial163.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-18T08:09:20.996 INFO:teuthology.orchestra.run.trial163.stdout:[client.mirror.1] 2026-02-18T08:09:20.996 INFO:teuthology.orchestra.run.trial163.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-18T08:09:20.996 INFO:teuthology.orchestra.run.trial163.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-18T08:09:20.996 INFO:teuthology.orchestra.run.trial163.stdout:[client.mirror.2] 2026-02-18T08:09:20.996 INFO:teuthology.orchestra.run.trial163.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-18T08:09:20.996 INFO:teuthology.orchestra.run.trial163.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-18T08:09:20.996 INFO:teuthology.orchestra.run.trial163.stdout:[client.mirror.3] 2026-02-18T08:09:20.996 INFO:teuthology.orchestra.run.trial163.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-18T08:09:20.997 INFO:teuthology.orchestra.run.trial163.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-18T08:09:20.997 INFO:tasks.ceph:Creating admin key on cluster2.mon.a... 2026-02-18T08:09:20.997 DEBUG:teuthology.orchestra.run.trial163:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --gen-key --name=client.admin --cap mon 'allow *' --cap osd 'allow *' --cap mds 'allow *' --cap mgr 'allow *' /etc/ceph/cluster2.keyring 2026-02-18T08:09:21.061 INFO:tasks.ceph:Copying monmap to all nodes... 2026-02-18T08:09:21.061 DEBUG:teuthology.orchestra.run.trial163:> set -ex 2026-02-18T08:09:21.061 DEBUG:teuthology.orchestra.run.trial163:> dd if=/etc/ceph/cluster2.keyring of=/dev/stdout 2026-02-18T08:09:21.109 DEBUG:teuthology.orchestra.run.trial163:> set -ex 2026-02-18T08:09:21.109 DEBUG:teuthology.orchestra.run.trial163:> dd if=/home/ubuntu/cephtest/cluster2.monmap of=/dev/stdout 2026-02-18T08:09:21.153 INFO:tasks.ceph:Sending monmap to node ubuntu@trial034.front.sepia.ceph.com 2026-02-18T08:09:21.153 DEBUG:teuthology.orchestra.run.trial034:> set -ex 2026-02-18T08:09:21.153 DEBUG:teuthology.orchestra.run.trial034:> sudo dd of=/etc/ceph/cluster2.keyring 2026-02-18T08:09:21.153 DEBUG:teuthology.orchestra.run.trial034:> sudo chmod 0644 /etc/ceph/cluster2.keyring 2026-02-18T08:09:21.168 DEBUG:teuthology.orchestra.run.trial034:> set -ex 2026-02-18T08:09:21.168 DEBUG:teuthology.orchestra.run.trial034:> dd of=/home/ubuntu/cephtest/cluster2.monmap 2026-02-18T08:09:21.212 INFO:tasks.ceph:Sending monmap to node ubuntu@trial163.front.sepia.ceph.com 2026-02-18T08:09:21.212 DEBUG:teuthology.orchestra.run.trial163:> set -ex 2026-02-18T08:09:21.212 DEBUG:teuthology.orchestra.run.trial163:> sudo dd of=/etc/ceph/cluster2.keyring 2026-02-18T08:09:21.212 DEBUG:teuthology.orchestra.run.trial163:> sudo chmod 0644 /etc/ceph/cluster2.keyring 2026-02-18T08:09:21.227 DEBUG:teuthology.orchestra.run.trial163:> set -ex 2026-02-18T08:09:21.227 DEBUG:teuthology.orchestra.run.trial163:> dd of=/home/ubuntu/cephtest/cluster2.monmap 2026-02-18T08:09:21.273 INFO:tasks.ceph:Setting up mon nodes... 2026-02-18T08:09:21.273 INFO:tasks.ceph:Setting up mgr nodes... 2026-02-18T08:09:21.273 DEBUG:teuthology.orchestra.run.trial034:> sudo mkdir -p /var/lib/ceph/mgr/cluster2-x && sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=mgr.x /var/lib/ceph/mgr/cluster2-x/keyring 2026-02-18T08:09:21.295 INFO:teuthology.orchestra.run.trial034.stdout:creating /var/lib/ceph/mgr/cluster2-x/keyring 2026-02-18T08:09:21.297 INFO:tasks.ceph:Setting up mds nodes... 2026-02-18T08:09:21.297 INFO:tasks.ceph_client:Setting up client nodes... 2026-02-18T08:09:21.297 DEBUG:teuthology.orchestra.run.trial034:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.0 /etc/ceph/cluster2.client.0.keyring && sudo chmod 0644 /etc/ceph/cluster2.client.0.keyring 2026-02-18T08:09:21.357 INFO:teuthology.orchestra.run.trial034.stdout:creating /etc/ceph/cluster2.client.0.keyring 2026-02-18T08:09:21.364 DEBUG:teuthology.orchestra.run.trial163:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.mirror /etc/ceph/cluster2.client.mirror.keyring && sudo chmod 0644 /etc/ceph/cluster2.client.mirror.keyring 2026-02-18T08:09:21.383 INFO:teuthology.orchestra.run.trial163.stdout:creating /etc/ceph/cluster2.client.mirror.keyring 2026-02-18T08:09:21.390 DEBUG:teuthology.orchestra.run.trial163:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.mirror.0 /etc/ceph/cluster2.client.mirror.0.keyring && sudo chmod 0644 /etc/ceph/cluster2.client.mirror.0.keyring 2026-02-18T08:09:21.450 INFO:teuthology.orchestra.run.trial163.stdout:creating /etc/ceph/cluster2.client.mirror.0.keyring 2026-02-18T08:09:21.457 DEBUG:teuthology.orchestra.run.trial163:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.mirror.1 /etc/ceph/cluster2.client.mirror.1.keyring && sudo chmod 0644 /etc/ceph/cluster2.client.mirror.1.keyring 2026-02-18T08:09:21.514 INFO:teuthology.orchestra.run.trial163.stdout:creating /etc/ceph/cluster2.client.mirror.1.keyring 2026-02-18T08:09:21.521 DEBUG:teuthology.orchestra.run.trial163:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.mirror.2 /etc/ceph/cluster2.client.mirror.2.keyring && sudo chmod 0644 /etc/ceph/cluster2.client.mirror.2.keyring 2026-02-18T08:09:21.577 INFO:teuthology.orchestra.run.trial163.stdout:creating /etc/ceph/cluster2.client.mirror.2.keyring 2026-02-18T08:09:21.584 DEBUG:teuthology.orchestra.run.trial163:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.mirror.3 /etc/ceph/cluster2.client.mirror.3.keyring && sudo chmod 0644 /etc/ceph/cluster2.client.mirror.3.keyring 2026-02-18T08:09:21.642 INFO:teuthology.orchestra.run.trial163.stdout:creating /etc/ceph/cluster2.client.mirror.3.keyring 2026-02-18T08:09:21.649 DEBUG:teuthology.orchestra.run.trial163:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.mirror.4 /etc/ceph/cluster2.client.mirror.4.keyring && sudo chmod 0644 /etc/ceph/cluster2.client.mirror.4.keyring 2026-02-18T08:09:21.706 INFO:teuthology.orchestra.run.trial163.stdout:creating /etc/ceph/cluster2.client.mirror.4.keyring 2026-02-18T08:09:21.713 DEBUG:teuthology.orchestra.run.trial163:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.mirror.5 /etc/ceph/cluster2.client.mirror.5.keyring && sudo chmod 0644 /etc/ceph/cluster2.client.mirror.5.keyring 2026-02-18T08:09:21.770 INFO:teuthology.orchestra.run.trial163.stdout:creating /etc/ceph/cluster2.client.mirror.5.keyring 2026-02-18T08:09:21.777 DEBUG:teuthology.orchestra.run.trial163:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.mirror.6 /etc/ceph/cluster2.client.mirror.6.keyring && sudo chmod 0644 /etc/ceph/cluster2.client.mirror.6.keyring 2026-02-18T08:09:21.833 INFO:teuthology.orchestra.run.trial163.stdout:creating /etc/ceph/cluster2.client.mirror.6.keyring 2026-02-18T08:09:21.840 INFO:tasks.ceph:Running mkfs on osd nodes... 2026-02-18T08:09:21.840 INFO:tasks.ceph:ctx.disk_config.remote_to_roles_to_dev: {Remote(name='ubuntu@trial034.front.sepia.ceph.com'): {'cluster1.osd.0': '/dev/vg_nvme/lv_1', 'cluster1.osd.1': '/dev/vg_nvme/lv_2', 'cluster1.osd.2': '/dev/vg_nvme/lv_3'}, Remote(name='ubuntu@trial163.front.sepia.ceph.com'): {'cluster2.osd.0': '/dev/vg_nvme/lv_1', 'cluster2.osd.1': '/dev/vg_nvme/lv_2', 'cluster2.osd.2': '/dev/vg_nvme/lv_3'}} 2026-02-18T08:09:21.841 DEBUG:teuthology.orchestra.run.trial163:> sudo mkdir -p /var/lib/ceph/osd/cluster2-0 2026-02-18T08:09:21.889 INFO:tasks.ceph:roles_to_devs: {'cluster2.osd.0': '/dev/vg_nvme/lv_1', 'cluster2.osd.1': '/dev/vg_nvme/lv_2', 'cluster2.osd.2': '/dev/vg_nvme/lv_3'} 2026-02-18T08:09:21.889 INFO:tasks.ceph:role: cluster2.osd.0 2026-02-18T08:09:21.890 INFO:tasks.ceph:['mkfs.xfs', '-f', '-i', 'size=2048'] on /dev/vg_nvme/lv_1 on ubuntu@trial163.front.sepia.ceph.com 2026-02-18T08:09:21.890 DEBUG:teuthology.orchestra.run.trial163:> yes | sudo mkfs.xfs -f -i size=2048 /dev/vg_nvme/lv_1 2026-02-18T08:09:21.939 INFO:teuthology.orchestra.run.trial163.stdout:meta-data=/dev/vg_nvme/lv_1 isize=2048 agcount=4, agsize=45875200 blks 2026-02-18T08:09:21.939 INFO:teuthology.orchestra.run.trial163.stdout: = sectsz=4096 attr=2, projid32bit=1 2026-02-18T08:09:21.939 INFO:teuthology.orchestra.run.trial163.stdout: = crc=1 finobt=1, sparse=1, rmapbt=0 2026-02-18T08:09:21.940 INFO:teuthology.orchestra.run.trial163.stdout: = reflink=1 bigtime=0 inobtcount=0 2026-02-18T08:09:21.940 INFO:teuthology.orchestra.run.trial163.stdout:data = bsize=4096 blocks=183500800, imaxpct=25 2026-02-18T08:09:21.940 INFO:teuthology.orchestra.run.trial163.stdout: = sunit=0 swidth=0 blks 2026-02-18T08:09:21.940 INFO:teuthology.orchestra.run.trial163.stdout:naming =version 2 bsize=4096 ascii-ci=0, ftype=1 2026-02-18T08:09:21.940 INFO:teuthology.orchestra.run.trial163.stdout:log =internal log bsize=4096 blocks=89600, version=2 2026-02-18T08:09:21.940 INFO:teuthology.orchestra.run.trial163.stdout: = sectsz=4096 sunit=1 blks, lazy-count=1 2026-02-18T08:09:21.940 INFO:teuthology.orchestra.run.trial163.stdout:realtime =none extsz=4096 blocks=0, rtextents=0 2026-02-18T08:09:22.515 INFO:teuthology.orchestra.run.trial163.stdout:Discarding blocks...Done. 2026-02-18T08:09:22.516 INFO:tasks.ceph:mount /dev/vg_nvme/lv_1 on ubuntu@trial163.front.sepia.ceph.com -o noatime 2026-02-18T08:09:22.517 DEBUG:teuthology.orchestra.run.trial163:> sudo mount -t xfs -o noatime /dev/vg_nvme/lv_1 /var/lib/ceph/osd/cluster2-0 2026-02-18T08:09:22.569 DEBUG:teuthology.orchestra.run.trial163:> sudo /sbin/restorecon /var/lib/ceph/osd/cluster2-0 2026-02-18T08:09:22.615 INFO:teuthology.orchestra.run.trial163.stderr:sudo: /sbin/restorecon: command not found 2026-02-18T08:09:22.615 DEBUG:teuthology.orchestra.run:got remote process result: 1 2026-02-18T08:09:22.615 DEBUG:teuthology.orchestra.run.trial163:> sudo mkdir -p /var/lib/ceph/osd/cluster2-1 2026-02-18T08:09:22.665 INFO:tasks.ceph:roles_to_devs: {'cluster2.osd.0': '/dev/vg_nvme/lv_1', 'cluster2.osd.1': '/dev/vg_nvme/lv_2', 'cluster2.osd.2': '/dev/vg_nvme/lv_3'} 2026-02-18T08:09:22.665 INFO:tasks.ceph:role: cluster2.osd.1 2026-02-18T08:09:22.665 INFO:tasks.ceph:['mkfs.xfs', '-f', '-i', 'size=2048'] on /dev/vg_nvme/lv_2 on ubuntu@trial163.front.sepia.ceph.com 2026-02-18T08:09:22.666 DEBUG:teuthology.orchestra.run.trial163:> yes | sudo mkfs.xfs -f -i size=2048 /dev/vg_nvme/lv_2 2026-02-18T08:09:22.715 INFO:teuthology.orchestra.run.trial163.stdout:meta-data=/dev/vg_nvme/lv_2 isize=2048 agcount=4, agsize=45875200 blks 2026-02-18T08:09:22.715 INFO:teuthology.orchestra.run.trial163.stdout: = sectsz=4096 attr=2, projid32bit=1 2026-02-18T08:09:22.715 INFO:teuthology.orchestra.run.trial163.stdout: = crc=1 finobt=1, sparse=1, rmapbt=0 2026-02-18T08:09:22.715 INFO:teuthology.orchestra.run.trial163.stdout: = reflink=1 bigtime=0 inobtcount=0 2026-02-18T08:09:22.715 INFO:teuthology.orchestra.run.trial163.stdout:data = bsize=4096 blocks=183500800, imaxpct=25 2026-02-18T08:09:22.715 INFO:teuthology.orchestra.run.trial163.stdout: = sunit=0 swidth=0 blks 2026-02-18T08:09:22.715 INFO:teuthology.orchestra.run.trial163.stdout:naming =version 2 bsize=4096 ascii-ci=0, ftype=1 2026-02-18T08:09:22.715 INFO:teuthology.orchestra.run.trial163.stdout:log =internal log bsize=4096 blocks=89600, version=2 2026-02-18T08:09:22.716 INFO:teuthology.orchestra.run.trial163.stdout: = sectsz=4096 sunit=1 blks, lazy-count=1 2026-02-18T08:09:22.716 INFO:teuthology.orchestra.run.trial163.stdout:realtime =none extsz=4096 blocks=0, rtextents=0 2026-02-18T08:09:23.281 INFO:teuthology.orchestra.run.trial163.stdout:Discarding blocks...Done. 2026-02-18T08:09:23.283 INFO:tasks.ceph:mount /dev/vg_nvme/lv_2 on ubuntu@trial163.front.sepia.ceph.com -o noatime 2026-02-18T08:09:23.284 DEBUG:teuthology.orchestra.run.trial163:> sudo mount -t xfs -o noatime /dev/vg_nvme/lv_2 /var/lib/ceph/osd/cluster2-1 2026-02-18T08:09:23.348 DEBUG:teuthology.orchestra.run.trial163:> sudo /sbin/restorecon /var/lib/ceph/osd/cluster2-1 2026-02-18T08:09:23.399 INFO:teuthology.orchestra.run.trial163.stderr:sudo: /sbin/restorecon: command not found 2026-02-18T08:09:23.400 DEBUG:teuthology.orchestra.run:got remote process result: 1 2026-02-18T08:09:23.400 DEBUG:teuthology.orchestra.run.trial163:> sudo mkdir -p /var/lib/ceph/osd/cluster2-2 2026-02-18T08:09:23.451 INFO:tasks.ceph:roles_to_devs: {'cluster2.osd.0': '/dev/vg_nvme/lv_1', 'cluster2.osd.1': '/dev/vg_nvme/lv_2', 'cluster2.osd.2': '/dev/vg_nvme/lv_3'} 2026-02-18T08:09:23.451 INFO:tasks.ceph:role: cluster2.osd.2 2026-02-18T08:09:23.451 INFO:tasks.ceph:['mkfs.xfs', '-f', '-i', 'size=2048'] on /dev/vg_nvme/lv_3 on ubuntu@trial163.front.sepia.ceph.com 2026-02-18T08:09:23.451 DEBUG:teuthology.orchestra.run.trial163:> yes | sudo mkfs.xfs -f -i size=2048 /dev/vg_nvme/lv_3 2026-02-18T08:09:23.503 INFO:teuthology.orchestra.run.trial163.stdout:meta-data=/dev/vg_nvme/lv_3 isize=2048 agcount=4, agsize=45875200 blks 2026-02-18T08:09:23.503 INFO:teuthology.orchestra.run.trial163.stdout: = sectsz=4096 attr=2, projid32bit=1 2026-02-18T08:09:23.503 INFO:teuthology.orchestra.run.trial163.stdout: = crc=1 finobt=1, sparse=1, rmapbt=0 2026-02-18T08:09:23.503 INFO:teuthology.orchestra.run.trial163.stdout: = reflink=1 bigtime=0 inobtcount=0 2026-02-18T08:09:23.503 INFO:teuthology.orchestra.run.trial163.stdout:data = bsize=4096 blocks=183500800, imaxpct=25 2026-02-18T08:09:23.503 INFO:teuthology.orchestra.run.trial163.stdout: = sunit=0 swidth=0 blks 2026-02-18T08:09:23.503 INFO:teuthology.orchestra.run.trial163.stdout:naming =version 2 bsize=4096 ascii-ci=0, ftype=1 2026-02-18T08:09:23.504 INFO:teuthology.orchestra.run.trial163.stdout:log =internal log bsize=4096 blocks=89600, version=2 2026-02-18T08:09:23.504 INFO:teuthology.orchestra.run.trial163.stdout: = sectsz=4096 sunit=1 blks, lazy-count=1 2026-02-18T08:09:23.504 INFO:teuthology.orchestra.run.trial163.stdout:realtime =none extsz=4096 blocks=0, rtextents=0 2026-02-18T08:09:24.073 INFO:teuthology.orchestra.run.trial163.stdout:Discarding blocks...Done. 2026-02-18T08:09:24.074 INFO:tasks.ceph:mount /dev/vg_nvme/lv_3 on ubuntu@trial163.front.sepia.ceph.com -o noatime 2026-02-18T08:09:24.075 DEBUG:teuthology.orchestra.run.trial163:> sudo mount -t xfs -o noatime /dev/vg_nvme/lv_3 /var/lib/ceph/osd/cluster2-2 2026-02-18T08:09:24.135 DEBUG:teuthology.orchestra.run.trial163:> sudo /sbin/restorecon /var/lib/ceph/osd/cluster2-2 2026-02-18T08:09:24.183 INFO:teuthology.orchestra.run.trial163.stderr:sudo: /sbin/restorecon: command not found 2026-02-18T08:09:24.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2026-02-18T08:09:24.184 DEBUG:teuthology.orchestra.run.trial163:> sudo MALLOC_CHECK_=3 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-osd --no-mon-config --cluster cluster2 --mkfs --mkkey -i 0 --monmap /home/ubuntu/cephtest/cluster2.monmap 2026-02-18T08:09:24.247 INFO:teuthology.orchestra.run.trial163.stderr:2026-02-18T08:09:24.241+0000 7fa017608a40 -1 auth: error reading file: /var/lib/ceph/osd/cluster2-0/keyring: can't open /var/lib/ceph/osd/cluster2-0/keyring: (2) No such file or directory 2026-02-18T08:09:24.247 INFO:teuthology.orchestra.run.trial163.stderr:2026-02-18T08:09:24.241+0000 7fa017608a40 -1 created new key in keyring /var/lib/ceph/osd/cluster2-0/keyring 2026-02-18T08:09:24.247 INFO:teuthology.orchestra.run.trial163.stderr:2026-02-18T08:09:24.241+0000 7fa017608a40 -1 bdev(0x55ddc5551800 /var/lib/ceph/osd/cluster2-0/block) open stat got: (1) Operation not permitted 2026-02-18T08:09:24.247 INFO:teuthology.orchestra.run.trial163.stderr:2026-02-18T08:09:24.241+0000 7fa017608a40 -1 bluestore(/var/lib/ceph/osd/cluster2-0) _read_fsid unparsable uuid 2026-02-18T08:09:25.093 DEBUG:teuthology.orchestra.run.trial163:> sudo chown -R ceph:ceph /var/lib/ceph/osd/cluster2-0 2026-02-18T08:09:25.141 DEBUG:teuthology.orchestra.run.trial163:> sudo MALLOC_CHECK_=3 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-osd --no-mon-config --cluster cluster2 --mkfs --mkkey -i 1 --monmap /home/ubuntu/cephtest/cluster2.monmap 2026-02-18T08:09:25.206 INFO:teuthology.orchestra.run.trial163.stderr:2026-02-18T08:09:25.201+0000 7f5858e3da40 -1 auth: error reading file: /var/lib/ceph/osd/cluster2-1/keyring: can't open /var/lib/ceph/osd/cluster2-1/keyring: (2) No such file or directory 2026-02-18T08:09:25.207 INFO:teuthology.orchestra.run.trial163.stderr:2026-02-18T08:09:25.201+0000 7f5858e3da40 -1 created new key in keyring /var/lib/ceph/osd/cluster2-1/keyring 2026-02-18T08:09:25.207 INFO:teuthology.orchestra.run.trial163.stderr:2026-02-18T08:09:25.201+0000 7f5858e3da40 -1 bdev(0x557793b8d800 /var/lib/ceph/osd/cluster2-1/block) open stat got: (1) Operation not permitted 2026-02-18T08:09:25.207 INFO:teuthology.orchestra.run.trial163.stderr:2026-02-18T08:09:25.201+0000 7f5858e3da40 -1 bluestore(/var/lib/ceph/osd/cluster2-1) _read_fsid unparsable uuid 2026-02-18T08:09:26.102 DEBUG:teuthology.orchestra.run.trial163:> sudo chown -R ceph:ceph /var/lib/ceph/osd/cluster2-1 2026-02-18T08:09:26.154 DEBUG:teuthology.orchestra.run.trial163:> sudo MALLOC_CHECK_=3 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-osd --no-mon-config --cluster cluster2 --mkfs --mkkey -i 2 --monmap /home/ubuntu/cephtest/cluster2.monmap 2026-02-18T08:09:26.217 INFO:teuthology.orchestra.run.trial163.stderr:2026-02-18T08:09:26.209+0000 7fd2e7714a40 -1 auth: error reading file: /var/lib/ceph/osd/cluster2-2/keyring: can't open /var/lib/ceph/osd/cluster2-2/keyring: (2) No such file or directory 2026-02-18T08:09:26.218 INFO:teuthology.orchestra.run.trial163.stderr:2026-02-18T08:09:26.213+0000 7fd2e7714a40 -1 created new key in keyring /var/lib/ceph/osd/cluster2-2/keyring 2026-02-18T08:09:26.219 INFO:teuthology.orchestra.run.trial163.stderr:2026-02-18T08:09:26.213+0000 7fd2e7714a40 -1 bdev(0x563c111db800 /var/lib/ceph/osd/cluster2-2/block) open stat got: (1) Operation not permitted 2026-02-18T08:09:26.219 INFO:teuthology.orchestra.run.trial163.stderr:2026-02-18T08:09:26.213+0000 7fd2e7714a40 -1 bluestore(/var/lib/ceph/osd/cluster2-2) _read_fsid unparsable uuid 2026-02-18T08:09:27.094 DEBUG:teuthology.orchestra.run.trial163:> sudo chown -R ceph:ceph /var/lib/ceph/osd/cluster2-2 2026-02-18T08:09:27.146 INFO:tasks.ceph:Reading keys from all nodes... 2026-02-18T08:09:27.146 DEBUG:teuthology.orchestra.run.trial034:> set -ex 2026-02-18T08:09:27.146 DEBUG:teuthology.orchestra.run.trial034:> sudo dd if=/var/lib/ceph/mgr/cluster2-x/keyring of=/dev/stdout 2026-02-18T08:09:27.157 DEBUG:teuthology.orchestra.run.trial163:> set -ex 2026-02-18T08:09:27.157 DEBUG:teuthology.orchestra.run.trial163:> sudo dd if=/var/lib/ceph/osd/cluster2-0/keyring of=/dev/stdout 2026-02-18T08:09:27.198 DEBUG:teuthology.orchestra.run.trial163:> set -ex 2026-02-18T08:09:27.198 DEBUG:teuthology.orchestra.run.trial163:> sudo dd if=/var/lib/ceph/osd/cluster2-1/keyring of=/dev/stdout 2026-02-18T08:09:27.249 DEBUG:teuthology.orchestra.run.trial163:> set -ex 2026-02-18T08:09:27.249 DEBUG:teuthology.orchestra.run.trial163:> sudo dd if=/var/lib/ceph/osd/cluster2-2/keyring of=/dev/stdout 2026-02-18T08:09:27.296 DEBUG:teuthology.orchestra.run.trial034:> set -ex 2026-02-18T08:09:27.297 DEBUG:teuthology.orchestra.run.trial034:> dd if=/etc/ceph/cluster2.client.0.keyring of=/dev/stdout 2026-02-18T08:09:27.302 DEBUG:teuthology.orchestra.run.trial163:> set -ex 2026-02-18T08:09:27.302 DEBUG:teuthology.orchestra.run.trial163:> dd if=/etc/ceph/cluster2.client.mirror.keyring of=/dev/stdout 2026-02-18T08:09:27.341 DEBUG:teuthology.orchestra.run.trial163:> set -ex 2026-02-18T08:09:27.341 DEBUG:teuthology.orchestra.run.trial163:> dd if=/etc/ceph/cluster2.client.mirror.0.keyring of=/dev/stdout 2026-02-18T08:09:27.385 DEBUG:teuthology.orchestra.run.trial163:> set -ex 2026-02-18T08:09:27.385 DEBUG:teuthology.orchestra.run.trial163:> dd if=/etc/ceph/cluster2.client.mirror.1.keyring of=/dev/stdout 2026-02-18T08:09:27.429 DEBUG:teuthology.orchestra.run.trial163:> set -ex 2026-02-18T08:09:27.429 DEBUG:teuthology.orchestra.run.trial163:> dd if=/etc/ceph/cluster2.client.mirror.2.keyring of=/dev/stdout 2026-02-18T08:09:27.473 DEBUG:teuthology.orchestra.run.trial163:> set -ex 2026-02-18T08:09:27.473 DEBUG:teuthology.orchestra.run.trial163:> dd if=/etc/ceph/cluster2.client.mirror.3.keyring of=/dev/stdout 2026-02-18T08:09:27.517 DEBUG:teuthology.orchestra.run.trial163:> set -ex 2026-02-18T08:09:27.517 DEBUG:teuthology.orchestra.run.trial163:> dd if=/etc/ceph/cluster2.client.mirror.4.keyring of=/dev/stdout 2026-02-18T08:09:27.561 DEBUG:teuthology.orchestra.run.trial163:> set -ex 2026-02-18T08:09:27.561 DEBUG:teuthology.orchestra.run.trial163:> dd if=/etc/ceph/cluster2.client.mirror.5.keyring of=/dev/stdout 2026-02-18T08:09:27.605 DEBUG:teuthology.orchestra.run.trial163:> set -ex 2026-02-18T08:09:27.605 DEBUG:teuthology.orchestra.run.trial163:> dd if=/etc/ceph/cluster2.client.mirror.6.keyring of=/dev/stdout 2026-02-18T08:09:27.649 INFO:tasks.ceph:Adding keys to all mons... 2026-02-18T08:09:27.649 DEBUG:teuthology.orchestra.run.trial163:> sudo tee -a /etc/ceph/cluster2.keyring 2026-02-18T08:09:27.696 INFO:teuthology.orchestra.run.trial163.stdout:[mgr.x] 2026-02-18T08:09:27.696 INFO:teuthology.orchestra.run.trial163.stdout: key = AQCxc5VpMZiVERAAHbQ3Uf5r6NLMtGWWJjJCTA== 2026-02-18T08:09:27.697 INFO:teuthology.orchestra.run.trial163.stdout:[osd.0] 2026-02-18T08:09:27.697 INFO:teuthology.orchestra.run.trial163.stdout: key = AQC0c5VpTfKaDhAAW4IHwCPUZHBS3fHrwTqUnA== 2026-02-18T08:09:27.697 INFO:teuthology.orchestra.run.trial163.stdout:[osd.1] 2026-02-18T08:09:27.697 INFO:teuthology.orchestra.run.trial163.stdout: key = AQC1c5Vp7j4wDBAA1JlsC5Ud/FTr75QiJHIqjg== 2026-02-18T08:09:27.697 INFO:teuthology.orchestra.run.trial163.stdout:[osd.2] 2026-02-18T08:09:27.697 INFO:teuthology.orchestra.run.trial163.stdout: key = AQC2c5Vp8fPbDBAAcZ6ydzVBTd+0vwkpkwGQYg== 2026-02-18T08:09:27.697 INFO:teuthology.orchestra.run.trial163.stdout:[client.0] 2026-02-18T08:09:27.697 INFO:teuthology.orchestra.run.trial163.stdout: key = AQCxc5VpYDFIFRAAYcB6Cl6io7aJQ+EKvBrc2A== 2026-02-18T08:09:27.697 INFO:teuthology.orchestra.run.trial163.stdout:[client.mirror] 2026-02-18T08:09:27.697 INFO:teuthology.orchestra.run.trial163.stdout: key = AQCxc5Vp+ba7FhAAUDOVjQYTJTHKeKjrSYceZw== 2026-02-18T08:09:27.697 INFO:teuthology.orchestra.run.trial163.stdout:[client.mirror.0] 2026-02-18T08:09:27.697 INFO:teuthology.orchestra.run.trial163.stdout: key = AQCxc5VpaB7BGhAAGOanPLvvNVlEqJwlZukPNQ== 2026-02-18T08:09:27.697 INFO:teuthology.orchestra.run.trial163.stdout:[client.mirror.1] 2026-02-18T08:09:27.697 INFO:teuthology.orchestra.run.trial163.stdout: key = AQCxc5VptseJHhAAeDJEY7SUTM372z+kYWiCXg== 2026-02-18T08:09:27.697 INFO:teuthology.orchestra.run.trial163.stdout:[client.mirror.2] 2026-02-18T08:09:27.697 INFO:teuthology.orchestra.run.trial163.stdout: key = AQCxc5VpOCBVIhAAGB4DXxBYl8tIxX1TtcG9sg== 2026-02-18T08:09:27.697 INFO:teuthology.orchestra.run.trial163.stdout:[client.mirror.3] 2026-02-18T08:09:27.698 INFO:teuthology.orchestra.run.trial163.stdout: key = AQCxc5VpaSonJhAA2BHCinkac3fk2FTSuGaE1Q== 2026-02-18T08:09:27.698 INFO:teuthology.orchestra.run.trial163.stdout:[client.mirror.4] 2026-02-18T08:09:27.698 INFO:teuthology.orchestra.run.trial163.stdout: key = AQCxc5VpJcH2KRAAmXFcG1VizGXTd1U4IOR8uA== 2026-02-18T08:09:27.698 INFO:teuthology.orchestra.run.trial163.stdout:[client.mirror.5] 2026-02-18T08:09:27.698 INFO:teuthology.orchestra.run.trial163.stdout: key = AQCxc5Vp/DLKLRAATeGZ31Ti5RsO8ulHxtL+rw== 2026-02-18T08:09:27.698 INFO:teuthology.orchestra.run.trial163.stdout:[client.mirror.6] 2026-02-18T08:09:27.698 INFO:teuthology.orchestra.run.trial163.stdout: key = AQCxc5VpMHOVMRAAWXCpevTrzpW4MEBIooT9QQ== 2026-02-18T08:09:27.699 DEBUG:teuthology.orchestra.run.trial163:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster2.keyring --name=mgr.x --cap mon 'allow profile mgr' --cap osd 'allow *' --cap mds 'allow *' 2026-02-18T08:09:27.763 DEBUG:teuthology.orchestra.run.trial163:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster2.keyring --name=osd.0 --cap mon 'allow profile osd' --cap mgr 'allow profile osd' --cap osd 'allow *' 2026-02-18T08:09:27.827 DEBUG:teuthology.orchestra.run.trial163:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster2.keyring --name=osd.1 --cap mon 'allow profile osd' --cap mgr 'allow profile osd' --cap osd 'allow *' 2026-02-18T08:09:27.890 DEBUG:teuthology.orchestra.run.trial163:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster2.keyring --name=osd.2 --cap mon 'allow profile osd' --cap mgr 'allow profile osd' --cap osd 'allow *' 2026-02-18T08:09:27.954 DEBUG:teuthology.orchestra.run.trial163:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster2.keyring --name=client.0 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2026-02-18T08:09:28.018 DEBUG:teuthology.orchestra.run.trial163:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster2.keyring --name=client.mirror --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2026-02-18T08:09:28.083 DEBUG:teuthology.orchestra.run.trial163:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster2.keyring --name=client.mirror.0 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2026-02-18T08:09:28.147 DEBUG:teuthology.orchestra.run.trial163:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster2.keyring --name=client.mirror.1 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2026-02-18T08:09:28.210 DEBUG:teuthology.orchestra.run.trial163:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster2.keyring --name=client.mirror.2 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2026-02-18T08:09:28.274 DEBUG:teuthology.orchestra.run.trial163:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster2.keyring --name=client.mirror.3 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2026-02-18T08:09:28.340 DEBUG:teuthology.orchestra.run.trial163:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster2.keyring --name=client.mirror.4 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2026-02-18T08:09:28.403 DEBUG:teuthology.orchestra.run.trial163:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster2.keyring --name=client.mirror.5 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2026-02-18T08:09:28.466 DEBUG:teuthology.orchestra.run.trial163:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster2.keyring --name=client.mirror.6 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2026-02-18T08:09:28.531 INFO:tasks.ceph:Running mkfs on mon nodes... 2026-02-18T08:09:28.531 DEBUG:teuthology.orchestra.run.trial163:> sudo mkdir -p /var/lib/ceph/mon/cluster2-a 2026-02-18T08:09:28.581 DEBUG:teuthology.orchestra.run.trial163:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-mon --cluster cluster2 --mkfs -i a --monmap /home/ubuntu/cephtest/cluster2.monmap --keyring /etc/ceph/cluster2.keyring 2026-02-18T08:09:28.647 DEBUG:teuthology.orchestra.run.trial163:> sudo chown -R ceph:ceph /var/lib/ceph/mon/cluster2-a 2026-02-18T08:09:28.697 DEBUG:teuthology.orchestra.run.trial163:> rm -- /home/ubuntu/cephtest/cluster2.monmap 2026-02-18T08:09:28.752 INFO:tasks.ceph:Starting mon daemons in cluster cluster2... 2026-02-18T08:09:28.752 INFO:tasks.ceph.cluster2.mon.a:Restarting daemon 2026-02-18T08:09:28.752 DEBUG:teuthology.orchestra.run.trial163:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-mon -f --cluster cluster2 -i a 2026-02-18T08:09:28.798 INFO:tasks.ceph.cluster2.mon.a:Started 2026-02-18T08:09:28.798 INFO:tasks.ceph:Starting mgr daemons in cluster cluster2... 2026-02-18T08:09:28.799 INFO:tasks.ceph.cluster2.mgr.x:Restarting daemon 2026-02-18T08:09:28.799 DEBUG:teuthology.orchestra.run.trial034:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-mgr -f --cluster cluster2 -i x 2026-02-18T08:09:28.800 INFO:tasks.ceph.cluster2.mgr.x:Started 2026-02-18T08:09:28.801 DEBUG:tasks.ceph:set 0 configs 2026-02-18T08:09:28.801 DEBUG:teuthology.orchestra.run.trial163:> sudo ceph --cluster cluster2 config dump 2026-02-18T08:09:28.918 INFO:teuthology.orchestra.run.trial163.stdout:WHO MASK LEVEL OPTION VALUE RO 2026-02-18T08:09:28.927 INFO:tasks.ceph:Setting crush tunables to default 2026-02-18T08:09:28.928 DEBUG:teuthology.orchestra.run.trial163:> sudo ceph --cluster cluster2 osd crush tunables default 2026-02-18T08:09:29.025 INFO:teuthology.orchestra.run.trial163.stderr:adjusted tunables profile to default 2026-02-18T08:09:29.037 INFO:tasks.ceph:check_enable_crimson: False 2026-02-18T08:09:29.037 INFO:tasks.ceph:Starting osd daemons in cluster cluster2... 2026-02-18T08:09:29.037 DEBUG:teuthology.orchestra.run.trial163:> set -ex 2026-02-18T08:09:29.037 DEBUG:teuthology.orchestra.run.trial163:> sudo dd if=/var/lib/ceph/osd/cluster2-0/fsid of=/dev/stdout 2026-02-18T08:09:29.046 DEBUG:teuthology.orchestra.run.trial163:> set -ex 2026-02-18T08:09:29.046 DEBUG:teuthology.orchestra.run.trial163:> sudo dd if=/var/lib/ceph/osd/cluster2-1/fsid of=/dev/stdout 2026-02-18T08:09:29.098 DEBUG:teuthology.orchestra.run.trial163:> set -ex 2026-02-18T08:09:29.098 DEBUG:teuthology.orchestra.run.trial163:> sudo dd if=/var/lib/ceph/osd/cluster2-2/fsid of=/dev/stdout 2026-02-18T08:09:29.150 DEBUG:teuthology.orchestra.run.trial163:> sudo ceph --cluster cluster2 osd new 6e23dc77-aa8c-4655-9ca3-7777a9882c2c 0 2026-02-18T08:09:29.295 INFO:teuthology.orchestra.run.trial163.stdout:0 2026-02-18T08:09:29.304 DEBUG:teuthology.orchestra.run.trial163:> sudo ceph --cluster cluster2 osd new e04d91f9-3710-4d1d-9161-4c60f2c21444 1 2026-02-18T08:09:29.404 INFO:teuthology.orchestra.run.trial163.stdout:1 2026-02-18T08:09:29.414 DEBUG:teuthology.orchestra.run.trial163:> sudo ceph --cluster cluster2 osd new 4d746e5b-ce93-4d1b-89d7-e25977e57605 2 2026-02-18T08:09:29.513 INFO:teuthology.orchestra.run.trial163.stdout:2 2026-02-18T08:09:29.525 INFO:tasks.ceph.cluster2.osd.0:Restarting daemon 2026-02-18T08:09:29.525 DEBUG:teuthology.orchestra.run.trial163:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-osd -f --cluster cluster2 -i 0 2026-02-18T08:09:29.526 INFO:tasks.ceph.cluster2.osd.0:Started 2026-02-18T08:09:29.527 INFO:tasks.ceph.cluster2.osd.1:Restarting daemon 2026-02-18T08:09:29.527 DEBUG:teuthology.orchestra.run.trial163:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-osd -f --cluster cluster2 -i 1 2026-02-18T08:09:29.528 INFO:tasks.ceph.cluster2.osd.1:Started 2026-02-18T08:09:29.528 INFO:tasks.ceph.cluster2.osd.2:Restarting daemon 2026-02-18T08:09:29.528 DEBUG:teuthology.orchestra.run.trial163:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-osd -f --cluster cluster2 -i 2 2026-02-18T08:09:29.529 INFO:tasks.ceph.cluster2.osd.2:Started 2026-02-18T08:09:29.530 DEBUG:teuthology.orchestra.run.trial163:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 osd dump --format=json 2026-02-18T08:09:29.622 INFO:teuthology.orchestra.run.trial163.stdout: 2026-02-18T08:09:29.622 INFO:teuthology.orchestra.run.trial163.stdout:{"epoch":5,"fsid":"892b71e6-8019-418b-97e7-5a7b63a9d866","created":"2026-02-18T08:09:28.873380+0000","modified":"2026-02-18T08:09:29.510528+0000","last_up_change":"0.000000","last_in_change":"2026-02-18T08:09:29.510528+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":2,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":0,"max_osd":3,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"tentacle","allow_crimson":false,"pools":[],"osds":[{"osd":0,"uuid":"6e23dc77-aa8c-4655-9ca3-7777a9882c2c","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":1,"uuid":"e04d91f9-3710-4d1d-9161-4c60f2c21444","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":2,"uuid":"4d746e5b-ce93-4d1b-89d7-e25977e57605","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"isa","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2026-02-18T08:09:29.631 INFO:tasks.ceph.ceph_manager.cluster2:[] 2026-02-18T08:09:29.631 INFO:tasks.ceph:Waiting for OSDs to come up 2026-02-18T08:09:29.886 INFO:tasks.ceph.cluster2.osd.0.trial163.stderr:2026-02-18T08:09:29.881+0000 7f213a7eaa40 -1 Falling back to public interface 2026-02-18T08:09:29.902 INFO:tasks.ceph.cluster2.osd.1.trial163.stderr:2026-02-18T08:09:29.897+0000 7f6f86d9da40 -1 Falling back to public interface 2026-02-18T08:09:29.933 DEBUG:teuthology.orchestra.run.trial163:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph --cluster cluster2 osd dump --format=json 2026-02-18T08:09:30.023 INFO:tasks.ceph.cluster2.mgr.x.trial034.stderr:ERROR:root:Module 'jmespath' is not installed. 2026-02-18T08:09:30.034 INFO:teuthology.misc.health.trial163.stdout: 2026-02-18T08:09:30.035 INFO:teuthology.misc.health.trial163.stdout:{"epoch":5,"fsid":"892b71e6-8019-418b-97e7-5a7b63a9d866","created":"2026-02-18T08:09:28.873380+0000","modified":"2026-02-18T08:09:29.510528+0000","last_up_change":"0.000000","last_in_change":"2026-02-18T08:09:29.510528+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":2,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":0,"max_osd":3,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"tentacle","allow_crimson":false,"pools":[],"osds":[{"osd":0,"uuid":"6e23dc77-aa8c-4655-9ca3-7777a9882c2c","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":1,"uuid":"e04d91f9-3710-4d1d-9161-4c60f2c21444","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":2,"uuid":"4d746e5b-ce93-4d1b-89d7-e25977e57605","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"isa","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2026-02-18T08:09:30.043 DEBUG:teuthology.misc:0 of 3 OSDs are up 2026-02-18T08:09:30.061 INFO:tasks.ceph.cluster2.mgr.x.trial034.stderr:ERROR:dashboard.controllers.nvmeof:Failed to import NVMeoFClient and related components: cannot import name 'NVMeoFClient' from 'dashboard.services.nvmeof_client' (/usr/share/ceph/mgr/dashboard/services/nvmeof_client.py) 2026-02-18T08:09:30.062 INFO:tasks.ceph.cluster2.mgr.x.trial034.stderr:ERROR:sso:Module 'jmespath' is not installed. 2026-02-18T08:09:30.074 INFO:tasks.ceph.cluster2.mgr.x.trial034.stderr:ERROR:root:Module 'xmltodict' is not installed. 2026-02-18T08:09:30.158 INFO:tasks.ceph.cluster2.osd.2.trial163.stderr:2026-02-18T08:09:30.153+0000 7f2b71deba40 -1 Falling back to public interface 2026-02-18T08:09:30.421 INFO:tasks.ceph.cluster2.mgr.x.trial034.stderr:/usr/lib/python3/dist-packages/scipy/__init__.py:67: UserWarning: NumPy was imported from a Python sub-interpreter but NumPy does not properly support sub-interpreters. This will likely work for most users but might cause hard to track down issues or subtle bugs. A common user of the rare sub-interpreter feature is wsgi which also allows single-interpreter mode. 2026-02-18T08:09:30.422 INFO:tasks.ceph.cluster2.mgr.x.trial034.stderr:Improvements in the case of bugs are welcome, but is not on the NumPy roadmap, and full support may require significant effort to achieve. 2026-02-18T08:09:30.422 INFO:tasks.ceph.cluster2.mgr.x.trial034.stderr: from numpy import show_config as show_numpy_config 2026-02-18T08:09:31.002 INFO:tasks.ceph.cluster2.osd.1.trial163.stderr:2026-02-18T08:09:30.993+0000 7f6f86d9da40 -1 osd.1 0 log_to_monitors true 2026-02-18T08:09:31.004 INFO:tasks.ceph.cluster2.osd.0.trial163.stderr:2026-02-18T08:09:30.997+0000 7f213a7eaa40 -1 osd.0 0 log_to_monitors true 2026-02-18T08:09:31.254 INFO:tasks.ceph.cluster2.osd.2.trial163.stderr:2026-02-18T08:09:31.249+0000 7f2b71deba40 -1 osd.2 0 log_to_monitors true 2026-02-18T08:09:31.482 INFO:tasks.ceph.cluster2.mgr.x.trial034.stderr:2026-02-18T08:09:31.478+0000 7f6d07329640 -1 mgr.server handle_report got status from non-daemon mon.a 2026-02-18T08:09:32.898 INFO:tasks.ceph.cluster2.osd.0.trial163.stderr:2026-02-18T08:09:32.893+0000 7f2136793640 -1 osd.0 0 waiting for initial osdmap 2026-02-18T08:09:32.900 INFO:tasks.ceph.cluster2.osd.0.trial163.stderr:2026-02-18T08:09:32.893+0000 7f212dd9a640 -1 osd.0 7 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2026-02-18T08:09:32.910 INFO:tasks.ceph.cluster2.osd.1.trial163.stderr:2026-02-18T08:09:32.905+0000 7f6f82d46640 -1 osd.1 0 waiting for initial osdmap 2026-02-18T08:09:32.910 INFO:tasks.ceph.cluster2.osd.2.trial163.stderr:2026-02-18T08:09:32.905+0000 7f2b6dd94640 -1 osd.2 0 waiting for initial osdmap 2026-02-18T08:09:32.912 INFO:tasks.ceph.cluster2.osd.2.trial163.stderr:2026-02-18T08:09:32.905+0000 7f2b6539b640 -1 osd.2 7 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2026-02-18T08:09:32.912 INFO:tasks.ceph.cluster2.osd.1.trial163.stderr:2026-02-18T08:09:32.905+0000 7f6f7a34d640 -1 osd.1 7 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2026-02-18T08:09:36.347 DEBUG:teuthology.orchestra.run.trial163:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph --cluster cluster2 osd dump --format=json 2026-02-18T08:09:36.493 INFO:teuthology.misc.health.trial163.stdout: 2026-02-18T08:09:36.494 INFO:teuthology.misc.health.trial163.stdout:{"epoch":10,"fsid":"892b71e6-8019-418b-97e7-5a7b63a9d866","created":"2026-02-18T08:09:28.873380+0000","modified":"2026-02-18T08:09:35.890362+0000","last_up_change":"2026-02-18T08:09:33.883650+0000","last_in_change":"2026-02-18T08:09:29.510528+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":4,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":3,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"tentacle","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2026-02-18T08:09:34.514392+0000","flags":32769,"flags_names":"hashpspool,creating","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"10","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"nonprimary_shards":"{}","options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":2.9900000095367432,"score_stable":2.9900000095367432,"optimal_score":0.67000001668930054,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"6e23dc77-aa8c-4655-9ca3-7777a9882c2c","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.163:6800","nonce":49861742},{"type":"v1","addr":"10.20.193.163:6801","nonce":49861742}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.163:6802","nonce":49861742},{"type":"v1","addr":"10.20.193.163:6803","nonce":49861742}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.163:6806","nonce":49861742},{"type":"v1","addr":"10.20.193.163:6807","nonce":49861742}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.163:6804","nonce":49861742},{"type":"v1","addr":"10.20.193.163:6805","nonce":49861742}]},"public_addr":"10.20.193.163:6801/49861742","cluster_addr":"10.20.193.163:6803/49861742","heartbeat_back_addr":"10.20.193.163:6807/49861742","heartbeat_front_addr":"10.20.193.163:6805/49861742","state":["exists","up"]},{"osd":1,"uuid":"e04d91f9-3710-4d1d-9161-4c60f2c21444","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":9,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.163:6808","nonce":1532368640},{"type":"v1","addr":"10.20.193.163:6809","nonce":1532368640}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.163:6810","nonce":1532368640},{"type":"v1","addr":"10.20.193.163:6811","nonce":1532368640}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.163:6814","nonce":1532368640},{"type":"v1","addr":"10.20.193.163:6815","nonce":1532368640}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.163:6812","nonce":1532368640},{"type":"v1","addr":"10.20.193.163:6813","nonce":1532368640}]},"public_addr":"10.20.193.163:6809/1532368640","cluster_addr":"10.20.193.163:6811/1532368640","heartbeat_back_addr":"10.20.193.163:6815/1532368640","heartbeat_front_addr":"10.20.193.163:6813/1532368640","state":["exists","up"]},{"osd":2,"uuid":"4d746e5b-ce93-4d1b-89d7-e25977e57605","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.163:6816","nonce":1781709176},{"type":"v1","addr":"10.20.193.163:6817","nonce":1781709176}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.163:6818","nonce":1781709176},{"type":"v1","addr":"10.20.193.163:6819","nonce":1781709176}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.163:6822","nonce":1781709176},{"type":"v1","addr":"10.20.193.163:6823","nonce":1781709176}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.163:6820","nonce":1781709176},{"type":"v1","addr":"10.20.193.163:6821","nonce":1781709176}]},"public_addr":"10.20.193.163:6817/1781709176","cluster_addr":"10.20.193.163:6819/1781709176","heartbeat_back_addr":"10.20.193.163:6823/1781709176","heartbeat_front_addr":"10.20.193.163:6821/1781709176","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4544132024016699391,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4544132024016699391,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4544132024016699391,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"isa","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2026-02-18T08:09:36.505 DEBUG:teuthology.misc:3 of 3 OSDs are up 2026-02-18T08:09:36.505 INFO:tasks.ceph:Creating RBD pool 2026-02-18T08:09:36.505 DEBUG:teuthology.orchestra.run.trial163:> sudo ceph --cluster cluster2 osd pool create rbd 8 2026-02-18T08:09:36.894 INFO:teuthology.orchestra.run.trial163.stderr:pool 'rbd' created 2026-02-18T08:09:36.908 DEBUG:teuthology.orchestra.run.trial163:> rbd --cluster cluster2 pool init rbd 2026-02-18T08:09:39.911 INFO:tasks.ceph:Starting mds daemons in cluster cluster2... 2026-02-18T08:09:39.912 DEBUG:teuthology.orchestra.run.trial163:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 config log 1 --format=json 2026-02-18T08:09:39.913 INFO:tasks.daemonwatchdog.daemon_watchdog:watchdog starting 2026-02-18T08:09:40.068 INFO:teuthology.orchestra.run.trial163.stdout: 2026-02-18T08:09:40.079 INFO:teuthology.orchestra.run.trial163.stdout:[{"version":1,"timestamp":"0.000000","name":"","changes":[]}] 2026-02-18T08:09:40.079 INFO:tasks.ceph_manager:config epoch is 1 2026-02-18T08:09:40.079 INFO:tasks.ceph:Waiting until cluster2 daemons up and pgs clean... 2026-02-18T08:09:40.079 INFO:tasks.ceph.ceph_manager.cluster2:waiting for mgr available 2026-02-18T08:09:40.080 DEBUG:teuthology.orchestra.run.trial163:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 mgr dump --format=json 2026-02-18T08:09:40.262 INFO:teuthology.orchestra.run.trial163.stdout: 2026-02-18T08:09:40.273 INFO:teuthology.orchestra.run.trial163.stdout:{"epoch":5,"flags":0,"active_gid":4099,"active_name":"x","active_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.34:6826","nonce":2506445334},{"type":"v1","addr":"10.20.193.34:6827","nonce":2506445334}]},"active_addr":"10.20.193.34:6827/2506445334","active_change":"2026-02-18T08:09:30.471118+0000","active_mgr_features":4544132024016699391,"available":true,"standbys":[],"modules":["iostat","nfs"],"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to, use commas to separate multiple","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2359","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","read","upmap","upmap-read"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"update_pg_upmap_activity":{"name":"update_pg_upmap_activity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Updates pg_upmap activity stats to be used in `balancer status detail`","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cephadm_log_destination":{"name":"cephadm_log_destination","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":["file","file,syslog","syslog"],"desc":"Destination for cephadm command's persistent logging","long_desc":"","tags":[],"see_also":[]},"certificate_automated_rotation_enabled":{"name":"certificate_automated_rotation_enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"This flag controls whether cephadm automatically rotates certificates upon expiration.","long_desc":"","tags":[],"see_also":[]},"certificate_check_debug_mode":{"name":"certificate_check_debug_mode","type":"bool","level":"dev","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"FOR TESTING ONLY: This flag forces the certificate check instead of waiting for certificate_check_period.","long_desc":"","tags":[],"see_also":[]},"certificate_check_period":{"name":"certificate_check_period","type":"int","level":"advanced","flags":0,"default_value":"1","min":"0","max":"30","enum_allowed":[],"desc":"Specifies how often (in days) the certificate should be checked for validity.","long_desc":"","tags":[],"see_also":[]},"certificate_duration_days":{"name":"certificate_duration_days","type":"int","level":"advanced","flags":0,"default_value":"1095","min":"90","max":"3650","enum_allowed":[],"desc":"Specifies the duration of self certificates generated and signed by cephadm root CA","long_desc":"","tags":[],"see_also":[]},"certificate_renewal_threshold_days":{"name":"certificate_renewal_threshold_days","type":"int","level":"advanced","flags":0,"default_value":"30","min":"10","max":"90","enum_allowed":[],"desc":"Specifies the lead time in days to initiate certificate renewal before expiration.","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.28.1","min":"","max":"","enum_allowed":[],"desc":"Alertmanager container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_elasticsearch":{"name":"container_image_elasticsearch","type":"str","level":"advanced","flags":0,"default_value":"quay.io/omrizeneva/elasticsearch:6.8.23","min":"","max":"","enum_allowed":[],"desc":"Elasticsearch container image","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/grafana:12.3.1","min":"","max":"","enum_allowed":[],"desc":"Grafana container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"Haproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_agent":{"name":"container_image_jaeger_agent","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-agent:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger agent container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_collector":{"name":"container_image_jaeger_collector","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-collector:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger collector container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_query":{"name":"container_image_jaeger_query","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-query:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger query container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.2.4","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:3.0.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_nginx":{"name":"container_image_nginx","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/nginx:sclorg-nginx-126","min":"","max":"","enum_allowed":[],"desc":"Nginx container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.9.1","min":"","max":"","enum_allowed":[],"desc":"Node exporter container image","long_desc":"","tags":[],"see_also":[]},"container_image_nvmeof":{"name":"container_image_nvmeof","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/nvmeof:1.5","min":"","max":"","enum_allowed":[],"desc":"Nvmeof container image","long_desc":"","tags":[],"see_also":[]},"container_image_oauth2_proxy":{"name":"container_image_oauth2_proxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/oauth2-proxy/oauth2-proxy:v7.6.0","min":"","max":"","enum_allowed":[],"desc":"Oauth2 proxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v3.6.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:3.0.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_samba":{"name":"container_image_samba","type":"str","level":"advanced","flags":0,"default_value":"quay.io/samba.org/samba-server:ceph20-centos-amd64","min":"","max":"","enum_allowed":[],"desc":"Samba container image","long_desc":"","tags":[],"see_also":[]},"container_image_samba_metrics":{"name":"container_image_samba_metrics","type":"str","level":"advanced","flags":0,"default_value":"quay.io/samba.org/samba-metrics:ceph20-centos-amd64","min":"","max":"","enum_allowed":[],"desc":"Samba metrics container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"Snmp gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"int","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"quay.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"grafana_dashboards_path":{"name":"grafana_dashboards_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/grafana/dashboards/ceph-dashboard/","min":"","max":"","enum_allowed":[],"desc":"location of dashboards to include in grafana deployments","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"hw_monitoring":{"name":"hw_monitoring","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Deploy hw monitoring daemon on every host.","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"oob_default_addr":{"name":"oob_default_addr","type":"str","level":"advanced","flags":0,"default_value":"169.254.1.1","min":"","max":"","enum_allowed":[],"desc":"Default address for RedFish API (oob management).","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"secure_monitoring_stack":{"name":"secure_monitoring_stack","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable TLS security for all the monitoring stack daemons","long_desc":"","tags":[],"see_also":[]},"service_discovery_port":{"name":"service_discovery_port","type":"int","level":"advanced","flags":0,"default_value":"8765","min":"","max":"","enum_allowed":[],"desc":"cephadm service discovery port","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"ssh_keepalive_count_max":{"name":"ssh_keepalive_count_max","type":"int","level":"advanced","flags":0,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"How many times ssh connections can fail liveness checks before the host is marked offline","long_desc":"","tags":[],"see_also":[]},"ssh_keepalive_interval":{"name":"ssh_keepalive_interval","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"How often ssh connections are checked for liveness","long_desc":"","tags":[],"see_also":[]},"stray_daemon_check_interval":{"name":"stray_daemon_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"how frequently cephadm should check for the presence of stray daemons","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"MANAGED_BY_CLUSTERS":{"name":"MANAGED_BY_CLUSTERS","type":"str","level":"advanced","flags":0,"default_value":"[]","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"MULTICLUSTER_CONFIG":{"name":"MULTICLUSTER_CONFIG","type":"str","level":"advanced","flags":0,"default_value":"{}","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROM_ALERT_CREDENTIAL_CACHE_TTL":{"name":"PROM_ALERT_CREDENTIAL_CACHE_TTL","type":"int","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_HOSTNAME_PER_DAEMON":{"name":"RGW_HOSTNAME_PER_DAEMON","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"UNSAFE_TLS_v1_2":{"name":"UNSAFE_TLS_v1_2","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crypto_caller":{"name":"crypto_caller","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sso_oauth2":{"name":"sso_oauth2","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"fail_fs":{"name":"fail_fs","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Fail filesystem for rapid multi-rank mds upgrade","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"exclude_perf_counters":{"name":"exclude_perf_counters","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Do not include perf-counters in the metrics output","long_desc":"Gathering perf-counters from a single Prometheus exporter can degrade ceph-mgr performance, especially in large clusters. Instead, Ceph-exporter daemons are now used by default for perf-counter gathering. This should only be disabled when no ceph-exporters are deployed.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"secondary_zone_period_retry_limit":{"name":"secondary_zone_period_retry_limit","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"RGW module period update retry limit for secondary site","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard_description":{"name":"leaderboard_description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"pause_cloning":{"name":"pause_cloning","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Pause asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"pause_purging":{"name":"pause_purging","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Pause asynchronous subvolume purge threads","long_desc":"","tags":[],"see_also":[]},"periodic_async_work":{"name":"periodic_async_work","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Periodically check for async work","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_no_wait":{"name":"snapshot_clone_no_wait","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Reject subvolume clone request when cloner threads are busy","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}],"services":{},"always_on_modules":{"octopus":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"pacific":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"quincy":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"reef":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"squid":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"tentacle":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"]},"force_disabled_modules":{},"last_failure_osd_epoch":0,"active_clients":[{"name":"devicehealth","addrvec":[{"type":"v2","addr":"10.20.193.34:0","nonce":3214226146}]},{"name":"libcephsqlite","addrvec":[{"type":"v2","addr":"10.20.193.34:0","nonce":2954431442}]},{"name":"rbd_support","addrvec":[{"type":"v2","addr":"10.20.193.34:0","nonce":3300721118}]},{"name":"volumes","addrvec":[{"type":"v2","addr":"10.20.193.34:0","nonce":3984239983}]}]} 2026-02-18T08:09:40.277 INFO:tasks.ceph.ceph_manager.cluster2:mgr available! 2026-02-18T08:09:40.277 INFO:tasks.ceph.ceph_manager.cluster2:waiting for all up 2026-02-18T08:09:40.277 DEBUG:teuthology.orchestra.run.trial163:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 osd dump --format=json 2026-02-18T08:09:40.428 INFO:teuthology.orchestra.run.trial163.stdout: 2026-02-18T08:09:40.428 INFO:teuthology.orchestra.run.trial163.stdout:{"epoch":14,"fsid":"892b71e6-8019-418b-97e7-5a7b63a9d866","created":"2026-02-18T08:09:28.873380+0000","modified":"2026-02-18T08:09:39.899282+0000","last_up_change":"2026-02-18T08:09:33.883650+0000","last_in_change":"2026-02-18T08:09:29.510528+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":4,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":2,"max_osd":3,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"tentacle","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2026-02-18T08:09:34.514392+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"11","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"nonprimary_shards":"{}","options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":2.9900000095367432,"score_stable":2.9900000095367432,"optimal_score":0.67000001668930054,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2026-02-18T08:09:36.663412+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"14","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":2,"snap_epoch":14,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"nonprimary_shards":"{}","options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_type":"Fair distribution","score_acting":1.8799999952316284,"score_stable":1.8799999952316284,"optimal_score":1,"raw_score_acting":1.8799999952316284,"raw_score_stable":1.8799999952316284,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"6e23dc77-aa8c-4655-9ca3-7777a9882c2c","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":11,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.163:6800","nonce":49861742},{"type":"v1","addr":"10.20.193.163:6801","nonce":49861742}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.163:6802","nonce":49861742},{"type":"v1","addr":"10.20.193.163:6803","nonce":49861742}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.163:6806","nonce":49861742},{"type":"v1","addr":"10.20.193.163:6807","nonce":49861742}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.163:6804","nonce":49861742},{"type":"v1","addr":"10.20.193.163:6805","nonce":49861742}]},"public_addr":"10.20.193.163:6801/49861742","cluster_addr":"10.20.193.163:6803/49861742","heartbeat_back_addr":"10.20.193.163:6807/49861742","heartbeat_front_addr":"10.20.193.163:6805/49861742","state":["exists","up"]},{"osd":1,"uuid":"e04d91f9-3710-4d1d-9161-4c60f2c21444","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":11,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.163:6808","nonce":1532368640},{"type":"v1","addr":"10.20.193.163:6809","nonce":1532368640}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.163:6810","nonce":1532368640},{"type":"v1","addr":"10.20.193.163:6811","nonce":1532368640}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.163:6814","nonce":1532368640},{"type":"v1","addr":"10.20.193.163:6815","nonce":1532368640}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.163:6812","nonce":1532368640},{"type":"v1","addr":"10.20.193.163:6813","nonce":1532368640}]},"public_addr":"10.20.193.163:6809/1532368640","cluster_addr":"10.20.193.163:6811/1532368640","heartbeat_back_addr":"10.20.193.163:6815/1532368640","heartbeat_front_addr":"10.20.193.163:6813/1532368640","state":["exists","up"]},{"osd":2,"uuid":"4d746e5b-ce93-4d1b-89d7-e25977e57605","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":11,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.163:6816","nonce":1781709176},{"type":"v1","addr":"10.20.193.163:6817","nonce":1781709176}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.163:6818","nonce":1781709176},{"type":"v1","addr":"10.20.193.163:6819","nonce":1781709176}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.163:6822","nonce":1781709176},{"type":"v1","addr":"10.20.193.163:6823","nonce":1781709176}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.163:6820","nonce":1781709176},{"type":"v1","addr":"10.20.193.163:6821","nonce":1781709176}]},"public_addr":"10.20.193.163:6817/1781709176","cluster_addr":"10.20.193.163:6819/1781709176","heartbeat_back_addr":"10.20.193.163:6823/1781709176","heartbeat_front_addr":"10.20.193.163:6821/1781709176","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4544132024016699391,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4544132024016699391,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4544132024016699391,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"isa","technique":"reed_sol_van"}},"removed_snaps_queue":[{"pool":2,"snaps":[{"begin":2,"length":1}]}],"new_removed_snaps":[{"pool":2,"snaps":[{"begin":2,"length":1}]}],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2026-02-18T08:09:40.439 INFO:tasks.ceph.ceph_manager.cluster2:all up! 2026-02-18T08:09:40.439 DEBUG:teuthology.orchestra.run.trial163:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 osd dump --format=json 2026-02-18T08:09:40.600 INFO:teuthology.orchestra.run.trial163.stdout: 2026-02-18T08:09:40.600 INFO:teuthology.orchestra.run.trial163.stdout:{"epoch":14,"fsid":"892b71e6-8019-418b-97e7-5a7b63a9d866","created":"2026-02-18T08:09:28.873380+0000","modified":"2026-02-18T08:09:39.899282+0000","last_up_change":"2026-02-18T08:09:33.883650+0000","last_in_change":"2026-02-18T08:09:29.510528+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":4,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":2,"max_osd":3,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"tentacle","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2026-02-18T08:09:34.514392+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"11","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"nonprimary_shards":"{}","options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":2.9900000095367432,"score_stable":2.9900000095367432,"optimal_score":0.67000001668930054,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2026-02-18T08:09:36.663412+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"14","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":2,"snap_epoch":14,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"nonprimary_shards":"{}","options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_type":"Fair distribution","score_acting":1.8799999952316284,"score_stable":1.8799999952316284,"optimal_score":1,"raw_score_acting":1.8799999952316284,"raw_score_stable":1.8799999952316284,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"6e23dc77-aa8c-4655-9ca3-7777a9882c2c","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":11,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.163:6800","nonce":49861742},{"type":"v1","addr":"10.20.193.163:6801","nonce":49861742}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.163:6802","nonce":49861742},{"type":"v1","addr":"10.20.193.163:6803","nonce":49861742}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.163:6806","nonce":49861742},{"type":"v1","addr":"10.20.193.163:6807","nonce":49861742}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.163:6804","nonce":49861742},{"type":"v1","addr":"10.20.193.163:6805","nonce":49861742}]},"public_addr":"10.20.193.163:6801/49861742","cluster_addr":"10.20.193.163:6803/49861742","heartbeat_back_addr":"10.20.193.163:6807/49861742","heartbeat_front_addr":"10.20.193.163:6805/49861742","state":["exists","up"]},{"osd":1,"uuid":"e04d91f9-3710-4d1d-9161-4c60f2c21444","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":11,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.163:6808","nonce":1532368640},{"type":"v1","addr":"10.20.193.163:6809","nonce":1532368640}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.163:6810","nonce":1532368640},{"type":"v1","addr":"10.20.193.163:6811","nonce":1532368640}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.163:6814","nonce":1532368640},{"type":"v1","addr":"10.20.193.163:6815","nonce":1532368640}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.163:6812","nonce":1532368640},{"type":"v1","addr":"10.20.193.163:6813","nonce":1532368640}]},"public_addr":"10.20.193.163:6809/1532368640","cluster_addr":"10.20.193.163:6811/1532368640","heartbeat_back_addr":"10.20.193.163:6815/1532368640","heartbeat_front_addr":"10.20.193.163:6813/1532368640","state":["exists","up"]},{"osd":2,"uuid":"4d746e5b-ce93-4d1b-89d7-e25977e57605","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":11,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.163:6816","nonce":1781709176},{"type":"v1","addr":"10.20.193.163:6817","nonce":1781709176}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.163:6818","nonce":1781709176},{"type":"v1","addr":"10.20.193.163:6819","nonce":1781709176}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.163:6822","nonce":1781709176},{"type":"v1","addr":"10.20.193.163:6823","nonce":1781709176}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.163:6820","nonce":1781709176},{"type":"v1","addr":"10.20.193.163:6821","nonce":1781709176}]},"public_addr":"10.20.193.163:6817/1781709176","cluster_addr":"10.20.193.163:6819/1781709176","heartbeat_back_addr":"10.20.193.163:6823/1781709176","heartbeat_front_addr":"10.20.193.163:6821/1781709176","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4544132024016699391,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4544132024016699391,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4544132024016699391,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"isa","technique":"reed_sol_van"}},"removed_snaps_queue":[{"pool":2,"snaps":[{"begin":2,"length":1}]}],"new_removed_snaps":[{"pool":2,"snaps":[{"begin":2,"length":1}]}],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2026-02-18T08:09:40.613 DEBUG:teuthology.orchestra.run.trial163:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 tell osd.0 flush_pg_stats 2026-02-18T08:09:40.614 DEBUG:teuthology.orchestra.run.trial163:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 tell osd.1 flush_pg_stats 2026-02-18T08:09:40.614 DEBUG:teuthology.orchestra.run.trial163:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 tell osd.2 flush_pg_stats 2026-02-18T08:09:40.699 INFO:teuthology.orchestra.run.trial163.stdout:34359738373 2026-02-18T08:09:40.699 DEBUG:teuthology.orchestra.run.trial163:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 osd last-stat-seq osd.1 2026-02-18T08:09:40.700 INFO:teuthology.orchestra.run.trial163.stdout:34359738373 2026-02-18T08:09:40.700 DEBUG:teuthology.orchestra.run.trial163:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 osd last-stat-seq osd.2 2026-02-18T08:09:40.704 INFO:teuthology.orchestra.run.trial163.stdout:34359738372 2026-02-18T08:09:40.704 DEBUG:teuthology.orchestra.run.trial163:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 osd last-stat-seq osd.0 2026-02-18T08:09:40.894 INFO:teuthology.orchestra.run.trial163.stdout:34359738371 2026-02-18T08:09:40.905 INFO:tasks.ceph.ceph_manager.cluster2:need seq 34359738373 got 34359738371 for osd.2 2026-02-18T08:09:40.923 INFO:teuthology.orchestra.run.trial163.stdout:34359738372 2026-02-18T08:09:40.935 INFO:tasks.ceph.ceph_manager.cluster2:need seq 34359738373 got 34359738372 for osd.1 2026-02-18T08:09:40.963 INFO:teuthology.orchestra.run.trial163.stdout:34359738371 2026-02-18T08:09:40.975 INFO:tasks.ceph.ceph_manager.cluster2:need seq 34359738372 got 34359738371 for osd.0 2026-02-18T08:09:41.906 DEBUG:teuthology.orchestra.run.trial163:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 osd last-stat-seq osd.2 2026-02-18T08:09:41.935 DEBUG:teuthology.orchestra.run.trial163:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 osd last-stat-seq osd.1 2026-02-18T08:09:41.976 DEBUG:teuthology.orchestra.run.trial163:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 osd last-stat-seq osd.0 2026-02-18T08:09:42.063 INFO:teuthology.orchestra.run.trial163.stdout:34359738372 2026-02-18T08:09:42.075 INFO:tasks.ceph.ceph_manager.cluster2:need seq 34359738373 got 34359738372 for osd.2 2026-02-18T08:09:42.104 INFO:teuthology.orchestra.run.trial163.stdout:34359738372 2026-02-18T08:09:42.116 INFO:tasks.ceph.ceph_manager.cluster2:need seq 34359738373 got 34359738372 for osd.1 2026-02-18T08:09:42.146 INFO:teuthology.orchestra.run.trial163.stdout:34359738371 2026-02-18T08:09:42.156 INFO:tasks.ceph.ceph_manager.cluster2:need seq 34359738372 got 34359738371 for osd.0 2026-02-18T08:09:43.076 DEBUG:teuthology.orchestra.run.trial163:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 osd last-stat-seq osd.2 2026-02-18T08:09:43.117 DEBUG:teuthology.orchestra.run.trial163:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 osd last-stat-seq osd.1 2026-02-18T08:09:43.157 DEBUG:teuthology.orchestra.run.trial163:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 osd last-stat-seq osd.0 2026-02-18T08:09:43.241 INFO:teuthology.orchestra.run.trial163.stdout:34359738373 2026-02-18T08:09:43.252 INFO:tasks.ceph.ceph_manager.cluster2:need seq 34359738373 got 34359738373 for osd.2 2026-02-18T08:09:43.252 DEBUG:teuthology.parallel:result is None 2026-02-18T08:09:43.276 INFO:teuthology.orchestra.run.trial163.stdout:34359738373 2026-02-18T08:09:43.288 INFO:tasks.ceph.ceph_manager.cluster2:need seq 34359738373 got 34359738373 for osd.1 2026-02-18T08:09:43.288 DEBUG:teuthology.parallel:result is None 2026-02-18T08:09:43.322 INFO:teuthology.orchestra.run.trial163.stdout:34359738372 2026-02-18T08:09:43.333 INFO:tasks.ceph.ceph_manager.cluster2:need seq 34359738372 got 34359738372 for osd.0 2026-02-18T08:09:43.334 DEBUG:teuthology.parallel:result is None 2026-02-18T08:09:43.334 INFO:tasks.ceph.ceph_manager.cluster2:waiting for clean 2026-02-18T08:09:43.334 DEBUG:teuthology.orchestra.run.trial163:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 pg dump --format=json 2026-02-18T08:09:43.525 INFO:teuthology.orchestra.run.trial163.stdout: 2026-02-18T08:09:43.525 INFO:teuthology.orchestra.run.trial163.stderr:dumped all 2026-02-18T08:09:43.537 INFO:teuthology.orchestra.run.trial163.stdout:{"pg_ready":true,"pg_map":{"version":16,"stamp":"2026-02-18T08:09:42.477283+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590387,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":103,"num_write_kb":1630,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":75,"ondisk_log_size":75,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":18,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":82768,"kb_used_data":1512,"kb_used_omap":21,"kb_used_meta":80810,"kb_avail":283032752,"statfs":{"total":289910292480,"available":289825538048,"internally_reserved":0,"allocated":1548288,"data_stored":1279978,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":22417,"internal_metadata":82749551},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":11,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"2.574930"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":20,"reported_epoch":14,"state":"active+clean","last_fresh":"2026-02-18T08:09:39.904376+0000","last_change":"2026-02-18T08:09:39.904504+0000","last_active":"2026-02-18T08:09:39.904376+0000","last_peered":"2026-02-18T08:09:39.904376+0000","last_clean":"2026-02-18T08:09:39.904376+0000","last_became_active":"2026-02-18T08:09:37.900809+0000","last_became_peered":"2026-02-18T08:09:37.900809+0000","last_unstale":"2026-02-18T08:09:39.904376+0000","last_undegraded":"2026-02-18T08:09:39.904376+0000","last_fullsized":"2026-02-18T08:09:39.904376+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-18T08:09:36.890905+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-18T08:09:36.890905+0000","last_clean_scrub_stamp":"2026-02-18T08:09:36.890905+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-19T18:39:43.531542+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00068838299999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.6","version":"0'0","reported_seq":20,"reported_epoch":14,"state":"active+clean","last_fresh":"2026-02-18T08:09:39.904123+0000","last_change":"2026-02-18T08:09:39.904331+0000","last_active":"2026-02-18T08:09:39.904123+0000","last_peered":"2026-02-18T08:09:39.904123+0000","last_clean":"2026-02-18T08:09:39.904123+0000","last_became_active":"2026-02-18T08:09:37.900594+0000","last_became_peered":"2026-02-18T08:09:37.900594+0000","last_unstale":"2026-02-18T08:09:39.904123+0000","last_undegraded":"2026-02-18T08:09:39.904123+0000","last_fullsized":"2026-02-18T08:09:39.904123+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-18T08:09:36.890905+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-18T08:09:36.890905+0000","last_clean_scrub_stamp":"2026-02-18T08:09:36.890905+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-19T19:59:21.557650+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00053109200000000004,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.5","version":"0'0","reported_seq":20,"reported_epoch":14,"state":"active+clean","last_fresh":"2026-02-18T08:09:39.904328+0000","last_change":"2026-02-18T08:09:39.904488+0000","last_active":"2026-02-18T08:09:39.904328+0000","last_peered":"2026-02-18T08:09:39.904328+0000","last_clean":"2026-02-18T08:09:39.904328+0000","last_became_active":"2026-02-18T08:09:37.900657+0000","last_became_peered":"2026-02-18T08:09:37.900657+0000","last_unstale":"2026-02-18T08:09:39.904328+0000","last_undegraded":"2026-02-18T08:09:39.904328+0000","last_fullsized":"2026-02-18T08:09:39.904328+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-18T08:09:36.890905+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-18T08:09:36.890905+0000","last_clean_scrub_stamp":"2026-02-18T08:09:36.890905+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-19T16:39:19.623528+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00067508900000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.4","version":"0'0","reported_seq":20,"reported_epoch":14,"state":"active+clean","last_fresh":"2026-02-18T08:09:39.904079+0000","last_change":"2026-02-18T08:09:39.904269+0000","last_active":"2026-02-18T08:09:39.904079+0000","last_peered":"2026-02-18T08:09:39.904079+0000","last_clean":"2026-02-18T08:09:39.904079+0000","last_became_active":"2026-02-18T08:09:37.901625+0000","last_became_peered":"2026-02-18T08:09:37.901625+0000","last_unstale":"2026-02-18T08:09:39.904079+0000","last_undegraded":"2026-02-18T08:09:39.904079+0000","last_fullsized":"2026-02-18T08:09:39.904079+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-18T08:09:36.890905+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-18T08:09:36.890905+0000","last_clean_scrub_stamp":"2026-02-18T08:09:36.890905+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-19T14:48:52.058850+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00056007599999999995,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.2","version":"14'2","reported_seq":22,"reported_epoch":14,"state":"active+clean","last_fresh":"2026-02-18T08:09:39.902339+0000","last_change":"2026-02-18T08:09:39.902339+0000","last_active":"2026-02-18T08:09:39.902339+0000","last_peered":"2026-02-18T08:09:39.902339+0000","last_clean":"2026-02-18T08:09:39.902339+0000","last_became_active":"2026-02-18T08:09:37.900679+0000","last_became_peered":"2026-02-18T08:09:37.900679+0000","last_unstale":"2026-02-18T08:09:39.902339+0000","last_undegraded":"2026-02-18T08:09:39.902339+0000","last_fullsized":"2026-02-18T08:09:39.902339+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-18T08:09:36.890905+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-18T08:09:36.890905+0000","last_clean_scrub_stamp":"2026-02-18T08:09:36.890905+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-19T14:23:03.463004+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00035240299999999999,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.1","version":"0'0","reported_seq":18,"reported_epoch":14,"state":"active+clean","last_fresh":"2026-02-18T08:09:40.252222+0000","last_change":"2026-02-18T08:09:40.252366+0000","last_active":"2026-02-18T08:09:40.252222+0000","last_peered":"2026-02-18T08:09:40.252222+0000","last_clean":"2026-02-18T08:09:40.252222+0000","last_became_active":"2026-02-18T08:09:37.900553+0000","last_became_peered":"2026-02-18T08:09:37.900553+0000","last_unstale":"2026-02-18T08:09:40.252222+0000","last_undegraded":"2026-02-18T08:09:40.252222+0000","last_fullsized":"2026-02-18T08:09:40.252222+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-18T08:09:36.890905+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-18T08:09:36.890905+0000","last_clean_scrub_stamp":"2026-02-18T08:09:36.890905+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-19T19:45:29.501652+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000311546,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.0","version":"0'0","reported_seq":18,"reported_epoch":14,"state":"active+clean","last_fresh":"2026-02-18T08:09:40.252253+0000","last_change":"2026-02-18T08:09:40.252404+0000","last_active":"2026-02-18T08:09:40.252253+0000","last_peered":"2026-02-18T08:09:40.252253+0000","last_clean":"2026-02-18T08:09:40.252253+0000","last_became_active":"2026-02-18T08:09:37.900556+0000","last_became_peered":"2026-02-18T08:09:37.900556+0000","last_unstale":"2026-02-18T08:09:40.252253+0000","last_undegraded":"2026-02-18T08:09:40.252253+0000","last_fullsized":"2026-02-18T08:09:40.252253+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-18T08:09:36.890905+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-18T08:09:36.890905+0000","last_clean_scrub_stamp":"2026-02-18T08:09:36.890905+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-19T15:36:20.626321+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032881799999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.3","version":"12'1","reported_seq":21,"reported_epoch":14,"state":"active+clean","last_fresh":"2026-02-18T08:09:39.904531+0000","last_change":"2026-02-18T08:09:39.904803+0000","last_active":"2026-02-18T08:09:39.904531+0000","last_peered":"2026-02-18T08:09:39.904531+0000","last_clean":"2026-02-18T08:09:39.904531+0000","last_became_active":"2026-02-18T08:09:37.900195+0000","last_became_peered":"2026-02-18T08:09:37.900195+0000","last_unstale":"2026-02-18T08:09:39.904531+0000","last_undegraded":"2026-02-18T08:09:39.904531+0000","last_fullsized":"2026-02-18T08:09:39.904531+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-18T08:09:36.890905+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-18T08:09:36.890905+0000","last_clean_scrub_stamp":"2026-02-18T08:09:36.890905+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-19T09:59:37.426943+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00096736000000000001,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"1.0","version":"10'72","reported_seq":109,"reported_epoch":14,"state":"active+clean","last_fresh":"2026-02-18T08:09:39.903541+0000","last_change":"2026-02-18T08:09:36.049944+0000","last_active":"2026-02-18T08:09:39.903541+0000","last_peered":"2026-02-18T08:09:39.903541+0000","last_clean":"2026-02-18T08:09:39.903541+0000","last_became_active":"2026-02-18T08:09:36.049266+0000","last_became_peered":"2026-02-18T08:09:36.049266+0000","last_unstale":"2026-02-18T08:09:39.903541+0000","last_undegraded":"2026-02-18T08:09:39.903541+0000","last_fullsized":"2026-02-18T08:09:39.903541+0000","mapping_epoch":9,"log_start":"0'0","ondisk_log_start":"0'0","created":9,"last_epoch_clean":10,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-18T08:09:34.886217+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-18T08:09:34.886217+0000","last_clean_scrub_stamp":"2026-02-18T08:09:34.886217+0000","objects_scrubbed":0,"log_size":72,"log_dups_size":0,"ondisk_log_size":72,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-19T08:12:47.784849+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":38,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":16,"acting":16,"num_store_stats":3},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1187840,"data_stored":1180736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":72,"ondisk_log_size":72,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":2,"up_from":8,"seq":34359738373,"num_pgs":3,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27064,"kb_used_data":104,"kb_used_omap":6,"kb_used_meta":26937,"kb_avail":94344776,"statfs":{"total":96636764160,"available":96609050624,"internally_reserved":0,"allocated":106496,"data_stored":28372,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6167,"internal_metadata":27584489},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":8,"seq":34359738373,"num_pgs":9,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27852,"kb_used_data":704,"kb_used_omap":7,"kb_used_meta":26936,"kb_avail":94343988,"statfs":{"total":96636764160,"available":96608243712,"internally_reserved":0,"allocated":720896,"data_stored":625803,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8125,"internal_metadata":27582531},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":8,"seq":34359738372,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27852,"kb_used_data":704,"kb_used_omap":7,"kb_used_meta":26936,"kb_avail":94343988,"statfs":{"total":96636764160,"available":96608243712,"internally_reserved":0,"allocated":720896,"data_stored":625803,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8125,"internal_metadata":27582531},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2026-02-18T08:09:43.538 DEBUG:teuthology.orchestra.run.trial163:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 pg dump --format=json 2026-02-18T08:09:43.688 INFO:teuthology.orchestra.run.trial163.stdout: 2026-02-18T08:09:43.689 INFO:teuthology.orchestra.run.trial163.stderr:dumped all 2026-02-18T08:09:43.700 INFO:teuthology.orchestra.run.trial163.stdout:{"pg_ready":true,"pg_map":{"version":16,"stamp":"2026-02-18T08:09:42.477283+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590387,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":103,"num_write_kb":1630,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":75,"ondisk_log_size":75,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":18,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":82768,"kb_used_data":1512,"kb_used_omap":21,"kb_used_meta":80810,"kb_avail":283032752,"statfs":{"total":289910292480,"available":289825538048,"internally_reserved":0,"allocated":1548288,"data_stored":1279978,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":22417,"internal_metadata":82749551},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":11,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"2.574930"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":20,"reported_epoch":14,"state":"active+clean","last_fresh":"2026-02-18T08:09:39.904376+0000","last_change":"2026-02-18T08:09:39.904504+0000","last_active":"2026-02-18T08:09:39.904376+0000","last_peered":"2026-02-18T08:09:39.904376+0000","last_clean":"2026-02-18T08:09:39.904376+0000","last_became_active":"2026-02-18T08:09:37.900809+0000","last_became_peered":"2026-02-18T08:09:37.900809+0000","last_unstale":"2026-02-18T08:09:39.904376+0000","last_undegraded":"2026-02-18T08:09:39.904376+0000","last_fullsized":"2026-02-18T08:09:39.904376+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-18T08:09:36.890905+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-18T08:09:36.890905+0000","last_clean_scrub_stamp":"2026-02-18T08:09:36.890905+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-19T18:39:43.531542+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00068838299999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.6","version":"0'0","reported_seq":20,"reported_epoch":14,"state":"active+clean","last_fresh":"2026-02-18T08:09:39.904123+0000","last_change":"2026-02-18T08:09:39.904331+0000","last_active":"2026-02-18T08:09:39.904123+0000","last_peered":"2026-02-18T08:09:39.904123+0000","last_clean":"2026-02-18T08:09:39.904123+0000","last_became_active":"2026-02-18T08:09:37.900594+0000","last_became_peered":"2026-02-18T08:09:37.900594+0000","last_unstale":"2026-02-18T08:09:39.904123+0000","last_undegraded":"2026-02-18T08:09:39.904123+0000","last_fullsized":"2026-02-18T08:09:39.904123+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-18T08:09:36.890905+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-18T08:09:36.890905+0000","last_clean_scrub_stamp":"2026-02-18T08:09:36.890905+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-19T19:59:21.557650+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00053109200000000004,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.5","version":"0'0","reported_seq":20,"reported_epoch":14,"state":"active+clean","last_fresh":"2026-02-18T08:09:39.904328+0000","last_change":"2026-02-18T08:09:39.904488+0000","last_active":"2026-02-18T08:09:39.904328+0000","last_peered":"2026-02-18T08:09:39.904328+0000","last_clean":"2026-02-18T08:09:39.904328+0000","last_became_active":"2026-02-18T08:09:37.900657+0000","last_became_peered":"2026-02-18T08:09:37.900657+0000","last_unstale":"2026-02-18T08:09:39.904328+0000","last_undegraded":"2026-02-18T08:09:39.904328+0000","last_fullsized":"2026-02-18T08:09:39.904328+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-18T08:09:36.890905+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-18T08:09:36.890905+0000","last_clean_scrub_stamp":"2026-02-18T08:09:36.890905+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-19T16:39:19.623528+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00067508900000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.4","version":"0'0","reported_seq":20,"reported_epoch":14,"state":"active+clean","last_fresh":"2026-02-18T08:09:39.904079+0000","last_change":"2026-02-18T08:09:39.904269+0000","last_active":"2026-02-18T08:09:39.904079+0000","last_peered":"2026-02-18T08:09:39.904079+0000","last_clean":"2026-02-18T08:09:39.904079+0000","last_became_active":"2026-02-18T08:09:37.901625+0000","last_became_peered":"2026-02-18T08:09:37.901625+0000","last_unstale":"2026-02-18T08:09:39.904079+0000","last_undegraded":"2026-02-18T08:09:39.904079+0000","last_fullsized":"2026-02-18T08:09:39.904079+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-18T08:09:36.890905+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-18T08:09:36.890905+0000","last_clean_scrub_stamp":"2026-02-18T08:09:36.890905+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-19T14:48:52.058850+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00056007599999999995,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.2","version":"14'2","reported_seq":22,"reported_epoch":14,"state":"active+clean","last_fresh":"2026-02-18T08:09:39.902339+0000","last_change":"2026-02-18T08:09:39.902339+0000","last_active":"2026-02-18T08:09:39.902339+0000","last_peered":"2026-02-18T08:09:39.902339+0000","last_clean":"2026-02-18T08:09:39.902339+0000","last_became_active":"2026-02-18T08:09:37.900679+0000","last_became_peered":"2026-02-18T08:09:37.900679+0000","last_unstale":"2026-02-18T08:09:39.902339+0000","last_undegraded":"2026-02-18T08:09:39.902339+0000","last_fullsized":"2026-02-18T08:09:39.902339+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-18T08:09:36.890905+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-18T08:09:36.890905+0000","last_clean_scrub_stamp":"2026-02-18T08:09:36.890905+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-19T14:23:03.463004+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00035240299999999999,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.1","version":"0'0","reported_seq":18,"reported_epoch":14,"state":"active+clean","last_fresh":"2026-02-18T08:09:40.252222+0000","last_change":"2026-02-18T08:09:40.252366+0000","last_active":"2026-02-18T08:09:40.252222+0000","last_peered":"2026-02-18T08:09:40.252222+0000","last_clean":"2026-02-18T08:09:40.252222+0000","last_became_active":"2026-02-18T08:09:37.900553+0000","last_became_peered":"2026-02-18T08:09:37.900553+0000","last_unstale":"2026-02-18T08:09:40.252222+0000","last_undegraded":"2026-02-18T08:09:40.252222+0000","last_fullsized":"2026-02-18T08:09:40.252222+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-18T08:09:36.890905+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-18T08:09:36.890905+0000","last_clean_scrub_stamp":"2026-02-18T08:09:36.890905+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-19T19:45:29.501652+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000311546,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.0","version":"0'0","reported_seq":18,"reported_epoch":14,"state":"active+clean","last_fresh":"2026-02-18T08:09:40.252253+0000","last_change":"2026-02-18T08:09:40.252404+0000","last_active":"2026-02-18T08:09:40.252253+0000","last_peered":"2026-02-18T08:09:40.252253+0000","last_clean":"2026-02-18T08:09:40.252253+0000","last_became_active":"2026-02-18T08:09:37.900556+0000","last_became_peered":"2026-02-18T08:09:37.900556+0000","last_unstale":"2026-02-18T08:09:40.252253+0000","last_undegraded":"2026-02-18T08:09:40.252253+0000","last_fullsized":"2026-02-18T08:09:40.252253+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-18T08:09:36.890905+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-18T08:09:36.890905+0000","last_clean_scrub_stamp":"2026-02-18T08:09:36.890905+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-19T15:36:20.626321+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032881799999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.3","version":"12'1","reported_seq":21,"reported_epoch":14,"state":"active+clean","last_fresh":"2026-02-18T08:09:39.904531+0000","last_change":"2026-02-18T08:09:39.904803+0000","last_active":"2026-02-18T08:09:39.904531+0000","last_peered":"2026-02-18T08:09:39.904531+0000","last_clean":"2026-02-18T08:09:39.904531+0000","last_became_active":"2026-02-18T08:09:37.900195+0000","last_became_peered":"2026-02-18T08:09:37.900195+0000","last_unstale":"2026-02-18T08:09:39.904531+0000","last_undegraded":"2026-02-18T08:09:39.904531+0000","last_fullsized":"2026-02-18T08:09:39.904531+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-18T08:09:36.890905+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-18T08:09:36.890905+0000","last_clean_scrub_stamp":"2026-02-18T08:09:36.890905+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-19T09:59:37.426943+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00096736000000000001,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"1.0","version":"10'72","reported_seq":109,"reported_epoch":14,"state":"active+clean","last_fresh":"2026-02-18T08:09:39.903541+0000","last_change":"2026-02-18T08:09:36.049944+0000","last_active":"2026-02-18T08:09:39.903541+0000","last_peered":"2026-02-18T08:09:39.903541+0000","last_clean":"2026-02-18T08:09:39.903541+0000","last_became_active":"2026-02-18T08:09:36.049266+0000","last_became_peered":"2026-02-18T08:09:36.049266+0000","last_unstale":"2026-02-18T08:09:39.903541+0000","last_undegraded":"2026-02-18T08:09:39.903541+0000","last_fullsized":"2026-02-18T08:09:39.903541+0000","mapping_epoch":9,"log_start":"0'0","ondisk_log_start":"0'0","created":9,"last_epoch_clean":10,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-18T08:09:34.886217+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-18T08:09:34.886217+0000","last_clean_scrub_stamp":"2026-02-18T08:09:34.886217+0000","objects_scrubbed":0,"log_size":72,"log_dups_size":0,"ondisk_log_size":72,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-19T08:12:47.784849+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":38,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":16,"acting":16,"num_store_stats":3},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1187840,"data_stored":1180736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":72,"ondisk_log_size":72,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":2,"up_from":8,"seq":34359738373,"num_pgs":3,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27064,"kb_used_data":104,"kb_used_omap":6,"kb_used_meta":26937,"kb_avail":94344776,"statfs":{"total":96636764160,"available":96609050624,"internally_reserved":0,"allocated":106496,"data_stored":28372,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6167,"internal_metadata":27584489},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":8,"seq":34359738373,"num_pgs":9,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27852,"kb_used_data":704,"kb_used_omap":7,"kb_used_meta":26936,"kb_avail":94343988,"statfs":{"total":96636764160,"available":96608243712,"internally_reserved":0,"allocated":720896,"data_stored":625803,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8125,"internal_metadata":27582531},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":8,"seq":34359738372,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27852,"kb_used_data":704,"kb_used_omap":7,"kb_used_meta":26936,"kb_avail":94343988,"statfs":{"total":96636764160,"available":96608243712,"internally_reserved":0,"allocated":720896,"data_stored":625803,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8125,"internal_metadata":27582531},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2026-02-18T08:09:43.701 INFO:tasks.ceph.ceph_manager.cluster2:clean! 2026-02-18T08:09:43.701 INFO:tasks.ceph:Waiting until ceph cluster cluster2 is healthy... 2026-02-18T08:09:43.701 INFO:tasks.ceph.ceph_manager.cluster2:wait_until_healthy 2026-02-18T08:09:43.701 DEBUG:teuthology.orchestra.run.trial163:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 health --format=json 2026-02-18T08:09:43.871 INFO:teuthology.orchestra.run.trial163.stdout: 2026-02-18T08:09:43.871 INFO:teuthology.orchestra.run.trial163.stdout:{"status":"HEALTH_OK","checks":{},"mutes":[]} 2026-02-18T08:09:43.882 INFO:tasks.ceph.ceph_manager.cluster2:wait_until_healthy done 2026-02-18T08:09:43.882 INFO:teuthology.run_tasks:Running task exec... 2026-02-18T08:09:43.895 INFO:teuthology.task.exec:Executing custom commands... 2026-02-18T08:09:43.895 INFO:teuthology.task.exec:Running commands on role cluster1.client.mirror.0 host ubuntu@trial163.front.sepia.ceph.com 2026-02-18T08:09:43.895 DEBUG:teuthology.orchestra.run.trial163:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'sudo ceph --cluster cluster1 auth caps client.mirror mon '"'"'profile rbd-mirror-peer'"'"' osd '"'"'profile rbd'"'"'' 2026-02-18T08:09:44.087 INFO:teuthology.orchestra.run.trial163.stdout:[client.mirror] 2026-02-18T08:09:44.087 INFO:teuthology.orchestra.run.trial163.stdout: key = AQCac5VptF2JGxAA8fwGPHy3Vimem1mhfATLqQ== 2026-02-18T08:09:44.087 INFO:teuthology.orchestra.run.trial163.stdout: caps mon = "profile rbd-mirror-peer" 2026-02-18T08:09:44.088 INFO:teuthology.orchestra.run.trial163.stdout: caps osd = "profile rbd" 2026-02-18T08:09:44.088 INFO:teuthology.orchestra.run.trial163.stderr:updated caps for client.mirror 2026-02-18T08:09:44.100 DEBUG:teuthology.orchestra.run.trial163:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'sudo ceph --cluster cluster1 auth caps client.mirror.0 mon '"'"'profile rbd-mirror'"'"' osd '"'"'profile rbd'"'"'' 2026-02-18T08:09:44.295 INFO:teuthology.orchestra.run.trial163.stdout:[client.mirror.0] 2026-02-18T08:09:44.296 INFO:teuthology.orchestra.run.trial163.stdout: key = AQCac5VpY4WIHxAAb8lF8oj1LHV4QwUL5CjlEA== 2026-02-18T08:09:44.296 INFO:teuthology.orchestra.run.trial163.stdout: caps mon = "profile rbd-mirror" 2026-02-18T08:09:44.296 INFO:teuthology.orchestra.run.trial163.stdout: caps osd = "profile rbd" 2026-02-18T08:09:44.296 INFO:teuthology.orchestra.run.trial163.stderr:updated caps for client.mirror.0 2026-02-18T08:09:44.309 DEBUG:teuthology.orchestra.run.trial163:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'sudo ceph --cluster cluster1 auth caps client.mirror.1 mon '"'"'profile rbd-mirror'"'"' osd '"'"'profile rbd'"'"'' 2026-02-18T08:09:44.502 INFO:teuthology.orchestra.run.trial163.stdout:[client.mirror.1] 2026-02-18T08:09:44.502 INFO:teuthology.orchestra.run.trial163.stdout: key = AQCac5VpXW/OIxAA/gkf9pv7/1PZCRjYBasG5g== 2026-02-18T08:09:44.502 INFO:teuthology.orchestra.run.trial163.stdout: caps mon = "profile rbd-mirror" 2026-02-18T08:09:44.502 INFO:teuthology.orchestra.run.trial163.stdout: caps osd = "profile rbd" 2026-02-18T08:09:44.503 INFO:teuthology.orchestra.run.trial163.stderr:updated caps for client.mirror.1 2026-02-18T08:09:44.514 DEBUG:teuthology.orchestra.run.trial163:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'sudo ceph --cluster cluster1 auth caps client.mirror.2 mon '"'"'profile rbd-mirror'"'"' osd '"'"'profile rbd'"'"'' 2026-02-18T08:09:44.705 INFO:teuthology.orchestra.run.trial163.stdout:[client.mirror.2] 2026-02-18T08:09:44.705 INFO:teuthology.orchestra.run.trial163.stdout: key = AQCac5VpxRvaJxAAqziyv6zUs3W9g1C4i6ZfSQ== 2026-02-18T08:09:44.705 INFO:teuthology.orchestra.run.trial163.stdout: caps mon = "profile rbd-mirror" 2026-02-18T08:09:44.705 INFO:teuthology.orchestra.run.trial163.stdout: caps osd = "profile rbd" 2026-02-18T08:09:44.705 INFO:teuthology.orchestra.run.trial163.stderr:updated caps for client.mirror.2 2026-02-18T08:09:44.718 DEBUG:teuthology.orchestra.run.trial163:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'sudo ceph --cluster cluster1 auth caps client.mirror.3 mon '"'"'profile rbd-mirror'"'"' osd '"'"'profile rbd'"'"'' 2026-02-18T08:09:44.913 INFO:teuthology.orchestra.run.trial163.stdout:[client.mirror.3] 2026-02-18T08:09:44.913 INFO:teuthology.orchestra.run.trial163.stdout: key = AQCac5Vp+RCqKxAAtMZG+CvONyOSXdhiObn7JA== 2026-02-18T08:09:44.913 INFO:teuthology.orchestra.run.trial163.stdout: caps mon = "profile rbd-mirror" 2026-02-18T08:09:44.913 INFO:teuthology.orchestra.run.trial163.stdout: caps osd = "profile rbd" 2026-02-18T08:09:44.913 INFO:teuthology.orchestra.run.trial163.stderr:updated caps for client.mirror.3 2026-02-18T08:09:44.926 DEBUG:teuthology.orchestra.run.trial163:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'sudo ceph --cluster cluster2 auth caps client.mirror mon '"'"'profile rbd-mirror-peer'"'"' osd '"'"'profile rbd'"'"'' 2026-02-18T08:09:45.109 INFO:teuthology.orchestra.run.trial163.stdout:[client.mirror] 2026-02-18T08:09:45.109 INFO:teuthology.orchestra.run.trial163.stdout: key = AQCxc5Vp+ba7FhAAUDOVjQYTJTHKeKjrSYceZw== 2026-02-18T08:09:45.109 INFO:teuthology.orchestra.run.trial163.stdout: caps mon = "profile rbd-mirror-peer" 2026-02-18T08:09:45.109 INFO:teuthology.orchestra.run.trial163.stdout: caps osd = "profile rbd" 2026-02-18T08:09:45.109 INFO:teuthology.orchestra.run.trial163.stderr:updated caps for client.mirror 2026-02-18T08:09:45.120 DEBUG:teuthology.orchestra.run.trial163:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'sudo ceph --cluster cluster2 auth caps client.mirror.0 mon '"'"'profile rbd-mirror'"'"' osd '"'"'profile rbd'"'"'' 2026-02-18T08:09:45.312 INFO:teuthology.orchestra.run.trial163.stdout:[client.mirror.0] 2026-02-18T08:09:45.313 INFO:teuthology.orchestra.run.trial163.stdout: key = AQCxc5VpaB7BGhAAGOanPLvvNVlEqJwlZukPNQ== 2026-02-18T08:09:45.313 INFO:teuthology.orchestra.run.trial163.stdout: caps mon = "profile rbd-mirror" 2026-02-18T08:09:45.313 INFO:teuthology.orchestra.run.trial163.stdout: caps osd = "profile rbd" 2026-02-18T08:09:45.313 INFO:teuthology.orchestra.run.trial163.stderr:updated caps for client.mirror.0 2026-02-18T08:09:45.324 DEBUG:teuthology.orchestra.run.trial163:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'sudo ceph --cluster cluster2 auth caps client.mirror.1 mon '"'"'profile rbd-mirror'"'"' osd '"'"'profile rbd'"'"'' 2026-02-18T08:09:45.516 INFO:teuthology.orchestra.run.trial163.stdout:[client.mirror.1] 2026-02-18T08:09:45.516 INFO:teuthology.orchestra.run.trial163.stdout: key = AQCxc5VptseJHhAAeDJEY7SUTM372z+kYWiCXg== 2026-02-18T08:09:45.516 INFO:teuthology.orchestra.run.trial163.stdout: caps mon = "profile rbd-mirror" 2026-02-18T08:09:45.516 INFO:teuthology.orchestra.run.trial163.stdout: caps osd = "profile rbd" 2026-02-18T08:09:45.516 INFO:teuthology.orchestra.run.trial163.stderr:updated caps for client.mirror.1 2026-02-18T08:09:45.528 DEBUG:teuthology.orchestra.run.trial163:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'sudo ceph --cluster cluster2 auth caps client.mirror.2 mon '"'"'profile rbd-mirror'"'"' osd '"'"'profile rbd'"'"'' 2026-02-18T08:09:45.719 INFO:teuthology.orchestra.run.trial163.stdout:[client.mirror.2] 2026-02-18T08:09:45.719 INFO:teuthology.orchestra.run.trial163.stdout: key = AQCxc5VpOCBVIhAAGB4DXxBYl8tIxX1TtcG9sg== 2026-02-18T08:09:45.720 INFO:teuthology.orchestra.run.trial163.stdout: caps mon = "profile rbd-mirror" 2026-02-18T08:09:45.720 INFO:teuthology.orchestra.run.trial163.stdout: caps osd = "profile rbd" 2026-02-18T08:09:45.720 INFO:teuthology.orchestra.run.trial163.stderr:updated caps for client.mirror.2 2026-02-18T08:09:45.733 DEBUG:teuthology.orchestra.run.trial163:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'sudo ceph --cluster cluster2 auth caps client.mirror.3 mon '"'"'profile rbd-mirror'"'"' osd '"'"'profile rbd'"'"'' 2026-02-18T08:09:45.928 INFO:teuthology.orchestra.run.trial163.stdout:[client.mirror.3] 2026-02-18T08:09:45.928 INFO:teuthology.orchestra.run.trial163.stdout: key = AQCxc5VpaSonJhAA2BHCinkac3fk2FTSuGaE1Q== 2026-02-18T08:09:45.928 INFO:teuthology.orchestra.run.trial163.stdout: caps mon = "profile rbd-mirror" 2026-02-18T08:09:45.928 INFO:teuthology.orchestra.run.trial163.stdout: caps osd = "profile rbd" 2026-02-18T08:09:45.929 INFO:teuthology.orchestra.run.trial163.stderr:updated caps for client.mirror.3 2026-02-18T08:09:45.941 INFO:teuthology.run_tasks:Running task rbd-mirror... 2026-02-18T08:09:45.963 INFO:tasks.rbd_mirror.cluster1.client.mirror.0:Restarting daemon 2026-02-18T08:09:45.963 DEBUG:teuthology.orchestra.run.trial163:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.0 2026-02-18T08:09:45.965 INFO:tasks.rbd_mirror.cluster1.client.mirror.0:Started 2026-02-18T08:09:45.966 INFO:teuthology.run_tasks:Running task rbd-mirror... 2026-02-18T08:09:45.980 INFO:tasks.rbd_mirror.cluster1.client.mirror.1:Restarting daemon 2026-02-18T08:09:45.980 DEBUG:teuthology.orchestra.run.trial163:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.1 2026-02-18T08:09:45.983 INFO:tasks.rbd_mirror.cluster1.client.mirror.1:Started 2026-02-18T08:09:45.983 INFO:teuthology.run_tasks:Running task rbd-mirror... 2026-02-18T08:09:45.998 INFO:tasks.rbd_mirror.cluster1.client.mirror.2:Restarting daemon 2026-02-18T08:09:45.998 DEBUG:teuthology.orchestra.run.trial163:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.2 2026-02-18T08:09:46.000 INFO:tasks.rbd_mirror.cluster1.client.mirror.2:Started 2026-02-18T08:09:46.000 INFO:teuthology.run_tasks:Running task rbd-mirror... 2026-02-18T08:09:46.015 INFO:tasks.rbd_mirror.cluster1.client.mirror.3:Restarting daemon 2026-02-18T08:09:46.015 DEBUG:teuthology.orchestra.run.trial163:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.3 2026-02-18T08:09:46.018 INFO:tasks.rbd_mirror.cluster1.client.mirror.3:Started 2026-02-18T08:09:46.018 INFO:teuthology.run_tasks:Running task rbd-mirror... 2026-02-18T08:09:46.036 INFO:tasks.rbd_mirror.cluster2.client.mirror.0:Restarting daemon 2026-02-18T08:09:46.036 DEBUG:teuthology.orchestra.run.trial163:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.0 2026-02-18T08:09:46.038 INFO:tasks.rbd_mirror.cluster2.client.mirror.0:Started 2026-02-18T08:09:46.038 INFO:teuthology.run_tasks:Running task rbd-mirror... 2026-02-18T08:09:46.053 INFO:tasks.rbd_mirror.cluster2.client.mirror.1:Restarting daemon 2026-02-18T08:09:46.053 DEBUG:teuthology.orchestra.run.trial163:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.1 2026-02-18T08:09:46.055 INFO:tasks.rbd_mirror.cluster2.client.mirror.1:Started 2026-02-18T08:09:46.055 INFO:teuthology.run_tasks:Running task rbd-mirror... 2026-02-18T08:09:46.070 INFO:tasks.rbd_mirror.cluster2.client.mirror.2:Restarting daemon 2026-02-18T08:09:46.071 DEBUG:teuthology.orchestra.run.trial163:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.2 2026-02-18T08:09:46.072 INFO:tasks.rbd_mirror.cluster2.client.mirror.2:Started 2026-02-18T08:09:46.073 INFO:teuthology.run_tasks:Running task rbd-mirror... 2026-02-18T08:09:46.088 INFO:tasks.rbd_mirror.cluster2.client.mirror.3:Restarting daemon 2026-02-18T08:09:46.088 DEBUG:teuthology.orchestra.run.trial163:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.3 2026-02-18T08:09:46.090 INFO:tasks.rbd_mirror.cluster2.client.mirror.3:Started 2026-02-18T08:09:46.090 INFO:teuthology.run_tasks:Running task rbd-mirror-thrash... 2026-02-18T08:09:46.118 INFO:tasks.rbd_mirror_thrash:rbd_mirror_thrash using random seed: 1771402186 2026-02-18T08:09:46.119 DEBUG:tasks.rbd_mirror_thrash:Yielding 2026-02-18T08:09:46.119 INFO:teuthology.run_tasks:Running task rbd-mirror-thrash... 2026-02-18T08:09:46.138 INFO:tasks.rbd_mirror_thrash:rbd_mirror_thrash using random seed: 1771402186 2026-02-18T08:09:46.138 DEBUG:tasks.rbd_mirror_thrash:Yielding 2026-02-18T08:09:46.138 INFO:teuthology.run_tasks:Running task workunit... 2026-02-18T08:09:46.166 INFO:tasks.workunit:Pulling workunits from ref adc1923395f2bcd6ae89b5d331d278364d1848ed 2026-02-18T08:09:46.166 INFO:tasks.workunit:Making a separate scratch dir for every client... 2026-02-18T08:09:46.166 DEBUG:teuthology.orchestra.run.trial163:> stat -- /home/ubuntu/cephtest/mnt.cluster1.mirror 2026-02-18T08:09:46.167 INFO:tasks.rbd_mirror_thrash:starting thrash for cluster cluster1 2026-02-18T08:09:46.167 INFO:tasks.rbd_mirror_thrash:waiting for 103.53557704096373 secs before thrashing 2026-02-18T08:09:46.167 INFO:tasks.rbd_mirror_thrash:starting thrash for cluster cluster2 2026-02-18T08:09:46.167 INFO:tasks.rbd_mirror_thrash:waiting for 60.08863322657584 secs before thrashing 2026-02-18T08:09:46.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2026-02-18T08:09:46.171 INFO:teuthology.orchestra.run.trial163.stderr:stat: cannot statx '/home/ubuntu/cephtest/mnt.cluster1.mirror': No such file or directory 2026-02-18T08:09:46.172 DEBUG:teuthology.orchestra.run.trial163:> mkdir -- /home/ubuntu/cephtest/mnt.cluster1.mirror 2026-02-18T08:09:46.217 INFO:tasks.workunit:Created dir /home/ubuntu/cephtest/mnt.cluster1.mirror 2026-02-18T08:09:46.217 DEBUG:teuthology.orchestra.run.trial163:> cd -- /home/ubuntu/cephtest/mnt.cluster1.mirror && mkdir -- client.mirror 2026-02-18T08:09:46.260 INFO:tasks.workunit:timeout=6h 2026-02-18T08:09:46.260 INFO:tasks.workunit:cleanup=True 2026-02-18T08:09:46.261 DEBUG:teuthology.orchestra.run.trial163:> rm -rf /home/ubuntu/cephtest/clone.cluster1.client.mirror && git clone https://github.com/ceph/ceph-ci.git /home/ubuntu/cephtest/clone.cluster1.client.mirror && cd /home/ubuntu/cephtest/clone.cluster1.client.mirror && git checkout adc1923395f2bcd6ae89b5d331d278364d1848ed 2026-02-18T08:09:46.312 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Cloning into '/home/ubuntu/cephtest/clone.cluster1.client.mirror'... 2026-02-18T08:10:08.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Note: switching to 'adc1923395f2bcd6ae89b5d331d278364d1848ed'. 2026-02-18T08:10:08.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: 2026-02-18T08:10:08.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:You are in 'detached HEAD' state. You can look around, make experimental 2026-02-18T08:10:08.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:changes and commit them, and you can discard any commits you make in this 2026-02-18T08:10:08.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:state without impacting any branches by switching back to a branch. 2026-02-18T08:10:08.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: 2026-02-18T08:10:08.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:If you want to create a new branch to retain commits you create, you may 2026-02-18T08:10:08.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:do so (now or later) by using -c with the switch command. Example: 2026-02-18T08:10:08.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: 2026-02-18T08:10:08.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: git switch -c 2026-02-18T08:10:08.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: 2026-02-18T08:10:08.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Or undo this operation with: 2026-02-18T08:10:08.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: 2026-02-18T08:10:08.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: git switch - 2026-02-18T08:10:08.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: 2026-02-18T08:10:08.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Turn off this advice by setting config variable advice.detachedHead to false 2026-02-18T08:10:08.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: 2026-02-18T08:10:08.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:HEAD is now at adc1923395f Merge PR #67279 into tentacle 2026-02-18T08:10:08.249 DEBUG:teuthology.orchestra.run.trial163:> cd -- /home/ubuntu/cephtest/clone.cluster1.client.mirror/qa/workunits && if test -e Makefile ; then make ; fi && find -executable -type f -printf '%P\0' >/home/ubuntu/cephtest/workunits.list.cluster1.client.mirror 2026-02-18T08:10:08.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:for d in direct_io fs ; do ( cd $d ; make all ) ; done 2026-02-18T08:10:08.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:make[1]: Entering directory '/home/ubuntu/cephtest/clone.cluster1.client.mirror/qa/workunits/direct_io' 2026-02-18T08:10:08.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:cc -Wall -Wextra -D_GNU_SOURCE direct_io_test.c -o direct_io_test 2026-02-18T08:10:08.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_sync_io.c -o test_sync_io 2026-02-18T08:10:08.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_short_dio_read.c -o test_short_dio_read 2026-02-18T08:10:08.449 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:make[1]: Leaving directory '/home/ubuntu/cephtest/clone.cluster1.client.mirror/qa/workunits/direct_io' 2026-02-18T08:10:08.451 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:make[1]: Entering directory '/home/ubuntu/cephtest/clone.cluster1.client.mirror/qa/workunits/fs' 2026-02-18T08:10:08.451 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_o_trunc.c -o test_o_trunc 2026-02-18T08:10:08.473 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:make[1]: Leaving directory '/home/ubuntu/cephtest/clone.cluster1.client.mirror/qa/workunits/fs' 2026-02-18T08:10:08.477 DEBUG:teuthology.orchestra.run.trial163:> set -ex 2026-02-18T08:10:08.477 DEBUG:teuthology.orchestra.run.trial163:> dd if=/home/ubuntu/cephtest/workunits.list.cluster1.client.mirror of=/dev/stdout 2026-02-18T08:10:08.521 INFO:tasks.workunit:Running workunits matching rbd/rbd_mirror_stress.sh on cluster1.client.mirror... 2026-02-18T08:10:08.522 INFO:tasks.workunit:Running workunit rbd/rbd_mirror_stress.sh... 2026-02-18T08:10:08.522 DEBUG:teuthology.orchestra.run.trial163:workunit test rbd/rbd_mirror_stress.sh> mkdir -p -- /home/ubuntu/cephtest/mnt.cluster1.mirror/client.mirror/tmp && cd -- /home/ubuntu/cephtest/mnt.cluster1.mirror/client.mirror/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=adc1923395f2bcd6ae89b5d331d278364d1848ed TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster cluster1" CEPH_ID="mirror" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.cluster1.client.mirror CEPH_ROOT=/home/ubuntu/cephtest/clone.cluster1.client.mirror CEPH_MNT=/home/ubuntu/cephtest/mnt.cluster1.mirror CEPH_ARGS='' MIRROR_POOL_MODE=pool RBD_MIRROR_INSTANCES=4 RBD_MIRROR_MODE=journal RBD_MIRROR_USE_EXISTING_CLUSTER=1 RBD_MIRROR_USE_RBD_MIRROR=1 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 6h /home/ubuntu/cephtest/clone.cluster1.client.mirror/qa/workunits/rbd/rbd_mirror_stress.sh 2026-02-18T08:10:08.569 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ IMAGE_COUNT=50 2026-02-18T08:10:08.569 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ export LOCKDEP=0 2026-02-18T08:10:08.569 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ LOCKDEP=0 2026-02-18T08:10:08.569 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ dirname /home/ubuntu/cephtest/clone.cluster1.client.mirror/qa/workunits/rbd/rbd_mirror_stress.sh 2026-02-18T08:10:08.571 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ . /home/ubuntu/cephtest/clone.cluster1.client.mirror/qa/workunits/rbd/rbd_mirror_helpers.sh 2026-02-18T08:10:08.571 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ RBD_MIRROR_INSTANCES=4 2026-02-18T08:10:08.571 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ CLUSTER1=cluster1 2026-02-18T08:10:08.571 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ CLUSTER2=cluster2 2026-02-18T08:10:08.571 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ PEER_CLUSTER_SUFFIX= 2026-02-18T08:10:08.571 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ POOL=mirror 2026-02-18T08:10:08.571 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ PARENT_POOL=mirror_parent 2026-02-18T08:10:08.571 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ NS1=ns1 2026-02-18T08:10:08.572 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ NS2=ns2 2026-02-18T08:10:08.572 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ TEMPDIR= 2026-02-18T08:10:08.572 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ CEPH_ID=mirror 2026-02-18T08:10:08.572 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ RBD_IMAGE_FEATURES=layering,exclusive-lock,journaling 2026-02-18T08:10:08.572 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ MIRROR_USER_ID_PREFIX=mirror. 2026-02-18T08:10:08.572 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ RBD_MIRROR_MODE=journal 2026-02-18T08:10:08.572 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ MIRROR_POOL_MODE=pool 2026-02-18T08:10:08.572 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' journal = snapshot ']' 2026-02-18T08:10:08.572 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ export 'CEPH_ARGS=--id mirror' 2026-02-18T08:10:08.572 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ CEPH_ARGS='--id mirror' 2026-02-18T08:10:08.572 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ LAST_MIRROR_INSTANCE=3 2026-02-18T08:10:08.573 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ dirname /home/ubuntu/cephtest/clone.cluster1.client.mirror/qa/workunits/rbd/rbd_mirror_stress.sh 2026-02-18T08:10:08.573 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ readlink -f /home/ubuntu/cephtest/clone.cluster1.client.mirror/qa/workunits/rbd/../../../src 2026-02-18T08:10:08.574 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ CEPH_ROOT=/home/ubuntu/cephtest/clone.cluster1.client.mirror/src 2026-02-18T08:10:08.574 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ CEPH_BIN=. 2026-02-18T08:10:08.574 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ CEPH_SRC=. 2026-02-18T08:10:08.574 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' -e CMakeCache.txt ']' 2026-02-18T08:10:08.575 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 -gt 0 ']' 2026-02-18T08:10:08.575 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ setup 2026-02-18T08:10:08.575 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local c 2026-02-18T08:10:08.576 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ trap 'cleanup $?' INT TERM EXIT 2026-02-18T08:10:08.576 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ setup_tempdir 2026-02-18T08:10:08.576 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n '' ']' 2026-02-18T08:10:08.576 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ mktemp -d 2026-02-18T08:10:08.577 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ TEMPDIR=/tmp/tmp.vybh1m9yJF 2026-02-18T08:10:08.577 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -z 1 ']' 2026-02-18T08:10:08.577 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ setup_pools cluster1 cluster2 2026-02-18T08:10:08.577 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:10:08.577 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local remote_cluster=cluster2 2026-02-18T08:10:08.577 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local mon_map_file 2026-02-18T08:10:08.577 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local mon_addr 2026-02-18T08:10:08.577 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local admin_key_file 2026-02-18T08:10:08.577 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local uuid 2026-02-18T08:10:08.577 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ CEPH_ARGS= 2026-02-18T08:10:08.577 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --cluster cluster1 osd pool create mirror 64 64 2026-02-18T08:10:09.255 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:pool 'mirror' already exists 2026-02-18T08:10:09.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ CEPH_ARGS= 2026-02-18T08:10:09.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --cluster cluster1 osd pool create mirror_parent 64 64 2026-02-18T08:10:10.259 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:pool 'mirror_parent' already exists 2026-02-18T08:10:10.270 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ CEPH_ARGS= 2026-02-18T08:10:10.270 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster1 pool init mirror 2026-02-18T08:10:13.234 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ CEPH_ARGS= 2026-02-18T08:10:13.234 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster1 pool init mirror_parent 2026-02-18T08:10:16.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n '' ']' 2026-02-18T08:10:16.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ CEPH_ARGS= 2026-02-18T08:10:16.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster1 mirror pool enable --site-name cluster1 mirror pool 2026-02-18T08:10:16.472 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster1 mirror pool enable mirror_parent image 2026-02-18T08:10:16.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster1 namespace create mirror/ns1 2026-02-18T08:10:16.529 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster1 namespace create mirror/ns2 2026-02-18T08:10:16.560 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster1 namespace create mirror_parent/ns1 2026-02-18T08:10:16.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster1 mirror pool enable mirror/ns1 pool 2026-02-18T08:10:16.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster1 mirror pool enable mirror/ns2 image 2026-02-18T08:10:16.659 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster1 mirror pool enable mirror_parent/ns1 pool 2026-02-18T08:10:16.694 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -z ']' 2026-02-18T08:10:16.694 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -z ']' 2026-02-18T08:10:16.694 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ peer_add cluster1 mirror cluster2 2026-02-18T08:10:16.694 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:10:16.694 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:10:16.694 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:10:16.695 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:10:16.695 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local client_cluster=cluster2 2026-02-18T08:10:16.695 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:10:16.695 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local remote_cluster=cluster2 2026-02-18T08:10:16.695 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local uuid_var_name 2026-02-18T08:10:16.695 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n '' ']' 2026-02-18T08:10:16.695 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local error_code 2026-02-18T08:10:16.695 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local peer_uuid 2026-02-18T08:10:16.695 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 1 2 4 8 16 32 2026-02-18T08:10:16.695 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set +e 2026-02-18T08:10:16.696 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster1 mirror pool peer add mirror cluster2 2026-02-18T08:10:16.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ peer_uuid=88a484c3-d7d0-4b28-a573-340fd9e0ca28 2026-02-18T08:10:16.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ error_code=0 2026-02-18T08:10:16.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set -e 2026-02-18T08:10:16.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 -eq 17 ']' 2026-02-18T08:10:16.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 0 -eq 0 2026-02-18T08:10:16.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n '' ']' 2026-02-18T08:10:16.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:10:16.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ peer_add cluster1 mirror_parent cluster2 2026-02-18T08:10:16.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:10:16.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:10:16.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror_parent 2026-02-18T08:10:16.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:10:16.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local client_cluster=cluster2 2026-02-18T08:10:16.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:10:16.728 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local remote_cluster=cluster2 2026-02-18T08:10:16.728 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local uuid_var_name 2026-02-18T08:10:16.728 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n '' ']' 2026-02-18T08:10:16.728 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local error_code 2026-02-18T08:10:16.728 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local peer_uuid 2026-02-18T08:10:16.728 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 1 2 4 8 16 32 2026-02-18T08:10:16.728 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set +e 2026-02-18T08:10:16.728 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster1 mirror pool peer add mirror_parent cluster2 2026-02-18T08:10:16.751 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ peer_uuid=c0062f70-58f6-48a2-aa08-93e79b831ecc 2026-02-18T08:10:16.751 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ error_code=0 2026-02-18T08:10:16.752 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set -e 2026-02-18T08:10:16.752 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 -eq 17 ']' 2026-02-18T08:10:16.752 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 0 -eq 0 2026-02-18T08:10:16.752 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n '' ']' 2026-02-18T08:10:16.752 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:10:16.752 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ setup_pools cluster2 cluster1 2026-02-18T08:10:16.752 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:10:16.752 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local remote_cluster=cluster1 2026-02-18T08:10:16.752 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local mon_map_file 2026-02-18T08:10:16.752 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local mon_addr 2026-02-18T08:10:16.752 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local admin_key_file 2026-02-18T08:10:16.753 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local uuid 2026-02-18T08:10:16.753 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ CEPH_ARGS= 2026-02-18T08:10:16.753 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --cluster cluster2 osd pool create mirror 64 64 2026-02-18T08:10:17.597 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:pool 'mirror' already exists 2026-02-18T08:10:17.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ CEPH_ARGS= 2026-02-18T08:10:17.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --cluster cluster2 osd pool create mirror_parent 64 64 2026-02-18T08:10:18.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:pool 'mirror_parent' already exists 2026-02-18T08:10:18.620 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ CEPH_ARGS= 2026-02-18T08:10:18.620 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 pool init mirror 2026-02-18T08:10:21.572 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ CEPH_ARGS= 2026-02-18T08:10:21.573 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 pool init mirror_parent 2026-02-18T08:10:24.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n '' ']' 2026-02-18T08:10:24.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ CEPH_ARGS= 2026-02-18T08:10:24.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 mirror pool enable --site-name cluster2 mirror pool 2026-02-18T08:10:24.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 mirror pool enable mirror_parent image 2026-02-18T08:10:24.641 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 namespace create mirror/ns1 2026-02-18T08:10:24.672 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 namespace create mirror/ns2 2026-02-18T08:10:24.702 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 namespace create mirror_parent/ns1 2026-02-18T08:10:24.731 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 mirror pool enable mirror/ns1 pool 2026-02-18T08:10:24.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 mirror pool enable mirror/ns2 image 2026-02-18T08:10:24.796 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 mirror pool enable mirror_parent/ns1 pool 2026-02-18T08:10:24.829 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -z ']' 2026-02-18T08:10:24.829 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -z ']' 2026-02-18T08:10:24.830 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ peer_add cluster2 mirror cluster1 2026-02-18T08:10:24.830 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:10:24.830 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:10:24.830 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:10:24.830 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:10:24.830 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local client_cluster=cluster1 2026-02-18T08:10:24.830 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:10:24.830 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local remote_cluster=cluster1 2026-02-18T08:10:24.830 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local uuid_var_name 2026-02-18T08:10:24.830 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n '' ']' 2026-02-18T08:10:24.831 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local error_code 2026-02-18T08:10:24.831 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local peer_uuid 2026-02-18T08:10:24.831 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 1 2 4 8 16 32 2026-02-18T08:10:24.831 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set +e 2026-02-18T08:10:24.831 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 mirror pool peer add mirror cluster1 2026-02-18T08:10:24.853 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ peer_uuid=7ee176e7-70a8-4f63-9600-72eac624cae8 2026-02-18T08:10:24.853 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ error_code=0 2026-02-18T08:10:24.853 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set -e 2026-02-18T08:10:24.853 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 -eq 17 ']' 2026-02-18T08:10:24.853 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 0 -eq 0 2026-02-18T08:10:24.853 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n '' ']' 2026-02-18T08:10:24.853 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:10:24.853 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ peer_add cluster2 mirror_parent cluster1 2026-02-18T08:10:24.853 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:10:24.853 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:10:24.854 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror_parent 2026-02-18T08:10:24.854 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:10:24.854 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local client_cluster=cluster1 2026-02-18T08:10:24.854 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:10:24.854 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local remote_cluster=cluster1 2026-02-18T08:10:24.854 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local uuid_var_name 2026-02-18T08:10:24.854 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n '' ']' 2026-02-18T08:10:24.854 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local error_code 2026-02-18T08:10:24.854 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local peer_uuid 2026-02-18T08:10:24.854 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 1 2 4 8 16 32 2026-02-18T08:10:24.854 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set +e 2026-02-18T08:10:24.854 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 mirror pool peer add mirror_parent cluster1 2026-02-18T08:10:24.874 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ peer_uuid=610def0a-3fad-4e70-89fd-858f507afba2 2026-02-18T08:10:24.874 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ error_code=0 2026-02-18T08:10:24.874 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set -e 2026-02-18T08:10:24.874 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 -eq 17 ']' 2026-02-18T08:10:24.875 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 0 -eq 0 2026-02-18T08:10:24.875 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n '' ']' 2026-02-18T08:10:24.875 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:10:24.875 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n '' ']' 2026-02-18T08:10:24.875 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ start_mirrors cluster1 2026-02-18T08:10:24.875 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:10:24.875 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:10:24.883 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:10:24.883 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ start_mirror cluster1:0 2026-02-18T08:10:24.883 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:10:24.883 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:10:24.883 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:10:24.883 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:10:24.883 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:10:24.883 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:10:24.884 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:10:24.884 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:10:24.884 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:10:24.884 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:10:24.884 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:10:24.884 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:10:24.884 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:10:24.884 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 1 2026-02-18T08:10:24.884 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 2026-02-18T08:10:24.884 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:10:24.884 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ start_mirror cluster1:1 2026-02-18T08:10:24.884 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:10:24.885 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:10:24.885 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:10:24.885 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:10:24.885 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:10:24.885 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:10:24.885 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:10:24.885 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:10:24.885 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:10:24.885 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:10:24.885 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:10:24.885 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:10:24.885 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:10:24.885 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 1 2026-02-18T08:10:24.886 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 2026-02-18T08:10:24.886 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:10:24.886 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ start_mirror cluster1:2 2026-02-18T08:10:24.886 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:10:24.886 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:10:24.886 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:10:24.886 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:10:24.886 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:10:24.886 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:10:24.886 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:10:24.886 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:10:24.886 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:10:24.887 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:10:24.887 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:10:24.887 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:10:24.887 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:10:24.887 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 1 2026-02-18T08:10:24.887 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 2026-02-18T08:10:24.887 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:10:24.887 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ start_mirror cluster1:3 2026-02-18T08:10:24.887 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:10:24.887 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:10:24.887 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:10:24.887 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:10:24.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:10:24.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:10:24.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:10:24.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:10:24.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:10:24.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:10:24.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:10:24.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:10:24.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:10:24.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 1 2026-02-18T08:10:24.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 2026-02-18T08:10:24.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ start_mirrors cluster2 2026-02-18T08:10:24.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:10:24.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:10:24.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:10:24.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ start_mirror cluster2:0 2026-02-18T08:10:24.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2:0 2026-02-18T08:10:24.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:10:24.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster2:0 cluster instance 2026-02-18T08:10:24.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster2:0 2026-02-18T08:10:24.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:10:24.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:10:24.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster2 2026-02-18T08:10:24.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:10:24.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster2:0 ']' 2026-02-18T08:10:24.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster2 2026-02-18T08:10:24.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster2 2026-02-18T08:10:24.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:10:24.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:10:24.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 1 2026-02-18T08:10:24.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 2026-02-18T08:10:24.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:10:24.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ start_mirror cluster2:1 2026-02-18T08:10:24.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2:1 2026-02-18T08:10:24.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:10:24.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster2:1 cluster instance 2026-02-18T08:10:24.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster2:1 2026-02-18T08:10:24.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:10:24.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:10:24.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster2 2026-02-18T08:10:24.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:10:24.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster2:1 ']' 2026-02-18T08:10:24.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster2 2026-02-18T08:10:24.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster2 2026-02-18T08:10:24.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:10:24.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:10:24.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 1 2026-02-18T08:10:24.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 2026-02-18T08:10:24.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:10:24.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ start_mirror cluster2:2 2026-02-18T08:10:24.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2:2 2026-02-18T08:10:24.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:10:24.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster2:2 cluster instance 2026-02-18T08:10:24.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster2:2 2026-02-18T08:10:24.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:10:24.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:10:24.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster2 2026-02-18T08:10:24.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:10:24.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster2:2 ']' 2026-02-18T08:10:24.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster2 2026-02-18T08:10:24.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster2 2026-02-18T08:10:24.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:10:24.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:10:24.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 1 2026-02-18T08:10:24.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 2026-02-18T08:10:24.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:10:24.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ start_mirror cluster2:3 2026-02-18T08:10:24.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2:3 2026-02-18T08:10:24.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:10:24.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster2:3 cluster instance 2026-02-18T08:10:24.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster2:3 2026-02-18T08:10:24.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:10:24.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:10:24.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster2 2026-02-18T08:10:24.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:10:24.894 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster2:3 ']' 2026-02-18T08:10:24.894 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster2 2026-02-18T08:10:24.894 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster2 2026-02-18T08:10:24.894 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:10:24.894 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:10:24.894 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 1 2026-02-18T08:10:24.894 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 2026-02-18T08:10:24.894 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ testlog 'TEST: add image and test replay after client crashes' 2026-02-18T08:10:24.894 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ tee -a /tmp/tmp.vybh1m9yJF/rbd-mirror.test.log 2026-02-18T08:10:24.894 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ date '+%F %T' 2026-02-18T08:10:24.894 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ echo 2026-02-18 08:10:24 TEST: add image and test replay after client crashes 2026-02-18T08:10:24.894 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:2026-02-18 08:10:24 TEST: add image and test replay after client crashes 2026-02-18T08:10:24.894 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=test 2026-02-18T08:10:24.895 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_image_and_enable_mirror cluster2 mirror test journal 512M 2026-02-18T08:10:24.895 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:10:24.895 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:10:24.895 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:10:24.895 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:10:24.895 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:10:24.895 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:10:24.895 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local mode=journal 2026-02-18T08:10:24.895 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n journal ']' 2026-02-18T08:10:24.895 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:10:24.895 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_image cluster2 mirror test 512M 2026-02-18T08:10:24.896 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:10:24.896 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:10:24.896 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:10:24.896 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:10:24.896 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:10:24.896 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:10:24.896 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local size=128 2026-02-18T08:10:24.896 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n 512M ']' 2026-02-18T08:10:24.896 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ size=512M 2026-02-18T08:10:24.896 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:10:24.896 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 create --size 512M --image-feature layering,exclusive-lock,journaling mirror/test 2026-02-18T08:10:24.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ is_pool_mirror_mode_image mirror 2026-02-18T08:10:24.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:10:24.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' pool = image ']' 2026-02-18T08:10:24.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ case "${pool}" in 2026-02-18T08:10:24.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 1 2026-02-18T08:10:24.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_replay_started cluster1 mirror test 2026-02-18T08:10:24.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:10:24.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:10:24.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:10:24.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_replay_state cluster1 mirror test started 2026-02-18T08:10:24.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:10:24.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:10:24.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:10:24.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local state=started 2026-02-18T08:10:24.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-18T08:10:24.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:10:24.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1 2026-02-18T08:10:25.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test_image_replay_state cluster1 mirror test started 2026-02-18T08:10:25.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:10:25.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:10:25.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:10:25.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local test_state=started 2026-02-18T08:10:25.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local status_result 2026-02-18T08:10:25.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_state=stopped 2026-02-18T08:10:25.922 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-18T08:10:25.923 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -i state 2026-02-18T08:10:25.923 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_instance=cluster1 2026-02-18T08:10:25.923 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-18T08:10:25.923 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:10:25.923 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance=cluster1 2026-02-18T08:10:25.923 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local loop_instance 2026-02-18T08:10:25.923 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:10:25.923 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sleep 0 2026-02-18T08:10:25.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:10:25.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ seq 0 3 2026-02-18T08:10:25.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:10:25.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-18T08:10:25.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:0 2026-02-18T08:10:25.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-18T08:10:25.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:10:25.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:10:25.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:10:25.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-18T08:10:25.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:10:25.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:10:25.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:10:25.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:10:25.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-18T08:10:25.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:10:25.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:10:25.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:10:25.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:10:25.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:0 2026-02-18T08:10:25.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-18T08:10:25.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-18T08:10:25.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:10:25.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:0 2026-02-18T08:10:25.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T08:10:25.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-18T08:10:25.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:10:25.928 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:10:25.928 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-18T08:10:25.928 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-18T08:10:25.928 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-18T08:10:25.928 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=0 2026-02-18T08:10:25.928 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=0 2026-02-18T08:10:25.928 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:10:25.936 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:10:25.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:10:25.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:10:25.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-18T08:10:25.992 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:10:25.992 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status : get status for rbd mirror 2026-02-18T08:10:25.992 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:10:25.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:10:25.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test 2026-02-18T08:10:25.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:1 2026-02-18T08:10:25.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-18T08:10:25.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:10:25.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:10:25.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:10:25.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-18T08:10:25.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:10:25.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:10:25.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:10:25.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:10:25.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-18T08:10:25.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:10:25.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:10:25.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:10:25.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:10:25.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:1 2026-02-18T08:10:25.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-18T08:10:25.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-18T08:10:25.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:10:25.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:1 2026-02-18T08:10:25.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T08:10:25.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-18T08:10:25.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:10:25.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:10:25.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-18T08:10:25.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-18T08:10:25.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-18T08:10:25.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=1 2026-02-18T08:10:25.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=1 2026-02-18T08:10:25.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:10:26.004 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:10:26.005 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:10:26.005 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:10:26.005 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test 2026-02-18T08:10:26.059 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:10:26.060 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status : get status for rbd mirror 2026-02-18T08:10:26.060 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:10:26.062 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:10:26.062 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/test 2026-02-18T08:10:26.062 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:2 2026-02-18T08:10:26.062 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-18T08:10:26.062 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:10:26.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:10:26.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:10:26.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-18T08:10:26.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:10:26.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:10:26.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:10:26.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:10:26.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-18T08:10:26.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:10:26.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:10:26.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:10:26.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:10:26.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:2 2026-02-18T08:10:26.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-18T08:10:26.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-18T08:10:26.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:10:26.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:2 2026-02-18T08:10:26.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T08:10:26.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-18T08:10:26.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:10:26.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:10:26.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-18T08:10:26.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-18T08:10:26.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-18T08:10:26.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=2 2026-02-18T08:10:26.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=2 2026-02-18T08:10:26.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:10:26.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:10:26.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:10:26.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:10:26.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/test 2026-02-18T08:10:26.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:10:26.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status : get status for rbd mirror 2026-02-18T08:10:26.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:10:26.132 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:10:26.132 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/test 2026-02-18T08:10:26.132 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:3 2026-02-18T08:10:26.132 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-18T08:10:26.132 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:10:26.132 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:10:26.132 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:10:26.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-18T08:10:26.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:10:26.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:10:26.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:10:26.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:10:26.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-18T08:10:26.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:10:26.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:10:26.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:10:26.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:10:26.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:3 2026-02-18T08:10:26.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-18T08:10:26.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-18T08:10:26.134 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:10:26.134 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:3 2026-02-18T08:10:26.134 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T08:10:26.134 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-18T08:10:26.134 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:10:26.134 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:10:26.134 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-18T08:10:26.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-18T08:10:26.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-18T08:10:26.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=3 2026-02-18T08:10:26.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=3 2026-02-18T08:10:26.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:10:26.140 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:10:26.140 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:10:26.141 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:10:26.141 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/test 2026-02-18T08:10:26.195 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:10:26.196 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status : get status for rbd mirror 2026-02-18T08:10:26.196 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:10:26.197 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:10:26.197 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sleep 1 2026-02-18T08:10:27.198 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:10:27.198 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ seq 0 3 2026-02-18T08:10:27.201 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:10:27.201 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-18T08:10:27.201 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:0 2026-02-18T08:10:27.201 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-18T08:10:27.201 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:10:27.201 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:10:27.201 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:10:27.201 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-18T08:10:27.202 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:10:27.202 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:10:27.202 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:10:27.202 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:10:27.202 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-18T08:10:27.202 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:10:27.202 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:10:27.202 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:10:27.202 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:10:27.202 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:0 2026-02-18T08:10:27.202 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-18T08:10:27.202 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-18T08:10:27.203 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:10:27.203 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:0 2026-02-18T08:10:27.203 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T08:10:27.203 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-18T08:10:27.203 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:10:27.203 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:10:27.203 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-18T08:10:27.203 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-18T08:10:27.203 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-18T08:10:27.203 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=0 2026-02-18T08:10:27.203 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=0 2026-02-18T08:10:27.203 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:10:27.219 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:10:27.219 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:10:27.219 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:10:27.219 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-18T08:10:27.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:10:27.269 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status : get status for rbd mirror 2026-02-18T08:10:27.269 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:10:27.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:10:27.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test 2026-02-18T08:10:27.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:1 2026-02-18T08:10:27.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-18T08:10:27.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:10:27.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:10:27.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:10:27.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-18T08:10:27.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:10:27.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:10:27.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:10:27.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:10:27.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-18T08:10:27.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:10:27.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:10:27.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:10:27.273 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:10:27.273 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:1 2026-02-18T08:10:27.273 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-18T08:10:27.273 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-18T08:10:27.273 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:10:27.273 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:1 2026-02-18T08:10:27.273 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T08:10:27.273 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-18T08:10:27.273 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:10:27.273 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:10:27.273 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-18T08:10:27.273 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-18T08:10:27.274 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-18T08:10:27.274 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=1 2026-02-18T08:10:27.274 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=1 2026-02-18T08:10:27.274 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:10:27.283 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:10:27.283 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:10:27.283 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:10:27.283 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test 2026-02-18T08:10:27.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:10:27.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status : get status for rbd mirror 2026-02-18T08:10:27.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:10:27.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:10:27.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/test 2026-02-18T08:10:27.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:2 2026-02-18T08:10:27.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-18T08:10:27.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:10:27.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:10:27.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:10:27.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-18T08:10:27.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:10:27.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:10:27.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:10:27.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:10:27.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-18T08:10:27.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:10:27.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:10:27.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:10:27.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:10:27.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:2 2026-02-18T08:10:27.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-18T08:10:27.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-18T08:10:27.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:10:27.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:2 2026-02-18T08:10:27.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T08:10:27.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-18T08:10:27.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:10:27.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:10:27.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-18T08:10:27.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-18T08:10:27.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-18T08:10:27.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=2 2026-02-18T08:10:27.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=2 2026-02-18T08:10:27.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:10:27.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:10:27.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:10:27.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:10:27.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/test 2026-02-18T08:10:27.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:10:27.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status : get status for rbd mirror 2026-02-18T08:10:27.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:10:27.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:10:27.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/test 2026-02-18T08:10:27.405 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:3 2026-02-18T08:10:27.405 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-18T08:10:27.405 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:10:27.405 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:10:27.405 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:10:27.405 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-18T08:10:27.405 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:10:27.405 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:10:27.406 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:10:27.406 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:10:27.406 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-18T08:10:27.406 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:10:27.406 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:10:27.406 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:10:27.406 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:10:27.406 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:3 2026-02-18T08:10:27.406 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-18T08:10:27.407 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-18T08:10:27.407 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:10:27.407 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:3 2026-02-18T08:10:27.407 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T08:10:27.407 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-18T08:10:27.407 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:10:27.407 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:10:27.407 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-18T08:10:27.407 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-18T08:10:27.408 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-18T08:10:27.408 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=3 2026-02-18T08:10:27.408 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=3 2026-02-18T08:10:27.408 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:10:27.415 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:10:27.415 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:10:27.415 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:10:27.415 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/test 2026-02-18T08:10:27.462 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:10:27.463 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status : get status for rbd mirror 2026-02-18T08:10:27.463 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:10:27.463 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:10:27.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sleep 2 2026-02-18T08:10:29.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:10:29.465 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ seq 0 3 2026-02-18T08:10:29.468 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:10:29.468 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-18T08:10:29.468 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:0 2026-02-18T08:10:29.468 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-18T08:10:29.468 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:10:29.468 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:10:29.468 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:10:29.468 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-18T08:10:29.468 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:10:29.468 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:10:29.468 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:10:29.468 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:10:29.468 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-18T08:10:29.468 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:10:29.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:10:29.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:10:29.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:10:29.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:0 2026-02-18T08:10:29.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-18T08:10:29.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-18T08:10:29.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:10:29.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:0 2026-02-18T08:10:29.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T08:10:29.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-18T08:10:29.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:10:29.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:10:29.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-18T08:10:29.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-18T08:10:29.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-18T08:10:29.470 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=0 2026-02-18T08:10:29.470 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=0 2026-02-18T08:10:29.470 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:10:29.480 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:10:29.481 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:10:29.481 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:10:29.481 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-18T08:10:29.528 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:10:29.528 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status : get status for rbd mirror 2026-02-18T08:10:29.528 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:10:29.530 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:10:29.530 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test 2026-02-18T08:10:29.530 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:1 2026-02-18T08:10:29.530 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-18T08:10:29.530 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:10:29.530 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:10:29.531 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:10:29.531 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-18T08:10:29.531 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:10:29.531 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:10:29.531 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:10:29.531 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:10:29.531 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-18T08:10:29.531 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:10:29.531 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:10:29.531 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:10:29.531 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:10:29.531 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:1 2026-02-18T08:10:29.531 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-18T08:10:29.531 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-18T08:10:29.531 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:10:29.531 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:1 2026-02-18T08:10:29.531 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T08:10:29.531 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-18T08:10:29.531 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:10:29.531 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:10:29.531 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-18T08:10:29.531 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-18T08:10:29.532 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-18T08:10:29.532 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=1 2026-02-18T08:10:29.532 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=1 2026-02-18T08:10:29.532 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:10:29.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:10:29.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:10:29.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:10:29.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test 2026-02-18T08:10:29.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:10:29.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status : get status for rbd mirror 2026-02-18T08:10:29.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:10:29.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:10:29.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/test 2026-02-18T08:10:29.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:2 2026-02-18T08:10:29.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-18T08:10:29.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:10:29.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:10:29.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:10:29.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-18T08:10:29.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:10:29.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:10:29.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:10:29.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:10:29.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-18T08:10:29.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:10:29.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:10:29.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:10:29.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:10:29.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:2 2026-02-18T08:10:29.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-18T08:10:29.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-18T08:10:29.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:10:29.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:2 2026-02-18T08:10:29.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T08:10:29.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-18T08:10:29.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:10:29.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:10:29.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-18T08:10:29.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-18T08:10:29.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-18T08:10:29.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=2 2026-02-18T08:10:29.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=2 2026-02-18T08:10:29.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:10:29.603 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:10:29.604 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:10:29.604 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:10:29.604 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/test 2026-02-18T08:10:29.659 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:10:29.659 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status : get status for rbd mirror 2026-02-18T08:10:29.659 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:10:29.661 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:10:29.661 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/test 2026-02-18T08:10:29.661 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:3 2026-02-18T08:10:29.661 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-18T08:10:29.661 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:10:29.661 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:10:29.661 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:10:29.661 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-18T08:10:29.661 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:10:29.661 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:10:29.661 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:10:29.661 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:10:29.661 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-18T08:10:29.661 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:10:29.661 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:10:29.661 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:10:29.662 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:10:29.662 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:3 2026-02-18T08:10:29.662 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-18T08:10:29.662 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-18T08:10:29.662 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:10:29.662 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:3 2026-02-18T08:10:29.662 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T08:10:29.662 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-18T08:10:29.662 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:10:29.662 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:10:29.662 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-18T08:10:29.662 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-18T08:10:29.662 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-18T08:10:29.662 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=3 2026-02-18T08:10:29.662 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=3 2026-02-18T08:10:29.662 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:10:29.672 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:10:29.672 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:10:29.672 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:10:29.672 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/test 2026-02-18T08:10:29.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:10:29.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status : get status for rbd mirror 2026-02-18T08:10:29.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:10:29.728 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:10:29.728 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sleep 4 2026-02-18T08:10:33.729 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:10:33.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ seq 0 3 2026-02-18T08:10:33.731 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:10:33.731 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-18T08:10:33.731 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:0 2026-02-18T08:10:33.731 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-18T08:10:33.732 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:10:33.732 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:10:33.732 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:10:33.732 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-18T08:10:33.732 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:10:33.732 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:10:33.732 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:10:33.732 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:10:33.732 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-18T08:10:33.732 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:10:33.732 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:10:33.732 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:10:33.733 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:10:33.733 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:0 2026-02-18T08:10:33.733 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-18T08:10:33.733 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-18T08:10:33.733 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:10:33.733 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:0 2026-02-18T08:10:33.733 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T08:10:33.733 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-18T08:10:33.733 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:10:33.733 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:10:33.733 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-18T08:10:33.733 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-18T08:10:33.733 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-18T08:10:33.733 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=0 2026-02-18T08:10:33.733 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=0 2026-02-18T08:10:33.733 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:10:33.745 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:10:33.745 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:10:33.746 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:10:33.746 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-18T08:10:33.804 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:10:33.805 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status : get status for rbd mirror 2026-02-18T08:10:33.805 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:10:33.807 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:10:33.807 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test 2026-02-18T08:10:33.807 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:1 2026-02-18T08:10:33.807 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-18T08:10:33.807 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:10:33.807 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:10:33.807 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:10:33.808 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-18T08:10:33.808 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:10:33.808 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:10:33.808 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:10:33.808 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:10:33.808 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-18T08:10:33.808 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:10:33.808 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:10:33.808 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:10:33.808 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:10:33.808 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:1 2026-02-18T08:10:33.808 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-18T08:10:33.808 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-18T08:10:33.808 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:10:33.808 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:1 2026-02-18T08:10:33.808 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T08:10:33.808 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-18T08:10:33.808 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:10:33.808 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:10:33.808 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-18T08:10:33.809 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-18T08:10:33.809 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-18T08:10:33.809 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=1 2026-02-18T08:10:33.809 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=1 2026-02-18T08:10:33.809 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:10:33.819 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:10:33.819 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:10:33.819 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:10:33.819 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test 2026-02-18T08:10:33.868 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:10:33.868 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status : get status for rbd mirror 2026-02-18T08:10:33.868 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:10:33.870 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:10:33.870 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/test 2026-02-18T08:10:33.870 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:2 2026-02-18T08:10:33.870 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-18T08:10:33.870 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:10:33.870 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:10:33.870 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:10:33.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-18T08:10:33.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:10:33.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:10:33.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:10:33.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:10:33.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-18T08:10:33.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:10:33.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:10:33.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:10:33.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:10:33.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:2 2026-02-18T08:10:33.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-18T08:10:33.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-18T08:10:33.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:10:33.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:2 2026-02-18T08:10:33.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T08:10:33.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-18T08:10:33.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:10:33.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:10:33.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-18T08:10:33.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-18T08:10:33.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-18T08:10:33.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=2 2026-02-18T08:10:33.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=2 2026-02-18T08:10:33.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:10:33.880 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:10:33.881 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:10:33.881 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:10:33.881 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/test 2026-02-18T08:10:33.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:10:33.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status : get status for rbd mirror 2026-02-18T08:10:33.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:10:33.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:10:33.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/test 2026-02-18T08:10:33.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:3 2026-02-18T08:10:33.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-18T08:10:33.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:10:33.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:10:33.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:10:33.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-18T08:10:33.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:10:33.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:10:33.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:10:33.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:10:33.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-18T08:10:33.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:10:33.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:10:33.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:10:33.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:10:33.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:3 2026-02-18T08:10:33.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-18T08:10:33.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-18T08:10:33.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:10:33.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:3 2026-02-18T08:10:33.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T08:10:33.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-18T08:10:33.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:10:33.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:10:33.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-18T08:10:33.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-18T08:10:33.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-18T08:10:33.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=3 2026-02-18T08:10:33.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=3 2026-02-18T08:10:33.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:10:33.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:10:33.952 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:10:33.952 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:10:33.952 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/test 2026-02-18T08:10:34.008 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:10:34.008 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status : get status for rbd mirror 2026-02-18T08:10:34.008 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:10:34.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:10:34.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sleep 8 2026-02-18T08:10:42.010 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:10:42.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ seq 0 3 2026-02-18T08:10:42.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:10:42.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-18T08:10:42.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:0 2026-02-18T08:10:42.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-18T08:10:42.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:10:42.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:10:42.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:10:42.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-18T08:10:42.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:10:42.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:10:42.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:10:42.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:10:42.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-18T08:10:42.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:10:42.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:10:42.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:10:42.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:10:42.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:0 2026-02-18T08:10:42.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-18T08:10:42.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-18T08:10:42.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:10:42.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:0 2026-02-18T08:10:42.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T08:10:42.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-18T08:10:42.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:10:42.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:10:42.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-18T08:10:42.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-18T08:10:42.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-18T08:10:42.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=0 2026-02-18T08:10:42.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=0 2026-02-18T08:10:42.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:10:42.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:10:42.027 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:10:42.027 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:10:42.027 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-18T08:10:42.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:10:42.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status : get status for rbd mirror 2026-02-18T08:10:42.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:10:42.076 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:10:42.076 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test 2026-02-18T08:10:42.076 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:1 2026-02-18T08:10:42.076 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-18T08:10:42.076 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:10:42.076 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:10:42.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:10:42.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-18T08:10:42.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:10:42.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:10:42.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:10:42.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:10:42.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-18T08:10:42.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:10:42.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:10:42.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:10:42.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:10:42.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:1 2026-02-18T08:10:42.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-18T08:10:42.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-18T08:10:42.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:10:42.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:1 2026-02-18T08:10:42.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T08:10:42.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-18T08:10:42.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:10:42.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:10:42.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-18T08:10:42.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-18T08:10:42.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-18T08:10:42.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=1 2026-02-18T08:10:42.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=1 2026-02-18T08:10:42.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:10:42.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:10:42.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:10:42.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:10:42.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test 2026-02-18T08:10:42.146 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:10:42.146 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status : get status for rbd mirror 2026-02-18T08:10:42.146 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:10:42.149 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:10:42.149 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/test 2026-02-18T08:10:42.149 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:2 2026-02-18T08:10:42.149 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-18T08:10:42.149 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:10:42.149 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:10:42.149 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:10:42.149 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-18T08:10:42.149 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:10:42.149 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:10:42.149 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:10:42.150 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:10:42.150 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-18T08:10:42.150 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:10:42.150 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:10:42.150 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:10:42.150 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:10:42.150 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:2 2026-02-18T08:10:42.150 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-18T08:10:42.150 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-18T08:10:42.150 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:10:42.150 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:2 2026-02-18T08:10:42.150 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T08:10:42.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-18T08:10:42.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:10:42.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:10:42.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-18T08:10:42.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-18T08:10:42.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-18T08:10:42.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=2 2026-02-18T08:10:42.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=2 2026-02-18T08:10:42.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:10:42.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:10:42.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:10:42.161 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:10:42.161 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/test 2026-02-18T08:10:42.215 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:10:42.215 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status : get status for rbd mirror 2026-02-18T08:10:42.215 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:10:42.217 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:10:42.217 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/test 2026-02-18T08:10:42.218 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:3 2026-02-18T08:10:42.218 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-18T08:10:42.218 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:10:42.218 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:10:42.218 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:10:42.218 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-18T08:10:42.218 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:10:42.218 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:10:42.218 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:10:42.218 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:10:42.218 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-18T08:10:42.218 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:10:42.219 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:10:42.219 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:10:42.219 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:10:42.219 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:3 2026-02-18T08:10:42.219 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-18T08:10:42.219 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-18T08:10:42.219 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:10:42.219 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:3 2026-02-18T08:10:42.219 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T08:10:42.219 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-18T08:10:42.219 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:10:42.219 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:10:42.220 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-18T08:10:42.220 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-18T08:10:42.220 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-18T08:10:42.220 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=3 2026-02-18T08:10:42.220 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=3 2026-02-18T08:10:42.220 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:10:42.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:10:42.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:10:42.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:10:42.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/test 2026-02-18T08:10:42.283 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:10:42.283 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status : get status for rbd mirror 2026-02-18T08:10:42.283 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:10:42.284 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:10:42.284 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sleep 8 2026-02-18T08:10:46.257 INFO:tasks.rbd_mirror_thrash:kill cluster2.client.mirror.0 2026-02-18T08:10:46.258 INFO:tasks.rbd_mirror.cluster2.client.mirror.0:Sent signal 15 2026-02-18T08:10:46.258 INFO:tasks.rbd_mirror_thrash:waiting for 4.836555540581676 secs before reviving daemons 2026-02-18T08:10:46.258 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.trial163.stderr:2026-02-18T08:10:46.255+0000 7f362d57e640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.0 (PID: 14978) UID: 1000 2026-02-18T08:10:50.285 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:10:50.286 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ seq 0 3 2026-02-18T08:10:50.289 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:10:50.290 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-18T08:10:50.290 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:0 2026-02-18T08:10:50.290 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-18T08:10:50.290 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:10:50.290 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:10:50.290 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:10:50.290 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-18T08:10:50.290 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:10:50.290 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:10:50.290 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:10:50.290 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:10:50.291 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-18T08:10:50.291 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:10:50.291 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:10:50.291 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:10:50.291 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:10:50.291 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:0 2026-02-18T08:10:50.291 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-18T08:10:50.291 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-18T08:10:50.291 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:10:50.291 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:0 2026-02-18T08:10:50.291 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T08:10:50.291 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-18T08:10:50.292 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:10:50.292 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:10:50.292 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-18T08:10:50.292 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-18T08:10:50.292 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-18T08:10:50.292 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=0 2026-02-18T08:10:50.292 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=0 2026-02-18T08:10:50.292 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:10:50.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:10:50.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:10:50.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:10:50.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-18T08:10:50.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ return 0 2026-02-18T08:10:50.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ status_result=' "state": "Replaying"' 2026-02-18T08:10:50.360 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ echo ' "state": "Replaying"' 2026-02-18T08:10:50.360 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ grep -i Replaying 2026-02-18T08:10:50.361 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: "state": "Replaying" 2026-02-18T08:10:50.362 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ current_state=started 2026-02-18T08:10:50.362 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test started = started 2026-02-18T08:10:50.362 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:10:50.362 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ clean_snap_name= 2026-02-18T08:10:50.362 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 1 10 2026-02-18T08:10:50.363 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 10` 2026-02-18T08:10:50.363 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ stress_write_image cluster2 mirror test 2026-02-18T08:10:50.363 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:10:50.364 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:10:50.364 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:10:50.364 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-18T08:10:50.364 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local duration=12 2026-02-18T08:10:50.364 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set +e 2026-02-18T08:10:50.365 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ timeout 12s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror test --debug-rbd=20 --debug-journaler=20 2026-02-18T08:10:51.095 INFO:tasks.rbd_mirror_thrash:waiting for cluster2.client.mirror.0 2026-02-18T08:10:51.095 INFO:teuthology.orchestra.run:waiting for 600 2026-02-18T08:10:51.096 INFO:tasks.rbd_mirror_thrash:reviving cluster2.client.mirror.0 2026-02-18T08:10:51.096 INFO:tasks.rbd_mirror.cluster2.client.mirror.0:Restarting daemon 2026-02-18T08:10:51.096 DEBUG:teuthology.orchestra.run.trial163:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.0 2026-02-18T08:10:51.099 INFO:tasks.rbd_mirror.cluster2.client.mirror.0:Started 2026-02-18T08:10:51.099 INFO:tasks.rbd_mirror_thrash:waiting for 81.48728246316406 secs before thrashing 2026-02-18T08:10:51.121 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:100 IOs, 89593856 bytes 2026-02-18T08:10:51.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:200 IOs, 128939008 bytes 2026-02-18T08:10:52.728 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:300 IOs, 210783232 bytes 2026-02-18T08:10:52.929 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:400 IOs, 237128704 bytes 2026-02-18T08:10:53.578 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:500 IOs, 290501632 bytes 2026-02-18T08:10:53.862 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:600 IOs, 335373312 bytes 2026-02-18T08:10:54.585 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:700 IOs, 362221568 bytes 2026-02-18T08:10:55.481 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:800 IOs, 424163328 bytes 2026-02-18T08:10:56.531 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:900 IOs, 487220224 bytes 2026-02-18T08:10:56.613 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1000 IOs, 491686912 bytes 2026-02-18T08:10:57.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1100 IOs, 539992064 bytes 2026-02-18T08:10:58.208 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1200 IOs, 602560512 bytes 2026-02-18T08:10:59.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1300 IOs, 663207936 bytes 2026-02-18T08:10:59.156 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1400 IOs, 670205952 bytes 2026-02-18T08:10:59.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1500 IOs, 729671680 bytes 2026-02-18T08:11:00.370 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1600 IOs, 763117568 bytes 2026-02-18T08:11:01.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1700 IOs, 847519744 bytes 2026-02-18T08:11:01.844 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1800 IOs, 907694080 bytes 2026-02-18T08:11:02.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1900 IOs, 931602432 bytes 2026-02-18T08:11:02.372 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ error_code=124 2026-02-18T08:11:02.372 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set -e 2026-02-18T08:11:02.372 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 124 -eq 124 ']' 2026-02-18T08:11:02.372 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:11:02.372 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+replaying 2026-02-18T08:11:02.373 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:11:02.373 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:11:02.373 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:11:02.373 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local state_pattern=up+replaying 2026-02-18T08:11:02.373 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local description_pattern= 2026-02-18T08:11:02.373 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local service_pattern= 2026-02-18T08:11:02.373 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:11:02.373 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1 2026-02-18T08:11:03.373 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test_status_in_pool_dir cluster1 mirror test up+replaying '' '' 2026-02-18T08:11:03.373 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:11:03.373 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:11:03.373 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:11:03.373 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local state_pattern=up+replaying 2026-02-18T08:11:03.373 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local description_pattern= 2026-02-18T08:11:03.373 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local service_pattern= 2026-02-18T08:11:03.374 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local status 2026-02-18T08:11:03.374 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ CEPH_ARGS= 2026-02-18T08:11:03.374 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-18T08:11:03.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ status='test: 2026-02-18T08:11:03.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: global_id: 0b548823-d751-43df-a076-4efac2da4d3d 2026-02-18T08:11:03.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: state: up+replaying 2026-02-18T08:11:03.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: description: replaying, {"bytes_per_second":0.0,"entries_behind_primary":3,"entries_per_second":0.0,"non_primary_position":{},"primary_position":{"entry_tid":3,"object_number":3,"tag_tid":1}} 2026-02-18T08:11:03.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: service: mirror.0 on trial163 2026-02-18T08:11:03.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: last_update: 2026-02-18 08:10:49 2026-02-18T08:11:03.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: peer_sites: 2026-02-18T08:11:03.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: name: cluster2 2026-02-18T08:11:03.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: state: up+stopped 2026-02-18T08:11:03.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: description: local image is primary 2026-02-18T08:11:03.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: last_update: 2026-02-18 08:10:48' 2026-02-18T08:11:03.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ grep '^ state: .*up+replaying' 2026-02-18T08:11:03.412 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: state: up+replaying 2026-02-18T08:11:03.412 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ grep '^ description: .*' 2026-02-18T08:11:03.412 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: description: replaying, {"bytes_per_second":0.0,"entries_behind_primary":3,"entries_per_second":0.0,"non_primary_position":{},"primary_position":{"entry_tid":3,"object_number":3,"tag_tid":1}} 2026-02-18T08:11:03.413 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n '' ']' 2026-02-18T08:11:03.413 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ echo up+replaying 2026-02-18T08:11:03.413 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ grep '^up+' 2026-02-18T08:11:03.414 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:up+replaying 2026-02-18T08:11:03.414 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ grep 'service: *mirror..* on ' 2026-02-18T08:11:03.415 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: service: mirror.0 on trial163 2026-02-18T08:11:03.415 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local last_update 2026-02-18T08:11:03.415 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-18T08:11:03.417 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_update='2026-02-18 08:10:49' 2026-02-18T08:11:03.417 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+replaying '2026-02-18 08:10:49' 2026-02-18T08:11:03.418 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:11:03.418 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:11:03.418 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:11:03.418 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local state_pattern=up+replaying 2026-02-18T08:11:03.418 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'prev_last_update=2026-02-18 08:10:49' 2026-02-18T08:11:03.418 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local status 2026-02-18T08:11:03.418 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ CEPH_ARGS= 2026-02-18T08:11:03.418 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-18T08:11:03.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ status='OKOKOK142074263mirror.0trial16320.2.0-670-gadc19233falseOK42114265mirror.1trial16320.2.0-670-gadc19233trueOK42154297mirror.2trial16320.2.0-670-gadc19233falseOK42194268mirror.3trial16320.2.0-670-gadc19233falseOKtest0b548823-d751-43df-a076-4efac2da4d3dup+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":3,"entries_per_second":0.0,"non_primary_position":{},"primary_position":{"entry_tid":3,"object_number":3,"tag_tid":1}}42074263mirror.0trial1632026-02-18 08:10:49cluster2c5e4f85e-ec31-430b-a71d-56e000d0de0bup+stoppedlocal image is primary2026-02-18 08:10:48' 2026-02-18T08:11:03.462 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local last_update state 2026-02-18T08:11:03.462 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-18T08:11:03.465 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_update='2026-02-18 08:10:49' 2026-02-18T08:11:03.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-18T08:11:03.468 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ state=up+replaying 2026-02-18T08:11:03.468 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ echo up+replaying 2026-02-18T08:11:03.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ grep up+replaying 2026-02-18T08:11:03.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:up+replaying 2026-02-18T08:11:03.470 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:11:03.470 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:11:03.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ snap_name=snap1 2026-02-18T08:11:03.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_snap cluster2 mirror test snap1 2026-02-18T08:11:03.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:11:03.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:11:03.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:11:03.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap1 2026-02-18T08:11:03.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 -p mirror snap create test@snap1 --debug-rbd=20 --debug-journaler=20 2026-02-18T08:11:29.703 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.0 with skip (0.33707678827963783) > weight (0.25) 2026-02-18T08:11:29.703 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.1 with skip (0.6155524249158479) > weight (0.25) 2026-02-18T08:11:29.704 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.2 with skip (0.25885697355374315) > weight (0.25) 2026-02-18T08:11:29.704 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.3 with skip (0.5330863417495375) > weight (0.25) 2026-02-18T08:11:29.704 INFO:tasks.rbd_mirror_thrash:waiting for 82.1684793385561 secs before thrashing 2026-02-18T08:11:33.914 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_replay_started cluster1 mirror test 2026-02-18T08:11:33.915 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:11:33.915 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:11:33.915 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:11:33.915 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_replay_state cluster1 mirror test started 2026-02-18T08:11:33.915 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:11:33.915 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:11:33.915 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:11:33.915 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local state=started 2026-02-18T08:11:33.915 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-18T08:11:33.915 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:11:33.916 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1 2026-02-18T08:11:34.915 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test_image_replay_state cluster1 mirror test started 2026-02-18T08:11:34.915 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:11:34.915 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:11:34.915 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:11:34.915 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local test_state=started 2026-02-18T08:11:34.916 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local status_result 2026-02-18T08:11:34.916 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_state=stopped 2026-02-18T08:11:34.916 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-18T08:11:34.916 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_instance=cluster1 2026-02-18T08:11:34.916 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -i state 2026-02-18T08:11:34.916 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-18T08:11:34.916 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:11:34.916 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance=cluster1 2026-02-18T08:11:34.916 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local loop_instance 2026-02-18T08:11:34.916 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:11:34.917 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sleep 0 2026-02-18T08:11:34.917 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:11:34.917 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ seq 0 3 2026-02-18T08:11:34.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:11:34.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-18T08:11:34.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:0 2026-02-18T08:11:34.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-18T08:11:34.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:11:34.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:11:34.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:11:34.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-18T08:11:34.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:11:34.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:11:34.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:11:34.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:11:34.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-18T08:11:34.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:11:34.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:11:34.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:11:34.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:11:34.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:0 2026-02-18T08:11:34.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-18T08:11:34.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-18T08:11:34.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:11:34.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:0 2026-02-18T08:11:34.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T08:11:34.922 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-18T08:11:34.922 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:11:34.922 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:11:34.922 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-18T08:11:34.922 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-18T08:11:34.922 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-18T08:11:34.922 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=0 2026-02-18T08:11:34.922 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=0 2026-02-18T08:11:34.922 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:11:34.930 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:11:34.931 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:11:34.931 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:11:34.931 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-18T08:11:34.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ return 0 2026-02-18T08:11:34.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ status_result=' "state": "Replaying"' 2026-02-18T08:11:34.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ echo ' "state": "Replaying"' 2026-02-18T08:11:34.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ grep -i Replaying 2026-02-18T08:11:34.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: "state": "Replaying" 2026-02-18T08:11:34.999 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ current_state=started 2026-02-18T08:11:34.999 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test started = started 2026-02-18T08:11:34.999 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:11:34.999 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror test 2026-02-18T08:11:34.999 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_cluster=cluster1 2026-02-18T08:11:34.999 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:11:34.999 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_pool=mirror 2026-02-18T08:11:34.999 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local remote_pool=mirror 2026-02-18T08:11:34.999 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:11:34.999 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' journal = journal ']' 2026-02-18T08:11:34.999 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror mirror test 2026-02-18T08:11:35.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_cluster=cluster1 2026-02-18T08:11:35.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:11:35.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_pool=mirror 2026-02-18T08:11:35.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local remote_pool=mirror 2026-02-18T08:11:35.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:11:35.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-02-18T08:11:35.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-02-18T08:11:35.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:11:35.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:11:35.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:11:35.201 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:11:35.201 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:11:35.201 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:11:35.201 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:11:35.201 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:11:35.201 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:11:35.201 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:11:35.201 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:11:35.201 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:11:35.202 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:11:35.202 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:11:35.202 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:11:35.202 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:11:35.202 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:11:35.202 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:11:35.206 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:11:35.206 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:11:35.207 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:11:35.207 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:11:35.207 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:11:35.207 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:11:35.207 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:11:35.207 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:11:35.208 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:11:35.208 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:11:35.208 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:11:35.208 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:11:35.208 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:11:35.208 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:11:35.208 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:11:35.209 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:11:35.209 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:11:35.209 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:11:35.209 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:11:35.209 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:11:35.209 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:11:35.209 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:11:35.209 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:11:35.210 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:11:35.210 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:11:35.210 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:11:35.210 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:11:35.210 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:11:35.210 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:11:35.210 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:11:35.211 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:11:35.211 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:11:35.211 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:11:35.211 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:11:35.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:11:35.217 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:11:35.217 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:11:35.217 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:11:35.283 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:11:35.284 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:11:35.284 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:11:35.284 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:11:35.284 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:11:35.284 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:11:35.284 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:11:35.284 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:11:35.284 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:11:35.284 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:11:35.284 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:11:35.285 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:11:35.285 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:11:35.285 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:35.285 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:11:35.286 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:35.312 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 8 2026-02-18T08:11:35.312 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 14 2026-02-18T08:11:35.312 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:11:35.312 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=57, tag_tid=3, entry_tid=1], [object_number=56, tag_tid=3, entry_tid=0], [object_number=59, tag_tid=2, entry_tid=59619], [object_number=58, tag_tid=2, entry_tid=59618]]], state=connected] 2026-02-18T08:11:35.312 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=34, tag_tid=2, entry_tid=35858], [object_number=33, tag_tid=2, entry_tid=35857], [object_number=32, tag_tid=2, entry_tid=35856], [object_number=35, tag_tid=2, entry_tid=35855]]], state=connected] 2026-02-18T08:11:35.314 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:35.316 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=57, tag_tid=3, entry_tid=1' 2026-02-18T08:11:35.317 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:11:35.317 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:11:35.317 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:11:35.317 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:11:35.317 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:11:35.317 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:11:35.317 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:11:35.318 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:11:35.318 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:11:35.318 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:11:35.318 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:11:35.318 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:11:35.318 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:35.319 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:11:35.319 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:35.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 8 2026-02-18T08:11:35.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 14 2026-02-18T08:11:35.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:11:35.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=57, tag_tid=3, entry_tid=1], [object_number=56, tag_tid=3, entry_tid=0], [object_number=59, tag_tid=2, entry_tid=59619], [object_number=58, tag_tid=2, entry_tid=59618]]], state=connected] 2026-02-18T08:11:35.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=34, tag_tid=2, entry_tid=35858], [object_number=33, tag_tid=2, entry_tid=35857], [object_number=32, tag_tid=2, entry_tid=35856], [object_number=35, tag_tid=2, entry_tid=35855]]], state=connected] 2026-02-18T08:11:35.348 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:35.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=34, tag_tid=2, entry_tid=35858' 2026-02-18T08:11:35.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=57, tag_tid=3, entry_tid=1' -a 'object_number=57, tag_tid=3, entry_tid=1' = 'object_number=34, tag_tid=2, entry_tid=35858' 2026-02-18T08:11:35.350 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=34, tag_tid=2, entry_tid=35858' '!=' '' 2026-02-18T08:11:35.350 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:11:35.350 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=34, tag_tid=2, entry_tid=35858' = '' 2026-02-18T08:11:35.350 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=34, tag_tid=2, entry_tid=35858' 2026-02-18T08:11:35.350 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=57, tag_tid=3, entry_tid=1' 2026-02-18T08:11:35.350 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:11:35.350 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:11:35.352 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=3 2026-02-18T08:11:35.352 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=34, tag_tid=2, entry_tid=35858' 2026-02-18T08:11:35.352 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:11:35.352 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:11:35.354 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=2 2026-02-18T08:11:35.355 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=57, tag_tid=3, entry_tid=1' 2026-02-18T08:11:35.355 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:11:35.355 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:11:35.356 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:11:35.356 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=34, tag_tid=2, entry_tid=35858' 2026-02-18T08:11:35.357 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:11:35.357 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:11:35.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=35858 2026-02-18T08:11:35.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 3 = 2 -a 1 -le 35858 2026-02-18T08:11:35.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:11:35.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:11:35.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:11:35.560 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:11:35.560 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:11:35.560 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:11:35.560 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:11:35.560 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:11:35.560 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:11:35.560 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:11:35.560 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:11:35.560 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:11:35.560 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:11:35.560 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:11:35.560 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:11:35.560 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:11:35.560 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:11:35.561 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:11:35.561 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:11:35.561 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:11:35.561 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:11:35.561 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:11:35.562 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:11:35.562 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:11:35.562 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:11:35.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:11:35.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:11:35.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:11:35.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:11:35.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:11:35.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:11:35.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:11:35.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:11:35.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:11:35.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:11:35.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:11:35.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:11:35.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:11:35.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:11:35.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:11:35.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:11:35.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:11:35.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:11:35.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:11:35.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:11:35.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:11:35.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:11:35.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:11:35.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:11:35.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:11:35.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:11:35.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:11:35.573 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:11:35.574 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:11:35.574 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:11:35.574 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:11:35.642 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:11:35.643 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:11:35.643 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:11:35.643 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:11:35.643 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:11:35.643 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:11:35.643 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:11:35.643 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:11:35.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:11:35.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:11:35.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:11:35.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:11:35.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:11:35.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:35.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:11:35.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:35.681 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 8 2026-02-18T08:11:35.681 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 14 2026-02-18T08:11:35.682 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:11:35.682 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=57, tag_tid=3, entry_tid=1], [object_number=56, tag_tid=3, entry_tid=0], [object_number=59, tag_tid=2, entry_tid=59619], [object_number=58, tag_tid=2, entry_tid=59618]]], state=connected] 2026-02-18T08:11:35.682 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=34, tag_tid=2, entry_tid=36126], [object_number=33, tag_tid=2, entry_tid=36125], [object_number=32, tag_tid=2, entry_tid=36124], [object_number=35, tag_tid=2, entry_tid=36123]]], state=connected] 2026-02-18T08:11:35.685 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:35.686 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=57, tag_tid=3, entry_tid=1' 2026-02-18T08:11:35.688 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:11:35.688 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:11:35.688 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:11:35.688 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:11:35.688 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:11:35.688 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:11:35.688 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:11:35.688 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:11:35.688 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:11:35.688 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:11:35.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:11:35.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:11:35.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:35.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:11:35.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:35.725 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 8 2026-02-18T08:11:35.725 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 14 2026-02-18T08:11:35.725 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:11:35.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=57, tag_tid=3, entry_tid=1], [object_number=56, tag_tid=3, entry_tid=0], [object_number=59, tag_tid=2, entry_tid=59619], [object_number=58, tag_tid=2, entry_tid=59618]]], state=connected] 2026-02-18T08:11:35.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=34, tag_tid=2, entry_tid=36126], [object_number=33, tag_tid=2, entry_tid=36125], [object_number=32, tag_tid=2, entry_tid=36124], [object_number=35, tag_tid=2, entry_tid=36123]]], state=connected] 2026-02-18T08:11:35.729 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:35.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=34, tag_tid=2, entry_tid=36126' 2026-02-18T08:11:35.731 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=57, tag_tid=3, entry_tid=1' -a 'object_number=57, tag_tid=3, entry_tid=1' = 'object_number=34, tag_tid=2, entry_tid=36126' 2026-02-18T08:11:35.731 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=34, tag_tid=2, entry_tid=36126' '!=' 'object_number=34, tag_tid=2, entry_tid=35858' 2026-02-18T08:11:35.731 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:11:35.731 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=34, tag_tid=2, entry_tid=36126' = 'object_number=34, tag_tid=2, entry_tid=35858' 2026-02-18T08:11:35.731 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=34, tag_tid=2, entry_tid=36126' 2026-02-18T08:11:35.732 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=57, tag_tid=3, entry_tid=1' 2026-02-18T08:11:35.732 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:11:35.732 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:11:35.732 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=3 2026-02-18T08:11:35.733 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=34, tag_tid=2, entry_tid=36126' 2026-02-18T08:11:35.733 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:11:35.733 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:11:35.735 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=2 2026-02-18T08:11:35.735 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=57, tag_tid=3, entry_tid=1' 2026-02-18T08:11:35.735 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:11:35.736 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:11:35.737 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:11:35.737 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=34, tag_tid=2, entry_tid=36126' 2026-02-18T08:11:35.737 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:11:35.738 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:11:35.739 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=36126 2026-02-18T08:11:35.739 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 3 = 2 -a 1 -le 36126 2026-02-18T08:11:35.739 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:11:35.739 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:11:35.740 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:11:35.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:11:35.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:11:35.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:11:35.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:11:35.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:11:35.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:11:35.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:11:35.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:11:35.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:11:35.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:11:35.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:11:35.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:11:35.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:11:35.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:11:35.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:11:35.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:11:35.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:11:35.946 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:11:35.946 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:11:35.946 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:11:35.946 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:11:35.946 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:11:35.946 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:11:35.946 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:11:35.946 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:11:35.946 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:11:35.946 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:11:35.946 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:11:35.946 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:11:35.947 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:11:35.947 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:11:35.947 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:11:35.947 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:11:35.947 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:11:35.947 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:11:35.947 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:11:35.947 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:11:35.947 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:11:35.947 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:11:35.947 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:11:35.947 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:11:35.947 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:11:35.947 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:11:35.948 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:11:35.948 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:11:35.948 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:11:35.948 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:11:35.948 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:11:35.948 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:11:35.956 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:11:35.956 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:11:35.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:11:35.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:11:36.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:11:36.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:11:36.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:11:36.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:11:36.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:11:36.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:11:36.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:11:36.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:11:36.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:11:36.018 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:11:36.018 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:11:36.018 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:11:36.018 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:11:36.019 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:36.019 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:11:36.020 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:36.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 8 2026-02-18T08:11:36.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 14 2026-02-18T08:11:36.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:11:36.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=57, tag_tid=3, entry_tid=1], [object_number=56, tag_tid=3, entry_tid=0], [object_number=59, tag_tid=2, entry_tid=59619], [object_number=58, tag_tid=2, entry_tid=59618]]], state=connected] 2026-02-18T08:11:36.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=33, tag_tid=2, entry_tid=36453], [object_number=32, tag_tid=2, entry_tid=36452], [object_number=35, tag_tid=2, entry_tid=36451], [object_number=34, tag_tid=2, entry_tid=36450]]], state=connected] 2026-02-18T08:11:36.055 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:36.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=57, tag_tid=3, entry_tid=1' 2026-02-18T08:11:36.058 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:11:36.058 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:11:36.058 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:11:36.058 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:11:36.058 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:11:36.058 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:11:36.058 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:11:36.058 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:11:36.058 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:11:36.058 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:11:36.059 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:11:36.059 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:11:36.059 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:36.060 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:11:36.060 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:36.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 8 2026-02-18T08:11:36.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 14 2026-02-18T08:11:36.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:11:36.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=57, tag_tid=3, entry_tid=1], [object_number=56, tag_tid=3, entry_tid=0], [object_number=59, tag_tid=2, entry_tid=59619], [object_number=58, tag_tid=2, entry_tid=59618]]], state=connected] 2026-02-18T08:11:36.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=33, tag_tid=2, entry_tid=36453], [object_number=32, tag_tid=2, entry_tid=36452], [object_number=35, tag_tid=2, entry_tid=36451], [object_number=34, tag_tid=2, entry_tid=36450]]], state=connected] 2026-02-18T08:11:36.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:36.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=33, tag_tid=2, entry_tid=36453' 2026-02-18T08:11:36.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=57, tag_tid=3, entry_tid=1' -a 'object_number=57, tag_tid=3, entry_tid=1' = 'object_number=33, tag_tid=2, entry_tid=36453' 2026-02-18T08:11:36.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=33, tag_tid=2, entry_tid=36453' '!=' 'object_number=34, tag_tid=2, entry_tid=36126' 2026-02-18T08:11:36.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:11:36.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=33, tag_tid=2, entry_tid=36453' = 'object_number=34, tag_tid=2, entry_tid=36126' 2026-02-18T08:11:36.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=33, tag_tid=2, entry_tid=36453' 2026-02-18T08:11:36.092 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=57, tag_tid=3, entry_tid=1' 2026-02-18T08:11:36.092 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:11:36.092 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:11:36.093 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=3 2026-02-18T08:11:36.094 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=33, tag_tid=2, entry_tid=36453' 2026-02-18T08:11:36.094 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:11:36.094 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:11:36.095 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=2 2026-02-18T08:11:36.096 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=57, tag_tid=3, entry_tid=1' 2026-02-18T08:11:36.096 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:11:36.096 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:11:36.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:11:36.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=33, tag_tid=2, entry_tid=36453' 2026-02-18T08:11:36.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:11:36.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:11:36.100 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=36453 2026-02-18T08:11:36.100 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 3 = 2 -a 1 -le 36453 2026-02-18T08:11:36.100 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:11:36.100 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:11:36.100 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:11:36.301 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:11:36.301 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:11:36.302 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:11:36.302 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:11:36.302 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:11:36.302 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:11:36.302 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:11:36.302 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:11:36.302 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:11:36.302 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:11:36.302 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:11:36.302 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:11:36.302 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:11:36.302 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:11:36.302 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:11:36.304 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:11:36.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:11:36.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:11:36.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:11:36.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:11:36.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:11:36.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:11:36.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:11:36.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:11:36.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:11:36.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:11:36.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:11:36.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:11:36.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:11:36.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:11:36.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:11:36.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:11:36.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:11:36.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:11:36.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:11:36.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:11:36.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:11:36.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:11:36.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:11:36.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:11:36.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:11:36.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:11:36.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:11:36.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:11:36.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:11:36.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:11:36.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:11:36.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:11:36.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:11:36.316 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:11:36.317 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:11:36.317 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:11:36.317 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:11:36.384 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:11:36.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:11:36.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:11:36.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:11:36.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:11:36.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:11:36.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:11:36.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:11:36.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:11:36.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:11:36.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:11:36.387 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:11:36.387 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:11:36.388 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:36.388 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:11:36.388 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:36.421 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 8 2026-02-18T08:11:36.421 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 14 2026-02-18T08:11:36.422 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:11:36.422 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=57, tag_tid=3, entry_tid=1], [object_number=56, tag_tid=3, entry_tid=0], [object_number=59, tag_tid=2, entry_tid=59619], [object_number=58, tag_tid=2, entry_tid=59618]]], state=connected] 2026-02-18T08:11:36.422 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=33, tag_tid=2, entry_tid=36597], [object_number=32, tag_tid=2, entry_tid=36596], [object_number=35, tag_tid=2, entry_tid=36595], [object_number=34, tag_tid=2, entry_tid=36594]]], state=connected] 2026-02-18T08:11:36.425 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:36.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=57, tag_tid=3, entry_tid=1' 2026-02-18T08:11:36.427 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:11:36.427 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:11:36.427 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:11:36.428 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:11:36.428 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:11:36.428 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:11:36.428 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:11:36.428 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:11:36.428 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:11:36.428 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:11:36.429 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:11:36.429 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:11:36.429 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:36.430 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:11:36.430 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:36.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 8 2026-02-18T08:11:36.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 14 2026-02-18T08:11:36.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:11:36.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=57, tag_tid=3, entry_tid=1], [object_number=56, tag_tid=3, entry_tid=0], [object_number=59, tag_tid=2, entry_tid=59619], [object_number=58, tag_tid=2, entry_tid=59618]]], state=connected] 2026-02-18T08:11:36.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=33, tag_tid=2, entry_tid=36597], [object_number=32, tag_tid=2, entry_tid=36596], [object_number=35, tag_tid=2, entry_tid=36595], [object_number=34, tag_tid=2, entry_tid=36594]]], state=connected] 2026-02-18T08:11:36.460 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:36.462 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=33, tag_tid=2, entry_tid=36597' 2026-02-18T08:11:36.462 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=57, tag_tid=3, entry_tid=1' -a 'object_number=57, tag_tid=3, entry_tid=1' = 'object_number=33, tag_tid=2, entry_tid=36597' 2026-02-18T08:11:36.462 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=33, tag_tid=2, entry_tid=36597' '!=' 'object_number=33, tag_tid=2, entry_tid=36453' 2026-02-18T08:11:36.462 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:11:36.462 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=33, tag_tid=2, entry_tid=36597' = 'object_number=33, tag_tid=2, entry_tid=36453' 2026-02-18T08:11:36.462 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=33, tag_tid=2, entry_tid=36597' 2026-02-18T08:11:36.463 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=57, tag_tid=3, entry_tid=1' 2026-02-18T08:11:36.463 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:11:36.463 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:11:36.463 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=3 2026-02-18T08:11:36.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=33, tag_tid=2, entry_tid=36597' 2026-02-18T08:11:36.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:11:36.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:11:36.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=2 2026-02-18T08:11:36.467 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=57, tag_tid=3, entry_tid=1' 2026-02-18T08:11:36.467 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:11:36.467 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:11:36.468 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:11:36.468 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=33, tag_tid=2, entry_tid=36597' 2026-02-18T08:11:36.468 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:11:36.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:11:36.470 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=36597 2026-02-18T08:11:36.470 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 3 = 2 -a 1 -le 36597 2026-02-18T08:11:36.470 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:11:36.470 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:11:36.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:11:36.672 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:11:36.672 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:11:36.673 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:11:36.673 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:11:36.673 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:11:36.673 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:11:36.673 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:11:36.673 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:11:36.673 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:11:36.673 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:11:36.673 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:11:36.673 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:11:36.673 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:11:36.674 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:11:36.674 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:11:36.674 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:11:36.676 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:11:36.677 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:11:36.677 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:11:36.677 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:11:36.677 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:11:36.677 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:11:36.677 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:11:36.677 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:11:36.677 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:11:36.677 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:11:36.677 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:11:36.678 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:11:36.678 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:11:36.678 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:11:36.678 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:11:36.678 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:11:36.678 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:11:36.678 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:11:36.678 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:11:36.678 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:11:36.678 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:11:36.678 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:11:36.678 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:11:36.679 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:11:36.679 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:11:36.679 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:11:36.679 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:11:36.679 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:11:36.679 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:11:36.679 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:11:36.679 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:11:36.679 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:11:36.679 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:11:36.686 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:11:36.687 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:11:36.687 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:11:36.687 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:11:36.754 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:11:36.755 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:11:36.755 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:11:36.755 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:11:36.755 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:11:36.755 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:11:36.755 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:11:36.755 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:11:36.755 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:11:36.755 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:11:36.755 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:11:36.755 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:11:36.755 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:11:36.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:36.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:11:36.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:36.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 8 2026-02-18T08:11:36.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 14 2026-02-18T08:11:36.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:11:36.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=57, tag_tid=3, entry_tid=1], [object_number=56, tag_tid=3, entry_tid=0], [object_number=59, tag_tid=2, entry_tid=59619], [object_number=58, tag_tid=2, entry_tid=59618]]], state=connected] 2026-02-18T08:11:36.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=34, tag_tid=2, entry_tid=36914], [object_number=33, tag_tid=2, entry_tid=36913], [object_number=32, tag_tid=2, entry_tid=36912], [object_number=35, tag_tid=2, entry_tid=36911]]], state=connected] 2026-02-18T08:11:36.796 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:36.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=57, tag_tid=3, entry_tid=1' 2026-02-18T08:11:36.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:11:36.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:11:36.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:11:36.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:11:36.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:11:36.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:11:36.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:11:36.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:11:36.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:11:36.798 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:11:36.798 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:11:36.798 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:11:36.799 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:36.800 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:11:36.800 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:36.827 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 8 2026-02-18T08:11:36.827 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 14 2026-02-18T08:11:36.827 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:11:36.827 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=57, tag_tid=3, entry_tid=1], [object_number=56, tag_tid=3, entry_tid=0], [object_number=59, tag_tid=2, entry_tid=59619], [object_number=58, tag_tid=2, entry_tid=59618]]], state=connected] 2026-02-18T08:11:36.827 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=34, tag_tid=2, entry_tid=36914], [object_number=33, tag_tid=2, entry_tid=36913], [object_number=32, tag_tid=2, entry_tid=36912], [object_number=35, tag_tid=2, entry_tid=36911]]], state=connected] 2026-02-18T08:11:36.829 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:36.831 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=34, tag_tid=2, entry_tid=36914' 2026-02-18T08:11:36.831 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=57, tag_tid=3, entry_tid=1' -a 'object_number=57, tag_tid=3, entry_tid=1' = 'object_number=34, tag_tid=2, entry_tid=36914' 2026-02-18T08:11:36.831 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=34, tag_tid=2, entry_tid=36914' '!=' 'object_number=33, tag_tid=2, entry_tid=36597' 2026-02-18T08:11:36.831 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:11:36.831 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=34, tag_tid=2, entry_tid=36914' = 'object_number=33, tag_tid=2, entry_tid=36597' 2026-02-18T08:11:36.832 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=34, tag_tid=2, entry_tid=36914' 2026-02-18T08:11:36.832 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=57, tag_tid=3, entry_tid=1' 2026-02-18T08:11:36.832 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:11:36.832 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:11:36.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=3 2026-02-18T08:11:36.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=34, tag_tid=2, entry_tid=36914' 2026-02-18T08:11:36.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:11:36.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:11:36.835 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=2 2026-02-18T08:11:36.836 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=57, tag_tid=3, entry_tid=1' 2026-02-18T08:11:36.836 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:11:36.836 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:11:36.837 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:11:36.838 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=34, tag_tid=2, entry_tid=36914' 2026-02-18T08:11:36.838 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:11:36.838 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:11:36.840 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=36914 2026-02-18T08:11:36.840 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 3 = 2 -a 1 -le 36914 2026-02-18T08:11:36.840 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:11:36.840 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:11:36.840 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:11:37.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:11:37.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:11:37.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:11:37.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:11:37.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:11:37.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:11:37.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:11:37.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:11:37.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:11:37.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:11:37.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:11:37.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:11:37.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:11:37.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:11:37.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:11:37.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:11:37.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:11:37.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:11:37.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:11:37.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:11:37.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:11:37.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:11:37.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:11:37.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:11:37.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:11:37.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:11:37.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:11:37.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:11:37.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:11:37.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:11:37.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:11:37.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:11:37.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:11:37.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:11:37.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:11:37.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:11:37.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:11:37.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:11:37.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:11:37.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:11:37.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:11:37.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:11:37.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:11:37.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:11:37.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:11:37.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:11:37.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:11:37.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:11:37.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:11:37.056 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:11:37.056 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:11:37.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:11:37.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:11:37.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:11:37.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:11:37.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:11:37.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:11:37.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:11:37.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:11:37.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:11:37.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:11:37.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:11:37.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:11:37.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:11:37.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:11:37.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:11:37.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:37.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:11:37.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:37.163 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 8 2026-02-18T08:11:37.163 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 14 2026-02-18T08:11:37.163 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:11:37.163 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=57, tag_tid=3, entry_tid=1], [object_number=56, tag_tid=3, entry_tid=0], [object_number=59, tag_tid=2, entry_tid=59619], [object_number=58, tag_tid=2, entry_tid=59618]]], state=connected] 2026-02-18T08:11:37.164 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=34, tag_tid=2, entry_tid=37234], [object_number=33, tag_tid=2, entry_tid=37233], [object_number=32, tag_tid=2, entry_tid=37232], [object_number=35, tag_tid=2, entry_tid=37231]]], state=connected] 2026-02-18T08:11:37.166 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:37.167 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=57, tag_tid=3, entry_tid=1' 2026-02-18T08:11:37.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:11:37.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:11:37.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:11:37.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:11:37.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:11:37.169 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:11:37.169 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:11:37.169 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:11:37.169 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:11:37.169 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:11:37.170 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:11:37.170 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:11:37.170 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:37.171 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:11:37.171 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:37.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 8 2026-02-18T08:11:37.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 14 2026-02-18T08:11:37.206 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:11:37.206 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=57, tag_tid=3, entry_tid=1], [object_number=56, tag_tid=3, entry_tid=0], [object_number=59, tag_tid=2, entry_tid=59619], [object_number=58, tag_tid=2, entry_tid=59618]]], state=connected] 2026-02-18T08:11:37.206 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=34, tag_tid=2, entry_tid=37234], [object_number=33, tag_tid=2, entry_tid=37233], [object_number=32, tag_tid=2, entry_tid=37232], [object_number=35, tag_tid=2, entry_tid=37231]]], state=connected] 2026-02-18T08:11:37.209 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:37.211 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=34, tag_tid=2, entry_tid=37234' 2026-02-18T08:11:37.211 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=57, tag_tid=3, entry_tid=1' -a 'object_number=57, tag_tid=3, entry_tid=1' = 'object_number=34, tag_tid=2, entry_tid=37234' 2026-02-18T08:11:37.211 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=34, tag_tid=2, entry_tid=37234' '!=' 'object_number=34, tag_tid=2, entry_tid=36914' 2026-02-18T08:11:37.211 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:11:37.211 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=34, tag_tid=2, entry_tid=37234' = 'object_number=34, tag_tid=2, entry_tid=36914' 2026-02-18T08:11:37.211 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=34, tag_tid=2, entry_tid=37234' 2026-02-18T08:11:37.212 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=57, tag_tid=3, entry_tid=1' 2026-02-18T08:11:37.212 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:11:37.212 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:11:37.213 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=3 2026-02-18T08:11:37.214 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=34, tag_tid=2, entry_tid=37234' 2026-02-18T08:11:37.214 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:11:37.214 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:11:37.215 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=2 2026-02-18T08:11:37.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=57, tag_tid=3, entry_tid=1' 2026-02-18T08:11:37.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:11:37.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:11:37.217 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:11:37.218 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=34, tag_tid=2, entry_tid=37234' 2026-02-18T08:11:37.218 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:11:37.218 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:11:37.220 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=37234 2026-02-18T08:11:37.220 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 3 = 2 -a 1 -le 37234 2026-02-18T08:11:37.220 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:11:37.220 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:11:37.220 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:11:37.422 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:11:37.422 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:11:37.422 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:11:37.422 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:11:37.422 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:11:37.423 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:11:37.423 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:11:37.423 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:11:37.423 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:11:37.423 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:11:37.423 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:11:37.423 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:11:37.423 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:11:37.423 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:11:37.423 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:11:37.423 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:11:37.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:11:37.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:11:37.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:11:37.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:11:37.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:11:37.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:11:37.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:11:37.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:11:37.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:11:37.427 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:11:37.427 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:11:37.427 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:11:37.427 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:11:37.427 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:11:37.427 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:11:37.427 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:11:37.427 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:11:37.427 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:11:37.427 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:11:37.427 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:11:37.427 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:11:37.427 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:11:37.428 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:11:37.428 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:11:37.428 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:11:37.428 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:11:37.428 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:11:37.428 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:11:37.428 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:11:37.428 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:11:37.428 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:11:37.428 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:11:37.428 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:11:37.437 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:11:37.437 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:11:37.437 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:11:37.437 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:11:37.501 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:11:37.502 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:11:37.503 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:11:37.503 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:11:37.503 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:11:37.503 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:11:37.503 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:11:37.503 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:11:37.503 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:11:37.503 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:11:37.503 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:11:37.503 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:11:37.504 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:11:37.504 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:37.505 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:11:37.505 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:37.534 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 8 2026-02-18T08:11:37.534 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 14 2026-02-18T08:11:37.534 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:11:37.535 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=57, tag_tid=3, entry_tid=1], [object_number=56, tag_tid=3, entry_tid=0], [object_number=59, tag_tid=2, entry_tid=59619], [object_number=58, tag_tid=2, entry_tid=59618]]], state=connected] 2026-02-18T08:11:37.535 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=37, tag_tid=2, entry_tid=37457], [object_number=36, tag_tid=2, entry_tid=37456], [object_number=35, tag_tid=2, entry_tid=37455], [object_number=34, tag_tid=2, entry_tid=37454]]], state=connected] 2026-02-18T08:11:37.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:37.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=57, tag_tid=3, entry_tid=1' 2026-02-18T08:11:37.539 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:11:37.539 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:11:37.539 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:11:37.539 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:11:37.539 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:11:37.539 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:11:37.540 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:11:37.540 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:11:37.540 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:11:37.540 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:11:37.540 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:11:37.540 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:11:37.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:37.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:11:37.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:37.569 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 8 2026-02-18T08:11:37.569 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 14 2026-02-18T08:11:37.569 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:11:37.569 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=57, tag_tid=3, entry_tid=1], [object_number=56, tag_tid=3, entry_tid=0], [object_number=59, tag_tid=2, entry_tid=59619], [object_number=58, tag_tid=2, entry_tid=59618]]], state=connected] 2026-02-18T08:11:37.569 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=37, tag_tid=2, entry_tid=37457], [object_number=36, tag_tid=2, entry_tid=37456], [object_number=35, tag_tid=2, entry_tid=37455], [object_number=34, tag_tid=2, entry_tid=37454]]], state=connected] 2026-02-18T08:11:37.572 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:37.573 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=37, tag_tid=2, entry_tid=37457' 2026-02-18T08:11:37.573 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=57, tag_tid=3, entry_tid=1' -a 'object_number=57, tag_tid=3, entry_tid=1' = 'object_number=37, tag_tid=2, entry_tid=37457' 2026-02-18T08:11:37.573 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=37, tag_tid=2, entry_tid=37457' '!=' 'object_number=34, tag_tid=2, entry_tid=37234' 2026-02-18T08:11:37.573 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:11:37.573 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=37, tag_tid=2, entry_tid=37457' = 'object_number=34, tag_tid=2, entry_tid=37234' 2026-02-18T08:11:37.574 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=37, tag_tid=2, entry_tid=37457' 2026-02-18T08:11:37.574 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=57, tag_tid=3, entry_tid=1' 2026-02-18T08:11:37.574 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:11:37.574 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:11:37.575 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=3 2026-02-18T08:11:37.576 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=37, tag_tid=2, entry_tid=37457' 2026-02-18T08:11:37.576 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:11:37.576 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:11:37.577 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=2 2026-02-18T08:11:37.578 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=57, tag_tid=3, entry_tid=1' 2026-02-18T08:11:37.578 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:11:37.578 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:11:37.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:11:37.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=37, tag_tid=2, entry_tid=37457' 2026-02-18T08:11:37.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:11:37.582 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:11:37.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=37457 2026-02-18T08:11:37.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 3 = 2 -a 1 -le 37457 2026-02-18T08:11:37.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:11:37.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:11:37.584 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:11:37.785 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:11:37.785 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:11:37.785 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:11:37.786 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:11:37.786 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:11:37.786 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:11:37.786 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:11:37.786 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:11:37.786 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:11:37.786 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:11:37.786 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:11:37.786 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:11:37.786 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:11:37.787 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:11:37.787 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:11:37.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:11:37.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:11:37.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:11:37.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:11:37.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:11:37.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:11:37.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:11:37.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:11:37.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:11:37.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:11:37.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:11:37.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:11:37.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:11:37.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:11:37.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:11:37.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:11:37.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:11:37.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:11:37.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:11:37.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:11:37.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:11:37.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:11:37.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:11:37.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:11:37.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:11:37.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:11:37.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:11:37.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:11:37.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:11:37.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:11:37.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:11:37.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:11:37.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:11:37.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:11:37.801 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:11:37.801 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:11:37.801 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:11:37.801 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:11:37.860 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:11:37.861 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:11:37.861 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:11:37.862 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:11:37.862 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:11:37.862 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:11:37.862 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:11:37.862 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:11:37.862 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:11:37.862 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:11:37.862 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:11:37.862 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:11:37.862 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:11:37.863 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:37.863 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:11:37.863 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:37.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 9 2026-02-18T08:11:37.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 14 2026-02-18T08:11:37.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:11:37.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=57, tag_tid=3, entry_tid=1], [object_number=56, tag_tid=3, entry_tid=0], [object_number=59, tag_tid=2, entry_tid=59619], [object_number=58, tag_tid=2, entry_tid=59618]]], state=connected] 2026-02-18T08:11:37.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=38, tag_tid=2, entry_tid=37718], [object_number=37, tag_tid=2, entry_tid=37717], [object_number=36, tag_tid=2, entry_tid=37716], [object_number=39, tag_tid=2, entry_tid=37715]]], state=connected] 2026-02-18T08:11:37.894 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:37.895 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=57, tag_tid=3, entry_tid=1' 2026-02-18T08:11:37.896 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:11:37.896 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:11:37.897 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:11:37.897 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:11:37.897 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:11:37.897 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:11:37.897 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:11:37.897 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:11:37.897 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:11:37.897 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:11:37.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:11:37.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:11:37.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:37.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:37.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:11:37.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 9 2026-02-18T08:11:37.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 14 2026-02-18T08:11:37.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:11:37.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=57, tag_tid=3, entry_tid=1], [object_number=56, tag_tid=3, entry_tid=0], [object_number=59, tag_tid=2, entry_tid=59619], [object_number=58, tag_tid=2, entry_tid=59618]]], state=connected] 2026-02-18T08:11:37.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=38, tag_tid=2, entry_tid=37718], [object_number=37, tag_tid=2, entry_tid=37717], [object_number=36, tag_tid=2, entry_tid=37716], [object_number=39, tag_tid=2, entry_tid=37715]]], state=connected] 2026-02-18T08:11:37.929 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:37.931 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=38, tag_tid=2, entry_tid=37718' 2026-02-18T08:11:37.931 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=57, tag_tid=3, entry_tid=1' -a 'object_number=57, tag_tid=3, entry_tid=1' = 'object_number=38, tag_tid=2, entry_tid=37718' 2026-02-18T08:11:37.931 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=38, tag_tid=2, entry_tid=37718' '!=' 'object_number=37, tag_tid=2, entry_tid=37457' 2026-02-18T08:11:37.931 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:11:37.932 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=38, tag_tid=2, entry_tid=37718' = 'object_number=37, tag_tid=2, entry_tid=37457' 2026-02-18T08:11:37.932 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=38, tag_tid=2, entry_tid=37718' 2026-02-18T08:11:37.932 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=57, tag_tid=3, entry_tid=1' 2026-02-18T08:11:37.933 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:11:37.933 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:11:37.933 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=3 2026-02-18T08:11:37.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=38, tag_tid=2, entry_tid=37718' 2026-02-18T08:11:37.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:11:37.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:11:37.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=2 2026-02-18T08:11:37.936 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=57, tag_tid=3, entry_tid=1' 2026-02-18T08:11:37.936 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:11:37.936 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:11:37.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:11:37.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=38, tag_tid=2, entry_tid=37718' 2026-02-18T08:11:37.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:11:37.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:11:37.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=37718 2026-02-18T08:11:37.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 3 = 2 -a 1 -le 37718 2026-02-18T08:11:37.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:11:37.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:11:37.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:11:38.142 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:11:38.142 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:11:38.142 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:11:38.142 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:11:38.142 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:11:38.142 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:11:38.142 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:11:38.143 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:11:38.143 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:11:38.143 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:11:38.143 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:11:38.143 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:11:38.143 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:11:38.143 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:11:38.143 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:11:38.143 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:11:38.146 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:11:38.146 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:11:38.146 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:11:38.146 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:11:38.146 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:11:38.146 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:11:38.147 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:11:38.147 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:11:38.147 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:11:38.147 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:11:38.147 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:11:38.147 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:11:38.147 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:11:38.147 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:11:38.147 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:11:38.147 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:11:38.147 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:11:38.147 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:11:38.147 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:11:38.148 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:11:38.148 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:11:38.148 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:11:38.148 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:11:38.148 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:11:38.148 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:11:38.148 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:11:38.148 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:11:38.148 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:11:38.148 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:11:38.148 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:11:38.148 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:11:38.149 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:11:38.149 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:11:38.156 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:11:38.156 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:11:38.157 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:11:38.157 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:11:38.214 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:11:38.215 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:11:38.215 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:11:38.215 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:11:38.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:11:38.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:11:38.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:11:38.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:11:38.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:11:38.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:11:38.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:11:38.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:11:38.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:11:38.217 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:38.217 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:11:38.218 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:38.244 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 9 2026-02-18T08:11:38.244 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 14 2026-02-18T08:11:38.244 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:11:38.244 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=57, tag_tid=3, entry_tid=1], [object_number=56, tag_tid=3, entry_tid=0], [object_number=59, tag_tid=2, entry_tid=59619], [object_number=58, tag_tid=2, entry_tid=59618]]], state=connected] 2026-02-18T08:11:38.244 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=39, tag_tid=2, entry_tid=37855], [object_number=38, tag_tid=2, entry_tid=37854], [object_number=37, tag_tid=2, entry_tid=37853], [object_number=36, tag_tid=2, entry_tid=37852]]], state=connected] 2026-02-18T08:11:38.247 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:38.248 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=57, tag_tid=3, entry_tid=1' 2026-02-18T08:11:38.250 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:11:38.250 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:11:38.250 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:11:38.250 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:11:38.250 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:11:38.250 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:11:38.250 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:11:38.250 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:11:38.250 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:11:38.250 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:11:38.251 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:11:38.251 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:11:38.251 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:38.252 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:11:38.252 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:38.488 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 9 2026-02-18T08:11:38.488 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 14 2026-02-18T08:11:38.489 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:11:38.489 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=57, tag_tid=3, entry_tid=1], [object_number=56, tag_tid=3, entry_tid=0], [object_number=59, tag_tid=2, entry_tid=59619], [object_number=58, tag_tid=2, entry_tid=59618]]], state=connected] 2026-02-18T08:11:38.489 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=39, tag_tid=2, entry_tid=37855], [object_number=38, tag_tid=2, entry_tid=37854], [object_number=37, tag_tid=2, entry_tid=37853], [object_number=36, tag_tid=2, entry_tid=37852]]], state=connected] 2026-02-18T08:11:38.492 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:38.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=39, tag_tid=2, entry_tid=37855' 2026-02-18T08:11:38.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=57, tag_tid=3, entry_tid=1' -a 'object_number=57, tag_tid=3, entry_tid=1' = 'object_number=39, tag_tid=2, entry_tid=37855' 2026-02-18T08:11:38.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=39, tag_tid=2, entry_tid=37855' '!=' 'object_number=38, tag_tid=2, entry_tid=37718' 2026-02-18T08:11:38.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:11:38.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=39, tag_tid=2, entry_tid=37855' = 'object_number=38, tag_tid=2, entry_tid=37718' 2026-02-18T08:11:38.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=39, tag_tid=2, entry_tid=37855' 2026-02-18T08:11:38.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=57, tag_tid=3, entry_tid=1' 2026-02-18T08:11:38.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:11:38.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:11:38.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=3 2026-02-18T08:11:38.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=39, tag_tid=2, entry_tid=37855' 2026-02-18T08:11:38.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:11:38.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:11:38.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=2 2026-02-18T08:11:38.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=57, tag_tid=3, entry_tid=1' 2026-02-18T08:11:38.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:11:38.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:11:38.500 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:11:38.501 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=39, tag_tid=2, entry_tid=37855' 2026-02-18T08:11:38.501 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:11:38.501 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:11:38.502 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=37855 2026-02-18T08:11:38.502 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 3 = 2 -a 1 -le 37855 2026-02-18T08:11:38.502 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:11:38.503 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:11:38.503 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:11:38.704 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:11:38.705 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:11:38.705 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:11:38.705 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:11:38.705 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:11:38.705 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:11:38.705 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:11:38.705 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:11:38.705 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:11:38.705 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:11:38.705 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:11:38.706 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:11:38.706 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:11:38.706 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:11:38.706 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:11:38.706 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:11:38.709 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:11:38.709 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:11:38.709 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:11:38.709 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:11:38.709 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:11:38.709 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:11:38.709 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:11:38.709 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:11:38.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:11:38.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:11:38.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:11:38.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:11:38.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:11:38.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:11:38.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:11:38.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:11:38.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:11:38.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:11:38.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:11:38.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:11:38.711 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:11:38.711 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:11:38.711 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:11:38.711 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:11:38.711 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:11:38.711 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:11:38.711 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:11:38.711 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:11:38.711 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:11:38.711 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:11:38.711 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:11:38.711 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:11:38.712 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:11:38.718 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:11:38.718 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:11:38.718 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:11:38.719 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:11:38.787 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:11:38.788 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:11:38.788 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:11:38.788 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:11:38.788 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:11:38.788 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:11:38.788 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:11:38.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:11:38.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:11:38.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:11:38.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:11:38.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:11:38.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:11:38.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:38.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:11:38.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:38.826 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 9 2026-02-18T08:11:38.826 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 14 2026-02-18T08:11:38.826 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:11:38.826 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=57, tag_tid=3, entry_tid=1], [object_number=56, tag_tid=3, entry_tid=0], [object_number=59, tag_tid=2, entry_tid=59619], [object_number=58, tag_tid=2, entry_tid=59618]]], state=connected] 2026-02-18T08:11:38.826 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=37, tag_tid=2, entry_tid=38349], [object_number=36, tag_tid=2, entry_tid=38348], [object_number=39, tag_tid=2, entry_tid=38347], [object_number=38, tag_tid=2, entry_tid=38346]]], state=connected] 2026-02-18T08:11:38.830 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:38.831 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=57, tag_tid=3, entry_tid=1' 2026-02-18T08:11:38.832 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:11:38.832 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:11:38.832 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:11:38.832 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:11:38.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:11:38.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:11:38.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:11:38.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:11:38.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:11:38.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:11:38.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:11:38.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:11:38.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:38.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:11:38.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:38.860 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 9 2026-02-18T08:11:38.860 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 14 2026-02-18T08:11:38.860 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:11:38.860 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=57, tag_tid=3, entry_tid=1], [object_number=56, tag_tid=3, entry_tid=0], [object_number=59, tag_tid=2, entry_tid=59619], [object_number=58, tag_tid=2, entry_tid=59618]]], state=connected] 2026-02-18T08:11:38.860 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=37, tag_tid=2, entry_tid=38349], [object_number=36, tag_tid=2, entry_tid=38348], [object_number=39, tag_tid=2, entry_tid=38347], [object_number=38, tag_tid=2, entry_tid=38346]]], state=connected] 2026-02-18T08:11:38.863 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:38.865 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=37, tag_tid=2, entry_tid=38349' 2026-02-18T08:11:38.865 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=57, tag_tid=3, entry_tid=1' -a 'object_number=57, tag_tid=3, entry_tid=1' = 'object_number=37, tag_tid=2, entry_tid=38349' 2026-02-18T08:11:38.865 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=37, tag_tid=2, entry_tid=38349' '!=' 'object_number=39, tag_tid=2, entry_tid=37855' 2026-02-18T08:11:38.865 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:11:38.865 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=37, tag_tid=2, entry_tid=38349' = 'object_number=39, tag_tid=2, entry_tid=37855' 2026-02-18T08:11:38.866 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=37, tag_tid=2, entry_tid=38349' 2026-02-18T08:11:38.866 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=57, tag_tid=3, entry_tid=1' 2026-02-18T08:11:38.866 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:11:38.866 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:11:38.867 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=3 2026-02-18T08:11:38.868 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=37, tag_tid=2, entry_tid=38349' 2026-02-18T08:11:38.868 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:11:38.868 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:11:38.869 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=2 2026-02-18T08:11:38.870 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=57, tag_tid=3, entry_tid=1' 2026-02-18T08:11:38.870 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:11:38.870 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:11:38.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:11:38.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=37, tag_tid=2, entry_tid=38349' 2026-02-18T08:11:38.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:11:38.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:11:38.874 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=38349 2026-02-18T08:11:38.874 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 3 = 2 -a 1 -le 38349 2026-02-18T08:11:38.874 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:11:38.874 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:11:38.874 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:11:39.075 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:11:39.075 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:11:39.075 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:11:39.075 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:11:39.075 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:11:39.075 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:11:39.075 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:11:39.075 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:11:39.075 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:11:39.075 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:11:39.075 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:11:39.075 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:11:39.075 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:11:39.075 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:11:39.075 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:11:39.076 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:11:39.076 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:11:39.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:11:39.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:11:39.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:11:39.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:11:39.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:11:39.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:11:39.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:11:39.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:11:39.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:11:39.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:11:39.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:11:39.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:11:39.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:11:39.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:11:39.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:11:39.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:11:39.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:11:39.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:11:39.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:11:39.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:11:39.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:11:39.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:11:39.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:11:39.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:11:39.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:11:39.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:11:39.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:11:39.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:11:39.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:11:39.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:11:39.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:11:39.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:11:39.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:11:39.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:11:39.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:11:39.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:11:39.158 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:11:39.158 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:11:39.158 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:11:39.158 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:11:39.158 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:11:39.158 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:11:39.158 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:11:39.158 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:11:39.158 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:11:39.158 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:11:39.158 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:11:39.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:11:39.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:11:39.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:39.161 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:11:39.161 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:39.188 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 9 2026-02-18T08:11:39.188 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 14 2026-02-18T08:11:39.188 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:11:39.188 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=57, tag_tid=3, entry_tid=1], [object_number=56, tag_tid=3, entry_tid=0], [object_number=59, tag_tid=2, entry_tid=59619], [object_number=58, tag_tid=2, entry_tid=59618]]], state=connected] 2026-02-18T08:11:39.188 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=36, tag_tid=2, entry_tid=38640], [object_number=39, tag_tid=2, entry_tid=38639], [object_number=38, tag_tid=2, entry_tid=38638], [object_number=37, tag_tid=2, entry_tid=38637]]], state=connected] 2026-02-18T08:11:39.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:39.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=57, tag_tid=3, entry_tid=1' 2026-02-18T08:11:39.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:11:39.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:11:39.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:11:39.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:11:39.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:11:39.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:11:39.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:11:39.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:11:39.194 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:11:39.194 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:11:39.194 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:11:39.194 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:11:39.195 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:39.195 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:11:39.195 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:39.223 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 9 2026-02-18T08:11:39.223 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 14 2026-02-18T08:11:39.223 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:11:39.223 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=57, tag_tid=3, entry_tid=1], [object_number=56, tag_tid=3, entry_tid=0], [object_number=59, tag_tid=2, entry_tid=59619], [object_number=58, tag_tid=2, entry_tid=59618]]], state=connected] 2026-02-18T08:11:39.224 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=36, tag_tid=2, entry_tid=38640], [object_number=39, tag_tid=2, entry_tid=38639], [object_number=38, tag_tid=2, entry_tid=38638], [object_number=37, tag_tid=2, entry_tid=38637]]], state=connected] 2026-02-18T08:11:39.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:39.227 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=36, tag_tid=2, entry_tid=38640' 2026-02-18T08:11:39.227 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=57, tag_tid=3, entry_tid=1' -a 'object_number=57, tag_tid=3, entry_tid=1' = 'object_number=36, tag_tid=2, entry_tid=38640' 2026-02-18T08:11:39.227 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=36, tag_tid=2, entry_tid=38640' '!=' 'object_number=37, tag_tid=2, entry_tid=38349' 2026-02-18T08:11:39.227 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:11:39.227 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=36, tag_tid=2, entry_tid=38640' = 'object_number=37, tag_tid=2, entry_tid=38349' 2026-02-18T08:11:39.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=36, tag_tid=2, entry_tid=38640' 2026-02-18T08:11:39.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=57, tag_tid=3, entry_tid=1' 2026-02-18T08:11:39.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:11:39.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:11:39.229 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=3 2026-02-18T08:11:39.230 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=36, tag_tid=2, entry_tid=38640' 2026-02-18T08:11:39.230 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:11:39.230 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:11:39.231 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=2 2026-02-18T08:11:39.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=57, tag_tid=3, entry_tid=1' 2026-02-18T08:11:39.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:11:39.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:11:39.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:11:39.234 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=36, tag_tid=2, entry_tid=38640' 2026-02-18T08:11:39.234 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:11:39.234 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:11:39.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=38640 2026-02-18T08:11:39.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 3 = 2 -a 1 -le 38640 2026-02-18T08:11:39.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:11:39.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:11:39.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:11:39.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:11:39.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:11:39.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:11:39.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:11:39.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:11:39.439 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:11:39.439 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:11:39.439 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:11:39.439 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:11:39.439 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:11:39.439 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:11:39.439 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:11:39.439 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:11:39.439 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:11:39.439 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:11:39.439 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:11:39.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:11:39.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:11:39.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:11:39.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:11:39.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:11:39.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:11:39.443 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:11:39.443 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:11:39.443 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:11:39.443 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:11:39.443 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:11:39.443 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:11:39.443 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:11:39.443 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:11:39.443 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:11:39.443 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:11:39.443 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:11:39.443 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:11:39.443 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:11:39.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:11:39.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:11:39.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:11:39.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:11:39.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:11:39.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:11:39.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:11:39.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:11:39.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:11:39.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:11:39.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:11:39.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:11:39.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:11:39.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:11:39.452 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:11:39.453 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:11:39.453 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:11:39.453 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:11:39.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:11:39.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:11:39.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:11:39.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:11:39.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:11:39.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:11:39.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:11:39.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:11:39.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:11:39.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:11:39.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:11:39.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:11:39.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:11:39.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:39.523 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:11:39.523 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:39.551 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 9 2026-02-18T08:11:39.551 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 14 2026-02-18T08:11:39.551 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:11:39.551 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=57, tag_tid=3, entry_tid=1], [object_number=56, tag_tid=3, entry_tid=0], [object_number=59, tag_tid=2, entry_tid=59619], [object_number=58, tag_tid=2, entry_tid=59618]]], state=connected] 2026-02-18T08:11:39.551 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=37, tag_tid=2, entry_tid=38933], [object_number=36, tag_tid=2, entry_tid=38932], [object_number=39, tag_tid=2, entry_tid=38931], [object_number=38, tag_tid=2, entry_tid=38930]]], state=connected] 2026-02-18T08:11:39.553 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:39.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=57, tag_tid=3, entry_tid=1' 2026-02-18T08:11:39.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:11:39.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:11:39.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:11:39.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:11:39.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:11:39.557 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:11:39.557 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:11:39.557 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:11:39.557 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:11:39.557 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:11:39.557 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:11:39.557 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:11:39.557 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:39.558 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:11:39.558 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:39.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 9 2026-02-18T08:11:39.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 14 2026-02-18T08:11:39.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:11:39.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=57, tag_tid=3, entry_tid=1], [object_number=56, tag_tid=3, entry_tid=0], [object_number=59, tag_tid=2, entry_tid=59619], [object_number=58, tag_tid=2, entry_tid=59618]]], state=connected] 2026-02-18T08:11:39.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=37, tag_tid=2, entry_tid=38933], [object_number=36, tag_tid=2, entry_tid=38932], [object_number=39, tag_tid=2, entry_tid=38931], [object_number=38, tag_tid=2, entry_tid=38930]]], state=connected] 2026-02-18T08:11:39.585 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:39.587 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=37, tag_tid=2, entry_tid=38933' 2026-02-18T08:11:39.587 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=57, tag_tid=3, entry_tid=1' -a 'object_number=57, tag_tid=3, entry_tid=1' = 'object_number=37, tag_tid=2, entry_tid=38933' 2026-02-18T08:11:39.587 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=37, tag_tid=2, entry_tid=38933' '!=' 'object_number=36, tag_tid=2, entry_tid=38640' 2026-02-18T08:11:39.587 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:11:39.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=37, tag_tid=2, entry_tid=38933' = 'object_number=36, tag_tid=2, entry_tid=38640' 2026-02-18T08:11:39.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=37, tag_tid=2, entry_tid=38933' 2026-02-18T08:11:39.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=57, tag_tid=3, entry_tid=1' 2026-02-18T08:11:39.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:11:39.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:11:39.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=3 2026-02-18T08:11:39.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=37, tag_tid=2, entry_tid=38933' 2026-02-18T08:11:39.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:11:39.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:11:39.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=2 2026-02-18T08:11:39.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=57, tag_tid=3, entry_tid=1' 2026-02-18T08:11:39.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:11:39.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:11:39.594 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:11:39.595 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=37, tag_tid=2, entry_tid=38933' 2026-02-18T08:11:39.595 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:11:39.595 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:11:39.597 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=38933 2026-02-18T08:11:39.597 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 3 = 2 -a 1 -le 38933 2026-02-18T08:11:39.597 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:11:39.597 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:11:39.597 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:11:39.799 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:11:39.799 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:11:39.799 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:11:39.799 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:11:39.799 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:11:39.799 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:11:39.799 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:11:39.799 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:11:39.800 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:11:39.800 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:11:39.800 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:11:39.800 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:11:39.800 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:11:39.800 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:11:39.800 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:11:39.800 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:11:39.803 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:11:39.803 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:11:39.803 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:11:39.803 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:11:39.803 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:11:39.803 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:11:39.803 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:11:39.804 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:11:39.804 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:11:39.804 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:11:39.804 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:11:39.804 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:11:39.804 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:11:39.804 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:11:39.804 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:11:39.804 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:11:39.804 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:11:39.804 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:11:39.804 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:11:39.805 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:11:39.805 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:11:39.805 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:11:39.805 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:11:39.805 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:11:39.805 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:11:39.805 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:11:39.805 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:11:39.805 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:11:39.805 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:11:39.805 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:11:39.805 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:11:39.806 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:11:39.806 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:11:39.813 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:11:39.813 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:11:39.813 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:11:39.813 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:11:39.880 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:11:39.881 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:11:39.881 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:11:39.881 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:11:39.882 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:11:39.882 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:11:39.882 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:11:39.882 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:11:39.882 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:11:39.882 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:11:39.882 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:11:39.883 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:11:39.883 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:11:39.883 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:39.884 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:11:39.884 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:39.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 9 2026-02-18T08:11:39.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 14 2026-02-18T08:11:39.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:11:39.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=57, tag_tid=3, entry_tid=1], [object_number=56, tag_tid=3, entry_tid=0], [object_number=59, tag_tid=2, entry_tid=59619], [object_number=58, tag_tid=2, entry_tid=59618]]], state=connected] 2026-02-18T08:11:39.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=39, tag_tid=2, entry_tid=39239], [object_number=38, tag_tid=2, entry_tid=39238], [object_number=37, tag_tid=2, entry_tid=39237], [object_number=36, tag_tid=2, entry_tid=39236]]], state=connected] 2026-02-18T08:11:39.923 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:39.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=57, tag_tid=3, entry_tid=1' 2026-02-18T08:11:39.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:11:39.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:11:39.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:11:39.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:11:39.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:11:39.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:11:39.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:11:39.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:11:39.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:11:39.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:11:39.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:11:39.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:11:39.928 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:39.928 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:11:39.928 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:40.166 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 9 2026-02-18T08:11:40.166 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 14 2026-02-18T08:11:40.167 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:11:40.167 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=57, tag_tid=3, entry_tid=1], [object_number=56, tag_tid=3, entry_tid=0], [object_number=59, tag_tid=2, entry_tid=59619], [object_number=58, tag_tid=2, entry_tid=59618]]], state=connected] 2026-02-18T08:11:40.167 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=39, tag_tid=2, entry_tid=39239], [object_number=38, tag_tid=2, entry_tid=39238], [object_number=37, tag_tid=2, entry_tid=39237], [object_number=36, tag_tid=2, entry_tid=39236]]], state=connected] 2026-02-18T08:11:40.170 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:40.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=39, tag_tid=2, entry_tid=39239' 2026-02-18T08:11:40.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=57, tag_tid=3, entry_tid=1' -a 'object_number=57, tag_tid=3, entry_tid=1' = 'object_number=39, tag_tid=2, entry_tid=39239' 2026-02-18T08:11:40.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=39, tag_tid=2, entry_tid=39239' '!=' 'object_number=37, tag_tid=2, entry_tid=38933' 2026-02-18T08:11:40.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:11:40.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=39, tag_tid=2, entry_tid=39239' = 'object_number=37, tag_tid=2, entry_tid=38933' 2026-02-18T08:11:40.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=39, tag_tid=2, entry_tid=39239' 2026-02-18T08:11:40.173 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=57, tag_tid=3, entry_tid=1' 2026-02-18T08:11:40.173 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:11:40.173 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:11:40.174 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=3 2026-02-18T08:11:40.175 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=39, tag_tid=2, entry_tid=39239' 2026-02-18T08:11:40.175 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:11:40.175 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:11:40.176 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=2 2026-02-18T08:11:40.177 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=57, tag_tid=3, entry_tid=1' 2026-02-18T08:11:40.177 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:11:40.177 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:11:40.178 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:11:40.179 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=39, tag_tid=2, entry_tid=39239' 2026-02-18T08:11:40.179 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:11:40.180 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:11:40.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=39239 2026-02-18T08:11:40.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 3 = 2 -a 1 -le 39239 2026-02-18T08:11:40.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:11:40.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:11:40.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:11:40.383 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:11:40.383 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:11:40.383 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:11:40.383 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:11:40.383 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:11:40.384 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:11:40.384 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:11:40.384 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:11:40.384 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:11:40.384 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:11:40.384 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:11:40.384 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:11:40.384 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:11:40.384 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:11:40.384 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:11:40.387 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:11:40.387 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:11:40.387 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:11:40.387 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:11:40.387 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:11:40.388 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:11:40.388 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:11:40.388 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:11:40.388 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:11:40.388 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:11:40.388 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:11:40.388 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:11:40.388 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:11:40.388 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:11:40.388 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:11:40.388 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:11:40.388 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:11:40.389 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:11:40.389 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:11:40.389 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:11:40.389 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:11:40.389 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:11:40.389 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:11:40.389 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:11:40.389 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:11:40.389 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:11:40.389 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:11:40.389 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:11:40.389 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:11:40.390 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:11:40.390 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:11:40.390 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:11:40.390 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:11:40.390 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:11:40.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:11:40.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:11:40.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:11:40.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:11:40.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:11:40.465 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:11:40.465 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:11:40.465 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:11:40.465 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:11:40.465 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:11:40.465 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:11:40.465 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:11:40.465 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:11:40.465 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:11:40.465 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:11:40.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:11:40.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:11:40.467 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:40.467 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:11:40.467 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:40.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 9 2026-02-18T08:11:40.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 14 2026-02-18T08:11:40.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:11:40.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=57, tag_tid=3, entry_tid=1], [object_number=56, tag_tid=3, entry_tid=0], [object_number=59, tag_tid=2, entry_tid=59619], [object_number=58, tag_tid=2, entry_tid=59618]]], state=connected] 2026-02-18T08:11:40.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=37, tag_tid=2, entry_tid=39781], [object_number=36, tag_tid=2, entry_tid=39780], [object_number=39, tag_tid=2, entry_tid=39779], [object_number=38, tag_tid=2, entry_tid=39778]]], state=connected] 2026-02-18T08:11:40.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:40.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=57, tag_tid=3, entry_tid=1' 2026-02-18T08:11:40.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:11:40.500 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:11:40.500 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:11:40.500 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:11:40.500 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:11:40.500 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:11:40.500 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:11:40.500 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:11:40.500 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:11:40.500 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:11:40.501 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:11:40.501 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:11:40.502 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:40.502 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:11:40.502 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:40.527 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 9 2026-02-18T08:11:40.527 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 14 2026-02-18T08:11:40.527 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:11:40.527 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=57, tag_tid=3, entry_tid=1], [object_number=56, tag_tid=3, entry_tid=0], [object_number=59, tag_tid=2, entry_tid=59619], [object_number=58, tag_tid=2, entry_tid=59618]]], state=connected] 2026-02-18T08:11:40.528 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=37, tag_tid=2, entry_tid=39781], [object_number=36, tag_tid=2, entry_tid=39780], [object_number=39, tag_tid=2, entry_tid=39779], [object_number=38, tag_tid=2, entry_tid=39778]]], state=connected] 2026-02-18T08:11:40.529 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:40.532 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=37, tag_tid=2, entry_tid=39781' 2026-02-18T08:11:40.532 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=57, tag_tid=3, entry_tid=1' -a 'object_number=57, tag_tid=3, entry_tid=1' = 'object_number=37, tag_tid=2, entry_tid=39781' 2026-02-18T08:11:40.532 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=37, tag_tid=2, entry_tid=39781' '!=' 'object_number=39, tag_tid=2, entry_tid=39239' 2026-02-18T08:11:40.532 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:11:40.532 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=37, tag_tid=2, entry_tid=39781' = 'object_number=39, tag_tid=2, entry_tid=39239' 2026-02-18T08:11:40.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=37, tag_tid=2, entry_tid=39781' 2026-02-18T08:11:40.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=57, tag_tid=3, entry_tid=1' 2026-02-18T08:11:40.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:11:40.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:11:40.534 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=3 2026-02-18T08:11:40.534 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=37, tag_tid=2, entry_tid=39781' 2026-02-18T08:11:40.535 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:11:40.535 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:11:40.536 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=2 2026-02-18T08:11:40.536 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=57, tag_tid=3, entry_tid=1' 2026-02-18T08:11:40.536 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:11:40.536 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:11:40.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:11:40.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=37, tag_tid=2, entry_tid=39781' 2026-02-18T08:11:40.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:11:40.539 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:11:40.540 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=39781 2026-02-18T08:11:40.540 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 3 = 2 -a 1 -le 39781 2026-02-18T08:11:40.540 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:11:40.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:11:40.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:11:40.741 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:11:40.742 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:11:40.742 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:11:40.742 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:11:40.742 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:11:40.742 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:11:40.742 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:11:40.742 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:11:40.742 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:11:40.742 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:11:40.743 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:11:40.743 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:11:40.743 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:11:40.743 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:11:40.743 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:11:40.743 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:11:40.745 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:11:40.746 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:11:40.746 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:11:40.746 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:11:40.746 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:11:40.746 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:11:40.746 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:11:40.746 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:11:40.746 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:11:40.746 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:11:40.746 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:11:40.746 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:11:40.747 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:11:40.747 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:11:40.747 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:11:40.747 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:11:40.747 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:11:40.747 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:11:40.747 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:11:40.747 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:11:40.747 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:11:40.747 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:11:40.747 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:11:40.747 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:11:40.748 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:11:40.748 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:11:40.748 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:11:40.748 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:11:40.748 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:11:40.748 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:11:40.748 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:11:40.748 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:11:40.748 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:11:40.756 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:11:40.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:11:40.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:11:40.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:11:40.815 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:11:40.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:11:40.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:11:40.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:11:40.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:11:40.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:11:40.817 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:11:40.817 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:11:40.817 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:11:40.817 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:11:40.817 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:11:40.817 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:11:40.817 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:11:40.818 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:40.818 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:11:40.818 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:40.844 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 9 2026-02-18T08:11:40.844 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 14 2026-02-18T08:11:40.844 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:11:40.844 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=57, tag_tid=3, entry_tid=1], [object_number=56, tag_tid=3, entry_tid=0], [object_number=59, tag_tid=2, entry_tid=59619], [object_number=58, tag_tid=2, entry_tid=59618]]], state=connected] 2026-02-18T08:11:40.844 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=38, tag_tid=2, entry_tid=40042], [object_number=37, tag_tid=2, entry_tid=40041], [object_number=36, tag_tid=2, entry_tid=40040], [object_number=39, tag_tid=2, entry_tid=40039]]], state=connected] 2026-02-18T08:11:40.846 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:40.847 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=57, tag_tid=3, entry_tid=1' 2026-02-18T08:11:40.848 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:11:40.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:11:40.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:11:40.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:11:40.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:11:40.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:11:40.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:11:40.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:11:40.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:11:40.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:11:40.850 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:11:40.850 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:11:40.850 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:40.851 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:11:40.851 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:40.876 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 9 2026-02-18T08:11:40.876 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 14 2026-02-18T08:11:40.877 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:11:40.877 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=57, tag_tid=3, entry_tid=1], [object_number=56, tag_tid=3, entry_tid=0], [object_number=59, tag_tid=2, entry_tid=59619], [object_number=58, tag_tid=2, entry_tid=59618]]], state=connected] 2026-02-18T08:11:40.877 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=38, tag_tid=2, entry_tid=40042], [object_number=37, tag_tid=2, entry_tid=40041], [object_number=36, tag_tid=2, entry_tid=40040], [object_number=39, tag_tid=2, entry_tid=40039]]], state=connected] 2026-02-18T08:11:40.880 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:40.882 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=38, tag_tid=2, entry_tid=40042' 2026-02-18T08:11:40.882 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=57, tag_tid=3, entry_tid=1' -a 'object_number=57, tag_tid=3, entry_tid=1' = 'object_number=38, tag_tid=2, entry_tid=40042' 2026-02-18T08:11:40.882 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=38, tag_tid=2, entry_tid=40042' '!=' 'object_number=37, tag_tid=2, entry_tid=39781' 2026-02-18T08:11:40.882 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:11:40.882 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=38, tag_tid=2, entry_tid=40042' = 'object_number=37, tag_tid=2, entry_tid=39781' 2026-02-18T08:11:40.882 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=38, tag_tid=2, entry_tid=40042' 2026-02-18T08:11:40.883 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=57, tag_tid=3, entry_tid=1' 2026-02-18T08:11:40.883 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:11:40.883 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:11:40.884 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=3 2026-02-18T08:11:40.884 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=38, tag_tid=2, entry_tid=40042' 2026-02-18T08:11:40.885 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:11:40.885 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:11:40.886 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=2 2026-02-18T08:11:40.887 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=57, tag_tid=3, entry_tid=1' 2026-02-18T08:11:40.887 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:11:40.887 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:11:40.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:11:40.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=38, tag_tid=2, entry_tid=40042' 2026-02-18T08:11:40.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:11:40.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:11:40.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=40042 2026-02-18T08:11:40.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 3 = 2 -a 1 -le 40042 2026-02-18T08:11:40.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:11:40.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:11:40.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:11:41.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:11:41.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:11:41.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:11:41.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:11:41.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:11:41.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:11:41.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:11:41.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:11:41.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:11:41.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:11:41.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:11:41.100 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:11:41.100 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:11:41.100 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:11:41.100 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:11:41.100 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:11:41.100 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:11:41.100 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:11:41.101 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:11:41.101 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:11:41.101 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:11:41.101 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:11:41.101 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:11:41.101 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:11:41.101 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:11:41.102 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:11:41.102 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:11:41.102 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:11:41.102 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:11:41.102 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:11:41.102 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:11:41.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:11:41.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:11:41.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:11:41.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:11:41.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:11:41.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:11:41.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:11:41.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:11:41.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:11:41.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:11:41.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:11:41.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:11:41.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:11:41.105 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:11:41.105 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:11:41.105 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:11:41.105 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:11:41.105 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:11:41.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:11:41.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:11:41.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:11:41.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:11:41.171 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:11:41.171 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:11:41.171 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:11:41.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:11:41.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:11:41.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:11:41.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:11:41.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:11:41.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:11:41.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:11:41.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:11:41.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:11:41.173 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:11:41.173 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:41.174 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:11:41.174 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:41.199 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 9 2026-02-18T08:11:41.199 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 14 2026-02-18T08:11:41.199 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:11:41.200 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=57, tag_tid=3, entry_tid=1], [object_number=56, tag_tid=3, entry_tid=0], [object_number=59, tag_tid=2, entry_tid=59619], [object_number=58, tag_tid=2, entry_tid=59618]]], state=connected] 2026-02-18T08:11:41.200 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=36, tag_tid=2, entry_tid=40328], [object_number=39, tag_tid=2, entry_tid=40327], [object_number=38, tag_tid=2, entry_tid=40326], [object_number=37, tag_tid=2, entry_tid=40325]]], state=connected] 2026-02-18T08:11:41.202 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:41.203 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=57, tag_tid=3, entry_tid=1' 2026-02-18T08:11:41.204 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:11:41.204 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:11:41.204 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:11:41.204 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:11:41.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:11:41.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:11:41.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:11:41.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:11:41.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:11:41.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:11:41.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:11:41.206 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:11:41.206 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:41.206 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:11:41.207 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:41.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 9 2026-02-18T08:11:41.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 14 2026-02-18T08:11:41.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:11:41.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=57, tag_tid=3, entry_tid=1], [object_number=56, tag_tid=3, entry_tid=0], [object_number=59, tag_tid=2, entry_tid=59619], [object_number=58, tag_tid=2, entry_tid=59618]]], state=connected] 2026-02-18T08:11:41.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=36, tag_tid=2, entry_tid=40328], [object_number=39, tag_tid=2, entry_tid=40327], [object_number=38, tag_tid=2, entry_tid=40326], [object_number=37, tag_tid=2, entry_tid=40325]]], state=connected] 2026-02-18T08:11:41.245 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:41.246 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=36, tag_tid=2, entry_tid=40328' 2026-02-18T08:11:41.247 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=57, tag_tid=3, entry_tid=1' -a 'object_number=57, tag_tid=3, entry_tid=1' = 'object_number=36, tag_tid=2, entry_tid=40328' 2026-02-18T08:11:41.247 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=36, tag_tid=2, entry_tid=40328' '!=' 'object_number=38, tag_tid=2, entry_tid=40042' 2026-02-18T08:11:41.247 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:11:41.247 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=36, tag_tid=2, entry_tid=40328' = 'object_number=38, tag_tid=2, entry_tid=40042' 2026-02-18T08:11:41.247 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=36, tag_tid=2, entry_tid=40328' 2026-02-18T08:11:41.247 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=57, tag_tid=3, entry_tid=1' 2026-02-18T08:11:41.247 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:11:41.247 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:11:41.248 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=3 2026-02-18T08:11:41.249 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=36, tag_tid=2, entry_tid=40328' 2026-02-18T08:11:41.249 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:11:41.249 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:11:41.251 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=2 2026-02-18T08:11:41.251 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=57, tag_tid=3, entry_tid=1' 2026-02-18T08:11:41.251 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:11:41.251 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:11:41.253 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:11:41.253 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=36, tag_tid=2, entry_tid=40328' 2026-02-18T08:11:41.254 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:11:41.254 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:11:41.255 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=40328 2026-02-18T08:11:41.255 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 3 = 2 -a 1 -le 40328 2026-02-18T08:11:41.255 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:11:41.255 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:11:41.256 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:11:41.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:11:41.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:11:41.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:11:41.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:11:41.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:11:41.458 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:11:41.458 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:11:41.458 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:11:41.458 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:11:41.458 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:11:41.458 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:11:41.458 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:11:41.458 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:11:41.458 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:11:41.458 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:11:41.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:11:41.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:11:41.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:11:41.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:11:41.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:11:41.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:11:41.462 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:11:41.462 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:11:41.462 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:11:41.462 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:11:41.462 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:11:41.462 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:11:41.462 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:11:41.462 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:11:41.462 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:11:41.462 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:11:41.462 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:11:41.463 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:11:41.463 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:11:41.463 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:11:41.463 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:11:41.463 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:11:41.463 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:11:41.463 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:11:41.463 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:11:41.463 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:11:41.463 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:11:41.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:11:41.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:11:41.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:11:41.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:11:41.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:11:41.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:11:41.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:11:41.473 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:11:41.473 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:11:41.473 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:11:41.473 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:11:41.532 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:11:41.532 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:11:41.532 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:11:41.532 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:11:41.532 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:11:41.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:11:41.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:11:41.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:11:41.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:11:41.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:11:41.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:11:41.534 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:11:41.534 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:11:41.535 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:41.535 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:11:41.535 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:41.559 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 9 2026-02-18T08:11:41.559 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 14 2026-02-18T08:11:41.560 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:11:41.560 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=57, tag_tid=3, entry_tid=1], [object_number=56, tag_tid=3, entry_tid=0], [object_number=59, tag_tid=2, entry_tid=59619], [object_number=58, tag_tid=2, entry_tid=59618]]], state=connected] 2026-02-18T08:11:41.560 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=38, tag_tid=2, entry_tid=40450], [object_number=37, tag_tid=2, entry_tid=40449], [object_number=36, tag_tid=2, entry_tid=40448], [object_number=39, tag_tid=2, entry_tid=40447]]], state=connected] 2026-02-18T08:11:41.562 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:41.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=57, tag_tid=3, entry_tid=1' 2026-02-18T08:11:41.565 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:11:41.565 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:11:41.566 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:11:41.566 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:11:41.566 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:11:41.566 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:11:41.566 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:11:41.566 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:11:41.566 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:11:41.567 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:11:41.567 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:11:41.567 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:11:41.568 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:41.568 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:11:41.568 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:41.602 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 9 2026-02-18T08:11:41.602 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 14 2026-02-18T08:11:41.602 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:11:41.602 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=57, tag_tid=3, entry_tid=1], [object_number=56, tag_tid=3, entry_tid=0], [object_number=59, tag_tid=2, entry_tid=59619], [object_number=58, tag_tid=2, entry_tid=59618]]], state=connected] 2026-02-18T08:11:41.602 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=38, tag_tid=2, entry_tid=40450], [object_number=37, tag_tid=2, entry_tid=40449], [object_number=36, tag_tid=2, entry_tid=40448], [object_number=39, tag_tid=2, entry_tid=40447]]], state=connected] 2026-02-18T08:11:41.605 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:41.606 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=38, tag_tid=2, entry_tid=40450' 2026-02-18T08:11:41.606 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=57, tag_tid=3, entry_tid=1' -a 'object_number=57, tag_tid=3, entry_tid=1' = 'object_number=38, tag_tid=2, entry_tid=40450' 2026-02-18T08:11:41.606 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=38, tag_tid=2, entry_tid=40450' '!=' 'object_number=36, tag_tid=2, entry_tid=40328' 2026-02-18T08:11:41.607 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:11:41.607 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=38, tag_tid=2, entry_tid=40450' = 'object_number=36, tag_tid=2, entry_tid=40328' 2026-02-18T08:11:41.607 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=38, tag_tid=2, entry_tid=40450' 2026-02-18T08:11:41.607 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=57, tag_tid=3, entry_tid=1' 2026-02-18T08:11:41.607 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:11:41.607 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:11:41.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=3 2026-02-18T08:11:41.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=38, tag_tid=2, entry_tid=40450' 2026-02-18T08:11:41.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:11:41.610 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:11:41.611 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=2 2026-02-18T08:11:41.612 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=57, tag_tid=3, entry_tid=1' 2026-02-18T08:11:41.612 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:11:41.612 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:11:41.613 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:11:41.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=38, tag_tid=2, entry_tid=40450' 2026-02-18T08:11:41.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:11:41.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:11:41.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=40450 2026-02-18T08:11:41.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 3 = 2 -a 1 -le 40450 2026-02-18T08:11:41.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:11:41.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:11:41.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:11:41.817 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:11:41.817 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:11:41.817 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:11:41.817 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:11:41.817 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:11:41.817 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:11:41.817 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:11:41.817 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:11:41.817 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:11:41.818 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:11:41.818 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:11:41.818 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:11:41.818 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:11:41.818 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:11:41.818 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:11:41.818 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:11:41.818 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:11:41.819 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:11:41.819 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:11:41.819 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:11:41.819 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:11:41.819 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:11:41.819 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:11:41.819 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:11:41.819 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:11:41.819 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:11:41.819 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:11:41.819 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:11:41.819 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:11:41.819 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:11:41.819 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:11:41.819 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:11:41.819 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:11:41.820 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:11:41.820 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:11:41.820 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:11:41.820 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:11:41.820 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:11:41.820 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:11:41.820 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:11:41.820 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:11:41.820 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:11:41.820 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:11:41.820 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:11:41.820 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:11:41.820 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:11:41.820 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:11:41.820 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:11:41.820 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:11:41.832 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:11:41.832 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:11:41.832 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:11:41.832 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:11:41.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:11:41.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:11:41.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:11:41.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:11:41.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:11:41.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:11:41.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:11:41.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:11:41.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:11:41.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:11:41.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:11:41.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:11:41.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:11:41.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:41.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:11:41.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:41.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 9 2026-02-18T08:11:41.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 14 2026-02-18T08:11:41.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:11:41.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=57, tag_tid=3, entry_tid=1], [object_number=56, tag_tid=3, entry_tid=0], [object_number=59, tag_tid=2, entry_tid=59619], [object_number=58, tag_tid=2, entry_tid=59618]]], state=connected] 2026-02-18T08:11:41.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=36, tag_tid=2, entry_tid=40736], [object_number=39, tag_tid=2, entry_tid=40735], [object_number=38, tag_tid=2, entry_tid=40734], [object_number=37, tag_tid=2, entry_tid=40733]]], state=connected] 2026-02-18T08:11:41.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:41.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=57, tag_tid=3, entry_tid=1' 2026-02-18T08:11:41.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:11:41.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:11:41.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:11:41.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:11:41.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:11:41.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:11:41.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:11:41.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:11:41.944 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:11:41.944 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:11:41.944 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:11:41.944 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:11:41.946 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:41.946 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:11:41.946 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:41.971 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 9 2026-02-18T08:11:41.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 14 2026-02-18T08:11:41.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:11:41.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=57, tag_tid=3, entry_tid=1], [object_number=56, tag_tid=3, entry_tid=0], [object_number=59, tag_tid=2, entry_tid=59619], [object_number=58, tag_tid=2, entry_tid=59618]]], state=connected] 2026-02-18T08:11:41.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=36, tag_tid=2, entry_tid=40736], [object_number=39, tag_tid=2, entry_tid=40735], [object_number=38, tag_tid=2, entry_tid=40734], [object_number=37, tag_tid=2, entry_tid=40733]]], state=connected] 2026-02-18T08:11:41.974 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:41.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=36, tag_tid=2, entry_tid=40736' 2026-02-18T08:11:41.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=57, tag_tid=3, entry_tid=1' -a 'object_number=57, tag_tid=3, entry_tid=1' = 'object_number=36, tag_tid=2, entry_tid=40736' 2026-02-18T08:11:41.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=36, tag_tid=2, entry_tid=40736' '!=' 'object_number=38, tag_tid=2, entry_tid=40450' 2026-02-18T08:11:41.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:11:41.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=36, tag_tid=2, entry_tid=40736' = 'object_number=38, tag_tid=2, entry_tid=40450' 2026-02-18T08:11:41.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=36, tag_tid=2, entry_tid=40736' 2026-02-18T08:11:41.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=57, tag_tid=3, entry_tid=1' 2026-02-18T08:11:41.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:11:41.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:11:41.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=3 2026-02-18T08:11:41.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=36, tag_tid=2, entry_tid=40736' 2026-02-18T08:11:41.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:11:41.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:11:41.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=2 2026-02-18T08:11:41.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=57, tag_tid=3, entry_tid=1' 2026-02-18T08:11:41.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:11:41.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:11:41.982 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:11:41.982 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=36, tag_tid=2, entry_tid=40736' 2026-02-18T08:11:41.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:11:41.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:11:41.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=40736 2026-02-18T08:11:41.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 3 = 2 -a 1 -le 40736 2026-02-18T08:11:41.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:11:41.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:11:41.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:11:42.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:11:42.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:11:42.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:11:42.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:11:42.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:11:42.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:11:42.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:11:42.187 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:11:42.187 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:11:42.187 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:11:42.187 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:11:42.187 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:11:42.187 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:11:42.187 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:11:42.187 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:11:42.190 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:11:42.190 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:11:42.190 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:11:42.190 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:11:42.190 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:11:42.190 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:11:42.190 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:11:42.190 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:11:42.190 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:11:42.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:11:42.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:11:42.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:11:42.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:11:42.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:11:42.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:11:42.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:11:42.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:11:42.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:11:42.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:11:42.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:11:42.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:11:42.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:11:42.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:11:42.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:11:42.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:11:42.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:11:42.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:11:42.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:11:42.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:11:42.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:11:42.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:11:42.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:11:42.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:11:42.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:11:42.204 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:11:42.204 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:11:42.204 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:11:42.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:11:42.269 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:11:42.270 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:11:42.270 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:11:42.270 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:11:42.270 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:11:42.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:11:42.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:11:42.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:11:42.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:11:42.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:11:42.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:11:42.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:11:42.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:11:42.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:42.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:11:42.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:42.298 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 9 2026-02-18T08:11:42.298 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 14 2026-02-18T08:11:42.298 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:11:42.298 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=57, tag_tid=3, entry_tid=1], [object_number=56, tag_tid=3, entry_tid=0], [object_number=59, tag_tid=2, entry_tid=59619], [object_number=58, tag_tid=2, entry_tid=59618]]], state=connected] 2026-02-18T08:11:42.298 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=37, tag_tid=2, entry_tid=41005], [object_number=36, tag_tid=2, entry_tid=41004], [object_number=39, tag_tid=2, entry_tid=41003], [object_number=38, tag_tid=2, entry_tid=41002]]], state=connected] 2026-02-18T08:11:42.300 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:42.301 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=57, tag_tid=3, entry_tid=1' 2026-02-18T08:11:42.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:11:42.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:11:42.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:11:42.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:11:42.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:11:42.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:11:42.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:11:42.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:11:42.304 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:11:42.304 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:11:42.304 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:11:42.304 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:11:42.304 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:42.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:11:42.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:42.329 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 9 2026-02-18T08:11:42.329 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 14 2026-02-18T08:11:42.329 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:11:42.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=57, tag_tid=3, entry_tid=1], [object_number=56, tag_tid=3, entry_tid=0], [object_number=59, tag_tid=2, entry_tid=59619], [object_number=58, tag_tid=2, entry_tid=59618]]], state=connected] 2026-02-18T08:11:42.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=37, tag_tid=2, entry_tid=41005], [object_number=36, tag_tid=2, entry_tid=41004], [object_number=39, tag_tid=2, entry_tid=41003], [object_number=38, tag_tid=2, entry_tid=41002]]], state=connected] 2026-02-18T08:11:42.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:42.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=37, tag_tid=2, entry_tid=41005' 2026-02-18T08:11:42.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=57, tag_tid=3, entry_tid=1' -a 'object_number=57, tag_tid=3, entry_tid=1' = 'object_number=37, tag_tid=2, entry_tid=41005' 2026-02-18T08:11:42.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=37, tag_tid=2, entry_tid=41005' '!=' 'object_number=36, tag_tid=2, entry_tid=40736' 2026-02-18T08:11:42.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:11:42.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=37, tag_tid=2, entry_tid=41005' = 'object_number=36, tag_tid=2, entry_tid=40736' 2026-02-18T08:11:42.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=37, tag_tid=2, entry_tid=41005' 2026-02-18T08:11:42.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=57, tag_tid=3, entry_tid=1' 2026-02-18T08:11:42.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:11:42.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:11:42.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=3 2026-02-18T08:11:42.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=37, tag_tid=2, entry_tid=41005' 2026-02-18T08:11:42.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:11:42.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:11:42.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=2 2026-02-18T08:11:42.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=57, tag_tid=3, entry_tid=1' 2026-02-18T08:11:42.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:11:42.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:11:42.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:11:42.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=37, tag_tid=2, entry_tid=41005' 2026-02-18T08:11:42.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:11:42.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:11:42.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=41005 2026-02-18T08:11:42.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 3 = 2 -a 1 -le 41005 2026-02-18T08:11:42.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:11:42.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:11:42.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:11:42.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:11:42.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:11:42.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:11:42.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:11:42.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:11:42.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:11:42.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:11:42.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:11:42.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:11:42.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:11:42.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:11:42.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:11:42.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:11:42.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:11:42.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:11:42.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:11:42.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:11:42.545 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:11:42.545 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:11:42.545 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:11:42.545 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:11:42.545 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:11:42.545 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:11:42.545 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:11:42.545 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:11:42.545 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:11:42.545 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:11:42.545 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:11:42.545 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:11:42.545 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:11:42.545 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:11:42.545 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:11:42.545 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:11:42.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:11:42.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:11:42.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:11:42.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:11:42.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:11:42.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:11:42.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:11:42.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:11:42.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:11:42.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:11:42.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:11:42.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:11:42.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:11:42.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:11:42.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:11:42.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:11:42.557 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:11:42.557 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:11:42.557 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:11:42.557 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:11:42.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:11:42.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:11:42.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:11:42.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:11:42.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:11:42.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:11:42.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:11:42.618 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:11:42.618 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:11:42.618 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:11:42.618 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:11:42.618 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:11:42.618 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:11:42.619 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:42.619 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:11:42.619 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:42.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 9 2026-02-18T08:11:42.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 14 2026-02-18T08:11:42.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:11:42.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=57, tag_tid=3, entry_tid=1], [object_number=56, tag_tid=3, entry_tid=0], [object_number=59, tag_tid=2, entry_tid=59619], [object_number=58, tag_tid=2, entry_tid=59618]]], state=connected] 2026-02-18T08:11:42.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=38, tag_tid=2, entry_tid=41242], [object_number=37, tag_tid=2, entry_tid=41241], [object_number=36, tag_tid=2, entry_tid=41240], [object_number=39, tag_tid=2, entry_tid=41239]]], state=connected] 2026-02-18T08:11:42.648 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:42.649 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=57, tag_tid=3, entry_tid=1' 2026-02-18T08:11:42.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:11:42.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:11:42.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:11:42.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:11:42.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:11:42.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:11:42.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:11:42.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:11:42.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:11:42.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:11:42.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:11:42.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:11:42.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:42.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:11:42.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:42.887 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 9 2026-02-18T08:11:42.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 14 2026-02-18T08:11:42.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:11:42.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=57, tag_tid=3, entry_tid=1], [object_number=56, tag_tid=3, entry_tid=0], [object_number=59, tag_tid=2, entry_tid=59619], [object_number=58, tag_tid=2, entry_tid=59618]]], state=connected] 2026-02-18T08:11:42.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=38, tag_tid=2, entry_tid=41242], [object_number=37, tag_tid=2, entry_tid=41241], [object_number=36, tag_tid=2, entry_tid=41240], [object_number=39, tag_tid=2, entry_tid=41239]]], state=connected] 2026-02-18T08:11:42.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:42.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=38, tag_tid=2, entry_tid=41242' 2026-02-18T08:11:42.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=57, tag_tid=3, entry_tid=1' -a 'object_number=57, tag_tid=3, entry_tid=1' = 'object_number=38, tag_tid=2, entry_tid=41242' 2026-02-18T08:11:42.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=38, tag_tid=2, entry_tid=41242' '!=' 'object_number=37, tag_tid=2, entry_tid=41005' 2026-02-18T08:11:42.894 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:11:42.894 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=38, tag_tid=2, entry_tid=41242' = 'object_number=37, tag_tid=2, entry_tid=41005' 2026-02-18T08:11:42.894 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=38, tag_tid=2, entry_tid=41242' 2026-02-18T08:11:42.894 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=57, tag_tid=3, entry_tid=1' 2026-02-18T08:11:42.894 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:11:42.895 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:11:42.895 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=3 2026-02-18T08:11:42.896 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=38, tag_tid=2, entry_tid=41242' 2026-02-18T08:11:42.896 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:11:42.896 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:11:42.897 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=2 2026-02-18T08:11:42.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=57, tag_tid=3, entry_tid=1' 2026-02-18T08:11:42.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:11:42.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:11:42.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:11:42.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=38, tag_tid=2, entry_tid=41242' 2026-02-18T08:11:42.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:11:42.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:11:42.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=41242 2026-02-18T08:11:42.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 3 = 2 -a 1 -le 41242 2026-02-18T08:11:42.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:11:42.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:11:42.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:11:43.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:11:43.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:11:43.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:11:43.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:11:43.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:11:43.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:11:43.105 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:11:43.105 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:11:43.105 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:11:43.105 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:11:43.105 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:11:43.105 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:11:43.105 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:11:43.105 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:11:43.105 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:11:43.105 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:11:43.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:11:43.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:11:43.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:11:43.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:11:43.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:11:43.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:11:43.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:11:43.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:11:43.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:11:43.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:11:43.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:11:43.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:11:43.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:11:43.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:11:43.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:11:43.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:11:43.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:11:43.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:11:43.110 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:11:43.110 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:11:43.110 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:11:43.110 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:11:43.110 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:11:43.110 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:11:43.110 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:11:43.110 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:11:43.110 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:11:43.110 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:11:43.110 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:11:43.110 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:11:43.111 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:11:43.111 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:11:43.111 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:11:43.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:11:43.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:11:43.118 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:11:43.118 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:11:43.180 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:11:43.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:11:43.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:11:43.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:11:43.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:11:43.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:11:43.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:11:43.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:11:43.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:11:43.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:11:43.183 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:11:43.183 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:11:43.183 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:11:43.183 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:43.183 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:11:43.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:43.211 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 9 2026-02-18T08:11:43.211 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 14 2026-02-18T08:11:43.211 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:11:43.211 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=57, tag_tid=3, entry_tid=1], [object_number=56, tag_tid=3, entry_tid=0], [object_number=59, tag_tid=2, entry_tid=59619], [object_number=58, tag_tid=2, entry_tid=59618]]], state=connected] 2026-02-18T08:11:43.211 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=43, tag_tid=2, entry_tid=41623], [object_number=38, tag_tid=2, entry_tid=41622], [object_number=37, tag_tid=2, entry_tid=41621], [object_number=36, tag_tid=2, entry_tid=41620]]], state=connected] 2026-02-18T08:11:43.214 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:43.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=57, tag_tid=3, entry_tid=1' 2026-02-18T08:11:43.217 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:11:43.217 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:11:43.217 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:11:43.217 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:11:43.217 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:11:43.217 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:11:43.217 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:11:43.217 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:11:43.218 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:11:43.218 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:11:43.218 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:11:43.218 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:11:43.219 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:43.219 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:11:43.219 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:43.246 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 9 2026-02-18T08:11:43.246 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 14 2026-02-18T08:11:43.246 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:11:43.246 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=57, tag_tid=3, entry_tid=1], [object_number=56, tag_tid=3, entry_tid=0], [object_number=59, tag_tid=2, entry_tid=59619], [object_number=58, tag_tid=2, entry_tid=59618]]], state=connected] 2026-02-18T08:11:43.246 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=43, tag_tid=2, entry_tid=41623], [object_number=38, tag_tid=2, entry_tid=41622], [object_number=37, tag_tid=2, entry_tid=41621], [object_number=36, tag_tid=2, entry_tid=41620]]], state=connected] 2026-02-18T08:11:43.248 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:43.250 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=43, tag_tid=2, entry_tid=41623' 2026-02-18T08:11:43.250 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=57, tag_tid=3, entry_tid=1' -a 'object_number=57, tag_tid=3, entry_tid=1' = 'object_number=43, tag_tid=2, entry_tid=41623' 2026-02-18T08:11:43.250 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=43, tag_tid=2, entry_tid=41623' '!=' 'object_number=38, tag_tid=2, entry_tid=41242' 2026-02-18T08:11:43.250 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:11:43.250 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=43, tag_tid=2, entry_tid=41623' = 'object_number=38, tag_tid=2, entry_tid=41242' 2026-02-18T08:11:43.250 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=43, tag_tid=2, entry_tid=41623' 2026-02-18T08:11:43.251 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=57, tag_tid=3, entry_tid=1' 2026-02-18T08:11:43.251 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:11:43.251 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:11:43.252 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=3 2026-02-18T08:11:43.252 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=43, tag_tid=2, entry_tid=41623' 2026-02-18T08:11:43.253 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:11:43.253 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:11:43.254 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=2 2026-02-18T08:11:43.255 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=57, tag_tid=3, entry_tid=1' 2026-02-18T08:11:43.255 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:11:43.255 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:11:43.256 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:11:43.257 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=43, tag_tid=2, entry_tid=41623' 2026-02-18T08:11:43.257 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:11:43.257 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:11:43.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=41623 2026-02-18T08:11:43.259 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 3 = 2 -a 1 -le 41623 2026-02-18T08:11:43.259 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:11:43.259 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:11:43.259 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:11:43.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:11:43.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:11:43.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:11:43.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:11:43.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:11:43.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:11:43.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:11:43.462 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:11:43.462 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:11:43.462 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:11:43.462 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:11:43.462 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:11:43.462 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:11:43.462 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:11:43.462 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:11:43.462 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:11:43.462 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:11:43.465 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:11:43.465 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:11:43.465 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:11:43.465 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:11:43.465 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:11:43.465 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:11:43.465 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:11:43.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:11:43.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:11:43.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:11:43.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:11:43.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:11:43.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:11:43.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:11:43.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:11:43.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:11:43.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:11:43.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:11:43.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:11:43.467 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:11:43.467 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:11:43.467 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:11:43.467 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:11:43.467 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:11:43.467 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:11:43.467 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:11:43.467 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:11:43.467 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:11:43.467 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:11:43.467 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:11:43.467 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:11:43.468 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:11:43.476 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:11:43.476 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:11:43.477 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:11:43.477 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:11:43.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:11:43.539 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:11:43.539 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:11:43.539 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:11:43.539 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:11:43.540 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:11:43.540 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:11:43.540 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:11:43.540 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:11:43.540 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:11:43.540 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:11:43.540 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:11:43.540 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:11:43.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:43.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:11:43.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:43.567 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 10 2026-02-18T08:11:43.567 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 14 2026-02-18T08:11:43.567 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:11:43.567 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=57, tag_tid=3, entry_tid=1], [object_number=56, tag_tid=3, entry_tid=0], [object_number=59, tag_tid=2, entry_tid=59619], [object_number=58, tag_tid=2, entry_tid=59618]]], state=connected] 2026-02-18T08:11:43.567 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=42, tag_tid=2, entry_tid=41874], [object_number=41, tag_tid=2, entry_tid=41873], [object_number=40, tag_tid=2, entry_tid=41872], [object_number=43, tag_tid=2, entry_tid=41871]]], state=connected] 2026-02-18T08:11:43.570 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:43.571 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=57, tag_tid=3, entry_tid=1' 2026-02-18T08:11:43.572 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:11:43.572 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:11:43.572 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:11:43.572 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:11:43.572 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:11:43.573 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:11:43.573 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:11:43.573 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:11:43.573 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:11:43.573 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:11:43.574 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:11:43.574 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:11:43.574 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:43.574 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:11:43.575 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:43.601 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 10 2026-02-18T08:11:43.601 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 14 2026-02-18T08:11:43.601 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:11:43.601 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=57, tag_tid=3, entry_tid=1], [object_number=56, tag_tid=3, entry_tid=0], [object_number=59, tag_tid=2, entry_tid=59619], [object_number=58, tag_tid=2, entry_tid=59618]]], state=connected] 2026-02-18T08:11:43.601 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=42, tag_tid=2, entry_tid=41874], [object_number=41, tag_tid=2, entry_tid=41873], [object_number=40, tag_tid=2, entry_tid=41872], [object_number=43, tag_tid=2, entry_tid=41871]]], state=connected] 2026-02-18T08:11:43.604 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:43.605 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=42, tag_tid=2, entry_tid=41874' 2026-02-18T08:11:43.605 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=57, tag_tid=3, entry_tid=1' -a 'object_number=57, tag_tid=3, entry_tid=1' = 'object_number=42, tag_tid=2, entry_tid=41874' 2026-02-18T08:11:43.606 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=42, tag_tid=2, entry_tid=41874' '!=' 'object_number=43, tag_tid=2, entry_tid=41623' 2026-02-18T08:11:43.606 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:11:43.606 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=42, tag_tid=2, entry_tid=41874' = 'object_number=43, tag_tid=2, entry_tid=41623' 2026-02-18T08:11:43.606 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=42, tag_tid=2, entry_tid=41874' 2026-02-18T08:11:43.606 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=57, tag_tid=3, entry_tid=1' 2026-02-18T08:11:43.607 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:11:43.607 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:11:43.607 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=3 2026-02-18T08:11:43.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=42, tag_tid=2, entry_tid=41874' 2026-02-18T08:11:43.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:11:43.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:11:43.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=2 2026-02-18T08:11:43.610 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=57, tag_tid=3, entry_tid=1' 2026-02-18T08:11:43.610 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:11:43.611 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:11:43.612 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:11:43.613 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=42, tag_tid=2, entry_tid=41874' 2026-02-18T08:11:43.613 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:11:43.613 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:11:43.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=41874 2026-02-18T08:11:43.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 3 = 2 -a 1 -le 41874 2026-02-18T08:11:43.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:11:43.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:11:43.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:11:43.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:11:43.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:11:43.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:11:43.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:11:43.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:11:43.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:11:43.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:11:43.817 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:11:43.817 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:11:43.817 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:11:43.817 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:11:43.817 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:11:43.817 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:11:43.817 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:11:43.817 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:11:43.820 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:11:43.820 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:11:43.820 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:11:43.820 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:11:43.820 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:11:43.820 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:11:43.820 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:11:43.820 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:11:43.821 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:11:43.821 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:11:43.821 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:11:43.821 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:11:43.821 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:11:43.821 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:11:43.821 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:11:43.821 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:11:43.821 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:11:43.821 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:11:43.821 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:11:43.821 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:11:43.821 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:11:43.822 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:11:43.822 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:11:43.822 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:11:43.822 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:11:43.822 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:11:43.822 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:11:43.822 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:11:43.822 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:11:43.822 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:11:43.822 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:11:43.822 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:11:43.822 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:11:43.823 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:11:43.829 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:11:43.830 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:11:43.830 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:11:43.830 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:11:43.887 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:11:43.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:11:43.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:11:43.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:11:43.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:11:43.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:11:43.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:11:43.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:11:43.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:11:43.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:11:43.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:11:43.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:11:43.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:11:43.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:43.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:11:43.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:43.918 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 10 2026-02-18T08:11:43.918 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 14 2026-02-18T08:11:43.918 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:11:43.918 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=57, tag_tid=3, entry_tid=1], [object_number=56, tag_tid=3, entry_tid=0], [object_number=59, tag_tid=2, entry_tid=59619], [object_number=58, tag_tid=2, entry_tid=59618]]], state=connected] 2026-02-18T08:11:43.918 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=43, tag_tid=2, entry_tid=42187], [object_number=42, tag_tid=2, entry_tid=42186], [object_number=41, tag_tid=2, entry_tid=42185], [object_number=40, tag_tid=2, entry_tid=42184]]], state=connected] 2026-02-18T08:11:43.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:43.922 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=57, tag_tid=3, entry_tid=1' 2026-02-18T08:11:43.923 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:11:43.924 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:11:43.924 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:11:43.924 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:11:43.924 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:11:43.924 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:11:43.924 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:11:43.924 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:11:43.924 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:11:43.924 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:11:43.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:11:43.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:11:43.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:43.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:11:43.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:43.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 10 2026-02-18T08:11:43.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 14 2026-02-18T08:11:43.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:11:43.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=57, tag_tid=3, entry_tid=1], [object_number=56, tag_tid=3, entry_tid=0], [object_number=59, tag_tid=2, entry_tid=59619], [object_number=58, tag_tid=2, entry_tid=59618]]], state=connected] 2026-02-18T08:11:43.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=43, tag_tid=2, entry_tid=42187], [object_number=42, tag_tid=2, entry_tid=42186], [object_number=41, tag_tid=2, entry_tid=42185], [object_number=40, tag_tid=2, entry_tid=42184]]], state=connected] 2026-02-18T08:11:43.953 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:43.955 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=43, tag_tid=2, entry_tid=42187' 2026-02-18T08:11:43.955 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=57, tag_tid=3, entry_tid=1' -a 'object_number=57, tag_tid=3, entry_tid=1' = 'object_number=43, tag_tid=2, entry_tid=42187' 2026-02-18T08:11:43.955 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=43, tag_tid=2, entry_tid=42187' '!=' 'object_number=42, tag_tid=2, entry_tid=41874' 2026-02-18T08:11:43.955 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:11:43.955 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=43, tag_tid=2, entry_tid=42187' = 'object_number=42, tag_tid=2, entry_tid=41874' 2026-02-18T08:11:43.956 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=43, tag_tid=2, entry_tid=42187' 2026-02-18T08:11:43.956 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=57, tag_tid=3, entry_tid=1' 2026-02-18T08:11:43.956 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:11:43.956 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:11:43.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=3 2026-02-18T08:11:43.958 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=43, tag_tid=2, entry_tid=42187' 2026-02-18T08:11:43.958 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:11:43.958 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:11:43.959 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=2 2026-02-18T08:11:43.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=57, tag_tid=3, entry_tid=1' 2026-02-18T08:11:43.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:11:43.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:11:43.962 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:11:43.963 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=43, tag_tid=2, entry_tid=42187' 2026-02-18T08:11:43.963 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:11:43.963 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:11:43.964 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=42187 2026-02-18T08:11:43.964 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 3 = 2 -a 1 -le 42187 2026-02-18T08:11:43.965 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:11:43.965 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:11:43.965 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:11:44.165 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:11:44.165 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:11:44.165 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:11:44.165 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:11:44.165 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:11:44.165 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:11:44.165 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:11:44.165 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:11:44.165 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:11:44.165 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:11:44.165 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:11:44.165 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:11:44.166 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:11:44.166 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:11:44.166 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:11:44.166 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:11:44.166 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:11:44.167 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:11:44.167 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:11:44.167 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:11:44.167 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:11:44.167 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:11:44.167 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:11:44.167 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:11:44.167 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:11:44.167 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:11:44.167 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:11:44.167 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:11:44.167 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:11:44.167 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:11:44.167 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:11:44.167 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:11:44.167 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:11:44.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:11:44.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:11:44.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:11:44.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:11:44.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:11:44.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:11:44.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:11:44.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:11:44.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:11:44.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:11:44.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:11:44.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:11:44.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:11:44.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:11:44.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:11:44.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:11:44.180 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:11:44.180 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:11:44.180 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:11:44.180 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:11:44.238 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:11:44.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:11:44.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:11:44.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:11:44.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:11:44.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:11:44.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:11:44.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:11:44.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:11:44.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:11:44.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:11:44.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:11:44.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:11:44.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:44.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:11:44.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:44.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 10 2026-02-18T08:11:44.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 14 2026-02-18T08:11:44.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:11:44.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=57, tag_tid=3, entry_tid=1], [object_number=56, tag_tid=3, entry_tid=0], [object_number=59, tag_tid=2, entry_tid=59619], [object_number=58, tag_tid=2, entry_tid=59618]]], state=connected] 2026-02-18T08:11:44.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=42, tag_tid=2, entry_tid=42478], [object_number=41, tag_tid=2, entry_tid=42477], [object_number=40, tag_tid=2, entry_tid=42476], [object_number=43, tag_tid=2, entry_tid=42475]]], state=connected] 2026-02-18T08:11:44.270 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:44.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=57, tag_tid=3, entry_tid=1' 2026-02-18T08:11:44.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:11:44.273 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:11:44.273 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:11:44.273 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:11:44.273 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:11:44.273 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:11:44.273 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:11:44.273 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:11:44.273 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:11:44.273 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:11:44.273 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:11:44.273 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:11:44.274 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:44.274 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:11:44.275 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:44.300 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 10 2026-02-18T08:11:44.300 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 14 2026-02-18T08:11:44.300 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:11:44.300 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=57, tag_tid=3, entry_tid=1], [object_number=56, tag_tid=3, entry_tid=0], [object_number=59, tag_tid=2, entry_tid=59619], [object_number=58, tag_tid=2, entry_tid=59618]]], state=connected] 2026-02-18T08:11:44.301 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=42, tag_tid=2, entry_tid=42478], [object_number=41, tag_tid=2, entry_tid=42477], [object_number=40, tag_tid=2, entry_tid=42476], [object_number=43, tag_tid=2, entry_tid=42475]]], state=connected] 2026-02-18T08:11:44.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:44.304 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=42, tag_tid=2, entry_tid=42478' 2026-02-18T08:11:44.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=57, tag_tid=3, entry_tid=1' -a 'object_number=57, tag_tid=3, entry_tid=1' = 'object_number=42, tag_tid=2, entry_tid=42478' 2026-02-18T08:11:44.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=42, tag_tid=2, entry_tid=42478' '!=' 'object_number=43, tag_tid=2, entry_tid=42187' 2026-02-18T08:11:44.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:11:44.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=42, tag_tid=2, entry_tid=42478' = 'object_number=43, tag_tid=2, entry_tid=42187' 2026-02-18T08:11:44.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=42, tag_tid=2, entry_tid=42478' 2026-02-18T08:11:44.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=57, tag_tid=3, entry_tid=1' 2026-02-18T08:11:44.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:11:44.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:11:44.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=3 2026-02-18T08:11:44.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=42, tag_tid=2, entry_tid=42478' 2026-02-18T08:11:44.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:11:44.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:11:44.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=2 2026-02-18T08:11:44.310 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=57, tag_tid=3, entry_tid=1' 2026-02-18T08:11:44.310 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:11:44.310 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:11:44.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:11:44.312 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=42, tag_tid=2, entry_tid=42478' 2026-02-18T08:11:44.312 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:11:44.312 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:11:44.314 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=42478 2026-02-18T08:11:44.314 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 3 = 2 -a 1 -le 42478 2026-02-18T08:11:44.314 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:11:44.314 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:11:44.314 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:11:44.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:11:44.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:11:44.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:11:44.516 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:11:44.516 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:11:44.516 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:11:44.516 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:11:44.516 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:11:44.516 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:11:44.516 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:11:44.516 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:11:44.516 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:11:44.516 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:11:44.516 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:11:44.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:11:44.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:11:44.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:11:44.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:11:44.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:11:44.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:11:44.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:11:44.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:11:44.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:11:44.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:11:44.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:11:44.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:11:44.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:11:44.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:11:44.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:11:44.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:11:44.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:11:44.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:11:44.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:11:44.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:11:44.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:11:44.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:11:44.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:11:44.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:11:44.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:11:44.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:11:44.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:11:44.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:11:44.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:11:44.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:11:44.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:11:44.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:11:44.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:11:44.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:11:44.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:11:44.529 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:11:44.529 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:11:44.529 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:11:44.530 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:11:44.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:11:44.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:11:44.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:11:44.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:11:44.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:11:44.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:11:44.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:11:44.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:11:44.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:11:44.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:11:44.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:11:44.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:11:44.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:11:44.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:44.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:11:44.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:44.618 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 10 2026-02-18T08:11:44.618 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 14 2026-02-18T08:11:44.619 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:11:44.619 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=57, tag_tid=3, entry_tid=1], [object_number=56, tag_tid=3, entry_tid=0], [object_number=59, tag_tid=2, entry_tid=59619], [object_number=58, tag_tid=2, entry_tid=59618]]], state=connected] 2026-02-18T08:11:44.619 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=41, tag_tid=2, entry_tid=42761], [object_number=40, tag_tid=2, entry_tid=42760], [object_number=43, tag_tid=2, entry_tid=42759], [object_number=42, tag_tid=2, entry_tid=42758]]], state=connected] 2026-02-18T08:11:44.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:44.623 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=57, tag_tid=3, entry_tid=1' 2026-02-18T08:11:44.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:11:44.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:11:44.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:11:44.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:11:44.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:11:44.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:11:44.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:11:44.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:11:44.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:11:44.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:11:44.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:11:44.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:11:44.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:44.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:11:44.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:44.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 10 2026-02-18T08:11:44.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 14 2026-02-18T08:11:44.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:11:44.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=57, tag_tid=3, entry_tid=1], [object_number=56, tag_tid=3, entry_tid=0], [object_number=59, tag_tid=2, entry_tid=59619], [object_number=58, tag_tid=2, entry_tid=59618]]], state=connected] 2026-02-18T08:11:44.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=41, tag_tid=2, entry_tid=42761], [object_number=40, tag_tid=2, entry_tid=42760], [object_number=43, tag_tid=2, entry_tid=42759], [object_number=42, tag_tid=2, entry_tid=42758]]], state=connected] 2026-02-18T08:11:44.654 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:44.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=41, tag_tid=2, entry_tid=42761' 2026-02-18T08:11:44.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=57, tag_tid=3, entry_tid=1' -a 'object_number=57, tag_tid=3, entry_tid=1' = 'object_number=41, tag_tid=2, entry_tid=42761' 2026-02-18T08:11:44.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=41, tag_tid=2, entry_tid=42761' '!=' 'object_number=42, tag_tid=2, entry_tid=42478' 2026-02-18T08:11:44.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:11:44.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=41, tag_tid=2, entry_tid=42761' = 'object_number=42, tag_tid=2, entry_tid=42478' 2026-02-18T08:11:44.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=41, tag_tid=2, entry_tid=42761' 2026-02-18T08:11:44.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=57, tag_tid=3, entry_tid=1' 2026-02-18T08:11:44.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:11:44.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:11:44.658 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=3 2026-02-18T08:11:44.659 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=41, tag_tid=2, entry_tid=42761' 2026-02-18T08:11:44.659 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:11:44.659 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:11:44.660 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=2 2026-02-18T08:11:44.660 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=57, tag_tid=3, entry_tid=1' 2026-02-18T08:11:44.661 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:11:44.661 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:11:44.662 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:11:44.663 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=41, tag_tid=2, entry_tid=42761' 2026-02-18T08:11:44.663 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:11:44.663 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:11:44.664 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=42761 2026-02-18T08:11:44.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 3 = 2 -a 1 -le 42761 2026-02-18T08:11:44.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:11:44.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:11:44.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:11:44.866 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:11:44.867 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:11:44.867 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:11:44.867 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:11:44.867 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:11:44.867 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:11:44.867 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:11:44.867 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:11:44.867 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:11:44.867 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:11:44.867 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:11:44.868 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:11:44.868 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:11:44.868 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:11:44.868 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:11:44.868 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:11:44.868 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:11:44.870 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:11:44.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:11:44.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:11:44.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:11:44.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:11:44.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:11:44.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:11:44.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:11:44.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:11:44.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:11:44.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:11:44.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:11:44.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:11:44.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:11:44.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:11:44.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:11:44.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:11:44.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:11:44.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:11:44.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:11:44.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:11:44.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:11:44.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:11:44.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:11:44.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:11:44.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:11:44.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:11:44.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:11:44.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:11:44.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:11:44.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:11:44.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:11:44.881 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:11:44.881 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:11:44.881 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:11:44.882 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:11:44.948 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:11:44.949 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:11:44.949 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:11:44.949 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:11:44.949 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:11:44.949 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:11:44.949 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:11:44.949 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:11:44.949 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:11:44.949 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:11:44.950 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:11:44.950 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:11:44.950 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:11:44.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:44.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:11:44.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:45.188 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 10 2026-02-18T08:11:45.189 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 14 2026-02-18T08:11:45.189 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:11:45.189 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=57, tag_tid=3, entry_tid=1], [object_number=56, tag_tid=3, entry_tid=0], [object_number=59, tag_tid=2, entry_tid=59619], [object_number=58, tag_tid=2, entry_tid=59618]]], state=connected] 2026-02-18T08:11:45.189 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=42, tag_tid=2, entry_tid=43042], [object_number=41, tag_tid=2, entry_tid=43041], [object_number=40, tag_tid=2, entry_tid=43040], [object_number=43, tag_tid=2, entry_tid=43039]]], state=connected] 2026-02-18T08:11:45.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:45.194 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=57, tag_tid=3, entry_tid=1' 2026-02-18T08:11:45.195 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:11:45.195 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:11:45.195 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:11:45.195 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:11:45.195 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:11:45.195 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:11:45.195 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:11:45.196 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:11:45.196 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:11:45.196 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:11:45.196 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:11:45.196 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:11:45.196 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:45.197 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:11:45.197 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:45.224 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 10 2026-02-18T08:11:45.224 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 14 2026-02-18T08:11:45.224 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:11:45.224 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=57, tag_tid=3, entry_tid=1], [object_number=56, tag_tid=3, entry_tid=0], [object_number=59, tag_tid=2, entry_tid=59619], [object_number=58, tag_tid=2, entry_tid=59618]]], state=connected] 2026-02-18T08:11:45.224 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=42, tag_tid=2, entry_tid=43042], [object_number=41, tag_tid=2, entry_tid=43041], [object_number=40, tag_tid=2, entry_tid=43040], [object_number=43, tag_tid=2, entry_tid=43039]]], state=connected] 2026-02-18T08:11:45.227 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:45.229 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=42, tag_tid=2, entry_tid=43042' 2026-02-18T08:11:45.229 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=57, tag_tid=3, entry_tid=1' -a 'object_number=57, tag_tid=3, entry_tid=1' = 'object_number=42, tag_tid=2, entry_tid=43042' 2026-02-18T08:11:45.229 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=42, tag_tid=2, entry_tid=43042' '!=' 'object_number=41, tag_tid=2, entry_tid=42761' 2026-02-18T08:11:45.229 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:11:45.229 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=42, tag_tid=2, entry_tid=43042' = 'object_number=41, tag_tid=2, entry_tid=42761' 2026-02-18T08:11:45.229 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=42, tag_tid=2, entry_tid=43042' 2026-02-18T08:11:45.230 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=57, tag_tid=3, entry_tid=1' 2026-02-18T08:11:45.230 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:11:45.230 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:11:45.231 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=3 2026-02-18T08:11:45.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=42, tag_tid=2, entry_tid=43042' 2026-02-18T08:11:45.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:11:45.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:11:45.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=2 2026-02-18T08:11:45.234 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=57, tag_tid=3, entry_tid=1' 2026-02-18T08:11:45.234 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:11:45.234 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:11:45.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:11:45.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=42, tag_tid=2, entry_tid=43042' 2026-02-18T08:11:45.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:11:45.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:11:45.238 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=43042 2026-02-18T08:11:45.238 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 3 = 2 -a 1 -le 43042 2026-02-18T08:11:45.238 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:11:45.238 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:11:45.238 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:11:45.440 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:11:45.440 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:11:45.440 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:11:45.440 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:11:45.440 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:11:45.440 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:11:45.440 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:11:45.440 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:11:45.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:11:45.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:11:45.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:11:45.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:11:45.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:11:45.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:11:45.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:11:45.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:11:45.443 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:11:45.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:11:45.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:11:45.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:11:45.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:11:45.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:11:45.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:11:45.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:11:45.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:11:45.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:11:45.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:11:45.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:11:45.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:11:45.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:11:45.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:11:45.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:11:45.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:11:45.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:11:45.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:11:45.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:11:45.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:11:45.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:11:45.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:11:45.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:11:45.446 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:11:45.446 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:11:45.446 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:11:45.446 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:11:45.446 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:11:45.446 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:11:45.446 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:11:45.446 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:11:45.446 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:11:45.454 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:11:45.454 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:11:45.454 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:11:45.454 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:11:45.513 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:11:45.514 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:11:45.514 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:11:45.514 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:11:45.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:11:45.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:11:45.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:11:45.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:11:45.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:11:45.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:11:45.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:11:45.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:11:45.516 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:11:45.516 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:45.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:11:45.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:45.551 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 10 2026-02-18T08:11:45.551 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 14 2026-02-18T08:11:45.551 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:11:45.551 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=57, tag_tid=3, entry_tid=1], [object_number=56, tag_tid=3, entry_tid=0], [object_number=59, tag_tid=2, entry_tid=59619], [object_number=58, tag_tid=2, entry_tid=59618]]], state=connected] 2026-02-18T08:11:45.551 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=41, tag_tid=2, entry_tid=43521], [object_number=40, tag_tid=2, entry_tid=43520], [object_number=43, tag_tid=2, entry_tid=43519], [object_number=42, tag_tid=2, entry_tid=43518]]], state=connected] 2026-02-18T08:11:45.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:45.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=57, tag_tid=3, entry_tid=1' 2026-02-18T08:11:45.557 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:11:45.558 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:11:45.558 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:11:45.558 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:11:45.558 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:11:45.558 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:11:45.558 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:11:45.558 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:11:45.558 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:11:45.558 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:11:45.559 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:11:45.559 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:11:45.559 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:45.559 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:11:45.559 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:45.587 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 10 2026-02-18T08:11:45.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 14 2026-02-18T08:11:45.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:11:45.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=57, tag_tid=3, entry_tid=1], [object_number=56, tag_tid=3, entry_tid=0], [object_number=59, tag_tid=2, entry_tid=59619], [object_number=58, tag_tid=2, entry_tid=59618]]], state=connected] 2026-02-18T08:11:45.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=41, tag_tid=2, entry_tid=43521], [object_number=40, tag_tid=2, entry_tid=43520], [object_number=43, tag_tid=2, entry_tid=43519], [object_number=42, tag_tid=2, entry_tid=43518]]], state=connected] 2026-02-18T08:11:45.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:45.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=41, tag_tid=2, entry_tid=43521' 2026-02-18T08:11:45.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=57, tag_tid=3, entry_tid=1' -a 'object_number=57, tag_tid=3, entry_tid=1' = 'object_number=41, tag_tid=2, entry_tid=43521' 2026-02-18T08:11:45.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=41, tag_tid=2, entry_tid=43521' '!=' 'object_number=42, tag_tid=2, entry_tid=43042' 2026-02-18T08:11:45.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:11:45.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=41, tag_tid=2, entry_tid=43521' = 'object_number=42, tag_tid=2, entry_tid=43042' 2026-02-18T08:11:45.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=41, tag_tid=2, entry_tid=43521' 2026-02-18T08:11:45.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=57, tag_tid=3, entry_tid=1' 2026-02-18T08:11:45.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:11:45.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:11:45.594 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=3 2026-02-18T08:11:45.595 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=41, tag_tid=2, entry_tid=43521' 2026-02-18T08:11:45.595 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:11:45.595 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:11:45.596 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=2 2026-02-18T08:11:45.597 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=57, tag_tid=3, entry_tid=1' 2026-02-18T08:11:45.597 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:11:45.597 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:11:45.599 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:11:45.599 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=41, tag_tid=2, entry_tid=43521' 2026-02-18T08:11:45.599 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:11:45.599 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:11:45.601 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=43521 2026-02-18T08:11:45.601 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 3 = 2 -a 1 -le 43521 2026-02-18T08:11:45.601 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:11:45.601 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:11:45.601 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:11:45.803 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:11:45.803 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:11:45.803 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:11:45.804 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:11:45.804 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:11:45.804 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:11:45.804 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:11:45.804 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:11:45.804 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:11:45.804 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:11:45.804 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:11:45.804 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:11:45.804 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:11:45.804 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:11:45.804 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:11:45.805 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:11:45.805 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:11:45.807 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:11:45.807 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:11:45.807 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:11:45.807 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:11:45.808 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:11:45.808 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:11:45.808 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:11:45.808 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:11:45.808 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:11:45.808 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:11:45.808 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:11:45.808 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:11:45.808 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:11:45.808 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:11:45.808 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:11:45.808 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:11:45.809 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:11:45.809 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:11:45.809 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:11:45.809 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:11:45.809 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:11:45.809 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:11:45.809 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:11:45.809 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:11:45.809 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:11:45.809 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:11:45.809 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:11:45.809 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:11:45.810 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:11:45.810 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:11:45.810 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:11:45.810 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:11:45.817 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:11:45.818 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:11:45.818 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:11:45.818 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:11:45.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:11:45.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:11:45.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:11:45.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:11:45.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:11:45.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:11:45.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:11:45.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:11:45.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:11:45.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:11:45.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:11:45.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:11:45.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:11:45.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:45.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:11:45.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:45.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 10 2026-02-18T08:11:45.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 14 2026-02-18T08:11:45.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:11:45.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=57, tag_tid=3, entry_tid=1], [object_number=56, tag_tid=3, entry_tid=0], [object_number=59, tag_tid=2, entry_tid=59619], [object_number=58, tag_tid=2, entry_tid=59618]]], state=connected] 2026-02-18T08:11:45.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=43, tag_tid=2, entry_tid=43799], [object_number=42, tag_tid=2, entry_tid=43798], [object_number=41, tag_tid=2, entry_tid=43797], [object_number=40, tag_tid=2, entry_tid=43796]]], state=connected] 2026-02-18T08:11:45.929 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:45.931 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=57, tag_tid=3, entry_tid=1' 2026-02-18T08:11:45.932 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:11:45.932 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:11:45.932 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:11:45.932 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:11:45.932 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:11:45.932 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:11:45.933 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:11:45.933 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:11:45.933 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:11:45.933 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:11:45.933 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:11:45.933 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:11:45.933 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:45.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:11:45.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:45.970 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 10 2026-02-18T08:11:45.970 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 14 2026-02-18T08:11:45.970 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:11:45.970 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=57, tag_tid=3, entry_tid=1], [object_number=56, tag_tid=3, entry_tid=0], [object_number=59, tag_tid=2, entry_tid=59619], [object_number=58, tag_tid=2, entry_tid=59618]]], state=connected] 2026-02-18T08:11:45.970 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=43, tag_tid=2, entry_tid=43799], [object_number=42, tag_tid=2, entry_tid=43798], [object_number=41, tag_tid=2, entry_tid=43797], [object_number=40, tag_tid=2, entry_tid=43796]]], state=connected] 2026-02-18T08:11:45.974 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:45.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=43, tag_tid=2, entry_tid=43799' 2026-02-18T08:11:45.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=57, tag_tid=3, entry_tid=1' -a 'object_number=57, tag_tid=3, entry_tid=1' = 'object_number=43, tag_tid=2, entry_tid=43799' 2026-02-18T08:11:45.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=43, tag_tid=2, entry_tid=43799' '!=' 'object_number=41, tag_tid=2, entry_tid=43521' 2026-02-18T08:11:45.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:11:45.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=43, tag_tid=2, entry_tid=43799' = 'object_number=41, tag_tid=2, entry_tid=43521' 2026-02-18T08:11:45.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=43, tag_tid=2, entry_tid=43799' 2026-02-18T08:11:45.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=57, tag_tid=3, entry_tid=1' 2026-02-18T08:11:45.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:11:45.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:11:45.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=3 2026-02-18T08:11:45.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=43, tag_tid=2, entry_tid=43799' 2026-02-18T08:11:45.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:11:45.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:11:45.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=2 2026-02-18T08:11:45.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=57, tag_tid=3, entry_tid=1' 2026-02-18T08:11:45.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:11:45.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:11:45.982 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:11:45.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=43, tag_tid=2, entry_tid=43799' 2026-02-18T08:11:45.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:11:45.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:11:45.985 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=43799 2026-02-18T08:11:45.985 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 3 = 2 -a 1 -le 43799 2026-02-18T08:11:45.985 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:11:45.985 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:11:45.985 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:11:46.187 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:11:46.187 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:11:46.187 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:11:46.187 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:11:46.187 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:11:46.187 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:11:46.187 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:11:46.187 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:11:46.187 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:11:46.188 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:11:46.188 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:11:46.188 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:11:46.188 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:11:46.188 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:11:46.188 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:11:46.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:11:46.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:11:46.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:11:46.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:11:46.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:11:46.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:11:46.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:11:46.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:11:46.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:11:46.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:11:46.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:11:46.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:11:46.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:11:46.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:11:46.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:11:46.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:11:46.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:11:46.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:11:46.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:11:46.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:11:46.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:11:46.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:11:46.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:11:46.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:11:46.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:11:46.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:11:46.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:11:46.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:11:46.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:11:46.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:11:46.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:11:46.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:11:46.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:11:46.194 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:11:46.201 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:11:46.202 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:11:46.202 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:11:46.202 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:11:46.260 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:11:46.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:11:46.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:11:46.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:11:46.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:11:46.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:11:46.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:11:46.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:11:46.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:11:46.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:11:46.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:11:46.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:11:46.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:11:46.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:46.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:11:46.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:46.292 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 10 2026-02-18T08:11:46.292 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 14 2026-02-18T08:11:46.292 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:11:46.292 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=57, tag_tid=3, entry_tid=1], [object_number=56, tag_tid=3, entry_tid=0], [object_number=59, tag_tid=2, entry_tid=59619], [object_number=58, tag_tid=2, entry_tid=59618]]], state=connected] 2026-02-18T08:11:46.292 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=41, tag_tid=2, entry_tid=44113], [object_number=40, tag_tid=2, entry_tid=44112], [object_number=43, tag_tid=2, entry_tid=44111], [object_number=42, tag_tid=2, entry_tid=44110]]], state=connected] 2026-02-18T08:11:46.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:46.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=57, tag_tid=3, entry_tid=1' 2026-02-18T08:11:46.297 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:11:46.298 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:11:46.298 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:11:46.298 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:11:46.298 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:11:46.298 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:11:46.298 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:11:46.298 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:11:46.298 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:11:46.298 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:11:46.299 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:11:46.299 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:11:46.300 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:46.300 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:11:46.300 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:46.326 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 10 2026-02-18T08:11:46.326 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 14 2026-02-18T08:11:46.326 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:11:46.326 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=57, tag_tid=3, entry_tid=1], [object_number=56, tag_tid=3, entry_tid=0], [object_number=59, tag_tid=2, entry_tid=59619], [object_number=58, tag_tid=2, entry_tid=59618]]], state=connected] 2026-02-18T08:11:46.326 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=41, tag_tid=2, entry_tid=44113], [object_number=40, tag_tid=2, entry_tid=44112], [object_number=43, tag_tid=2, entry_tid=44111], [object_number=42, tag_tid=2, entry_tid=44110]]], state=connected] 2026-02-18T08:11:46.329 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:46.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=41, tag_tid=2, entry_tid=44113' 2026-02-18T08:11:46.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=57, tag_tid=3, entry_tid=1' -a 'object_number=57, tag_tid=3, entry_tid=1' = 'object_number=41, tag_tid=2, entry_tid=44113' 2026-02-18T08:11:46.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=41, tag_tid=2, entry_tid=44113' '!=' 'object_number=43, tag_tid=2, entry_tid=43799' 2026-02-18T08:11:46.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:11:46.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=41, tag_tid=2, entry_tid=44113' = 'object_number=43, tag_tid=2, entry_tid=43799' 2026-02-18T08:11:46.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=41, tag_tid=2, entry_tid=44113' 2026-02-18T08:11:46.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=57, tag_tid=3, entry_tid=1' 2026-02-18T08:11:46.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:11:46.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:11:46.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=3 2026-02-18T08:11:46.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=41, tag_tid=2, entry_tid=44113' 2026-02-18T08:11:46.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:11:46.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:11:46.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=2 2026-02-18T08:11:46.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=57, tag_tid=3, entry_tid=1' 2026-02-18T08:11:46.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:11:46.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:11:46.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:11:46.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=41, tag_tid=2, entry_tid=44113' 2026-02-18T08:11:46.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:11:46.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:11:46.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=44113 2026-02-18T08:11:46.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 3 = 2 -a 1 -le 44113 2026-02-18T08:11:46.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:11:46.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:11:46.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:11:46.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:11:46.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:11:46.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:11:46.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:11:46.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:11:46.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:11:46.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:11:46.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:11:46.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:11:46.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:11:46.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:11:46.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:11:46.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:11:46.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:11:46.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:11:46.545 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:11:46.545 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:11:46.545 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:11:46.545 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:11:46.545 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:11:46.545 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:11:46.545 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:11:46.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:11:46.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:11:46.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:11:46.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:11:46.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:11:46.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:11:46.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:11:46.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:11:46.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:11:46.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:11:46.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:11:46.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:11:46.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:11:46.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:11:46.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:11:46.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:11:46.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:11:46.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:11:46.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:11:46.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:11:46.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:11:46.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:11:46.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:11:46.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:11:46.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:11:46.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:11:46.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:11:46.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:11:46.557 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:11:46.557 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:11:46.557 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:11:46.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:11:46.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:11:46.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:11:46.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:11:46.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:11:46.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:11:46.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:11:46.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:11:46.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:11:46.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:11:46.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:11:46.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:11:46.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:11:46.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:46.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:11:46.618 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:46.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 10 2026-02-18T08:11:46.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 14 2026-02-18T08:11:46.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:11:46.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=57, tag_tid=3, entry_tid=1], [object_number=56, tag_tid=3, entry_tid=0], [object_number=59, tag_tid=2, entry_tid=59619], [object_number=58, tag_tid=2, entry_tid=59618]]], state=connected] 2026-02-18T08:11:46.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=43, tag_tid=2, entry_tid=44403], [object_number=42, tag_tid=2, entry_tid=44402], [object_number=41, tag_tid=2, entry_tid=44401], [object_number=40, tag_tid=2, entry_tid=44400]]], state=connected] 2026-02-18T08:11:46.647 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:46.649 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=57, tag_tid=3, entry_tid=1' 2026-02-18T08:11:46.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:11:46.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:11:46.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:11:46.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:11:46.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:11:46.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:11:46.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:11:46.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:11:46.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:11:46.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:11:46.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:11:46.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:11:46.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:46.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:11:46.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:46.686 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 10 2026-02-18T08:11:46.686 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 14 2026-02-18T08:11:46.686 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:11:46.686 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=57, tag_tid=3, entry_tid=1], [object_number=56, tag_tid=3, entry_tid=0], [object_number=59, tag_tid=2, entry_tid=59619], [object_number=58, tag_tid=2, entry_tid=59618]]], state=connected] 2026-02-18T08:11:46.686 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=43, tag_tid=2, entry_tid=44403], [object_number=42, tag_tid=2, entry_tid=44402], [object_number=41, tag_tid=2, entry_tid=44401], [object_number=40, tag_tid=2, entry_tid=44400]]], state=connected] 2026-02-18T08:11:46.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:46.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=43, tag_tid=2, entry_tid=44403' 2026-02-18T08:11:46.691 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=57, tag_tid=3, entry_tid=1' -a 'object_number=57, tag_tid=3, entry_tid=1' = 'object_number=43, tag_tid=2, entry_tid=44403' 2026-02-18T08:11:46.691 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=43, tag_tid=2, entry_tid=44403' '!=' 'object_number=41, tag_tid=2, entry_tid=44113' 2026-02-18T08:11:46.691 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:11:46.691 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=43, tag_tid=2, entry_tid=44403' = 'object_number=41, tag_tid=2, entry_tid=44113' 2026-02-18T08:11:46.691 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=43, tag_tid=2, entry_tid=44403' 2026-02-18T08:11:46.692 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=57, tag_tid=3, entry_tid=1' 2026-02-18T08:11:46.692 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:11:46.692 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:11:46.692 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=3 2026-02-18T08:11:46.693 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=43, tag_tid=2, entry_tid=44403' 2026-02-18T08:11:46.693 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:11:46.693 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:11:46.695 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=2 2026-02-18T08:11:46.696 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=57, tag_tid=3, entry_tid=1' 2026-02-18T08:11:46.696 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:11:46.696 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:11:46.697 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:11:46.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=43, tag_tid=2, entry_tid=44403' 2026-02-18T08:11:46.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:11:46.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:11:46.700 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=44403 2026-02-18T08:11:46.700 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 3 = 2 -a 1 -le 44403 2026-02-18T08:11:46.700 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:11:46.700 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:11:46.700 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:11:46.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:11:46.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:11:46.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:11:46.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:11:46.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:11:46.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:11:46.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:11:46.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:11:46.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:11:46.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:11:46.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:11:46.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:11:46.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:11:46.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:11:46.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:11:46.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:11:46.905 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:11:46.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:11:46.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:11:46.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:11:46.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:11:46.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:11:46.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:11:46.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:11:46.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:11:46.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:11:46.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:11:46.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:11:46.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:11:46.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:11:46.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:11:46.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:11:46.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:11:46.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:11:46.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:11:46.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:11:46.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:11:46.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:11:46.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:11:46.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:11:46.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:11:46.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:11:46.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:11:46.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:11:46.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:11:46.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:11:46.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:11:46.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:11:46.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:11:46.916 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:11:46.916 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:11:46.916 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:11:46.916 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:11:46.974 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:11:46.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:11:46.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:11:46.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:11:46.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:11:46.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:11:46.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:11:46.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:11:46.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:11:46.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:11:46.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:11:46.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:11:46.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:11:46.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:46.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:11:46.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:47.005 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 10 2026-02-18T08:11:47.006 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 14 2026-02-18T08:11:47.006 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:11:47.006 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=57, tag_tid=3, entry_tid=1], [object_number=56, tag_tid=3, entry_tid=0], [object_number=59, tag_tid=2, entry_tid=59619], [object_number=58, tag_tid=2, entry_tid=59618]]], state=connected] 2026-02-18T08:11:47.006 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=40, tag_tid=2, entry_tid=44764], [object_number=43, tag_tid=2, entry_tid=44763], [object_number=42, tag_tid=2, entry_tid=44762], [object_number=41, tag_tid=2, entry_tid=44761]]], state=connected] 2026-02-18T08:11:47.008 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:47.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=57, tag_tid=3, entry_tid=1' 2026-02-18T08:11:47.010 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:11:47.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:11:47.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:11:47.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:11:47.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:11:47.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:11:47.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:11:47.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:11:47.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:11:47.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:11:47.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:11:47.012 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:11:47.012 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:47.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:11:47.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:47.039 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 10 2026-02-18T08:11:47.039 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 14 2026-02-18T08:11:47.039 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:11:47.039 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=57, tag_tid=3, entry_tid=1], [object_number=56, tag_tid=3, entry_tid=0], [object_number=59, tag_tid=2, entry_tid=59619], [object_number=58, tag_tid=2, entry_tid=59618]]], state=connected] 2026-02-18T08:11:47.039 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=40, tag_tid=2, entry_tid=44764], [object_number=43, tag_tid=2, entry_tid=44763], [object_number=42, tag_tid=2, entry_tid=44762], [object_number=41, tag_tid=2, entry_tid=44761]]], state=connected] 2026-02-18T08:11:47.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:47.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=40, tag_tid=2, entry_tid=44764' 2026-02-18T08:11:47.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=57, tag_tid=3, entry_tid=1' -a 'object_number=57, tag_tid=3, entry_tid=1' = 'object_number=40, tag_tid=2, entry_tid=44764' 2026-02-18T08:11:47.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=40, tag_tid=2, entry_tid=44764' '!=' 'object_number=43, tag_tid=2, entry_tid=44403' 2026-02-18T08:11:47.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:11:47.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=40, tag_tid=2, entry_tid=44764' = 'object_number=43, tag_tid=2, entry_tid=44403' 2026-02-18T08:11:47.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=40, tag_tid=2, entry_tid=44764' 2026-02-18T08:11:47.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=57, tag_tid=3, entry_tid=1' 2026-02-18T08:11:47.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:11:47.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:11:47.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=3 2026-02-18T08:11:47.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=40, tag_tid=2, entry_tid=44764' 2026-02-18T08:11:47.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:11:47.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:11:47.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=2 2026-02-18T08:11:47.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=57, tag_tid=3, entry_tid=1' 2026-02-18T08:11:47.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:11:47.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:11:47.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:11:47.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=40, tag_tid=2, entry_tid=44764' 2026-02-18T08:11:47.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:11:47.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:11:47.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=44764 2026-02-18T08:11:47.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 3 = 2 -a 1 -le 44764 2026-02-18T08:11:47.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:11:47.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:11:47.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:11:47.255 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:11:47.255 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:11:47.255 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:11:47.255 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:11:47.255 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:11:47.255 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:11:47.255 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:11:47.255 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:11:47.255 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:11:47.256 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:11:47.256 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:11:47.256 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:11:47.256 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:11:47.256 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:11:47.256 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:11:47.256 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:11:47.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:11:47.259 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:11:47.259 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:11:47.259 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:11:47.259 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:11:47.259 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:11:47.259 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:11:47.259 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:11:47.259 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:11:47.259 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:11:47.259 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:11:47.259 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:11:47.260 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:11:47.260 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:11:47.260 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:11:47.260 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:11:47.260 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:11:47.260 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:11:47.260 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:11:47.260 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:11:47.260 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:11:47.260 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:11:47.260 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:11:47.260 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:11:47.261 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:11:47.261 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:11:47.261 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:11:47.261 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:11:47.261 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:11:47.261 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:11:47.261 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:11:47.261 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:11:47.261 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:11:47.269 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:11:47.269 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:11:47.269 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:11:47.270 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:11:47.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:11:47.329 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:11:47.329 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:11:47.329 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:11:47.329 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:11:47.329 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:11:47.329 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:11:47.329 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:11:47.329 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:11:47.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:11:47.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:11:47.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:11:47.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:11:47.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:47.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:11:47.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:47.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 10 2026-02-18T08:11:47.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 14 2026-02-18T08:11:47.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:11:47.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=57, tag_tid=3, entry_tid=1], [object_number=56, tag_tid=3, entry_tid=0], [object_number=59, tag_tid=2, entry_tid=59619], [object_number=58, tag_tid=2, entry_tid=59618]]], state=connected] 2026-02-18T08:11:47.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=43, tag_tid=2, entry_tid=45067], [object_number=42, tag_tid=2, entry_tid=45066], [object_number=41, tag_tid=2, entry_tid=45065], [object_number=40, tag_tid=2, entry_tid=45064]]], state=connected] 2026-02-18T08:11:47.361 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:47.363 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=57, tag_tid=3, entry_tid=1' 2026-02-18T08:11:47.364 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:11:47.364 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:11:47.364 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:11:47.364 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:11:47.364 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:11:47.364 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:11:47.364 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:11:47.364 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:11:47.364 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:11:47.364 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:11:47.365 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:11:47.365 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:11:47.366 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:47.366 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:11:47.366 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:47.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 10 2026-02-18T08:11:47.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 14 2026-02-18T08:11:47.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:11:47.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=57, tag_tid=3, entry_tid=1], [object_number=56, tag_tid=3, entry_tid=0], [object_number=59, tag_tid=2, entry_tid=59619], [object_number=58, tag_tid=2, entry_tid=59618]]], state=connected] 2026-02-18T08:11:47.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=43, tag_tid=2, entry_tid=45067], [object_number=42, tag_tid=2, entry_tid=45066], [object_number=41, tag_tid=2, entry_tid=45065], [object_number=40, tag_tid=2, entry_tid=45064]]], state=connected] 2026-02-18T08:11:47.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:47.406 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=43, tag_tid=2, entry_tid=45067' 2026-02-18T08:11:47.406 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=57, tag_tid=3, entry_tid=1' -a 'object_number=57, tag_tid=3, entry_tid=1' = 'object_number=43, tag_tid=2, entry_tid=45067' 2026-02-18T08:11:47.406 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=43, tag_tid=2, entry_tid=45067' '!=' 'object_number=40, tag_tid=2, entry_tid=44764' 2026-02-18T08:11:47.406 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:11:47.406 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=43, tag_tid=2, entry_tid=45067' = 'object_number=40, tag_tid=2, entry_tid=44764' 2026-02-18T08:11:47.406 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=43, tag_tid=2, entry_tid=45067' 2026-02-18T08:11:47.407 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=57, tag_tid=3, entry_tid=1' 2026-02-18T08:11:47.407 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:11:47.407 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:11:47.407 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=3 2026-02-18T08:11:47.408 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=43, tag_tid=2, entry_tid=45067' 2026-02-18T08:11:47.408 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:11:47.408 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:11:47.409 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=2 2026-02-18T08:11:47.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=57, tag_tid=3, entry_tid=1' 2026-02-18T08:11:47.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:11:47.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:11:47.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:11:47.412 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=43, tag_tid=2, entry_tid=45067' 2026-02-18T08:11:47.412 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:11:47.412 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:11:47.414 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=45067 2026-02-18T08:11:47.414 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 3 = 2 -a 1 -le 45067 2026-02-18T08:11:47.414 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:11:47.414 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:11:47.414 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:11:47.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:11:47.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:11:47.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:11:47.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:11:47.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:11:47.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:11:47.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:11:47.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:11:47.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:11:47.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:11:47.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:11:47.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:11:47.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:11:47.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:11:47.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:11:47.619 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:11:47.620 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:11:47.620 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:11:47.620 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:11:47.620 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:11:47.620 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:11:47.620 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:11:47.620 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:11:47.620 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:11:47.620 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:11:47.620 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:11:47.620 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:11:47.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:11:47.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:11:47.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:11:47.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:11:47.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:11:47.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:11:47.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:11:47.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:11:47.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:11:47.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:11:47.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:11:47.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:11:47.622 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:11:47.622 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:11:47.622 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:11:47.622 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:11:47.622 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:11:47.622 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:11:47.622 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:11:47.622 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:11:47.622 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:11:47.622 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:11:47.629 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:11:47.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:11:47.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:11:47.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:11:47.691 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:11:47.692 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:11:47.692 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:11:47.692 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:11:47.692 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:11:47.693 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:11:47.693 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:11:47.693 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:11:47.693 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:11:47.693 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:11:47.693 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:11:47.693 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:11:47.693 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:11:47.694 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:47.694 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:11:47.694 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:47.721 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 10 2026-02-18T08:11:47.721 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 14 2026-02-18T08:11:47.721 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:11:47.721 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=57, tag_tid=3, entry_tid=1], [object_number=56, tag_tid=3, entry_tid=0], [object_number=59, tag_tid=2, entry_tid=59619], [object_number=58, tag_tid=2, entry_tid=59618]]], state=connected] 2026-02-18T08:11:47.721 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=41, tag_tid=2, entry_tid=45421], [object_number=40, tag_tid=2, entry_tid=45420], [object_number=43, tag_tid=2, entry_tid=45419], [object_number=42, tag_tid=2, entry_tid=45418]]], state=connected] 2026-02-18T08:11:47.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:47.725 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=57, tag_tid=3, entry_tid=1' 2026-02-18T08:11:47.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:11:47.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:11:47.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:11:47.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:11:47.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:11:47.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:11:47.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:11:47.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:11:47.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:11:47.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:11:47.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:11:47.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:11:47.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:47.728 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:11:47.728 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:47.754 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 10 2026-02-18T08:11:47.754 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 14 2026-02-18T08:11:47.755 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:11:47.755 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=57, tag_tid=3, entry_tid=1], [object_number=56, tag_tid=3, entry_tid=0], [object_number=59, tag_tid=2, entry_tid=59619], [object_number=58, tag_tid=2, entry_tid=59618]]], state=connected] 2026-02-18T08:11:47.755 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=41, tag_tid=2, entry_tid=45421], [object_number=40, tag_tid=2, entry_tid=45420], [object_number=43, tag_tid=2, entry_tid=45419], [object_number=42, tag_tid=2, entry_tid=45418]]], state=connected] 2026-02-18T08:11:47.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:47.759 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=41, tag_tid=2, entry_tid=45421' 2026-02-18T08:11:47.759 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=57, tag_tid=3, entry_tid=1' -a 'object_number=57, tag_tid=3, entry_tid=1' = 'object_number=41, tag_tid=2, entry_tid=45421' 2026-02-18T08:11:47.759 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=41, tag_tid=2, entry_tid=45421' '!=' 'object_number=43, tag_tid=2, entry_tid=45067' 2026-02-18T08:11:47.759 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:11:47.760 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=41, tag_tid=2, entry_tid=45421' = 'object_number=43, tag_tid=2, entry_tid=45067' 2026-02-18T08:11:47.760 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=41, tag_tid=2, entry_tid=45421' 2026-02-18T08:11:47.760 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=57, tag_tid=3, entry_tid=1' 2026-02-18T08:11:47.760 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:11:47.761 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:11:47.761 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=3 2026-02-18T08:11:47.762 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=41, tag_tid=2, entry_tid=45421' 2026-02-18T08:11:47.762 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:11:47.762 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:11:47.763 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=2 2026-02-18T08:11:47.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=57, tag_tid=3, entry_tid=1' 2026-02-18T08:11:47.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:11:47.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:11:47.766 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:11:47.766 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=41, tag_tid=2, entry_tid=45421' 2026-02-18T08:11:47.767 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:11:47.767 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:11:47.768 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=45421 2026-02-18T08:11:47.768 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 3 = 2 -a 1 -le 45421 2026-02-18T08:11:47.769 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:11:47.769 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:11:47.769 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:11:47.970 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:11:47.970 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:11:47.970 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:11:47.970 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:11:47.970 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:11:47.970 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:11:47.971 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:11:47.971 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:11:47.971 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:11:47.971 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:11:47.971 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:11:47.971 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:11:47.971 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:11:47.971 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:11:47.971 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:11:47.974 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:11:47.974 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:11:47.974 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:11:47.974 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:11:47.974 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:11:47.974 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:11:47.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:11:47.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:11:47.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:11:47.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:11:47.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:11:47.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:11:47.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:11:47.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:11:47.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:11:47.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:11:47.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:11:47.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:11:47.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:11:47.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:11:47.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:11:47.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:11:47.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:11:47.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:11:47.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:11:47.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:11:47.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:11:47.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:11:47.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:11:47.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:11:47.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:11:47.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:11:47.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:11:47.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:11:47.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:11:47.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:11:47.985 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:11:47.985 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:11:48.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:11:48.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:11:48.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:11:48.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:11:48.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:11:48.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:11:48.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:11:48.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:11:48.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:11:48.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:11:48.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:11:48.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:11:48.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:11:48.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:48.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:11:48.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:48.075 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 10 2026-02-18T08:11:48.075 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 14 2026-02-18T08:11:48.075 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:11:48.075 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=57, tag_tid=3, entry_tid=1], [object_number=56, tag_tid=3, entry_tid=0], [object_number=59, tag_tid=2, entry_tid=59619], [object_number=58, tag_tid=2, entry_tid=59618]]], state=connected] 2026-02-18T08:11:48.075 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=43, tag_tid=2, entry_tid=45739], [object_number=42, tag_tid=2, entry_tid=45738], [object_number=41, tag_tid=2, entry_tid=45737], [object_number=40, tag_tid=2, entry_tid=45736]]], state=connected] 2026-02-18T08:11:48.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:48.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=57, tag_tid=3, entry_tid=1' 2026-02-18T08:11:48.080 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:11:48.080 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:11:48.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:11:48.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:11:48.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:11:48.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:11:48.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:11:48.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:11:48.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:11:48.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:11:48.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:11:48.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:11:48.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:48.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:11:48.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:48.110 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 10 2026-02-18T08:11:48.110 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 14 2026-02-18T08:11:48.110 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:11:48.110 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=57, tag_tid=3, entry_tid=1], [object_number=56, tag_tid=3, entry_tid=0], [object_number=59, tag_tid=2, entry_tid=59619], [object_number=58, tag_tid=2, entry_tid=59618]]], state=connected] 2026-02-18T08:11:48.110 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=43, tag_tid=2, entry_tid=45739], [object_number=42, tag_tid=2, entry_tid=45738], [object_number=41, tag_tid=2, entry_tid=45737], [object_number=40, tag_tid=2, entry_tid=45736]]], state=connected] 2026-02-18T08:11:48.112 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:48.114 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=43, tag_tid=2, entry_tid=45739' 2026-02-18T08:11:48.114 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=57, tag_tid=3, entry_tid=1' -a 'object_number=57, tag_tid=3, entry_tid=1' = 'object_number=43, tag_tid=2, entry_tid=45739' 2026-02-18T08:11:48.114 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=43, tag_tid=2, entry_tid=45739' '!=' 'object_number=41, tag_tid=2, entry_tid=45421' 2026-02-18T08:11:48.114 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:11:48.114 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=43, tag_tid=2, entry_tid=45739' = 'object_number=41, tag_tid=2, entry_tid=45421' 2026-02-18T08:11:48.114 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=43, tag_tid=2, entry_tid=45739' 2026-02-18T08:11:48.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=57, tag_tid=3, entry_tid=1' 2026-02-18T08:11:48.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:11:48.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:11:48.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=3 2026-02-18T08:11:48.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=43, tag_tid=2, entry_tid=45739' 2026-02-18T08:11:48.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:11:48.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:11:48.118 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=2 2026-02-18T08:11:48.119 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=57, tag_tid=3, entry_tid=1' 2026-02-18T08:11:48.119 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:11:48.119 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:11:48.120 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:11:48.120 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=43, tag_tid=2, entry_tid=45739' 2026-02-18T08:11:48.121 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:11:48.121 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:11:48.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=45739 2026-02-18T08:11:48.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 3 = 2 -a 1 -le 45739 2026-02-18T08:11:48.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:11:48.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:11:48.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:11:48.325 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:11:48.325 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:11:48.325 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:11:48.325 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:11:48.325 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:11:48.325 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:11:48.325 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:11:48.325 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:11:48.326 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:11:48.326 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:11:48.326 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:11:48.326 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:11:48.326 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:11:48.326 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:11:48.326 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:11:48.329 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:11:48.329 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:11:48.329 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:11:48.329 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:11:48.329 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:11:48.329 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:11:48.329 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:11:48.329 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:11:48.329 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:11:48.329 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:11:48.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:11:48.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:11:48.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:11:48.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:11:48.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:11:48.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:11:48.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:11:48.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:11:48.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:11:48.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:11:48.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:11:48.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:11:48.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:11:48.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:11:48.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:11:48.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:11:48.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:11:48.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:11:48.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:11:48.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:11:48.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:11:48.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:11:48.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:11:48.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:11:48.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:11:48.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:11:48.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:11:48.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:11:48.409 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:11:48.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:11:48.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:11:48.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:11:48.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:11:48.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:11:48.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:11:48.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:11:48.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:11:48.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:11:48.412 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:11:48.412 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:11:48.412 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:11:48.412 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:48.413 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:11:48.413 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:48.439 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 11 2026-02-18T08:11:48.439 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 14 2026-02-18T08:11:48.439 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:11:48.440 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=57, tag_tid=3, entry_tid=1], [object_number=56, tag_tid=3, entry_tid=0], [object_number=59, tag_tid=2, entry_tid=59619], [object_number=58, tag_tid=2, entry_tid=59618]]], state=connected] 2026-02-18T08:11:48.440 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=47, tag_tid=2, entry_tid=45903], [object_number=46, tag_tid=2, entry_tid=45902], [object_number=45, tag_tid=2, entry_tid=45901], [object_number=44, tag_tid=2, entry_tid=45900]]], state=connected] 2026-02-18T08:11:48.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:48.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=57, tag_tid=3, entry_tid=1' 2026-02-18T08:11:48.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:11:48.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:11:48.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:11:48.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:11:48.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:11:48.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:11:48.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:11:48.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:11:48.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:11:48.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:11:48.446 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:11:48.446 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:11:48.446 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:48.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:11:48.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:48.477 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 11 2026-02-18T08:11:48.477 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 14 2026-02-18T08:11:48.477 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:11:48.477 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=57, tag_tid=3, entry_tid=1], [object_number=56, tag_tid=3, entry_tid=0], [object_number=59, tag_tid=2, entry_tid=59619], [object_number=58, tag_tid=2, entry_tid=59618]]], state=connected] 2026-02-18T08:11:48.477 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=47, tag_tid=2, entry_tid=45903], [object_number=46, tag_tid=2, entry_tid=45902], [object_number=45, tag_tid=2, entry_tid=45901], [object_number=44, tag_tid=2, entry_tid=45900]]], state=connected] 2026-02-18T08:11:48.481 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:48.483 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=47, tag_tid=2, entry_tid=45903' 2026-02-18T08:11:48.483 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=57, tag_tid=3, entry_tid=1' -a 'object_number=57, tag_tid=3, entry_tid=1' = 'object_number=47, tag_tid=2, entry_tid=45903' 2026-02-18T08:11:48.483 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=47, tag_tid=2, entry_tid=45903' '!=' 'object_number=43, tag_tid=2, entry_tid=45739' 2026-02-18T08:11:48.483 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:11:48.483 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=47, tag_tid=2, entry_tid=45903' = 'object_number=43, tag_tid=2, entry_tid=45739' 2026-02-18T08:11:48.483 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=47, tag_tid=2, entry_tid=45903' 2026-02-18T08:11:48.483 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=57, tag_tid=3, entry_tid=1' 2026-02-18T08:11:48.483 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:11:48.484 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:11:48.484 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=3 2026-02-18T08:11:48.485 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=47, tag_tid=2, entry_tid=45903' 2026-02-18T08:11:48.485 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:11:48.485 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:11:48.487 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=2 2026-02-18T08:11:48.487 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=57, tag_tid=3, entry_tid=1' 2026-02-18T08:11:48.488 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:11:48.488 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:11:48.489 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:11:48.490 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=47, tag_tid=2, entry_tid=45903' 2026-02-18T08:11:48.490 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:11:48.490 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:11:48.491 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=45903 2026-02-18T08:11:48.491 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 3 = 2 -a 1 -le 45903 2026-02-18T08:11:48.491 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:11:48.492 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:11:48.492 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:11:48.693 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:11:48.693 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:11:48.693 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:11:48.693 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:11:48.693 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:11:48.694 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:11:48.694 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:11:48.694 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:11:48.694 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:11:48.694 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:11:48.694 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:11:48.694 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:11:48.694 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:11:48.694 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:11:48.694 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:11:48.697 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:11:48.697 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:11:48.697 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:11:48.697 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:11:48.697 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:11:48.697 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:11:48.697 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:11:48.697 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:11:48.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:11:48.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:11:48.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:11:48.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:11:48.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:11:48.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:11:48.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:11:48.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:11:48.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:11:48.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:11:48.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:11:48.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:11:48.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:11:48.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:11:48.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:11:48.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:11:48.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:11:48.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:11:48.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:11:48.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:11:48.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:11:48.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:11:48.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:11:48.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:11:48.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:11:48.700 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:11:48.708 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:11:48.708 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:11:48.708 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:11:48.708 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:11:48.766 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:11:48.767 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:11:48.767 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:11:48.767 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:11:48.767 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:11:48.767 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:11:48.767 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:11:48.767 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:11:48.768 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:11:48.768 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:11:48.768 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:11:48.768 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:11:48.768 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:11:48.769 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:48.769 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:11:48.769 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:48.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 11 2026-02-18T08:11:48.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 14 2026-02-18T08:11:48.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:11:48.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=57, tag_tid=3, entry_tid=1], [object_number=56, tag_tid=3, entry_tid=0], [object_number=59, tag_tid=2, entry_tid=59619], [object_number=58, tag_tid=2, entry_tid=59618]]], state=connected] 2026-02-18T08:11:48.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=44, tag_tid=2, entry_tid=46228], [object_number=47, tag_tid=2, entry_tid=46227], [object_number=46, tag_tid=2, entry_tid=46226], [object_number=45, tag_tid=2, entry_tid=46225]]], state=connected] 2026-02-18T08:11:48.800 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:48.801 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=57, tag_tid=3, entry_tid=1' 2026-02-18T08:11:48.802 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:11:48.802 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:11:48.802 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:11:48.802 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:11:48.802 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:11:48.802 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:11:48.802 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:11:48.802 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:11:48.802 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:11:48.803 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:11:48.803 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:11:48.803 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:11:48.804 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:48.804 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:11:48.804 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:48.830 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 11 2026-02-18T08:11:48.830 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 14 2026-02-18T08:11:48.830 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:11:48.830 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=57, tag_tid=3, entry_tid=1], [object_number=56, tag_tid=3, entry_tid=0], [object_number=59, tag_tid=2, entry_tid=59619], [object_number=58, tag_tid=2, entry_tid=59618]]], state=connected] 2026-02-18T08:11:48.830 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=44, tag_tid=2, entry_tid=46228], [object_number=47, tag_tid=2, entry_tid=46227], [object_number=46, tag_tid=2, entry_tid=46226], [object_number=45, tag_tid=2, entry_tid=46225]]], state=connected] 2026-02-18T08:11:48.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:48.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=44, tag_tid=2, entry_tid=46228' 2026-02-18T08:11:48.835 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=57, tag_tid=3, entry_tid=1' -a 'object_number=57, tag_tid=3, entry_tid=1' = 'object_number=44, tag_tid=2, entry_tid=46228' 2026-02-18T08:11:48.835 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=44, tag_tid=2, entry_tid=46228' '!=' 'object_number=47, tag_tid=2, entry_tid=45903' 2026-02-18T08:11:48.835 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:11:48.835 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=44, tag_tid=2, entry_tid=46228' = 'object_number=47, tag_tid=2, entry_tid=45903' 2026-02-18T08:11:48.835 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=44, tag_tid=2, entry_tid=46228' 2026-02-18T08:11:48.836 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=57, tag_tid=3, entry_tid=1' 2026-02-18T08:11:48.836 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:11:48.836 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:11:48.837 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=3 2026-02-18T08:11:48.837 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=44, tag_tid=2, entry_tid=46228' 2026-02-18T08:11:48.838 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:11:48.838 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:11:48.839 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=2 2026-02-18T08:11:48.839 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=57, tag_tid=3, entry_tid=1' 2026-02-18T08:11:48.840 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:11:48.840 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:11:48.841 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:11:48.842 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=44, tag_tid=2, entry_tid=46228' 2026-02-18T08:11:48.842 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:11:48.842 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:11:48.844 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=46228 2026-02-18T08:11:48.844 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 3 = 2 -a 1 -le 46228 2026-02-18T08:11:48.844 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:11:48.844 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:11:48.844 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:11:49.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:11:49.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:11:49.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:11:49.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:11:49.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:11:49.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:11:49.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:11:49.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:11:49.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:11:49.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:11:49.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:11:49.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:11:49.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:11:49.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:11:49.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:11:49.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:11:49.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:11:49.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:11:49.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:11:49.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:11:49.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:11:49.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:11:49.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:11:49.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:11:49.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:11:49.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:11:49.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:11:49.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:11:49.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:11:49.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:11:49.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:11:49.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:11:49.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:11:49.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:11:49.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:11:49.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:11:49.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:11:49.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:11:49.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:11:49.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:11:49.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:11:49.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:11:49.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:11:49.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:11:49.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:11:49.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:11:49.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:11:49.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:11:49.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:11:49.060 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:11:49.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:11:49.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:11:49.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:11:49.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:11:49.118 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:11:49.118 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:11:49.118 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:11:49.118 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:11:49.119 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:11:49.119 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:11:49.119 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:11:49.119 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:11:49.119 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:11:49.119 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:11:49.119 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:11:49.120 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:11:49.120 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:49.120 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:11:49.120 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:49.147 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 11 2026-02-18T08:11:49.147 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 14 2026-02-18T08:11:49.147 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:11:49.147 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=57, tag_tid=3, entry_tid=1], [object_number=56, tag_tid=3, entry_tid=0], [object_number=59, tag_tid=2, entry_tid=59619], [object_number=58, tag_tid=2, entry_tid=59618]]], state=connected] 2026-02-18T08:11:49.147 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=45, tag_tid=2, entry_tid=46521], [object_number=44, tag_tid=2, entry_tid=46520], [object_number=47, tag_tid=2, entry_tid=46519], [object_number=46, tag_tid=2, entry_tid=46518]]], state=connected] 2026-02-18T08:11:49.149 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:49.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=57, tag_tid=3, entry_tid=1' 2026-02-18T08:11:49.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:11:49.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:11:49.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:11:49.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:11:49.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:11:49.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:11:49.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:11:49.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:11:49.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:11:49.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:11:49.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:11:49.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:11:49.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:49.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:11:49.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:49.179 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 11 2026-02-18T08:11:49.179 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 14 2026-02-18T08:11:49.179 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:11:49.179 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=57, tag_tid=3, entry_tid=1], [object_number=56, tag_tid=3, entry_tid=0], [object_number=59, tag_tid=2, entry_tid=59619], [object_number=58, tag_tid=2, entry_tid=59618]]], state=connected] 2026-02-18T08:11:49.179 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=45, tag_tid=2, entry_tid=46521], [object_number=44, tag_tid=2, entry_tid=46520], [object_number=47, tag_tid=2, entry_tid=46519], [object_number=46, tag_tid=2, entry_tid=46518]]], state=connected] 2026-02-18T08:11:49.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:49.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=45, tag_tid=2, entry_tid=46521' 2026-02-18T08:11:49.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=57, tag_tid=3, entry_tid=1' -a 'object_number=57, tag_tid=3, entry_tid=1' = 'object_number=45, tag_tid=2, entry_tid=46521' 2026-02-18T08:11:49.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=45, tag_tid=2, entry_tid=46521' '!=' 'object_number=44, tag_tid=2, entry_tid=46228' 2026-02-18T08:11:49.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:11:49.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=45, tag_tid=2, entry_tid=46521' = 'object_number=44, tag_tid=2, entry_tid=46228' 2026-02-18T08:11:49.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=45, tag_tid=2, entry_tid=46521' 2026-02-18T08:11:49.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=57, tag_tid=3, entry_tid=1' 2026-02-18T08:11:49.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:11:49.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:11:49.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=3 2026-02-18T08:11:49.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=45, tag_tid=2, entry_tid=46521' 2026-02-18T08:11:49.187 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:11:49.187 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:11:49.188 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=2 2026-02-18T08:11:49.189 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=57, tag_tid=3, entry_tid=1' 2026-02-18T08:11:49.189 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:11:49.189 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:11:49.190 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:11:49.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=45, tag_tid=2, entry_tid=46521' 2026-02-18T08:11:49.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:11:49.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:11:49.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=46521 2026-02-18T08:11:49.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 3 = 2 -a 1 -le 46521 2026-02-18T08:11:49.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:11:49.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:11:49.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:11:49.394 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:11:49.395 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:11:49.395 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:11:49.395 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:11:49.395 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:11:49.395 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:11:49.395 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:11:49.395 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:11:49.395 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:11:49.395 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:11:49.395 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:11:49.395 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:11:49.396 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:11:49.396 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:11:49.396 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:11:49.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:11:49.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:11:49.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:11:49.399 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:11:49.399 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:11:49.399 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:11:49.399 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:11:49.399 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:11:49.399 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:11:49.399 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:11:49.399 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:11:49.399 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:11:49.399 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:11:49.399 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:11:49.399 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:11:49.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:11:49.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:11:49.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:11:49.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:11:49.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:11:49.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:11:49.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:11:49.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:11:49.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:11:49.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:11:49.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:11:49.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:11:49.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:11:49.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:11:49.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:11:49.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:11:49.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:11:49.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:11:49.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:11:49.408 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:11:49.409 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:11:49.409 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:11:49.409 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:11:49.470 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:11:49.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:11:49.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:11:49.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:11:49.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:11:49.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:11:49.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:11:49.472 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:11:49.472 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:11:49.472 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:11:49.472 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:11:49.472 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:11:49.472 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:11:49.473 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:49.473 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:11:49.473 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:49.500 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 11 2026-02-18T08:11:49.500 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 14 2026-02-18T08:11:49.500 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:11:49.500 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=57, tag_tid=3, entry_tid=1], [object_number=56, tag_tid=3, entry_tid=0], [object_number=59, tag_tid=2, entry_tid=59619], [object_number=58, tag_tid=2, entry_tid=59618]]], state=connected] 2026-02-18T08:11:49.500 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=45, tag_tid=2, entry_tid=46813], [object_number=44, tag_tid=2, entry_tid=46812], [object_number=47, tag_tid=2, entry_tid=46811], [object_number=46, tag_tid=2, entry_tid=46810]]], state=connected] 2026-02-18T08:11:49.503 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:49.504 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=57, tag_tid=3, entry_tid=1' 2026-02-18T08:11:49.506 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:11:49.506 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:11:49.506 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:11:49.506 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:11:49.506 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:11:49.506 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:11:49.506 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:11:49.506 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:11:49.506 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:11:49.507 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:11:49.507 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:11:49.507 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:11:49.507 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:49.508 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:11:49.508 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:49.534 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 11 2026-02-18T08:11:49.535 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 14 2026-02-18T08:11:49.535 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:11:49.535 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=57, tag_tid=3, entry_tid=1], [object_number=56, tag_tid=3, entry_tid=0], [object_number=59, tag_tid=2, entry_tid=59619], [object_number=58, tag_tid=2, entry_tid=59618]]], state=connected] 2026-02-18T08:11:49.535 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=45, tag_tid=2, entry_tid=46813], [object_number=44, tag_tid=2, entry_tid=46812], [object_number=47, tag_tid=2, entry_tid=46811], [object_number=46, tag_tid=2, entry_tid=46810]]], state=connected] 2026-02-18T08:11:49.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:49.539 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=45, tag_tid=2, entry_tid=46813' 2026-02-18T08:11:49.539 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=57, tag_tid=3, entry_tid=1' -a 'object_number=57, tag_tid=3, entry_tid=1' = 'object_number=45, tag_tid=2, entry_tid=46813' 2026-02-18T08:11:49.539 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=45, tag_tid=2, entry_tid=46813' '!=' 'object_number=45, tag_tid=2, entry_tid=46521' 2026-02-18T08:11:49.540 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:11:49.540 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=45, tag_tid=2, entry_tid=46813' = 'object_number=45, tag_tid=2, entry_tid=46521' 2026-02-18T08:11:49.540 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=45, tag_tid=2, entry_tid=46813' 2026-02-18T08:11:49.540 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=57, tag_tid=3, entry_tid=1' 2026-02-18T08:11:49.540 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:11:49.540 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:11:49.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=3 2026-02-18T08:11:49.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=45, tag_tid=2, entry_tid=46813' 2026-02-18T08:11:49.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:11:49.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:11:49.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=2 2026-02-18T08:11:49.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=57, tag_tid=3, entry_tid=1' 2026-02-18T08:11:49.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:11:49.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:11:49.545 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:11:49.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=45, tag_tid=2, entry_tid=46813' 2026-02-18T08:11:49.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:11:49.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:11:49.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=46813 2026-02-18T08:11:49.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 3 = 2 -a 1 -le 46813 2026-02-18T08:11:49.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:11:49.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:11:49.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:11:49.750 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:11:49.750 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:11:49.750 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:11:49.750 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:11:49.750 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:11:49.750 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:11:49.750 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:11:49.750 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:11:49.750 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:11:49.751 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:11:49.751 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:11:49.751 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:11:49.751 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:11:49.751 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:11:49.751 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:11:49.753 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:11:49.753 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:11:49.754 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:11:49.754 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:11:49.754 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:11:49.754 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:11:49.754 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:11:49.754 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:11:49.754 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:11:49.754 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:11:49.754 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:11:49.754 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:11:49.754 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:11:49.754 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:11:49.755 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:11:49.755 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:11:49.755 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:11:49.755 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:11:49.755 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:11:49.755 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:11:49.755 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:11:49.755 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:11:49.755 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:11:49.755 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:11:49.755 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:11:49.755 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:11:49.756 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:11:49.756 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:11:49.756 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:11:49.756 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:11:49.756 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:11:49.756 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:11:49.756 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:11:49.756 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:11:49.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:11:49.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:11:49.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:11:49.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:11:49.822 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:11:49.824 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:11:49.824 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:11:49.824 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:11:49.824 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:11:49.824 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:11:49.824 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:11:49.824 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:11:49.824 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:11:49.824 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:11:49.824 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:11:49.825 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:11:49.825 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:11:49.825 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:49.826 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:11:49.826 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:49.863 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 11 2026-02-18T08:11:49.863 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 14 2026-02-18T08:11:49.863 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:11:49.863 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=57, tag_tid=3, entry_tid=1], [object_number=56, tag_tid=3, entry_tid=0], [object_number=59, tag_tid=2, entry_tid=59619], [object_number=58, tag_tid=2, entry_tid=59618]]], state=connected] 2026-02-18T08:11:49.863 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=46, tag_tid=2, entry_tid=47134], [object_number=45, tag_tid=2, entry_tid=47133], [object_number=44, tag_tid=2, entry_tid=47132], [object_number=47, tag_tid=2, entry_tid=47131]]], state=connected] 2026-02-18T08:11:49.865 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:49.867 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=57, tag_tid=3, entry_tid=1' 2026-02-18T08:11:49.868 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:11:49.868 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:11:49.868 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:11:49.868 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:11:49.868 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:11:49.869 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:11:49.869 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:11:49.869 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:11:49.869 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:11:49.869 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:11:49.869 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:11:49.870 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:11:49.870 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:49.870 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:11:49.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:49.896 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 11 2026-02-18T08:11:49.896 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 14 2026-02-18T08:11:49.896 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:11:49.896 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=57, tag_tid=3, entry_tid=1], [object_number=56, tag_tid=3, entry_tid=0], [object_number=59, tag_tid=2, entry_tid=59619], [object_number=58, tag_tid=2, entry_tid=59618]]], state=connected] 2026-02-18T08:11:49.896 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=46, tag_tid=2, entry_tid=47134], [object_number=45, tag_tid=2, entry_tid=47133], [object_number=44, tag_tid=2, entry_tid=47132], [object_number=47, tag_tid=2, entry_tid=47131]]], state=connected] 2026-02-18T08:11:49.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:49.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=46, tag_tid=2, entry_tid=47134' 2026-02-18T08:11:49.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=57, tag_tid=3, entry_tid=1' -a 'object_number=57, tag_tid=3, entry_tid=1' = 'object_number=46, tag_tid=2, entry_tid=47134' 2026-02-18T08:11:49.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=46, tag_tid=2, entry_tid=47134' '!=' 'object_number=45, tag_tid=2, entry_tid=46813' 2026-02-18T08:11:49.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:11:49.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=46, tag_tid=2, entry_tid=47134' = 'object_number=45, tag_tid=2, entry_tid=46813' 2026-02-18T08:11:49.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=46, tag_tid=2, entry_tid=47134' 2026-02-18T08:11:49.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=57, tag_tid=3, entry_tid=1' 2026-02-18T08:11:49.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:11:49.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:11:49.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=3 2026-02-18T08:11:49.904 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=46, tag_tid=2, entry_tid=47134' 2026-02-18T08:11:49.904 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:11:49.904 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:11:49.905 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=2 2026-02-18T08:11:49.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=57, tag_tid=3, entry_tid=1' 2026-02-18T08:11:49.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:11:49.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:11:49.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:11:49.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=46, tag_tid=2, entry_tid=47134' 2026-02-18T08:11:49.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:11:49.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:11:49.910 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=47134 2026-02-18T08:11:49.910 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 3 = 2 -a 1 -le 47134 2026-02-18T08:11:49.910 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:11:49.910 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:11:49.910 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:11:50.111 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:11:50.111 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:11:50.111 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:11:50.111 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:11:50.112 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:11:50.112 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:11:50.112 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:11:50.112 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:11:50.112 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:11:50.112 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:11:50.112 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:11:50.112 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:11:50.112 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:11:50.112 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:11:50.112 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:11:50.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:11:50.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:11:50.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:11:50.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:11:50.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:11:50.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:11:50.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:11:50.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:11:50.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:11:50.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:11:50.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:11:50.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:11:50.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:11:50.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:11:50.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:11:50.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:11:50.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:11:50.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:11:50.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:11:50.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:11:50.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:11:50.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:11:50.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:11:50.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:11:50.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:11:50.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:11:50.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:11:50.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:11:50.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:11:50.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:11:50.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:11:50.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:11:50.118 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:11:50.118 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:11:50.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:11:50.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:11:50.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:11:50.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:11:50.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:11:50.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:11:50.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:11:50.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:11:50.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:11:50.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:11:50.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:11:50.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:11:50.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:11:50.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:11:50.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:11:50.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:11:50.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:11:50.187 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:50.187 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:11:50.187 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:50.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 11 2026-02-18T08:11:50.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 14 2026-02-18T08:11:50.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:11:50.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=57, tag_tid=3, entry_tid=1], [object_number=56, tag_tid=3, entry_tid=0], [object_number=59, tag_tid=2, entry_tid=59619], [object_number=58, tag_tid=2, entry_tid=59618]]], state=connected] 2026-02-18T08:11:50.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=45, tag_tid=2, entry_tid=47421], [object_number=44, tag_tid=2, entry_tid=47420], [object_number=47, tag_tid=2, entry_tid=47419], [object_number=46, tag_tid=2, entry_tid=47418]]], state=connected] 2026-02-18T08:11:50.219 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:50.220 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=57, tag_tid=3, entry_tid=1' 2026-02-18T08:11:50.221 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:11:50.221 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:11:50.221 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:11:50.221 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:11:50.221 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:11:50.221 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:11:50.222 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:11:50.222 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:11:50.222 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:11:50.222 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:11:50.222 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:11:50.222 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:11:50.222 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:50.223 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:11:50.223 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:50.250 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 11 2026-02-18T08:11:50.250 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 14 2026-02-18T08:11:50.250 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:11:50.250 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=57, tag_tid=3, entry_tid=1], [object_number=56, tag_tid=3, entry_tid=0], [object_number=59, tag_tid=2, entry_tid=59619], [object_number=58, tag_tid=2, entry_tid=59618]]], state=connected] 2026-02-18T08:11:50.250 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=45, tag_tid=2, entry_tid=47421], [object_number=44, tag_tid=2, entry_tid=47420], [object_number=47, tag_tid=2, entry_tid=47419], [object_number=46, tag_tid=2, entry_tid=47418]]], state=connected] 2026-02-18T08:11:50.253 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:50.254 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=45, tag_tid=2, entry_tid=47421' 2026-02-18T08:11:50.254 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=57, tag_tid=3, entry_tid=1' -a 'object_number=57, tag_tid=3, entry_tid=1' = 'object_number=45, tag_tid=2, entry_tid=47421' 2026-02-18T08:11:50.254 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=45, tag_tid=2, entry_tid=47421' '!=' 'object_number=46, tag_tid=2, entry_tid=47134' 2026-02-18T08:11:50.255 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:11:50.255 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=45, tag_tid=2, entry_tid=47421' = 'object_number=46, tag_tid=2, entry_tid=47134' 2026-02-18T08:11:50.255 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=45, tag_tid=2, entry_tid=47421' 2026-02-18T08:11:50.255 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=57, tag_tid=3, entry_tid=1' 2026-02-18T08:11:50.255 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:11:50.256 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:11:50.256 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=3 2026-02-18T08:11:50.257 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=45, tag_tid=2, entry_tid=47421' 2026-02-18T08:11:50.257 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:11:50.257 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:11:50.259 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=2 2026-02-18T08:11:50.259 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=57, tag_tid=3, entry_tid=1' 2026-02-18T08:11:50.259 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:11:50.260 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:11:50.261 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:11:50.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=45, tag_tid=2, entry_tid=47421' 2026-02-18T08:11:50.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:11:50.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:11:50.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=47421 2026-02-18T08:11:50.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 3 = 2 -a 1 -le 47421 2026-02-18T08:11:50.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:11:50.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:11:50.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:11:50.465 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:11:50.465 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:11:50.465 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:11:50.465 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:11:50.465 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:11:50.465 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:11:50.465 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:11:50.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:11:50.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:11:50.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:11:50.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:11:50.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:11:50.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:11:50.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:11:50.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:11:50.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:11:50.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:11:50.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:11:50.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:11:50.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:11:50.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:11:50.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:11:50.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:11:50.470 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:11:50.470 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:11:50.470 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:11:50.470 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:11:50.470 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:11:50.470 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:11:50.470 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:11:50.470 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:11:50.470 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:11:50.470 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:11:50.470 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:11:50.470 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:11:50.470 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:11:50.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:11:50.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:11:50.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:11:50.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:11:50.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:11:50.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:11:50.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:11:50.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:11:50.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:11:50.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:11:50.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:11:50.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:11:50.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:11:50.479 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:11:50.480 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:11:50.480 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:11:50.480 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:11:50.539 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:11:50.540 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:11:50.540 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:11:50.540 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:11:50.540 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:11:50.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:11:50.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:11:50.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:11:50.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:11:50.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:11:50.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:11:50.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:11:50.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:11:50.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:50.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:11:50.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:50.571 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 11 2026-02-18T08:11:50.571 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 14 2026-02-18T08:11:50.571 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:11:50.571 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=57, tag_tid=3, entry_tid=1], [object_number=56, tag_tid=3, entry_tid=0], [object_number=59, tag_tid=2, entry_tid=59619], [object_number=58, tag_tid=2, entry_tid=59618]]], state=connected] 2026-02-18T08:11:50.571 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=44, tag_tid=2, entry_tid=47676], [object_number=47, tag_tid=2, entry_tid=47675], [object_number=46, tag_tid=2, entry_tid=47674], [object_number=45, tag_tid=2, entry_tid=47673]]], state=connected] 2026-02-18T08:11:50.574 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:50.575 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=57, tag_tid=3, entry_tid=1' 2026-02-18T08:11:50.577 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:11:50.577 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:11:50.577 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:11:50.577 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:11:50.577 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:11:50.577 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:11:50.577 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:11:50.577 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:11:50.577 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:11:50.577 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:11:50.578 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:11:50.578 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:11:50.578 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:50.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:11:50.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:50.605 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 11 2026-02-18T08:11:50.605 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 14 2026-02-18T08:11:50.606 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:11:50.606 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=57, tag_tid=3, entry_tid=1], [object_number=56, tag_tid=3, entry_tid=0], [object_number=59, tag_tid=2, entry_tid=59619], [object_number=58, tag_tid=2, entry_tid=59618]]], state=connected] 2026-02-18T08:11:50.606 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=44, tag_tid=2, entry_tid=47676], [object_number=47, tag_tid=2, entry_tid=47675], [object_number=46, tag_tid=2, entry_tid=47674], [object_number=45, tag_tid=2, entry_tid=47673]]], state=connected] 2026-02-18T08:11:50.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:50.610 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=44, tag_tid=2, entry_tid=47676' 2026-02-18T08:11:50.610 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=57, tag_tid=3, entry_tid=1' -a 'object_number=57, tag_tid=3, entry_tid=1' = 'object_number=44, tag_tid=2, entry_tid=47676' 2026-02-18T08:11:50.610 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=44, tag_tid=2, entry_tid=47676' '!=' 'object_number=45, tag_tid=2, entry_tid=47421' 2026-02-18T08:11:50.610 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:11:50.610 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=44, tag_tid=2, entry_tid=47676' = 'object_number=45, tag_tid=2, entry_tid=47421' 2026-02-18T08:11:50.610 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=44, tag_tid=2, entry_tid=47676' 2026-02-18T08:11:50.611 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=57, tag_tid=3, entry_tid=1' 2026-02-18T08:11:50.611 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:11:50.611 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:11:50.612 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=3 2026-02-18T08:11:50.613 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=44, tag_tid=2, entry_tid=47676' 2026-02-18T08:11:50.613 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:11:50.613 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:11:50.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=2 2026-02-18T08:11:50.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=57, tag_tid=3, entry_tid=1' 2026-02-18T08:11:50.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:11:50.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:11:50.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:11:50.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=44, tag_tid=2, entry_tid=47676' 2026-02-18T08:11:50.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:11:50.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:11:50.619 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=47676 2026-02-18T08:11:50.619 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 3 = 2 -a 1 -le 47676 2026-02-18T08:11:50.619 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:11:50.619 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:11:50.619 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:11:50.820 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:11:50.821 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:11:50.821 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:11:50.821 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:11:50.821 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:11:50.821 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:11:50.821 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:11:50.821 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:11:50.821 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:11:50.821 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:11:50.821 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:11:50.822 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:11:50.822 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:11:50.822 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:11:50.822 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:11:50.824 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:11:50.824 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:11:50.824 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:11:50.825 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:11:50.825 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:11:50.825 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:11:50.825 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:11:50.825 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:11:50.825 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:11:50.825 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:11:50.825 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:11:50.825 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:11:50.825 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:11:50.825 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:11:50.825 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:11:50.826 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:11:50.826 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:11:50.826 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:11:50.826 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:11:50.826 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:11:50.826 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:11:50.826 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:11:50.826 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:11:50.826 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:11:50.826 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:11:50.826 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:11:50.826 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:11:50.826 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:11:50.827 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:11:50.827 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:11:50.827 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:11:50.827 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:11:50.827 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:11:50.827 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:11:50.836 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:11:50.837 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:11:50.837 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:11:50.837 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:11:50.948 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:11:50.949 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:11:50.949 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:11:50.949 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:11:50.950 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:11:50.950 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:11:50.950 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:11:50.950 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:11:50.950 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:11:50.950 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:11:50.950 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:11:50.950 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:11:50.950 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:11:50.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:50.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:11:50.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:50.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 11 2026-02-18T08:11:50.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 14 2026-02-18T08:11:50.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:11:50.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=57, tag_tid=3, entry_tid=1], [object_number=56, tag_tid=3, entry_tid=0], [object_number=59, tag_tid=2, entry_tid=59619], [object_number=58, tag_tid=2, entry_tid=59618]]], state=connected] 2026-02-18T08:11:50.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=45, tag_tid=2, entry_tid=47853], [object_number=44, tag_tid=2, entry_tid=47852], [object_number=47, tag_tid=2, entry_tid=47851], [object_number=46, tag_tid=2, entry_tid=47850]]], state=connected] 2026-02-18T08:11:50.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:50.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=57, tag_tid=3, entry_tid=1' 2026-02-18T08:11:50.982 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:11:50.982 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:11:50.982 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:11:50.982 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:11:50.982 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:11:50.982 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:11:50.982 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:11:50.982 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:11:50.982 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:11:50.982 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:11:50.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:11:50.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:11:50.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:50.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:11:50.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:51.211 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 11 2026-02-18T08:11:51.211 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 14 2026-02-18T08:11:51.211 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:11:51.211 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=57, tag_tid=3, entry_tid=1], [object_number=56, tag_tid=3, entry_tid=0], [object_number=59, tag_tid=2, entry_tid=59619], [object_number=58, tag_tid=2, entry_tid=59618]]], state=connected] 2026-02-18T08:11:51.211 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=45, tag_tid=2, entry_tid=47853], [object_number=44, tag_tid=2, entry_tid=47852], [object_number=47, tag_tid=2, entry_tid=47851], [object_number=46, tag_tid=2, entry_tid=47850]]], state=connected] 2026-02-18T08:11:51.215 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:51.217 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=45, tag_tid=2, entry_tid=47853' 2026-02-18T08:11:51.217 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=57, tag_tid=3, entry_tid=1' -a 'object_number=57, tag_tid=3, entry_tid=1' = 'object_number=45, tag_tid=2, entry_tid=47853' 2026-02-18T08:11:51.218 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=45, tag_tid=2, entry_tid=47853' '!=' 'object_number=44, tag_tid=2, entry_tid=47676' 2026-02-18T08:11:51.218 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:11:51.218 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=45, tag_tid=2, entry_tid=47853' = 'object_number=44, tag_tid=2, entry_tid=47676' 2026-02-18T08:11:51.218 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=45, tag_tid=2, entry_tid=47853' 2026-02-18T08:11:51.218 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=57, tag_tid=3, entry_tid=1' 2026-02-18T08:11:51.219 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:11:51.219 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:11:51.219 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=3 2026-02-18T08:11:51.220 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=45, tag_tid=2, entry_tid=47853' 2026-02-18T08:11:51.220 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:11:51.220 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:11:51.221 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=2 2026-02-18T08:11:51.222 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=57, tag_tid=3, entry_tid=1' 2026-02-18T08:11:51.222 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:11:51.223 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:11:51.224 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:11:51.225 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=45, tag_tid=2, entry_tid=47853' 2026-02-18T08:11:51.225 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:11:51.225 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:11:51.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=47853 2026-02-18T08:11:51.227 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 3 = 2 -a 1 -le 47853 2026-02-18T08:11:51.227 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:11:51.227 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:11:51.227 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:11:51.428 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:11:51.429 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:11:51.429 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:11:51.429 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:11:51.429 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:11:51.429 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:11:51.429 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:11:51.429 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:11:51.429 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:11:51.429 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:11:51.429 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:11:51.429 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:11:51.430 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:11:51.430 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:11:51.430 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:11:51.430 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:11:51.432 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:11:51.432 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:11:51.433 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:11:51.433 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:11:51.433 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:11:51.433 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:11:51.433 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:11:51.433 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:11:51.433 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:11:51.433 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:11:51.433 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:11:51.433 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:11:51.433 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:11:51.433 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:11:51.434 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:11:51.434 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:11:51.434 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:11:51.434 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:11:51.434 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:11:51.434 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:11:51.434 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:11:51.434 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:11:51.434 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:11:51.434 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:11:51.434 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:11:51.434 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:11:51.435 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:11:51.435 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:11:51.435 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:11:51.435 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:11:51.435 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:11:51.435 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:11:51.435 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:11:51.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:11:51.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:11:51.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:11:51.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:11:51.503 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:11:51.504 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:11:51.504 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:11:51.504 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:11:51.504 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:11:51.504 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:11:51.504 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:11:51.504 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:11:51.504 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:11:51.504 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:11:51.504 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:11:51.505 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:11:51.505 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:11:51.505 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:51.506 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:11:51.506 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:51.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 11 2026-02-18T08:11:51.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 14 2026-02-18T08:11:51.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:11:51.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=57, tag_tid=3, entry_tid=1], [object_number=56, tag_tid=3, entry_tid=0], [object_number=59, tag_tid=2, entry_tid=59619], [object_number=58, tag_tid=2, entry_tid=59618]]], state=connected] 2026-02-18T08:11:51.534 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=45, tag_tid=2, entry_tid=48321], [object_number=44, tag_tid=2, entry_tid=48320], [object_number=47, tag_tid=2, entry_tid=48319], [object_number=46, tag_tid=2, entry_tid=48318]]], state=connected] 2026-02-18T08:11:51.536 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:51.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=57, tag_tid=3, entry_tid=1' 2026-02-18T08:11:51.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:11:51.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:11:51.539 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:11:51.539 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:11:51.539 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:11:51.539 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:11:51.539 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:11:51.539 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:11:51.539 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:11:51.539 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:11:51.540 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:11:51.540 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:11:51.540 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:51.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:11:51.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:51.576 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 11 2026-02-18T08:11:51.577 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 14 2026-02-18T08:11:51.577 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:11:51.577 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=57, tag_tid=3, entry_tid=1], [object_number=56, tag_tid=3, entry_tid=0], [object_number=59, tag_tid=2, entry_tid=59619], [object_number=58, tag_tid=2, entry_tid=59618]]], state=connected] 2026-02-18T08:11:51.577 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=45, tag_tid=2, entry_tid=48321], [object_number=44, tag_tid=2, entry_tid=48320], [object_number=47, tag_tid=2, entry_tid=48319], [object_number=46, tag_tid=2, entry_tid=48318]]], state=connected] 2026-02-18T08:11:51.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:51.582 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=45, tag_tid=2, entry_tid=48321' 2026-02-18T08:11:51.582 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=57, tag_tid=3, entry_tid=1' -a 'object_number=57, tag_tid=3, entry_tid=1' = 'object_number=45, tag_tid=2, entry_tid=48321' 2026-02-18T08:11:51.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=45, tag_tid=2, entry_tid=48321' '!=' 'object_number=45, tag_tid=2, entry_tid=47853' 2026-02-18T08:11:51.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:11:51.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=45, tag_tid=2, entry_tid=48321' = 'object_number=45, tag_tid=2, entry_tid=47853' 2026-02-18T08:11:51.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=45, tag_tid=2, entry_tid=48321' 2026-02-18T08:11:51.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=57, tag_tid=3, entry_tid=1' 2026-02-18T08:11:51.584 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:11:51.584 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:11:51.584 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=3 2026-02-18T08:11:51.585 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=45, tag_tid=2, entry_tid=48321' 2026-02-18T08:11:51.585 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:11:51.585 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:11:51.586 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=2 2026-02-18T08:11:51.587 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=57, tag_tid=3, entry_tid=1' 2026-02-18T08:11:51.587 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:11:51.587 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:11:51.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:11:51.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=45, tag_tid=2, entry_tid=48321' 2026-02-18T08:11:51.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:11:51.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:11:51.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=48321 2026-02-18T08:11:51.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 3 = 2 -a 1 -le 48321 2026-02-18T08:11:51.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:11:51.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:11:51.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:11:51.793 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:11:51.793 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:11:51.793 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:11:51.793 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:11:51.793 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:11:51.793 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:11:51.793 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:11:51.793 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:11:51.794 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:11:51.794 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:11:51.794 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:11:51.794 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:11:51.794 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:11:51.794 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:11:51.794 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:11:51.796 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:11:51.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:11:51.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:11:51.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:11:51.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:11:51.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:11:51.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:11:51.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:11:51.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:11:51.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:11:51.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:11:51.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:11:51.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:11:51.798 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:11:51.798 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:11:51.798 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:11:51.798 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:11:51.798 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:11:51.798 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:11:51.798 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:11:51.798 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:11:51.798 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:11:51.798 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:11:51.798 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:11:51.798 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:11:51.798 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:11:51.799 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:11:51.799 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:11:51.799 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:11:51.799 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:11:51.799 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:11:51.799 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:11:51.799 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:11:51.799 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:11:51.808 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:11:51.808 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:11:51.808 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:11:51.808 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:11:51.868 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:11:51.869 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:11:51.869 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:11:51.869 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:11:51.869 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:11:51.870 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:11:51.870 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:11:51.870 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:11:51.870 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:11:51.870 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:11:51.870 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:11:51.870 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:11:51.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:11:51.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:51.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:11:51.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:51.905 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 11 2026-02-18T08:11:51.905 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 14 2026-02-18T08:11:51.905 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:11:51.905 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=57, tag_tid=3, entry_tid=1], [object_number=56, tag_tid=3, entry_tid=0], [object_number=59, tag_tid=2, entry_tid=59619], [object_number=58, tag_tid=2, entry_tid=59618]]], state=connected] 2026-02-18T08:11:51.905 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=45, tag_tid=2, entry_tid=48633], [object_number=44, tag_tid=2, entry_tid=48632], [object_number=47, tag_tid=2, entry_tid=48631], [object_number=46, tag_tid=2, entry_tid=48630]]], state=connected] 2026-02-18T08:11:51.909 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:51.910 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=57, tag_tid=3, entry_tid=1' 2026-02-18T08:11:51.911 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:11:51.911 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:11:51.911 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:11:51.911 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:11:51.911 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:11:51.911 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:11:51.911 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:11:51.911 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:11:51.911 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:11:51.911 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:11:51.912 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:11:51.912 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:11:51.913 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:51.913 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:11:51.913 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:51.948 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 11 2026-02-18T08:11:51.948 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 14 2026-02-18T08:11:51.948 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:11:51.948 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=57, tag_tid=3, entry_tid=1], [object_number=56, tag_tid=3, entry_tid=0], [object_number=59, tag_tid=2, entry_tid=59619], [object_number=58, tag_tid=2, entry_tid=59618]]], state=connected] 2026-02-18T08:11:51.948 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=45, tag_tid=2, entry_tid=48633], [object_number=44, tag_tid=2, entry_tid=48632], [object_number=47, tag_tid=2, entry_tid=48631], [object_number=46, tag_tid=2, entry_tid=48630]]], state=connected] 2026-02-18T08:11:51.952 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:51.953 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=45, tag_tid=2, entry_tid=48633' 2026-02-18T08:11:51.953 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=57, tag_tid=3, entry_tid=1' -a 'object_number=57, tag_tid=3, entry_tid=1' = 'object_number=45, tag_tid=2, entry_tid=48633' 2026-02-18T08:11:51.953 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=45, tag_tid=2, entry_tid=48633' '!=' 'object_number=45, tag_tid=2, entry_tid=48321' 2026-02-18T08:11:51.953 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:11:51.954 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=45, tag_tid=2, entry_tid=48633' = 'object_number=45, tag_tid=2, entry_tid=48321' 2026-02-18T08:11:51.954 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=45, tag_tid=2, entry_tid=48633' 2026-02-18T08:11:51.954 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=57, tag_tid=3, entry_tid=1' 2026-02-18T08:11:51.954 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:11:51.954 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:11:51.955 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=3 2026-02-18T08:11:51.956 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=45, tag_tid=2, entry_tid=48633' 2026-02-18T08:11:51.956 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:11:51.956 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:11:51.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=2 2026-02-18T08:11:51.958 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=57, tag_tid=3, entry_tid=1' 2026-02-18T08:11:51.958 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:11:51.958 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:11:51.959 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:11:51.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=45, tag_tid=2, entry_tid=48633' 2026-02-18T08:11:51.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:11:51.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:11:51.962 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=48633 2026-02-18T08:11:51.962 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 3 = 2 -a 1 -le 48633 2026-02-18T08:11:51.962 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:11:51.962 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:11:51.962 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:11:52.164 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:11:52.164 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:11:52.164 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:11:52.164 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:11:52.164 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:11:52.164 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:11:52.165 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:11:52.165 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:11:52.165 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:11:52.165 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:11:52.165 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:11:52.165 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:11:52.165 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:11:52.165 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:11:52.165 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:11:52.165 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:11:52.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:11:52.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:11:52.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:11:52.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:11:52.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:11:52.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:11:52.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:11:52.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:11:52.169 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:11:52.169 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:11:52.169 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:11:52.169 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:11:52.169 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:11:52.169 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:11:52.169 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:11:52.169 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:11:52.169 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:11:52.169 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:11:52.169 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:11:52.169 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:11:52.169 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:11:52.170 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:11:52.170 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:11:52.170 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:11:52.170 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:11:52.170 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:11:52.170 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:11:52.170 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:11:52.170 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:11:52.170 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:11:52.170 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:11:52.170 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:11:52.170 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:11:52.178 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:11:52.178 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:11:52.178 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:11:52.178 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:11:52.238 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:11:52.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:11:52.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:11:52.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:11:52.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:11:52.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:11:52.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:11:52.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:11:52.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:11:52.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:11:52.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:11:52.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:11:52.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:11:52.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:52.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:11:52.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:52.277 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 11 2026-02-18T08:11:52.277 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 14 2026-02-18T08:11:52.277 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:11:52.278 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=57, tag_tid=3, entry_tid=1], [object_number=56, tag_tid=3, entry_tid=0], [object_number=59, tag_tid=2, entry_tid=59619], [object_number=58, tag_tid=2, entry_tid=59618]]], state=connected] 2026-02-18T08:11:52.278 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=44, tag_tid=2, entry_tid=48920], [object_number=47, tag_tid=2, entry_tid=48919], [object_number=46, tag_tid=2, entry_tid=48918], [object_number=45, tag_tid=2, entry_tid=48917]]], state=connected] 2026-02-18T08:11:52.281 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:52.282 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=57, tag_tid=3, entry_tid=1' 2026-02-18T08:11:52.283 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:11:52.283 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:11:52.284 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:11:52.284 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:11:52.284 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:11:52.284 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:11:52.284 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:11:52.284 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:11:52.284 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:11:52.284 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:11:52.285 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:11:52.285 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:11:52.285 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:52.285 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:11:52.286 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:52.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 11 2026-02-18T08:11:52.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 14 2026-02-18T08:11:52.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:11:52.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=57, tag_tid=3, entry_tid=1], [object_number=56, tag_tid=3, entry_tid=0], [object_number=59, tag_tid=2, entry_tid=59619], [object_number=58, tag_tid=2, entry_tid=59618]]], state=connected] 2026-02-18T08:11:52.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=44, tag_tid=2, entry_tid=48920], [object_number=47, tag_tid=2, entry_tid=48919], [object_number=46, tag_tid=2, entry_tid=48918], [object_number=45, tag_tid=2, entry_tid=48917]]], state=connected] 2026-02-18T08:11:52.525 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:52.527 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=44, tag_tid=2, entry_tid=48920' 2026-02-18T08:11:52.527 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=57, tag_tid=3, entry_tid=1' -a 'object_number=57, tag_tid=3, entry_tid=1' = 'object_number=44, tag_tid=2, entry_tid=48920' 2026-02-18T08:11:52.527 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=44, tag_tid=2, entry_tid=48920' '!=' 'object_number=45, tag_tid=2, entry_tid=48633' 2026-02-18T08:11:52.527 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:11:52.527 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=44, tag_tid=2, entry_tid=48920' = 'object_number=45, tag_tid=2, entry_tid=48633' 2026-02-18T08:11:52.527 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=44, tag_tid=2, entry_tid=48920' 2026-02-18T08:11:52.528 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=57, tag_tid=3, entry_tid=1' 2026-02-18T08:11:52.528 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:11:52.528 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:11:52.529 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=3 2026-02-18T08:11:52.529 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=44, tag_tid=2, entry_tid=48920' 2026-02-18T08:11:52.530 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:11:52.530 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:11:52.531 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=2 2026-02-18T08:11:52.531 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=57, tag_tid=3, entry_tid=1' 2026-02-18T08:11:52.532 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:11:52.532 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:11:52.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:11:52.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=44, tag_tid=2, entry_tid=48920' 2026-02-18T08:11:52.534 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:11:52.534 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:11:52.535 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=48920 2026-02-18T08:11:52.535 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 3 = 2 -a 1 -le 48920 2026-02-18T08:11:52.535 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:11:52.536 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:11:52.536 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:11:52.737 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:11:52.737 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:11:52.737 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:11:52.737 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:11:52.737 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:11:52.737 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:11:52.737 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:11:52.737 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:11:52.737 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:11:52.738 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:11:52.738 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:11:52.738 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:11:52.738 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:11:52.738 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:11:52.738 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:11:52.740 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:11:52.741 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:11:52.741 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:11:52.741 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:11:52.741 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:11:52.741 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:11:52.741 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:11:52.741 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:11:52.741 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:11:52.741 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:11:52.741 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:11:52.742 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:11:52.742 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:11:52.742 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:11:52.742 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:11:52.742 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:11:52.742 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:11:52.742 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:11:52.742 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:11:52.742 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:11:52.742 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:11:52.742 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:11:52.742 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:11:52.742 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:11:52.743 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:11:52.743 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:11:52.743 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:11:52.743 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:11:52.743 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:11:52.743 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:11:52.743 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:11:52.743 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:11:52.743 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:11:52.743 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:11:52.751 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:11:52.752 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:11:52.752 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:11:52.752 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:11:52.811 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:11:52.812 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:11:52.812 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:11:52.812 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:11:52.812 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:11:52.813 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:11:52.813 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:11:52.813 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:11:52.813 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:11:52.813 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:11:52.813 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:11:52.813 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:11:52.814 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:11:52.814 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:52.815 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:11:52.815 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:52.841 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 11 2026-02-18T08:11:52.841 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 14 2026-02-18T08:11:52.842 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:11:52.842 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=57, tag_tid=3, entry_tid=1], [object_number=56, tag_tid=3, entry_tid=0], [object_number=59, tag_tid=2, entry_tid=59619], [object_number=58, tag_tid=2, entry_tid=59618]]], state=connected] 2026-02-18T08:11:52.842 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=44, tag_tid=2, entry_tid=49436], [object_number=47, tag_tid=2, entry_tid=49435], [object_number=46, tag_tid=2, entry_tid=49434], [object_number=45, tag_tid=2, entry_tid=49433]]], state=connected] 2026-02-18T08:11:52.844 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:52.845 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=57, tag_tid=3, entry_tid=1' 2026-02-18T08:11:52.846 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:11:52.846 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:11:52.847 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:11:52.847 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:11:52.847 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:11:52.847 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:11:52.847 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:11:52.847 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:11:52.847 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:11:52.847 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:11:52.848 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:11:52.848 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:11:52.848 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:52.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:11:52.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:52.875 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 11 2026-02-18T08:11:52.875 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 14 2026-02-18T08:11:52.875 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:11:52.875 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=57, tag_tid=3, entry_tid=1], [object_number=56, tag_tid=3, entry_tid=0], [object_number=59, tag_tid=2, entry_tid=59619], [object_number=58, tag_tid=2, entry_tid=59618]]], state=connected] 2026-02-18T08:11:52.875 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=44, tag_tid=2, entry_tid=49436], [object_number=47, tag_tid=2, entry_tid=49435], [object_number=46, tag_tid=2, entry_tid=49434], [object_number=45, tag_tid=2, entry_tid=49433]]], state=connected] 2026-02-18T08:11:52.878 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:52.879 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=44, tag_tid=2, entry_tid=49436' 2026-02-18T08:11:52.879 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=57, tag_tid=3, entry_tid=1' -a 'object_number=57, tag_tid=3, entry_tid=1' = 'object_number=44, tag_tid=2, entry_tid=49436' 2026-02-18T08:11:52.879 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=44, tag_tid=2, entry_tid=49436' '!=' 'object_number=44, tag_tid=2, entry_tid=48920' 2026-02-18T08:11:52.879 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:11:52.880 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=44, tag_tid=2, entry_tid=49436' = 'object_number=44, tag_tid=2, entry_tid=48920' 2026-02-18T08:11:52.880 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=44, tag_tid=2, entry_tid=49436' 2026-02-18T08:11:52.880 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=57, tag_tid=3, entry_tid=1' 2026-02-18T08:11:52.880 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:11:52.881 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:11:52.881 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=3 2026-02-18T08:11:52.882 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=44, tag_tid=2, entry_tid=49436' 2026-02-18T08:11:52.882 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:11:52.882 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:11:52.883 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=2 2026-02-18T08:11:52.884 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=57, tag_tid=3, entry_tid=1' 2026-02-18T08:11:52.884 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:11:52.884 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:11:52.885 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:11:52.886 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=44, tag_tid=2, entry_tid=49436' 2026-02-18T08:11:52.886 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:11:52.886 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:11:52.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=49436 2026-02-18T08:11:52.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 3 = 2 -a 1 -le 49436 2026-02-18T08:11:52.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:11:52.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:11:52.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:11:53.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:11:53.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:11:53.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:11:53.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:11:53.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:11:53.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:11:53.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:11:53.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:11:53.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:11:53.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:11:53.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:11:53.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:11:53.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:11:53.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:11:53.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:11:53.093 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:11:53.093 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:11:53.093 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:11:53.094 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:11:53.094 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:11:53.094 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:11:53.094 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:11:53.094 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:11:53.094 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:11:53.094 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:11:53.094 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:11:53.094 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:11:53.094 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:11:53.094 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:11:53.094 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:11:53.095 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:11:53.095 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:11:53.095 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:11:53.095 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:11:53.095 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:11:53.095 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:11:53.095 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:11:53.095 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:11:53.095 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:11:53.095 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:11:53.095 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:11:53.095 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:11:53.095 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:11:53.096 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:11:53.096 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:11:53.096 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:11:53.096 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:11:53.096 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:11:53.096 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:11:53.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:11:53.105 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:11:53.105 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:11:53.105 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:11:53.170 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:11:53.171 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:11:53.171 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:11:53.171 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:11:53.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:11:53.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:11:53.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:11:53.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:11:53.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:11:53.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:11:53.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:11:53.173 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:11:53.173 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:11:53.173 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:53.174 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:11:53.174 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:53.200 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 11 2026-02-18T08:11:53.200 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 14 2026-02-18T08:11:53.200 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:11:53.200 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=57, tag_tid=3, entry_tid=1], [object_number=56, tag_tid=3, entry_tid=0], [object_number=59, tag_tid=2, entry_tid=59619], [object_number=58, tag_tid=2, entry_tid=59618]]], state=connected] 2026-02-18T08:11:53.200 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=46, tag_tid=2, entry_tid=49758], [object_number=45, tag_tid=2, entry_tid=49757], [object_number=44, tag_tid=2, entry_tid=49756], [object_number=47, tag_tid=2, entry_tid=49755]]], state=connected] 2026-02-18T08:11:53.203 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:53.204 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=57, tag_tid=3, entry_tid=1' 2026-02-18T08:11:53.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:11:53.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:11:53.206 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:11:53.206 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:11:53.206 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:11:53.206 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:11:53.206 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:11:53.206 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:11:53.206 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:11:53.206 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:11:53.207 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:11:53.207 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:11:53.207 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:53.207 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:11:53.208 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:53.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 11 2026-02-18T08:11:53.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 14 2026-02-18T08:11:53.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:11:53.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=57, tag_tid=3, entry_tid=1], [object_number=56, tag_tid=3, entry_tid=0], [object_number=59, tag_tid=2, entry_tid=59619], [object_number=58, tag_tid=2, entry_tid=59618]]], state=connected] 2026-02-18T08:11:53.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=46, tag_tid=2, entry_tid=49758], [object_number=45, tag_tid=2, entry_tid=49757], [object_number=44, tag_tid=2, entry_tid=49756], [object_number=47, tag_tid=2, entry_tid=49755]]], state=connected] 2026-02-18T08:11:53.246 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:53.248 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=46, tag_tid=2, entry_tid=49758' 2026-02-18T08:11:53.248 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=57, tag_tid=3, entry_tid=1' -a 'object_number=57, tag_tid=3, entry_tid=1' = 'object_number=46, tag_tid=2, entry_tid=49758' 2026-02-18T08:11:53.248 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=46, tag_tid=2, entry_tid=49758' '!=' 'object_number=44, tag_tid=2, entry_tid=49436' 2026-02-18T08:11:53.248 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:11:53.248 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=46, tag_tid=2, entry_tid=49758' = 'object_number=44, tag_tid=2, entry_tid=49436' 2026-02-18T08:11:53.248 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=46, tag_tid=2, entry_tid=49758' 2026-02-18T08:11:53.249 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=57, tag_tid=3, entry_tid=1' 2026-02-18T08:11:53.249 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:11:53.249 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:11:53.250 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=3 2026-02-18T08:11:53.250 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=46, tag_tid=2, entry_tid=49758' 2026-02-18T08:11:53.251 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:11:53.251 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:11:53.252 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=2 2026-02-18T08:11:53.253 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=57, tag_tid=3, entry_tid=1' 2026-02-18T08:11:53.253 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:11:53.253 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:11:53.254 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:11:53.255 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=46, tag_tid=2, entry_tid=49758' 2026-02-18T08:11:53.255 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:11:53.255 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:11:53.257 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=49758 2026-02-18T08:11:53.257 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 3 = 2 -a 1 -le 49758 2026-02-18T08:11:53.257 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:11:53.257 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:11:53.257 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:11:53.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:11:53.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:11:53.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:11:53.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:11:53.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:11:53.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:11:53.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:11:53.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:11:53.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:11:53.460 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:11:53.460 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:11:53.460 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:11:53.460 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:11:53.460 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:11:53.460 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:11:53.460 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:11:53.463 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:11:53.463 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:11:53.463 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:11:53.463 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:11:53.463 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:11:53.463 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:11:53.463 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:11:53.463 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:11:53.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:11:53.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:11:53.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:11:53.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:11:53.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:11:53.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:11:53.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:11:53.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:11:53.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:11:53.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:11:53.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:11:53.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:11:53.465 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:11:53.465 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:11:53.465 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:11:53.465 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:11:53.465 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:11:53.465 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:11:53.465 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:11:53.465 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:11:53.465 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:11:53.465 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:11:53.465 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:11:53.465 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:11:53.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:11:53.473 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:11:53.473 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:11:53.474 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:11:53.474 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:11:53.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:11:53.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:11:53.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:11:53.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:11:53.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:11:53.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:11:53.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:11:53.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:11:53.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:11:53.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:11:53.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:11:53.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:11:53.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:11:53.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:53.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:11:53.545 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:53.572 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 11 2026-02-18T08:11:53.573 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 14 2026-02-18T08:11:53.573 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:11:53.573 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=57, tag_tid=3, entry_tid=1], [object_number=56, tag_tid=3, entry_tid=0], [object_number=59, tag_tid=2, entry_tid=59619], [object_number=58, tag_tid=2, entry_tid=59618]]], state=connected] 2026-02-18T08:11:53.573 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=49, tag_tid=2, entry_tid=50005], [object_number=48, tag_tid=2, entry_tid=50004], [object_number=47, tag_tid=2, entry_tid=50003], [object_number=46, tag_tid=2, entry_tid=50002]]], state=connected] 2026-02-18T08:11:53.575 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:53.577 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=57, tag_tid=3, entry_tid=1' 2026-02-18T08:11:53.578 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:11:53.578 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:11:53.578 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:11:53.578 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:11:53.578 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:11:53.578 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:11:53.578 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:11:53.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:11:53.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:11:53.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:11:53.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:11:53.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:11:53.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:53.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:11:53.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:53.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 11 2026-02-18T08:11:53.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 14 2026-02-18T08:11:53.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:11:53.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=57, tag_tid=3, entry_tid=1], [object_number=56, tag_tid=3, entry_tid=0], [object_number=59, tag_tid=2, entry_tid=59619], [object_number=58, tag_tid=2, entry_tid=59618]]], state=connected] 2026-02-18T08:11:53.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=49, tag_tid=2, entry_tid=50005], [object_number=48, tag_tid=2, entry_tid=50004], [object_number=47, tag_tid=2, entry_tid=50003], [object_number=46, tag_tid=2, entry_tid=50002]]], state=connected] 2026-02-18T08:11:53.618 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:53.620 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=49, tag_tid=2, entry_tid=50005' 2026-02-18T08:11:53.620 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=57, tag_tid=3, entry_tid=1' -a 'object_number=57, tag_tid=3, entry_tid=1' = 'object_number=49, tag_tid=2, entry_tid=50005' 2026-02-18T08:11:53.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=49, tag_tid=2, entry_tid=50005' '!=' 'object_number=46, tag_tid=2, entry_tid=49758' 2026-02-18T08:11:53.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:11:53.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=49, tag_tid=2, entry_tid=50005' = 'object_number=46, tag_tid=2, entry_tid=49758' 2026-02-18T08:11:53.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=49, tag_tid=2, entry_tid=50005' 2026-02-18T08:11:53.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=57, tag_tid=3, entry_tid=1' 2026-02-18T08:11:53.622 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:11:53.622 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:11:53.622 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=3 2026-02-18T08:11:53.623 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=49, tag_tid=2, entry_tid=50005' 2026-02-18T08:11:53.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:11:53.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:11:53.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=2 2026-02-18T08:11:53.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=57, tag_tid=3, entry_tid=1' 2026-02-18T08:11:53.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:11:53.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:11:53.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:11:53.628 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=49, tag_tid=2, entry_tid=50005' 2026-02-18T08:11:53.628 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:11:53.628 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:11:53.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=50005 2026-02-18T08:11:53.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 3 = 2 -a 1 -le 50005 2026-02-18T08:11:53.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:11:53.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:11:53.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:11:53.832 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:11:53.832 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:11:53.832 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:11:53.832 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:11:53.832 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:11:53.832 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:11:53.832 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:11:53.832 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:11:53.832 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:11:53.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:11:53.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:11:53.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:11:53.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:11:53.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:11:53.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:11:53.836 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:11:53.836 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:11:53.836 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:11:53.836 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:11:53.836 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:11:53.836 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:11:53.836 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:11:53.836 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:11:53.837 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:11:53.837 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:11:53.837 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:11:53.837 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:11:53.837 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:11:53.837 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:11:53.837 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:11:53.837 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:11:53.837 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:11:53.837 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:11:53.838 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:11:53.838 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:11:53.838 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:11:53.838 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:11:53.838 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:11:53.838 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:11:53.838 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:11:53.838 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:11:53.838 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:11:53.839 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:11:53.839 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:11:53.839 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:11:53.839 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:11:53.839 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:11:53.839 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:11:53.839 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:11:53.847 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:11:53.847 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:11:53.847 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:11:53.847 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:11:53.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:11:53.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:11:53.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:11:53.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:11:53.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:11:53.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:11:53.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:11:53.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:11:53.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:11:53.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:11:53.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:11:53.909 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:11:53.909 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:11:53.909 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:53.910 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:11:53.910 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:53.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 12 2026-02-18T08:11:53.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 14 2026-02-18T08:11:53.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:11:53.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=57, tag_tid=3, entry_tid=1], [object_number=56, tag_tid=3, entry_tid=0], [object_number=59, tag_tid=2, entry_tid=59619], [object_number=58, tag_tid=2, entry_tid=59618]]], state=connected] 2026-02-18T08:11:53.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=50, tag_tid=2, entry_tid=50302], [object_number=49, tag_tid=2, entry_tid=50301], [object_number=48, tag_tid=2, entry_tid=50300], [object_number=51, tag_tid=2, entry_tid=50299]]], state=connected] 2026-02-18T08:11:53.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:53.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=57, tag_tid=3, entry_tid=1' 2026-02-18T08:11:53.944 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:11:53.944 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:11:53.944 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:11:53.944 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:11:53.944 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:11:53.944 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:11:53.944 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:11:53.944 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:11:53.944 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:11:53.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:11:53.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:11:53.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:11:53.946 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:53.946 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:11:53.946 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:53.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 12 2026-02-18T08:11:53.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 14 2026-02-18T08:11:53.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:11:53.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=57, tag_tid=3, entry_tid=1], [object_number=56, tag_tid=3, entry_tid=0], [object_number=59, tag_tid=2, entry_tid=59619], [object_number=58, tag_tid=2, entry_tid=59618]]], state=connected] 2026-02-18T08:11:53.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=50, tag_tid=2, entry_tid=50302], [object_number=49, tag_tid=2, entry_tid=50301], [object_number=48, tag_tid=2, entry_tid=50300], [object_number=51, tag_tid=2, entry_tid=50299]]], state=connected] 2026-02-18T08:11:53.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:53.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=50, tag_tid=2, entry_tid=50302' 2026-02-18T08:11:53.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=57, tag_tid=3, entry_tid=1' -a 'object_number=57, tag_tid=3, entry_tid=1' = 'object_number=50, tag_tid=2, entry_tid=50302' 2026-02-18T08:11:53.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=50, tag_tid=2, entry_tid=50302' '!=' 'object_number=49, tag_tid=2, entry_tid=50005' 2026-02-18T08:11:53.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:11:53.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=50, tag_tid=2, entry_tid=50302' = 'object_number=49, tag_tid=2, entry_tid=50005' 2026-02-18T08:11:53.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=50, tag_tid=2, entry_tid=50302' 2026-02-18T08:11:53.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=57, tag_tid=3, entry_tid=1' 2026-02-18T08:11:53.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:11:53.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:11:53.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=3 2026-02-18T08:11:53.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=50, tag_tid=2, entry_tid=50302' 2026-02-18T08:11:53.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:11:53.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:11:53.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=2 2026-02-18T08:11:53.982 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=57, tag_tid=3, entry_tid=1' 2026-02-18T08:11:53.982 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:11:53.982 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:11:53.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:11:53.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=50, tag_tid=2, entry_tid=50302' 2026-02-18T08:11:53.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:11:53.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:11:53.986 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=50302 2026-02-18T08:11:53.986 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 3 = 2 -a 1 -le 50302 2026-02-18T08:11:53.986 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:11:53.986 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:11:53.986 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:11:54.187 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:11:54.188 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:11:54.188 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:11:54.188 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:11:54.188 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:11:54.188 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:11:54.188 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:11:54.188 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:11:54.188 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:11:54.188 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:11:54.188 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:11:54.189 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:11:54.189 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:11:54.189 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:11:54.189 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:11:54.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:11:54.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:11:54.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:11:54.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:11:54.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:11:54.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:11:54.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:11:54.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:11:54.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:11:54.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:11:54.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:11:54.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:11:54.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:11:54.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:11:54.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:11:54.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:11:54.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:11:54.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:11:54.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:11:54.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:11:54.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:11:54.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:11:54.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:11:54.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:11:54.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:11:54.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:11:54.194 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:11:54.194 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:11:54.194 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:11:54.194 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:11:54.194 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:11:54.194 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:11:54.194 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:11:54.194 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:11:54.202 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:11:54.202 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:11:54.202 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:11:54.202 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:11:54.270 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:11:54.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:11:54.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:11:54.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:11:54.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:11:54.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:11:54.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:11:54.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:11:54.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:11:54.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:11:54.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:11:54.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:11:54.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:11:54.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:54.273 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:11:54.273 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:54.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 12 2026-02-18T08:11:54.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 14 2026-02-18T08:11:54.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:11:54.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=57, tag_tid=3, entry_tid=1], [object_number=56, tag_tid=3, entry_tid=0], [object_number=59, tag_tid=2, entry_tid=59619], [object_number=58, tag_tid=2, entry_tid=59618]]], state=connected] 2026-02-18T08:11:54.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=50, tag_tid=2, entry_tid=50566], [object_number=49, tag_tid=2, entry_tid=50565], [object_number=48, tag_tid=2, entry_tid=50564], [object_number=51, tag_tid=2, entry_tid=50563]]], state=connected] 2026-02-18T08:11:54.312 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:54.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=57, tag_tid=3, entry_tid=1' 2026-02-18T08:11:54.314 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:11:54.314 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:11:54.314 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:11:54.314 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:11:54.314 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:11:54.315 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:11:54.315 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:11:54.315 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:11:54.315 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:11:54.315 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:11:54.315 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:11:54.315 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:11:54.316 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:54.316 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:11:54.316 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:54.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 12 2026-02-18T08:11:54.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 14 2026-02-18T08:11:54.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:11:54.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=57, tag_tid=3, entry_tid=1], [object_number=56, tag_tid=3, entry_tid=0], [object_number=59, tag_tid=2, entry_tid=59619], [object_number=58, tag_tid=2, entry_tid=59618]]], state=connected] 2026-02-18T08:11:54.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=50, tag_tid=2, entry_tid=50566], [object_number=49, tag_tid=2, entry_tid=50565], [object_number=48, tag_tid=2, entry_tid=50564], [object_number=51, tag_tid=2, entry_tid=50563]]], state=connected] 2026-02-18T08:11:54.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:54.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=50, tag_tid=2, entry_tid=50566' 2026-02-18T08:11:54.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=57, tag_tid=3, entry_tid=1' -a 'object_number=57, tag_tid=3, entry_tid=1' = 'object_number=50, tag_tid=2, entry_tid=50566' 2026-02-18T08:11:54.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=50, tag_tid=2, entry_tid=50566' '!=' 'object_number=50, tag_tid=2, entry_tid=50302' 2026-02-18T08:11:54.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:11:54.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=50, tag_tid=2, entry_tid=50566' = 'object_number=50, tag_tid=2, entry_tid=50302' 2026-02-18T08:11:54.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=50, tag_tid=2, entry_tid=50566' 2026-02-18T08:11:54.348 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=57, tag_tid=3, entry_tid=1' 2026-02-18T08:11:54.348 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:11:54.348 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:11:54.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=3 2026-02-18T08:11:54.350 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=50, tag_tid=2, entry_tid=50566' 2026-02-18T08:11:54.350 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:11:54.350 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:11:54.351 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=2 2026-02-18T08:11:54.352 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=57, tag_tid=3, entry_tid=1' 2026-02-18T08:11:54.352 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:11:54.352 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:11:54.353 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:11:54.354 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=50, tag_tid=2, entry_tid=50566' 2026-02-18T08:11:54.354 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:11:54.354 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:11:54.356 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=50566 2026-02-18T08:11:54.356 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 3 = 2 -a 1 -le 50566 2026-02-18T08:11:54.356 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:11:54.356 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:11:54.356 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:11:54.557 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:11:54.558 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:11:54.558 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:11:54.558 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:11:54.558 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:11:54.558 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:11:54.558 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:11:54.558 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:11:54.558 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:11:54.558 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:11:54.558 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:11:54.558 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:11:54.558 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:11:54.559 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:11:54.559 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:11:54.561 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:11:54.561 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:11:54.561 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:11:54.562 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:11:54.562 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:11:54.562 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:11:54.562 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:11:54.562 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:11:54.562 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:11:54.562 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:11:54.562 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:11:54.562 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:11:54.562 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:11:54.562 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:11:54.562 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:11:54.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:11:54.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:11:54.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:11:54.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:11:54.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:11:54.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:11:54.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:11:54.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:11:54.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:11:54.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:11:54.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:11:54.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:11:54.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:11:54.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:11:54.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:11:54.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:11:54.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:11:54.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:11:54.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:11:54.571 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:11:54.572 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:11:54.572 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:11:54.572 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:11:54.629 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:11:54.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:11:54.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:11:54.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:11:54.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:11:54.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:11:54.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:11:54.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:11:54.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:11:54.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:11:54.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:11:54.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:11:54.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:11:54.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:54.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:11:54.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:54.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 12 2026-02-18T08:11:54.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 14 2026-02-18T08:11:54.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:11:54.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=57, tag_tid=3, entry_tid=1], [object_number=56, tag_tid=3, entry_tid=0], [object_number=59, tag_tid=2, entry_tid=59619], [object_number=58, tag_tid=2, entry_tid=59618]]], state=connected] 2026-02-18T08:11:54.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=50, tag_tid=2, entry_tid=50814], [object_number=49, tag_tid=2, entry_tid=50813], [object_number=48, tag_tid=2, entry_tid=50812], [object_number=51, tag_tid=2, entry_tid=50811]]], state=connected] 2026-02-18T08:11:54.672 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:54.673 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=57, tag_tid=3, entry_tid=1' 2026-02-18T08:11:54.674 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:11:54.674 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:11:54.674 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:11:54.674 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:11:54.674 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:11:54.675 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:11:54.675 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:11:54.675 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:11:54.675 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:11:54.675 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:11:54.675 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:11:54.675 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:11:54.676 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:54.676 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:11:54.676 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:54.702 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 12 2026-02-18T08:11:54.702 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 14 2026-02-18T08:11:54.702 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:11:54.702 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=57, tag_tid=3, entry_tid=1], [object_number=56, tag_tid=3, entry_tid=0], [object_number=59, tag_tid=2, entry_tid=59619], [object_number=58, tag_tid=2, entry_tid=59618]]], state=connected] 2026-02-18T08:11:54.703 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=50, tag_tid=2, entry_tid=50814], [object_number=49, tag_tid=2, entry_tid=50813], [object_number=48, tag_tid=2, entry_tid=50812], [object_number=51, tag_tid=2, entry_tid=50811]]], state=connected] 2026-02-18T08:11:54.705 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:54.706 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=50, tag_tid=2, entry_tid=50814' 2026-02-18T08:11:54.706 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=57, tag_tid=3, entry_tid=1' -a 'object_number=57, tag_tid=3, entry_tid=1' = 'object_number=50, tag_tid=2, entry_tid=50814' 2026-02-18T08:11:54.707 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=50, tag_tid=2, entry_tid=50814' '!=' 'object_number=50, tag_tid=2, entry_tid=50566' 2026-02-18T08:11:54.707 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:11:54.707 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=50, tag_tid=2, entry_tid=50814' = 'object_number=50, tag_tid=2, entry_tid=50566' 2026-02-18T08:11:54.707 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=50, tag_tid=2, entry_tid=50814' 2026-02-18T08:11:54.707 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=57, tag_tid=3, entry_tid=1' 2026-02-18T08:11:54.708 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:11:54.708 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:11:54.708 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=3 2026-02-18T08:11:54.709 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=50, tag_tid=2, entry_tid=50814' 2026-02-18T08:11:54.709 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:11:54.709 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:11:54.711 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=2 2026-02-18T08:11:54.711 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=57, tag_tid=3, entry_tid=1' 2026-02-18T08:11:54.712 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:11:54.712 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:11:54.713 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:11:54.714 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=50, tag_tid=2, entry_tid=50814' 2026-02-18T08:11:54.714 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:11:54.714 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:11:54.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=50814 2026-02-18T08:11:54.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 3 = 2 -a 1 -le 50814 2026-02-18T08:11:54.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:11:54.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:11:54.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:11:54.917 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:11:54.917 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:11:54.917 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:11:54.917 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:11:54.917 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:11:54.917 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:11:54.918 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:11:54.918 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:11:54.918 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:11:54.918 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:11:54.918 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:11:54.918 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:11:54.918 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:11:54.918 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:11:54.918 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:11:54.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:11:54.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:11:54.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:11:54.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:11:54.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:11:54.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:11:54.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:11:54.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:11:54.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:11:54.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:11:54.922 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:11:54.922 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:11:54.922 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:11:54.922 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:11:54.922 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:11:54.922 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:11:54.922 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:11:54.922 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:11:54.922 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:11:54.922 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:11:54.922 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:11:54.922 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:11:54.922 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:11:54.923 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:11:54.923 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:11:54.923 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:11:54.923 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:11:54.923 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:11:54.923 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:11:54.923 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:11:54.923 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:11:54.923 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:11:54.923 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:11:54.923 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:11:54.932 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:11:54.933 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:11:54.933 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:11:54.933 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:11:54.999 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:11:55.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:11:55.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:11:55.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:11:55.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:11:55.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:11:55.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:11:55.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:11:55.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:11:55.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:11:55.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:11:55.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:11:55.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:11:55.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:55.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:11:55.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:55.029 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 12 2026-02-18T08:11:55.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 14 2026-02-18T08:11:55.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:11:55.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=57, tag_tid=3, entry_tid=1], [object_number=56, tag_tid=3, entry_tid=0], [object_number=59, tag_tid=2, entry_tid=59619], [object_number=58, tag_tid=2, entry_tid=59618]]], state=connected] 2026-02-18T08:11:55.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=51, tag_tid=2, entry_tid=51103], [object_number=50, tag_tid=2, entry_tid=51102], [object_number=49, tag_tid=2, entry_tid=51101], [object_number=48, tag_tid=2, entry_tid=51100]]], state=connected] 2026-02-18T08:11:55.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:55.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=57, tag_tid=3, entry_tid=1' 2026-02-18T08:11:55.035 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:11:55.035 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:11:55.035 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:11:55.035 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:11:55.036 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:11:55.036 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:11:55.036 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:11:55.036 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:11:55.036 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:11:55.036 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:11:55.036 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:11:55.036 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:11:55.037 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:55.037 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:11:55.037 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:55.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 12 2026-02-18T08:11:55.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 14 2026-02-18T08:11:55.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:11:55.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=57, tag_tid=3, entry_tid=1], [object_number=56, tag_tid=3, entry_tid=0], [object_number=59, tag_tid=2, entry_tid=59619], [object_number=58, tag_tid=2, entry_tid=59618]]], state=connected] 2026-02-18T08:11:55.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=51, tag_tid=2, entry_tid=51103], [object_number=50, tag_tid=2, entry_tid=51102], [object_number=49, tag_tid=2, entry_tid=51101], [object_number=48, tag_tid=2, entry_tid=51100]]], state=connected] 2026-02-18T08:11:55.076 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:55.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=51, tag_tid=2, entry_tid=51103' 2026-02-18T08:11:55.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=57, tag_tid=3, entry_tid=1' -a 'object_number=57, tag_tid=3, entry_tid=1' = 'object_number=51, tag_tid=2, entry_tid=51103' 2026-02-18T08:11:55.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=51, tag_tid=2, entry_tid=51103' '!=' 'object_number=50, tag_tid=2, entry_tid=50814' 2026-02-18T08:11:55.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:11:55.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=51, tag_tid=2, entry_tid=51103' = 'object_number=50, tag_tid=2, entry_tid=50814' 2026-02-18T08:11:55.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=51, tag_tid=2, entry_tid=51103' 2026-02-18T08:11:55.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=57, tag_tid=3, entry_tid=1' 2026-02-18T08:11:55.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:11:55.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:11:55.080 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=3 2026-02-18T08:11:55.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=51, tag_tid=2, entry_tid=51103' 2026-02-18T08:11:55.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:11:55.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:11:55.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=2 2026-02-18T08:11:55.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=57, tag_tid=3, entry_tid=1' 2026-02-18T08:11:55.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:11:55.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:11:55.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:11:55.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=51, tag_tid=2, entry_tid=51103' 2026-02-18T08:11:55.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:11:55.086 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:11:55.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=51103 2026-02-18T08:11:55.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 3 = 2 -a 1 -le 51103 2026-02-18T08:11:55.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:11:55.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:11:55.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:11:55.289 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:11:55.289 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:11:55.289 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:11:55.289 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:11:55.289 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:11:55.289 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:11:55.289 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:11:55.289 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:11:55.290 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:11:55.290 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:11:55.290 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:11:55.290 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:11:55.290 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:11:55.290 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:11:55.290 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:11:55.290 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:11:55.292 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:11:55.293 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:11:55.293 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:11:55.293 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:11:55.293 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:11:55.293 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:11:55.293 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:11:55.293 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:11:55.293 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:11:55.293 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:11:55.293 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:11:55.293 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:11:55.294 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:11:55.294 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:11:55.294 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:11:55.294 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:11:55.294 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:11:55.294 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:11:55.294 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:11:55.294 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:11:55.294 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:11:55.294 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:11:55.294 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:11:55.294 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:11:55.294 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:11:55.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:11:55.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:11:55.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:11:55.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:11:55.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:11:55.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:11:55.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:11:55.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:11:55.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:11:55.304 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:11:55.304 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:11:55.304 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:11:55.364 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:11:55.365 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:11:55.365 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:11:55.365 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:11:55.365 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:11:55.365 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:11:55.366 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:11:55.366 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:11:55.366 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:11:55.366 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:11:55.366 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:11:55.366 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:11:55.366 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:11:55.367 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:55.367 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:11:55.367 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:55.395 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 12 2026-02-18T08:11:55.395 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 14 2026-02-18T08:11:55.395 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:11:55.395 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=57, tag_tid=3, entry_tid=1], [object_number=56, tag_tid=3, entry_tid=0], [object_number=59, tag_tid=2, entry_tid=59619], [object_number=58, tag_tid=2, entry_tid=59618]]], state=connected] 2026-02-18T08:11:55.395 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=50, tag_tid=2, entry_tid=51406], [object_number=49, tag_tid=2, entry_tid=51405], [object_number=48, tag_tid=2, entry_tid=51404], [object_number=51, tag_tid=2, entry_tid=51403]]], state=connected] 2026-02-18T08:11:55.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:55.399 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=57, tag_tid=3, entry_tid=1' 2026-02-18T08:11:55.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:11:55.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:11:55.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:11:55.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:11:55.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:11:55.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:11:55.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:11:55.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:11:55.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:11:55.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:11:55.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:11:55.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:11:55.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:55.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:11:55.403 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:55.430 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 12 2026-02-18T08:11:55.430 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 14 2026-02-18T08:11:55.430 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:11:55.430 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=57, tag_tid=3, entry_tid=1], [object_number=56, tag_tid=3, entry_tid=0], [object_number=59, tag_tid=2, entry_tid=59619], [object_number=58, tag_tid=2, entry_tid=59618]]], state=connected] 2026-02-18T08:11:55.430 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=50, tag_tid=2, entry_tid=51406], [object_number=49, tag_tid=2, entry_tid=51405], [object_number=48, tag_tid=2, entry_tid=51404], [object_number=51, tag_tid=2, entry_tid=51403]]], state=connected] 2026-02-18T08:11:55.432 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:55.434 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=50, tag_tid=2, entry_tid=51406' 2026-02-18T08:11:55.434 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=57, tag_tid=3, entry_tid=1' -a 'object_number=57, tag_tid=3, entry_tid=1' = 'object_number=50, tag_tid=2, entry_tid=51406' 2026-02-18T08:11:55.434 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=50, tag_tid=2, entry_tid=51406' '!=' 'object_number=51, tag_tid=2, entry_tid=51103' 2026-02-18T08:11:55.434 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:11:55.435 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=50, tag_tid=2, entry_tid=51406' = 'object_number=51, tag_tid=2, entry_tid=51103' 2026-02-18T08:11:55.435 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=50, tag_tid=2, entry_tid=51406' 2026-02-18T08:11:55.435 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=57, tag_tid=3, entry_tid=1' 2026-02-18T08:11:55.435 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:11:55.435 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:11:55.436 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=3 2026-02-18T08:11:55.437 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=50, tag_tid=2, entry_tid=51406' 2026-02-18T08:11:55.437 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:11:55.437 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:11:55.439 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=2 2026-02-18T08:11:55.439 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=57, tag_tid=3, entry_tid=1' 2026-02-18T08:11:55.440 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:11:55.440 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:11:55.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:11:55.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=50, tag_tid=2, entry_tid=51406' 2026-02-18T08:11:55.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:11:55.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:11:55.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=51406 2026-02-18T08:11:55.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 3 = 2 -a 1 -le 51406 2026-02-18T08:11:55.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:11:55.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:11:55.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:11:55.646 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:11:55.646 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:11:55.646 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:11:55.646 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:11:55.646 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:11:55.646 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:11:55.646 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:11:55.646 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:11:55.647 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:11:55.647 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:11:55.647 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:11:55.647 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:11:55.647 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:11:55.647 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:11:55.647 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:11:55.647 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:11:55.647 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:11:55.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:11:55.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:11:55.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:11:55.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:11:55.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:11:55.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:11:55.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:11:55.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:11:55.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:11:55.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:11:55.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:11:55.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:11:55.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:11:55.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:11:55.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:11:55.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:11:55.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:11:55.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:11:55.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:11:55.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:11:55.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:11:55.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:11:55.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:11:55.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:11:55.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:11:55.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:11:55.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:11:55.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:11:55.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:11:55.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:11:55.653 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:11:55.653 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:11:55.660 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:11:55.660 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:11:55.661 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:11:55.661 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:11:55.718 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:11:55.719 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:11:55.719 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:11:55.719 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:11:55.719 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:11:55.720 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:11:55.720 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:11:55.720 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:11:55.720 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:11:55.720 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:11:55.720 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:11:55.720 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:11:55.720 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:11:55.721 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:55.721 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:11:55.721 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:55.756 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 12 2026-02-18T08:11:55.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 14 2026-02-18T08:11:55.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:11:55.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=57, tag_tid=3, entry_tid=1], [object_number=56, tag_tid=3, entry_tid=0], [object_number=59, tag_tid=2, entry_tid=59619], [object_number=58, tag_tid=2, entry_tid=59618]]], state=connected] 2026-02-18T08:11:55.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=48, tag_tid=2, entry_tid=51680], [object_number=51, tag_tid=2, entry_tid=51679], [object_number=50, tag_tid=2, entry_tid=51678], [object_number=49, tag_tid=2, entry_tid=51677]]], state=connected] 2026-02-18T08:11:55.760 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:55.762 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=57, tag_tid=3, entry_tid=1' 2026-02-18T08:11:55.762 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:11:55.762 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:11:55.763 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:11:55.763 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:11:55.763 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:11:55.763 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:11:55.763 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:11:55.763 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:11:55.763 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:11:55.763 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:11:55.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:11:55.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:11:55.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:55.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:11:55.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:55.800 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 12 2026-02-18T08:11:55.800 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 14 2026-02-18T08:11:55.800 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:11:55.800 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=57, tag_tid=3, entry_tid=1], [object_number=56, tag_tid=3, entry_tid=0], [object_number=59, tag_tid=2, entry_tid=59619], [object_number=58, tag_tid=2, entry_tid=59618]]], state=connected] 2026-02-18T08:11:55.800 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=48, tag_tid=2, entry_tid=51680], [object_number=51, tag_tid=2, entry_tid=51679], [object_number=50, tag_tid=2, entry_tid=51678], [object_number=49, tag_tid=2, entry_tid=51677]]], state=connected] 2026-02-18T08:11:55.803 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:55.805 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=48, tag_tid=2, entry_tid=51680' 2026-02-18T08:11:55.806 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=57, tag_tid=3, entry_tid=1' -a 'object_number=57, tag_tid=3, entry_tid=1' = 'object_number=48, tag_tid=2, entry_tid=51680' 2026-02-18T08:11:55.806 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=48, tag_tid=2, entry_tid=51680' '!=' 'object_number=50, tag_tid=2, entry_tid=51406' 2026-02-18T08:11:55.806 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:11:55.806 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=48, tag_tid=2, entry_tid=51680' = 'object_number=50, tag_tid=2, entry_tid=51406' 2026-02-18T08:11:55.806 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=48, tag_tid=2, entry_tid=51680' 2026-02-18T08:11:55.807 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=57, tag_tid=3, entry_tid=1' 2026-02-18T08:11:55.807 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:11:55.807 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:11:55.808 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=3 2026-02-18T08:11:55.809 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=48, tag_tid=2, entry_tid=51680' 2026-02-18T08:11:55.809 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:11:55.809 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:11:55.810 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=2 2026-02-18T08:11:55.811 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=57, tag_tid=3, entry_tid=1' 2026-02-18T08:11:55.811 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:11:55.811 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:11:55.812 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:11:55.813 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=48, tag_tid=2, entry_tid=51680' 2026-02-18T08:11:55.813 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:11:55.813 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:11:55.815 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=51680 2026-02-18T08:11:55.815 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 3 = 2 -a 1 -le 51680 2026-02-18T08:11:55.815 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:11:55.815 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:11:55.815 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:11:56.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:11:56.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:11:56.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:11:56.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:11:56.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:11:56.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:11:56.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:11:56.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:11:56.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:11:56.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:11:56.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:11:56.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:11:56.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:11:56.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:11:56.018 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:11:56.020 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:11:56.021 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:11:56.021 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:11:56.021 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:11:56.021 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:11:56.021 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:11:56.021 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:11:56.021 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:11:56.021 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:11:56.021 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:11:56.021 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:11:56.021 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:11:56.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:11:56.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:11:56.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:11:56.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:11:56.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:11:56.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:11:56.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:11:56.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:11:56.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:11:56.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:11:56.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:11:56.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:11:56.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:11:56.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:11:56.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:11:56.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:11:56.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:11:56.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:11:56.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:11:56.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:11:56.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:11:56.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:11:56.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:11:56.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:11:56.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:11:56.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:11:56.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:11:56.092 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:11:56.092 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:11:56.092 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:11:56.093 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:11:56.093 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:11:56.093 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:11:56.093 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:11:56.093 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:11:56.093 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:11:56.093 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:11:56.093 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:11:56.093 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:11:56.094 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:56.094 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:11:56.094 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:56.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 12 2026-02-18T08:11:56.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 14 2026-02-18T08:11:56.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:11:56.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=57, tag_tid=3, entry_tid=1], [object_number=56, tag_tid=3, entry_tid=0], [object_number=59, tag_tid=2, entry_tid=59619], [object_number=58, tag_tid=2, entry_tid=59618]]], state=connected] 2026-02-18T08:11:56.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=49, tag_tid=2, entry_tid=52005], [object_number=48, tag_tid=2, entry_tid=52004], [object_number=51, tag_tid=2, entry_tid=52003], [object_number=50, tag_tid=2, entry_tid=52002]]], state=connected] 2026-02-18T08:11:56.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:56.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=57, tag_tid=3, entry_tid=1' 2026-02-18T08:11:56.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:11:56.129 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:11:56.129 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:11:56.129 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:11:56.129 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:11:56.129 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:11:56.129 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:11:56.129 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:11:56.129 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:11:56.129 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:11:56.129 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:11:56.130 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:11:56.130 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:56.130 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:11:56.131 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:56.158 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 12 2026-02-18T08:11:56.158 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 14 2026-02-18T08:11:56.158 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:11:56.158 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=57, tag_tid=3, entry_tid=1], [object_number=56, tag_tid=3, entry_tid=0], [object_number=59, tag_tid=2, entry_tid=59619], [object_number=58, tag_tid=2, entry_tid=59618]]], state=connected] 2026-02-18T08:11:56.158 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=49, tag_tid=2, entry_tid=52005], [object_number=48, tag_tid=2, entry_tid=52004], [object_number=51, tag_tid=2, entry_tid=52003], [object_number=50, tag_tid=2, entry_tid=52002]]], state=connected] 2026-02-18T08:11:56.162 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:56.163 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=49, tag_tid=2, entry_tid=52005' 2026-02-18T08:11:56.163 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=57, tag_tid=3, entry_tid=1' -a 'object_number=57, tag_tid=3, entry_tid=1' = 'object_number=49, tag_tid=2, entry_tid=52005' 2026-02-18T08:11:56.163 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=49, tag_tid=2, entry_tid=52005' '!=' 'object_number=48, tag_tid=2, entry_tid=51680' 2026-02-18T08:11:56.164 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:11:56.164 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=49, tag_tid=2, entry_tid=52005' = 'object_number=48, tag_tid=2, entry_tid=51680' 2026-02-18T08:11:56.164 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=49, tag_tid=2, entry_tid=52005' 2026-02-18T08:11:56.164 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=57, tag_tid=3, entry_tid=1' 2026-02-18T08:11:56.165 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:11:56.165 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:11:56.165 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=3 2026-02-18T08:11:56.166 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=49, tag_tid=2, entry_tid=52005' 2026-02-18T08:11:56.166 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:11:56.166 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:11:56.167 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=2 2026-02-18T08:11:56.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=57, tag_tid=3, entry_tid=1' 2026-02-18T08:11:56.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:11:56.169 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:11:56.170 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:11:56.170 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=49, tag_tid=2, entry_tid=52005' 2026-02-18T08:11:56.170 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:11:56.171 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:11:56.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=52005 2026-02-18T08:11:56.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 3 = 2 -a 1 -le 52005 2026-02-18T08:11:56.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:11:56.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:11:56.173 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:11:56.374 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:11:56.374 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:11:56.375 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:11:56.375 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:11:56.375 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:11:56.375 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:11:56.375 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:11:56.375 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:11:56.375 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:11:56.375 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:11:56.375 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:11:56.375 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:11:56.375 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:11:56.376 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:11:56.376 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:11:56.376 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:11:56.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:11:56.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:11:56.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:11:56.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:11:56.379 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:11:56.379 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:11:56.379 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:11:56.379 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:11:56.379 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:11:56.379 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:11:56.379 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:11:56.379 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:11:56.379 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:11:56.379 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:11:56.379 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:11:56.380 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:11:56.380 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:11:56.380 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:11:56.380 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:11:56.380 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:11:56.380 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:11:56.380 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:11:56.380 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:11:56.380 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:11:56.380 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:11:56.380 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:11:56.380 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:11:56.380 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:11:56.381 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:11:56.381 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:11:56.381 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:11:56.381 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:11:56.381 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:11:56.389 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:11:56.390 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:11:56.390 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:11:56.390 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:11:56.448 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:11:56.449 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:11:56.450 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:11:56.450 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:11:56.450 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:11:56.450 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:11:56.450 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:11:56.450 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:11:56.450 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:11:56.450 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:11:56.450 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:11:56.450 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:11:56.450 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:11:56.451 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:56.451 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:11:56.452 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:56.479 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 12 2026-02-18T08:11:56.479 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 14 2026-02-18T08:11:56.479 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:11:56.480 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=57, tag_tid=3, entry_tid=1], [object_number=56, tag_tid=3, entry_tid=0], [object_number=59, tag_tid=2, entry_tid=59619], [object_number=58, tag_tid=2, entry_tid=59618]]], state=connected] 2026-02-18T08:11:56.480 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=51, tag_tid=2, entry_tid=52287], [object_number=50, tag_tid=2, entry_tid=52286], [object_number=49, tag_tid=2, entry_tid=52285], [object_number=48, tag_tid=2, entry_tid=52284]]], state=connected] 2026-02-18T08:11:56.482 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:56.484 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=57, tag_tid=3, entry_tid=1' 2026-02-18T08:11:56.485 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:11:56.485 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:11:56.485 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:11:56.485 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:11:56.485 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:11:56.485 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:11:56.485 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:11:56.485 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:11:56.486 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:11:56.486 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:11:56.486 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:11:56.486 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:11:56.487 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:56.487 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:11:56.487 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:56.516 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 12 2026-02-18T08:11:56.516 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 14 2026-02-18T08:11:56.516 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:11:56.516 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=57, tag_tid=3, entry_tid=1], [object_number=56, tag_tid=3, entry_tid=0], [object_number=59, tag_tid=2, entry_tid=59619], [object_number=58, tag_tid=2, entry_tid=59618]]], state=connected] 2026-02-18T08:11:56.516 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=51, tag_tid=2, entry_tid=52287], [object_number=50, tag_tid=2, entry_tid=52286], [object_number=49, tag_tid=2, entry_tid=52285], [object_number=48, tag_tid=2, entry_tid=52284]]], state=connected] 2026-02-18T08:11:56.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:56.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=51, tag_tid=2, entry_tid=52287' 2026-02-18T08:11:56.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=57, tag_tid=3, entry_tid=1' -a 'object_number=57, tag_tid=3, entry_tid=1' = 'object_number=51, tag_tid=2, entry_tid=52287' 2026-02-18T08:11:56.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=51, tag_tid=2, entry_tid=52287' '!=' 'object_number=49, tag_tid=2, entry_tid=52005' 2026-02-18T08:11:56.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:11:56.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=51, tag_tid=2, entry_tid=52287' = 'object_number=49, tag_tid=2, entry_tid=52005' 2026-02-18T08:11:56.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=51, tag_tid=2, entry_tid=52287' 2026-02-18T08:11:56.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=57, tag_tid=3, entry_tid=1' 2026-02-18T08:11:56.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:11:56.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:11:56.523 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=3 2026-02-18T08:11:56.523 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=51, tag_tid=2, entry_tid=52287' 2026-02-18T08:11:56.524 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:11:56.524 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:11:56.525 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=2 2026-02-18T08:11:56.526 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=57, tag_tid=3, entry_tid=1' 2026-02-18T08:11:56.526 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:11:56.526 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:11:56.527 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:11:56.528 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=51, tag_tid=2, entry_tid=52287' 2026-02-18T08:11:56.528 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:11:56.528 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:11:56.529 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=52287 2026-02-18T08:11:56.530 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 3 = 2 -a 1 -le 52287 2026-02-18T08:11:56.530 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:11:56.530 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:11:56.530 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:11:56.731 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:11:56.732 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:11:56.732 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:11:56.732 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:11:56.732 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:11:56.732 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:11:56.732 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:11:56.732 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:11:56.732 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:11:56.732 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:11:56.732 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:11:56.733 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:11:56.733 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:11:56.733 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:11:56.733 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:11:56.733 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:11:56.735 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:11:56.735 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:11:56.735 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:11:56.736 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:11:56.736 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:11:56.736 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:11:56.736 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:11:56.736 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:11:56.736 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:11:56.736 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:11:56.736 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:11:56.736 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:11:56.736 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:11:56.736 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:11:56.736 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:11:56.737 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:11:56.737 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:11:56.737 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:11:56.737 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:11:56.737 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:11:56.737 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:11:56.737 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:11:56.737 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:11:56.737 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:11:56.737 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:11:56.737 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:11:56.737 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:11:56.738 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:11:56.738 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:11:56.738 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:11:56.738 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:11:56.738 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:11:56.738 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:11:56.745 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:11:56.746 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:11:56.746 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:11:56.746 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:11:56.814 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:11:56.815 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:11:56.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:11:56.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:11:56.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:11:56.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:11:56.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:11:56.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:11:56.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:11:56.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:11:56.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:11:56.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:11:56.817 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:11:56.817 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:56.818 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:11:56.818 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:56.853 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 12 2026-02-18T08:11:56.853 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 14 2026-02-18T08:11:56.853 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:11:56.853 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=57, tag_tid=3, entry_tid=1], [object_number=56, tag_tid=3, entry_tid=0], [object_number=59, tag_tid=2, entry_tid=59619], [object_number=58, tag_tid=2, entry_tid=59618]]], state=connected] 2026-02-18T08:11:56.853 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=49, tag_tid=2, entry_tid=52573], [object_number=48, tag_tid=2, entry_tid=52572], [object_number=51, tag_tid=2, entry_tid=52571], [object_number=50, tag_tid=2, entry_tid=52570]]], state=connected] 2026-02-18T08:11:56.857 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:56.859 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=57, tag_tid=3, entry_tid=1' 2026-02-18T08:11:56.859 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:11:56.860 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:11:56.860 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:11:56.860 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:11:56.860 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:11:56.860 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:11:56.860 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:11:56.860 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:11:56.860 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:11:56.860 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:11:56.861 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:11:56.861 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:11:56.861 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:56.862 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:11:56.862 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:56.887 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 12 2026-02-18T08:11:56.887 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 14 2026-02-18T08:11:56.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:11:56.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=57, tag_tid=3, entry_tid=1], [object_number=56, tag_tid=3, entry_tid=0], [object_number=59, tag_tid=2, entry_tid=59619], [object_number=58, tag_tid=2, entry_tid=59618]]], state=connected] 2026-02-18T08:11:56.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=49, tag_tid=2, entry_tid=52573], [object_number=48, tag_tid=2, entry_tid=52572], [object_number=51, tag_tid=2, entry_tid=52571], [object_number=50, tag_tid=2, entry_tid=52570]]], state=connected] 2026-02-18T08:11:56.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:56.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=49, tag_tid=2, entry_tid=52573' 2026-02-18T08:11:56.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=57, tag_tid=3, entry_tid=1' -a 'object_number=57, tag_tid=3, entry_tid=1' = 'object_number=49, tag_tid=2, entry_tid=52573' 2026-02-18T08:11:56.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=49, tag_tid=2, entry_tid=52573' '!=' 'object_number=51, tag_tid=2, entry_tid=52287' 2026-02-18T08:11:56.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:11:56.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=49, tag_tid=2, entry_tid=52573' = 'object_number=51, tag_tid=2, entry_tid=52287' 2026-02-18T08:11:56.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=49, tag_tid=2, entry_tid=52573' 2026-02-18T08:11:56.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=57, tag_tid=3, entry_tid=1' 2026-02-18T08:11:56.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:11:56.894 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:11:56.894 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=3 2026-02-18T08:11:56.895 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=49, tag_tid=2, entry_tid=52573' 2026-02-18T08:11:56.895 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:11:56.895 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:11:56.896 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=2 2026-02-18T08:11:56.897 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=57, tag_tid=3, entry_tid=1' 2026-02-18T08:11:56.897 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:11:56.897 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:11:56.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:11:56.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=49, tag_tid=2, entry_tid=52573' 2026-02-18T08:11:56.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:11:56.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:11:56.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=52573 2026-02-18T08:11:56.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 3 = 2 -a 1 -le 52573 2026-02-18T08:11:56.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:11:56.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:11:56.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:11:57.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:11:57.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:11:57.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:11:57.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:11:57.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:11:57.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:11:57.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:11:57.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:11:57.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:11:57.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:11:57.105 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:11:57.105 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:11:57.105 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:11:57.105 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:11:57.105 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:11:57.105 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:11:57.105 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:11:57.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:11:57.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:11:57.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:11:57.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:11:57.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:11:57.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:11:57.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:11:57.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:11:57.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:11:57.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:11:57.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:11:57.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:11:57.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:11:57.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:11:57.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:11:57.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:11:57.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:11:57.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:11:57.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:11:57.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:11:57.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:11:57.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:11:57.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:11:57.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:11:57.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:11:57.110 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:11:57.110 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:11:57.110 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:11:57.110 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:11:57.110 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:11:57.110 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:11:57.110 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:11:57.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:11:57.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:11:57.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:11:57.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:11:57.179 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:11:57.180 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:11:57.180 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:11:57.180 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:11:57.180 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:11:57.180 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:11:57.180 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:11:57.180 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:11:57.180 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:11:57.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:11:57.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:11:57.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:11:57.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:11:57.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:57.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:11:57.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:57.209 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 12 2026-02-18T08:11:57.209 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 14 2026-02-18T08:11:57.209 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:11:57.210 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=57, tag_tid=3, entry_tid=1], [object_number=56, tag_tid=3, entry_tid=0], [object_number=59, tag_tid=2, entry_tid=59619], [object_number=58, tag_tid=2, entry_tid=59618]]], state=connected] 2026-02-18T08:11:57.210 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=49, tag_tid=2, entry_tid=52841], [object_number=48, tag_tid=2, entry_tid=52840], [object_number=51, tag_tid=2, entry_tid=52839], [object_number=50, tag_tid=2, entry_tid=52838]]], state=connected] 2026-02-18T08:11:57.212 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:57.214 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=57, tag_tid=3, entry_tid=1' 2026-02-18T08:11:57.215 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:11:57.215 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:11:57.215 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:11:57.215 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:11:57.215 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:11:57.215 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:11:57.215 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:11:57.215 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:11:57.215 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:11:57.215 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:11:57.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:11:57.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:11:57.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:57.217 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:11:57.217 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:57.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 12 2026-02-18T08:11:57.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 14 2026-02-18T08:11:57.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:11:57.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=57, tag_tid=3, entry_tid=1], [object_number=56, tag_tid=3, entry_tid=0], [object_number=59, tag_tid=2, entry_tid=59619], [object_number=58, tag_tid=2, entry_tid=59618]]], state=connected] 2026-02-18T08:11:57.244 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=49, tag_tid=2, entry_tid=52841], [object_number=48, tag_tid=2, entry_tid=52840], [object_number=51, tag_tid=2, entry_tid=52839], [object_number=50, tag_tid=2, entry_tid=52838]]], state=connected] 2026-02-18T08:11:57.246 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:57.247 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=49, tag_tid=2, entry_tid=52841' 2026-02-18T08:11:57.248 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=57, tag_tid=3, entry_tid=1' -a 'object_number=57, tag_tid=3, entry_tid=1' = 'object_number=49, tag_tid=2, entry_tid=52841' 2026-02-18T08:11:57.248 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=49, tag_tid=2, entry_tid=52841' '!=' 'object_number=49, tag_tid=2, entry_tid=52573' 2026-02-18T08:11:57.248 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:11:57.248 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=49, tag_tid=2, entry_tid=52841' = 'object_number=49, tag_tid=2, entry_tid=52573' 2026-02-18T08:11:57.248 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=49, tag_tid=2, entry_tid=52841' 2026-02-18T08:11:57.249 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=57, tag_tid=3, entry_tid=1' 2026-02-18T08:11:57.249 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:11:57.249 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:11:57.249 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=3 2026-02-18T08:11:57.250 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=49, tag_tid=2, entry_tid=52841' 2026-02-18T08:11:57.250 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:11:57.250 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:11:57.252 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=2 2026-02-18T08:11:57.252 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=57, tag_tid=3, entry_tid=1' 2026-02-18T08:11:57.253 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:11:57.253 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:11:57.254 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:11:57.254 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=49, tag_tid=2, entry_tid=52841' 2026-02-18T08:11:57.255 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:11:57.255 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:11:57.256 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=52841 2026-02-18T08:11:57.256 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 3 = 2 -a 1 -le 52841 2026-02-18T08:11:57.256 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:11:57.256 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:11:57.256 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:11:57.458 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:11:57.458 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:11:57.458 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:11:57.458 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:11:57.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:11:57.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:11:57.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:11:57.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:11:57.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:11:57.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:11:57.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:11:57.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:11:57.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:11:57.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:11:57.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:11:57.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:11:57.462 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:11:57.462 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:11:57.462 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:11:57.463 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:11:57.463 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:11:57.463 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:11:57.463 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:11:57.463 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:11:57.463 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:11:57.463 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:11:57.463 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:11:57.463 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:11:57.463 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:11:57.463 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:11:57.463 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:11:57.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:11:57.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:11:57.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:11:57.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:11:57.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:11:57.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:11:57.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:11:57.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:11:57.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:11:57.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:11:57.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:11:57.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:11:57.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:11:57.465 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:11:57.465 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:11:57.465 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:11:57.465 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:11:57.465 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:11:57.473 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:11:57.473 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:11:57.473 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:11:57.473 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:11:57.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:11:57.545 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:11:57.545 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:11:57.545 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:11:57.545 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:11:57.545 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:11:57.545 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:11:57.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:11:57.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:11:57.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:11:57.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:11:57.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:11:57.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:11:57.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:57.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:11:57.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:57.575 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 12 2026-02-18T08:11:57.575 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 14 2026-02-18T08:11:57.575 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:11:57.575 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=57, tag_tid=3, entry_tid=1], [object_number=56, tag_tid=3, entry_tid=0], [object_number=59, tag_tid=2, entry_tid=59619], [object_number=58, tag_tid=2, entry_tid=59618]]], state=connected] 2026-02-18T08:11:57.575 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=51, tag_tid=2, entry_tid=53135], [object_number=50, tag_tid=2, entry_tid=53134], [object_number=49, tag_tid=2, entry_tid=53133], [object_number=48, tag_tid=2, entry_tid=53132]]], state=connected] 2026-02-18T08:11:57.578 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:57.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=57, tag_tid=3, entry_tid=1' 2026-02-18T08:11:57.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:11:57.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:11:57.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:11:57.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:11:57.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:11:57.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:11:57.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:11:57.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:11:57.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:11:57.582 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:11:57.582 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:11:57.582 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:11:57.582 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:57.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:11:57.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:57.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 12 2026-02-18T08:11:57.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 14 2026-02-18T08:11:57.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:11:57.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=57, tag_tid=3, entry_tid=1], [object_number=56, tag_tid=3, entry_tid=0], [object_number=59, tag_tid=2, entry_tid=59619], [object_number=58, tag_tid=2, entry_tid=59618]]], state=connected] 2026-02-18T08:11:57.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=51, tag_tid=2, entry_tid=53135], [object_number=50, tag_tid=2, entry_tid=53134], [object_number=49, tag_tid=2, entry_tid=53133], [object_number=48, tag_tid=2, entry_tid=53132]]], state=connected] 2026-02-18T08:11:57.620 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:57.622 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=51, tag_tid=2, entry_tid=53135' 2026-02-18T08:11:57.622 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=57, tag_tid=3, entry_tid=1' -a 'object_number=57, tag_tid=3, entry_tid=1' = 'object_number=51, tag_tid=2, entry_tid=53135' 2026-02-18T08:11:57.622 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=51, tag_tid=2, entry_tid=53135' '!=' 'object_number=49, tag_tid=2, entry_tid=52841' 2026-02-18T08:11:57.622 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:11:57.622 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=51, tag_tid=2, entry_tid=53135' = 'object_number=49, tag_tid=2, entry_tid=52841' 2026-02-18T08:11:57.622 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=51, tag_tid=2, entry_tid=53135' 2026-02-18T08:11:57.623 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=57, tag_tid=3, entry_tid=1' 2026-02-18T08:11:57.623 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:11:57.623 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:11:57.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=3 2026-02-18T08:11:57.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=51, tag_tid=2, entry_tid=53135' 2026-02-18T08:11:57.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:11:57.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:11:57.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=2 2026-02-18T08:11:57.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=57, tag_tid=3, entry_tid=1' 2026-02-18T08:11:57.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:11:57.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:11:57.629 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:11:57.629 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=51, tag_tid=2, entry_tid=53135' 2026-02-18T08:11:57.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:11:57.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:11:57.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=53135 2026-02-18T08:11:57.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 3 = 2 -a 1 -le 53135 2026-02-18T08:11:57.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:11:57.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:11:57.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:11:57.838 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:11:57.838 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:11:57.838 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:11:57.838 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:11:57.838 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:11:57.838 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:11:57.838 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:11:57.838 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:11:57.839 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:11:57.839 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:11:57.839 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:11:57.839 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:11:57.839 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:11:57.839 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:11:57.839 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:11:57.839 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:11:57.839 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:11:57.840 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:11:57.840 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:11:57.840 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:11:57.840 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:11:57.840 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:11:57.840 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:11:57.840 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:11:57.840 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:11:57.840 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:11:57.841 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:11:57.841 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:11:57.841 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:11:57.841 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:11:57.841 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:11:57.841 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:11:57.841 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:11:57.841 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:11:57.841 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:11:57.841 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:11:57.842 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:11:57.842 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:11:57.842 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:11:57.842 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:11:57.842 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:11:57.842 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:11:57.842 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:11:57.842 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:11:57.842 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:11:57.842 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:11:57.842 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:11:57.843 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:11:57.843 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:11:57.848 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:11:57.848 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:11:57.848 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:11:57.848 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:11:57.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:11:57.909 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:11:57.909 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:11:57.909 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:11:57.909 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:11:57.909 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:11:57.909 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:11:57.909 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:11:57.910 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:11:57.910 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:11:57.910 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:11:57.910 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:11:57.910 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:11:57.910 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:57.911 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:11:57.911 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:57.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 12 2026-02-18T08:11:57.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 14 2026-02-18T08:11:57.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:11:57.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=57, tag_tid=3, entry_tid=1], [object_number=56, tag_tid=3, entry_tid=0], [object_number=59, tag_tid=2, entry_tid=59619], [object_number=58, tag_tid=2, entry_tid=59618]]], state=connected] 2026-02-18T08:11:57.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=48, tag_tid=2, entry_tid=53484], [object_number=51, tag_tid=2, entry_tid=53483], [object_number=50, tag_tid=2, entry_tid=53482], [object_number=49, tag_tid=2, entry_tid=53481]]], state=connected] 2026-02-18T08:11:57.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:57.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=57, tag_tid=3, entry_tid=1' 2026-02-18T08:11:57.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:11:57.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:11:57.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:11:57.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:11:57.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:11:57.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:11:57.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:11:57.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:11:57.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:11:57.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:11:57.944 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:11:57.944 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:11:57.944 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:57.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:11:57.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:57.971 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 12 2026-02-18T08:11:57.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 14 2026-02-18T08:11:57.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:11:57.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=57, tag_tid=3, entry_tid=1], [object_number=56, tag_tid=3, entry_tid=0], [object_number=59, tag_tid=2, entry_tid=59619], [object_number=58, tag_tid=2, entry_tid=59618]]], state=connected] 2026-02-18T08:11:57.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=48, tag_tid=2, entry_tid=53484], [object_number=51, tag_tid=2, entry_tid=53483], [object_number=50, tag_tid=2, entry_tid=53482], [object_number=49, tag_tid=2, entry_tid=53481]]], state=connected] 2026-02-18T08:11:57.974 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:57.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=48, tag_tid=2, entry_tid=53484' 2026-02-18T08:11:57.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=57, tag_tid=3, entry_tid=1' -a 'object_number=57, tag_tid=3, entry_tid=1' = 'object_number=48, tag_tid=2, entry_tid=53484' 2026-02-18T08:11:57.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=48, tag_tid=2, entry_tid=53484' '!=' 'object_number=51, tag_tid=2, entry_tid=53135' 2026-02-18T08:11:57.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:11:57.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=48, tag_tid=2, entry_tid=53484' = 'object_number=51, tag_tid=2, entry_tid=53135' 2026-02-18T08:11:57.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=48, tag_tid=2, entry_tid=53484' 2026-02-18T08:11:57.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=57, tag_tid=3, entry_tid=1' 2026-02-18T08:11:57.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:11:57.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:11:57.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=3 2026-02-18T08:11:57.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=48, tag_tid=2, entry_tid=53484' 2026-02-18T08:11:57.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:11:57.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:11:57.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=2 2026-02-18T08:11:57.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=57, tag_tid=3, entry_tid=1' 2026-02-18T08:11:57.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:11:57.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:11:57.982 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:11:57.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=48, tag_tid=2, entry_tid=53484' 2026-02-18T08:11:57.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:11:57.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:11:57.985 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=53484 2026-02-18T08:11:57.985 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 3 = 2 -a 1 -le 53484 2026-02-18T08:11:57.985 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:11:57.985 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:11:57.985 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:11:58.187 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:11:58.187 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:11:58.187 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:11:58.187 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:11:58.187 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:11:58.187 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:11:58.188 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:11:58.188 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:11:58.188 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:11:58.188 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:11:58.188 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:11:58.188 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:11:58.188 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:11:58.188 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:11:58.188 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:11:58.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:11:58.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:11:58.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:11:58.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:11:58.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:11:58.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:11:58.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:11:58.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:11:58.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:11:58.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:11:58.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:11:58.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:11:58.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:11:58.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:11:58.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:11:58.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:11:58.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:11:58.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:11:58.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:11:58.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:11:58.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:11:58.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:11:58.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:11:58.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:11:58.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:11:58.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:11:58.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:11:58.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:11:58.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:11:58.194 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:11:58.194 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:11:58.194 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:11:58.194 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:11:58.194 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:11:58.201 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:11:58.201 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:11:58.201 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:11:58.201 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:11:58.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:11:58.260 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:11:58.260 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:11:58.260 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:11:58.260 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:11:58.260 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:11:58.260 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:11:58.260 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:11:58.261 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:11:58.261 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:11:58.261 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:11:58.261 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:11:58.261 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:11:58.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:58.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:11:58.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:58.288 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 12 2026-02-18T08:11:58.288 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 14 2026-02-18T08:11:58.288 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:11:58.288 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=57, tag_tid=3, entry_tid=1], [object_number=56, tag_tid=3, entry_tid=0], [object_number=59, tag_tid=2, entry_tid=59619], [object_number=58, tag_tid=2, entry_tid=59618]]], state=connected] 2026-02-18T08:11:58.288 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=49, tag_tid=2, entry_tid=53769], [object_number=48, tag_tid=2, entry_tid=53768], [object_number=51, tag_tid=2, entry_tid=53767], [object_number=50, tag_tid=2, entry_tid=53766]]], state=connected] 2026-02-18T08:11:58.290 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:58.292 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=57, tag_tid=3, entry_tid=1' 2026-02-18T08:11:58.293 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:11:58.293 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:11:58.293 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:11:58.293 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:11:58.293 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:11:58.294 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:11:58.294 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:11:58.294 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:11:58.294 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:11:58.294 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:11:58.294 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:11:58.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:11:58.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:58.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:11:58.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:58.321 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 12 2026-02-18T08:11:58.321 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 14 2026-02-18T08:11:58.321 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:11:58.322 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=57, tag_tid=3, entry_tid=1], [object_number=56, tag_tid=3, entry_tid=0], [object_number=59, tag_tid=2, entry_tid=59619], [object_number=58, tag_tid=2, entry_tid=59618]]], state=connected] 2026-02-18T08:11:58.322 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=49, tag_tid=2, entry_tid=53769], [object_number=48, tag_tid=2, entry_tid=53768], [object_number=51, tag_tid=2, entry_tid=53767], [object_number=50, tag_tid=2, entry_tid=53766]]], state=connected] 2026-02-18T08:11:58.324 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:58.326 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=49, tag_tid=2, entry_tid=53769' 2026-02-18T08:11:58.326 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=57, tag_tid=3, entry_tid=1' -a 'object_number=57, tag_tid=3, entry_tid=1' = 'object_number=49, tag_tid=2, entry_tid=53769' 2026-02-18T08:11:58.326 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=49, tag_tid=2, entry_tid=53769' '!=' 'object_number=48, tag_tid=2, entry_tid=53484' 2026-02-18T08:11:58.326 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:11:58.326 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=49, tag_tid=2, entry_tid=53769' = 'object_number=48, tag_tid=2, entry_tid=53484' 2026-02-18T08:11:58.326 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=49, tag_tid=2, entry_tid=53769' 2026-02-18T08:11:58.327 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=57, tag_tid=3, entry_tid=1' 2026-02-18T08:11:58.327 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:11:58.327 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:11:58.327 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=3 2026-02-18T08:11:58.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=49, tag_tid=2, entry_tid=53769' 2026-02-18T08:11:58.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:11:58.329 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:11:58.329 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=2 2026-02-18T08:11:58.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=57, tag_tid=3, entry_tid=1' 2026-02-18T08:11:58.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:11:58.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:11:58.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:11:58.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=49, tag_tid=2, entry_tid=53769' 2026-02-18T08:11:58.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:11:58.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:11:58.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=53769 2026-02-18T08:11:58.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 3 = 2 -a 1 -le 53769 2026-02-18T08:11:58.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:11:58.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:11:58.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:11:58.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:11:58.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:11:58.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:11:58.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:11:58.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:11:58.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:11:58.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:11:58.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:11:58.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:11:58.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:11:58.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:11:58.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:11:58.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:11:58.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:11:58.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:11:58.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:11:58.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:11:58.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:11:58.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:11:58.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:11:58.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:11:58.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:11:58.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:11:58.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:11:58.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:11:58.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:11:58.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:11:58.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:11:58.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:11:58.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:11:58.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:11:58.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:11:58.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:11:58.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:11:58.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:11:58.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:11:58.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:11:58.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:11:58.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:11:58.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:11:58.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:11:58.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:11:58.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:11:58.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:11:58.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:11:58.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:11:58.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:11:58.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:11:58.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:11:58.551 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:11:58.551 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:11:58.552 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:11:58.552 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:11:58.618 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:11:58.620 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:11:58.620 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:11:58.620 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:11:58.620 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:11:58.620 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:11:58.620 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:11:58.620 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:11:58.620 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:11:58.620 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:11:58.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:11:58.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:11:58.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:11:58.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:58.622 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:11:58.622 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:58.658 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 12 2026-02-18T08:11:58.658 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 14 2026-02-18T08:11:58.658 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:11:58.658 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=57, tag_tid=3, entry_tid=1], [object_number=56, tag_tid=3, entry_tid=0], [object_number=59, tag_tid=2, entry_tid=59619], [object_number=58, tag_tid=2, entry_tid=59618]]], state=connected] 2026-02-18T08:11:58.658 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=48, tag_tid=2, entry_tid=54040], [object_number=51, tag_tid=2, entry_tid=54039], [object_number=50, tag_tid=2, entry_tid=54038], [object_number=49, tag_tid=2, entry_tid=54037]]], state=connected] 2026-02-18T08:11:58.661 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:58.663 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=57, tag_tid=3, entry_tid=1' 2026-02-18T08:11:58.664 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:11:58.664 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:11:58.664 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:11:58.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:11:58.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:11:58.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:11:58.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:11:58.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:11:58.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:11:58.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:11:58.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:11:58.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:11:58.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:58.667 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:11:58.667 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:58.692 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 12 2026-02-18T08:11:58.692 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 14 2026-02-18T08:11:58.692 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:11:58.692 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=57, tag_tid=3, entry_tid=1], [object_number=56, tag_tid=3, entry_tid=0], [object_number=59, tag_tid=2, entry_tid=59619], [object_number=58, tag_tid=2, entry_tid=59618]]], state=connected] 2026-02-18T08:11:58.692 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=48, tag_tid=2, entry_tid=54040], [object_number=51, tag_tid=2, entry_tid=54039], [object_number=50, tag_tid=2, entry_tid=54038], [object_number=49, tag_tid=2, entry_tid=54037]]], state=connected] 2026-02-18T08:11:58.694 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:58.696 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=48, tag_tid=2, entry_tid=54040' 2026-02-18T08:11:58.696 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=57, tag_tid=3, entry_tid=1' -a 'object_number=57, tag_tid=3, entry_tid=1' = 'object_number=48, tag_tid=2, entry_tid=54040' 2026-02-18T08:11:58.696 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=48, tag_tid=2, entry_tid=54040' '!=' 'object_number=49, tag_tid=2, entry_tid=53769' 2026-02-18T08:11:58.696 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:11:58.697 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=48, tag_tid=2, entry_tid=54040' = 'object_number=49, tag_tid=2, entry_tid=53769' 2026-02-18T08:11:58.697 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=48, tag_tid=2, entry_tid=54040' 2026-02-18T08:11:58.697 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=57, tag_tid=3, entry_tid=1' 2026-02-18T08:11:58.697 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:11:58.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:11:58.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=3 2026-02-18T08:11:58.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=48, tag_tid=2, entry_tid=54040' 2026-02-18T08:11:58.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:11:58.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:11:58.700 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=2 2026-02-18T08:11:58.701 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=57, tag_tid=3, entry_tid=1' 2026-02-18T08:11:58.701 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:11:58.701 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:11:58.702 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:11:58.703 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=48, tag_tid=2, entry_tid=54040' 2026-02-18T08:11:58.703 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:11:58.703 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:11:58.705 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=54040 2026-02-18T08:11:58.705 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 3 = 2 -a 1 -le 54040 2026-02-18T08:11:58.706 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:11:58.706 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:11:58.706 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:11:58.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:11:58.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:11:58.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:11:58.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:11:58.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:11:58.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:11:58.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:11:58.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:11:58.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:11:58.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:11:58.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:11:58.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:11:58.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:11:58.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:11:58.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:11:58.911 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:11:58.911 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:11:58.911 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:11:58.911 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:11:58.911 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:11:58.912 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:11:58.912 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:11:58.912 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:11:58.912 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:11:58.912 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:11:58.912 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:11:58.912 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:11:58.912 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:11:58.912 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:11:58.912 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:11:58.912 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:11:58.912 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:11:58.913 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:11:58.913 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:11:58.913 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:11:58.913 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:11:58.913 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:11:58.913 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:11:58.913 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:11:58.913 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:11:58.913 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:11:58.913 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:11:58.913 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:11:58.914 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:11:58.914 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:11:58.914 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:11:58.914 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:11:58.914 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:11:58.914 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:11:58.922 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:11:58.922 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:11:58.922 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:11:58.922 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:11:58.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:11:58.982 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:11:58.982 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:11:58.982 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:11:58.982 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:11:58.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:11:58.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:11:58.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:11:58.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:11:58.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:11:58.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:11:58.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:11:58.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:11:58.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:58.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:11:58.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:59.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 13 2026-02-18T08:11:59.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 14 2026-02-18T08:11:59.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:11:59.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=57, tag_tid=3, entry_tid=1], [object_number=56, tag_tid=3, entry_tid=0], [object_number=59, tag_tid=2, entry_tid=59619], [object_number=58, tag_tid=2, entry_tid=59618]]], state=connected] 2026-02-18T08:11:59.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=53, tag_tid=2, entry_tid=54209], [object_number=52, tag_tid=2, entry_tid=54208], [object_number=55, tag_tid=2, entry_tid=54207], [object_number=54, tag_tid=2, entry_tid=54206]]], state=connected] 2026-02-18T08:11:59.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:59.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=57, tag_tid=3, entry_tid=1' 2026-02-18T08:11:59.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:11:59.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:11:59.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:11:59.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:11:59.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:11:59.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:11:59.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:11:59.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:11:59.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:11:59.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:11:59.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:11:59.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:11:59.018 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:59.018 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:11:59.019 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:59.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 13 2026-02-18T08:11:59.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 14 2026-02-18T08:11:59.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:11:59.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=57, tag_tid=3, entry_tid=1], [object_number=56, tag_tid=3, entry_tid=0], [object_number=59, tag_tid=2, entry_tid=59619], [object_number=58, tag_tid=2, entry_tid=59618]]], state=connected] 2026-02-18T08:11:59.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=53, tag_tid=2, entry_tid=54209], [object_number=52, tag_tid=2, entry_tid=54208], [object_number=55, tag_tid=2, entry_tid=54207], [object_number=54, tag_tid=2, entry_tid=54206]]], state=connected] 2026-02-18T08:11:59.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:59.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=53, tag_tid=2, entry_tid=54209' 2026-02-18T08:11:59.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=57, tag_tid=3, entry_tid=1' -a 'object_number=57, tag_tid=3, entry_tid=1' = 'object_number=53, tag_tid=2, entry_tid=54209' 2026-02-18T08:11:59.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=53, tag_tid=2, entry_tid=54209' '!=' 'object_number=48, tag_tid=2, entry_tid=54040' 2026-02-18T08:11:59.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:11:59.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=53, tag_tid=2, entry_tid=54209' = 'object_number=48, tag_tid=2, entry_tid=54040' 2026-02-18T08:11:59.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=53, tag_tid=2, entry_tid=54209' 2026-02-18T08:11:59.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=57, tag_tid=3, entry_tid=1' 2026-02-18T08:11:59.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:11:59.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:11:59.055 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=3 2026-02-18T08:11:59.055 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=53, tag_tid=2, entry_tid=54209' 2026-02-18T08:11:59.055 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:11:59.056 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:11:59.056 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=2 2026-02-18T08:11:59.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=57, tag_tid=3, entry_tid=1' 2026-02-18T08:11:59.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:11:59.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:11:59.059 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:11:59.060 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=53, tag_tid=2, entry_tid=54209' 2026-02-18T08:11:59.060 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:11:59.060 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:11:59.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=54209 2026-02-18T08:11:59.062 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 3 = 2 -a 1 -le 54209 2026-02-18T08:11:59.062 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:11:59.062 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:11:59.062 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:11:59.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:11:59.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:11:59.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:11:59.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:11:59.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:11:59.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:11:59.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:11:59.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:11:59.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:11:59.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:11:59.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:11:59.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:11:59.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:11:59.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:11:59.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:11:59.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:11:59.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:11:59.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:11:59.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:11:59.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:11:59.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:11:59.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:11:59.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:11:59.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:11:59.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:11:59.269 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:11:59.269 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:11:59.269 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:11:59.269 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:11:59.269 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:11:59.269 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:11:59.269 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:11:59.269 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:11:59.269 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:11:59.269 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:11:59.269 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:11:59.269 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:11:59.270 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:11:59.270 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:11:59.270 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:11:59.270 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:11:59.270 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:11:59.270 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:11:59.270 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:11:59.270 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:11:59.270 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:11:59.270 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:11:59.270 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:11:59.270 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:11:59.278 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:11:59.278 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:11:59.278 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:11:59.278 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:11:59.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:11:59.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:11:59.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:11:59.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:11:59.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:11:59.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:11:59.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:11:59.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:11:59.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:11:59.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:11:59.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:11:59.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:11:59.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:11:59.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:59.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:11:59.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:59.373 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 13 2026-02-18T08:11:59.373 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 14 2026-02-18T08:11:59.373 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:11:59.373 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=57, tag_tid=3, entry_tid=1], [object_number=56, tag_tid=3, entry_tid=0], [object_number=59, tag_tid=2, entry_tid=59619], [object_number=58, tag_tid=2, entry_tid=59618]]], state=connected] 2026-02-18T08:11:59.373 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=53, tag_tid=2, entry_tid=54489], [object_number=52, tag_tid=2, entry_tid=54488], [object_number=55, tag_tid=2, entry_tid=54487], [object_number=54, tag_tid=2, entry_tid=54486]]], state=connected] 2026-02-18T08:11:59.377 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:59.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=57, tag_tid=3, entry_tid=1' 2026-02-18T08:11:59.379 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:11:59.379 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:11:59.380 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:11:59.380 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:11:59.380 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:11:59.380 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:11:59.380 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:11:59.380 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:11:59.380 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:11:59.380 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:11:59.381 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:11:59.381 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:11:59.381 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:59.382 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:11:59.382 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:59.409 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 13 2026-02-18T08:11:59.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 14 2026-02-18T08:11:59.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:11:59.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=57, tag_tid=3, entry_tid=1], [object_number=56, tag_tid=3, entry_tid=0], [object_number=59, tag_tid=2, entry_tid=59619], [object_number=58, tag_tid=2, entry_tid=59618]]], state=connected] 2026-02-18T08:11:59.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=53, tag_tid=2, entry_tid=54489], [object_number=52, tag_tid=2, entry_tid=54488], [object_number=55, tag_tid=2, entry_tid=54487], [object_number=54, tag_tid=2, entry_tid=54486]]], state=connected] 2026-02-18T08:11:59.412 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:59.413 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=53, tag_tid=2, entry_tid=54489' 2026-02-18T08:11:59.414 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=57, tag_tid=3, entry_tid=1' -a 'object_number=57, tag_tid=3, entry_tid=1' = 'object_number=53, tag_tid=2, entry_tid=54489' 2026-02-18T08:11:59.414 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=53, tag_tid=2, entry_tid=54489' '!=' 'object_number=53, tag_tid=2, entry_tid=54209' 2026-02-18T08:11:59.414 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:11:59.414 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=53, tag_tid=2, entry_tid=54489' = 'object_number=53, tag_tid=2, entry_tid=54209' 2026-02-18T08:11:59.414 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=53, tag_tid=2, entry_tid=54489' 2026-02-18T08:11:59.415 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=57, tag_tid=3, entry_tid=1' 2026-02-18T08:11:59.415 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:11:59.415 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:11:59.416 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=3 2026-02-18T08:11:59.416 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=53, tag_tid=2, entry_tid=54489' 2026-02-18T08:11:59.417 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:11:59.417 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:11:59.418 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=2 2026-02-18T08:11:59.419 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=57, tag_tid=3, entry_tid=1' 2026-02-18T08:11:59.419 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:11:59.419 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:11:59.420 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:11:59.421 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=53, tag_tid=2, entry_tid=54489' 2026-02-18T08:11:59.421 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:11:59.421 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:11:59.423 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=54489 2026-02-18T08:11:59.423 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 3 = 2 -a 1 -le 54489 2026-02-18T08:11:59.423 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:11:59.423 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:11:59.423 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:11:59.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:11:59.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:11:59.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:11:59.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:11:59.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:11:59.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:11:59.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:11:59.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:11:59.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:11:59.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:11:59.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:11:59.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:11:59.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:11:59.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:11:59.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:11:59.629 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:11:59.629 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:11:59.629 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:11:59.629 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:11:59.629 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:11:59.629 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:11:59.629 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:11:59.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:11:59.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:11:59.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:11:59.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:11:59.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:11:59.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:11:59.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:11:59.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:11:59.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:11:59.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:11:59.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:11:59.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:11:59.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:11:59.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:11:59.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:11:59.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:11:59.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:11:59.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:11:59.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:11:59.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:11:59.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:11:59.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:11:59.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:11:59.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:11:59.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:11:59.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:11:59.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:11:59.639 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:11:59.640 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:11:59.640 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:11:59.640 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:11:59.708 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:11:59.709 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:11:59.709 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:11:59.709 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:11:59.709 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:11:59.709 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:11:59.709 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:11:59.709 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:11:59.709 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:11:59.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:11:59.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:11:59.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:11:59.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:11:59.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:59.711 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:11:59.711 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:59.739 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 13 2026-02-18T08:11:59.739 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 14 2026-02-18T08:11:59.739 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:11:59.739 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=57, tag_tid=3, entry_tid=1], [object_number=56, tag_tid=3, entry_tid=0], [object_number=59, tag_tid=2, entry_tid=59619], [object_number=58, tag_tid=2, entry_tid=59618]]], state=connected] 2026-02-18T08:11:59.739 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=52, tag_tid=2, entry_tid=54772], [object_number=55, tag_tid=2, entry_tid=54771], [object_number=54, tag_tid=2, entry_tid=54770], [object_number=53, tag_tid=2, entry_tid=54769]]], state=connected] 2026-02-18T08:11:59.741 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:59.743 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=57, tag_tid=3, entry_tid=1' 2026-02-18T08:11:59.744 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:11:59.744 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:11:59.744 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:11:59.744 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:11:59.744 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:11:59.745 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:11:59.745 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:11:59.745 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:11:59.745 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:11:59.745 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:11:59.745 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:11:59.745 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:11:59.746 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:59.746 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:11:59.747 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:59.773 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 13 2026-02-18T08:11:59.773 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 14 2026-02-18T08:11:59.773 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:11:59.773 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=57, tag_tid=3, entry_tid=1], [object_number=56, tag_tid=3, entry_tid=0], [object_number=59, tag_tid=2, entry_tid=59619], [object_number=58, tag_tid=2, entry_tid=59618]]], state=connected] 2026-02-18T08:11:59.773 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=52, tag_tid=2, entry_tid=54772], [object_number=55, tag_tid=2, entry_tid=54771], [object_number=54, tag_tid=2, entry_tid=54770], [object_number=53, tag_tid=2, entry_tid=54769]]], state=connected] 2026-02-18T08:11:59.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:11:59.777 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=52, tag_tid=2, entry_tid=54772' 2026-02-18T08:11:59.778 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=57, tag_tid=3, entry_tid=1' -a 'object_number=57, tag_tid=3, entry_tid=1' = 'object_number=52, tag_tid=2, entry_tid=54772' 2026-02-18T08:11:59.778 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=52, tag_tid=2, entry_tid=54772' '!=' 'object_number=53, tag_tid=2, entry_tid=54489' 2026-02-18T08:11:59.778 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:11:59.778 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=52, tag_tid=2, entry_tid=54772' = 'object_number=53, tag_tid=2, entry_tid=54489' 2026-02-18T08:11:59.778 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=52, tag_tid=2, entry_tid=54772' 2026-02-18T08:11:59.778 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=57, tag_tid=3, entry_tid=1' 2026-02-18T08:11:59.778 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:11:59.778 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:11:59.779 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=3 2026-02-18T08:11:59.779 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=52, tag_tid=2, entry_tid=54772' 2026-02-18T08:11:59.779 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:11:59.780 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:11:59.781 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=2 2026-02-18T08:11:59.782 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=57, tag_tid=3, entry_tid=1' 2026-02-18T08:11:59.782 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:11:59.782 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:11:59.783 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:11:59.783 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=52, tag_tid=2, entry_tid=54772' 2026-02-18T08:11:59.784 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:11:59.784 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:11:59.785 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=54772 2026-02-18T08:11:59.785 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 3 = 2 -a 1 -le 54772 2026-02-18T08:11:59.785 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:11:59.786 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:11:59.786 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:11:59.987 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:11:59.987 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:11:59.987 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:11:59.987 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:11:59.988 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:11:59.988 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:11:59.988 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:11:59.988 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:11:59.988 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:11:59.988 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:11:59.988 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:11:59.988 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:11:59.988 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:11:59.988 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:11:59.989 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:11:59.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:11:59.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:11:59.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:11:59.992 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:11:59.992 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:11:59.992 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:11:59.992 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:11:59.992 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:11:59.992 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:11:59.992 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:11:59.992 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:11:59.992 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:11:59.992 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:11:59.992 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:11:59.992 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:11:59.993 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:11:59.993 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:11:59.993 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:11:59.993 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:11:59.993 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:11:59.993 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:11:59.993 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:11:59.993 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:11:59.993 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:11:59.993 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:11:59.993 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:11:59.993 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:11:59.993 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:11:59.994 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:11:59.994 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:11:59.994 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:11:59.994 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:11:59.994 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:11:59.994 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:12:00.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:12:00.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:12:00.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:12:00.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:12:00.058 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:12:00.060 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:12:00.060 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:12:00.060 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:12:00.060 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:12:00.060 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:12:00.060 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:12:00.060 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:12:00.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:12:00.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:12:00.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:12:00.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:12:00.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:12:00.062 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:12:00.062 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:12:00.062 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:12:00.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 13 2026-02-18T08:12:00.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 14 2026-02-18T08:12:00.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:12:00.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=57, tag_tid=3, entry_tid=1], [object_number=56, tag_tid=3, entry_tid=0], [object_number=59, tag_tid=2, entry_tid=59619], [object_number=58, tag_tid=2, entry_tid=59618]]], state=connected] 2026-02-18T08:12:00.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=54, tag_tid=2, entry_tid=55022], [object_number=53, tag_tid=2, entry_tid=55021], [object_number=52, tag_tid=2, entry_tid=55020], [object_number=55, tag_tid=2, entry_tid=55019]]], state=connected] 2026-02-18T08:12:00.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:12:00.092 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=57, tag_tid=3, entry_tid=1' 2026-02-18T08:12:00.093 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:12:00.093 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:12:00.094 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:12:00.094 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:12:00.094 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:12:00.094 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:12:00.094 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:12:00.094 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:12:00.094 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:12:00.094 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:12:00.094 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:12:00.094 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:12:00.095 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:12:00.095 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:12:00.095 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:12:00.120 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 13 2026-02-18T08:12:00.120 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 14 2026-02-18T08:12:00.120 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:12:00.120 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=57, tag_tid=3, entry_tid=1], [object_number=56, tag_tid=3, entry_tid=0], [object_number=59, tag_tid=2, entry_tid=59619], [object_number=58, tag_tid=2, entry_tid=59618]]], state=connected] 2026-02-18T08:12:00.121 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=54, tag_tid=2, entry_tid=55022], [object_number=53, tag_tid=2, entry_tid=55021], [object_number=52, tag_tid=2, entry_tid=55020], [object_number=55, tag_tid=2, entry_tid=55019]]], state=connected] 2026-02-18T08:12:00.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:12:00.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=54, tag_tid=2, entry_tid=55022' 2026-02-18T08:12:00.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=57, tag_tid=3, entry_tid=1' -a 'object_number=57, tag_tid=3, entry_tid=1' = 'object_number=54, tag_tid=2, entry_tid=55022' 2026-02-18T08:12:00.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=54, tag_tid=2, entry_tid=55022' '!=' 'object_number=52, tag_tid=2, entry_tid=54772' 2026-02-18T08:12:00.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:12:00.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=54, tag_tid=2, entry_tid=55022' = 'object_number=52, tag_tid=2, entry_tid=54772' 2026-02-18T08:12:00.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=54, tag_tid=2, entry_tid=55022' 2026-02-18T08:12:00.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=57, tag_tid=3, entry_tid=1' 2026-02-18T08:12:00.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:12:00.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:12:00.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=3 2026-02-18T08:12:00.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=54, tag_tid=2, entry_tid=55022' 2026-02-18T08:12:00.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:12:00.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:12:00.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=2 2026-02-18T08:12:00.129 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=57, tag_tid=3, entry_tid=1' 2026-02-18T08:12:00.129 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:12:00.129 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:12:00.130 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:12:00.131 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=54, tag_tid=2, entry_tid=55022' 2026-02-18T08:12:00.131 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:12:00.131 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:12:00.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=55022 2026-02-18T08:12:00.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 3 = 2 -a 1 -le 55022 2026-02-18T08:12:00.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:12:00.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:12:00.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:12:00.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:12:00.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:12:00.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:12:00.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:12:00.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:12:00.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:12:00.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:12:00.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:12:00.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:12:00.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:12:00.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:12:00.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:12:00.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:12:00.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:12:00.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:12:00.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:12:00.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:12:00.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:12:00.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:12:00.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:12:00.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:12:00.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:12:00.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:12:00.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:12:00.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:12:00.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:12:00.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:12:00.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:12:00.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:12:00.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:12:00.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:12:00.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:12:00.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:12:00.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:12:00.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:12:00.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:12:00.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:12:00.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:12:00.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:12:00.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:12:00.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:12:00.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:12:00.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:12:00.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:12:00.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:12:00.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:12:00.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:12:00.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:12:00.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:12:00.348 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:12:00.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:12:00.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:12:00.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:12:00.416 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:12:00.417 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:12:00.417 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:12:00.417 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:12:00.417 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:12:00.418 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:12:00.418 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:12:00.418 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:12:00.418 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:12:00.418 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:12:00.418 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:12:00.418 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:12:00.419 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:12:00.419 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:12:00.419 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:12:00.419 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:12:00.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 13 2026-02-18T08:12:00.446 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 14 2026-02-18T08:12:00.446 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:12:00.446 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=57, tag_tid=3, entry_tid=1], [object_number=56, tag_tid=3, entry_tid=0], [object_number=59, tag_tid=2, entry_tid=59619], [object_number=58, tag_tid=2, entry_tid=59618]]], state=connected] 2026-02-18T08:12:00.446 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=54, tag_tid=2, entry_tid=55290], [object_number=53, tag_tid=2, entry_tid=55289], [object_number=52, tag_tid=2, entry_tid=55288], [object_number=55, tag_tid=2, entry_tid=55287]]], state=connected] 2026-02-18T08:12:00.448 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:12:00.450 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=57, tag_tid=3, entry_tid=1' 2026-02-18T08:12:00.451 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:12:00.451 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:12:00.451 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:12:00.451 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:12:00.451 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:12:00.451 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:12:00.451 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:12:00.452 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:12:00.452 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:12:00.452 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:12:00.452 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:12:00.452 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:12:00.453 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:12:00.453 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:12:00.453 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:12:00.478 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 13 2026-02-18T08:12:00.479 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 14 2026-02-18T08:12:00.479 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:12:00.479 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=57, tag_tid=3, entry_tid=1], [object_number=56, tag_tid=3, entry_tid=0], [object_number=59, tag_tid=2, entry_tid=59619], [object_number=58, tag_tid=2, entry_tid=59618]]], state=connected] 2026-02-18T08:12:00.479 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=54, tag_tid=2, entry_tid=55290], [object_number=53, tag_tid=2, entry_tid=55289], [object_number=52, tag_tid=2, entry_tid=55288], [object_number=55, tag_tid=2, entry_tid=55287]]], state=connected] 2026-02-18T08:12:00.481 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:12:00.483 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=54, tag_tid=2, entry_tid=55290' 2026-02-18T08:12:00.483 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=57, tag_tid=3, entry_tid=1' -a 'object_number=57, tag_tid=3, entry_tid=1' = 'object_number=54, tag_tid=2, entry_tid=55290' 2026-02-18T08:12:00.483 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=54, tag_tid=2, entry_tid=55290' '!=' 'object_number=54, tag_tid=2, entry_tid=55022' 2026-02-18T08:12:00.483 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:12:00.483 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=54, tag_tid=2, entry_tid=55290' = 'object_number=54, tag_tid=2, entry_tid=55022' 2026-02-18T08:12:00.483 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=54, tag_tid=2, entry_tid=55290' 2026-02-18T08:12:00.484 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=57, tag_tid=3, entry_tid=1' 2026-02-18T08:12:00.484 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:12:00.484 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:12:00.485 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=3 2026-02-18T08:12:00.486 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=54, tag_tid=2, entry_tid=55290' 2026-02-18T08:12:00.486 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:12:00.486 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:12:00.487 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=2 2026-02-18T08:12:00.488 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=57, tag_tid=3, entry_tid=1' 2026-02-18T08:12:00.488 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:12:00.488 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:12:00.489 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:12:00.490 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=54, tag_tid=2, entry_tid=55290' 2026-02-18T08:12:00.490 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:12:00.490 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:12:00.491 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=55290 2026-02-18T08:12:00.491 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 3 = 2 -a 1 -le 55290 2026-02-18T08:12:00.491 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:12:00.491 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:12:00.491 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:12:00.693 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:12:00.693 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:12:00.693 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:12:00.693 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:12:00.693 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:12:00.693 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:12:00.694 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:12:00.694 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:12:00.694 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:12:00.694 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:12:00.694 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:12:00.694 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:12:00.694 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:12:00.694 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:12:00.694 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:12:00.697 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:12:00.697 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:12:00.697 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:12:00.697 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:12:00.697 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:12:00.697 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:12:00.697 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:12:00.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:12:00.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:12:00.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:12:00.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:12:00.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:12:00.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:12:00.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:12:00.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:12:00.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:12:00.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:12:00.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:12:00.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:12:00.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:12:00.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:12:00.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:12:00.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:12:00.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:12:00.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:12:00.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:12:00.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:12:00.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:12:00.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:12:00.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:12:00.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:12:00.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:12:00.700 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:12:00.700 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:12:00.707 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:12:00.708 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:12:00.708 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:12:00.708 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:12:00.767 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:12:00.768 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:12:00.768 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:12:00.768 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:12:00.768 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:12:00.769 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:12:00.769 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:12:00.769 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:12:00.769 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:12:00.769 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:12:00.769 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:12:00.769 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:12:00.769 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:12:00.770 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:12:00.770 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:12:00.770 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:12:00.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 13 2026-02-18T08:12:00.798 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 14 2026-02-18T08:12:00.798 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:12:00.798 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=57, tag_tid=3, entry_tid=1], [object_number=56, tag_tid=3, entry_tid=0], [object_number=59, tag_tid=2, entry_tid=59619], [object_number=58, tag_tid=2, entry_tid=59618]]], state=connected] 2026-02-18T08:12:00.798 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=54, tag_tid=2, entry_tid=55538], [object_number=53, tag_tid=2, entry_tid=55537], [object_number=52, tag_tid=2, entry_tid=55536], [object_number=55, tag_tid=2, entry_tid=55535]]], state=connected] 2026-02-18T08:12:00.800 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:12:00.801 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=57, tag_tid=3, entry_tid=1' 2026-02-18T08:12:00.802 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:12:00.802 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:12:00.802 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:12:00.803 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:12:00.803 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:12:00.803 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:12:00.803 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:12:00.803 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:12:00.803 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:12:00.803 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:12:00.804 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:12:00.804 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:12:00.804 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:12:00.805 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:12:00.805 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:12:00.830 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 13 2026-02-18T08:12:00.831 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 14 2026-02-18T08:12:00.831 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:12:00.831 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=57, tag_tid=3, entry_tid=1], [object_number=56, tag_tid=3, entry_tid=0], [object_number=59, tag_tid=2, entry_tid=59619], [object_number=58, tag_tid=2, entry_tid=59618]]], state=connected] 2026-02-18T08:12:00.831 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=54, tag_tid=2, entry_tid=55538], [object_number=53, tag_tid=2, entry_tid=55537], [object_number=52, tag_tid=2, entry_tid=55536], [object_number=55, tag_tid=2, entry_tid=55535]]], state=connected] 2026-02-18T08:12:00.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:12:00.835 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=54, tag_tid=2, entry_tid=55538' 2026-02-18T08:12:00.835 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=57, tag_tid=3, entry_tid=1' -a 'object_number=57, tag_tid=3, entry_tid=1' = 'object_number=54, tag_tid=2, entry_tid=55538' 2026-02-18T08:12:00.835 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=54, tag_tid=2, entry_tid=55538' '!=' 'object_number=54, tag_tid=2, entry_tid=55290' 2026-02-18T08:12:00.835 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:12:00.835 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=54, tag_tid=2, entry_tid=55538' = 'object_number=54, tag_tid=2, entry_tid=55290' 2026-02-18T08:12:00.835 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=54, tag_tid=2, entry_tid=55538' 2026-02-18T08:12:00.836 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=57, tag_tid=3, entry_tid=1' 2026-02-18T08:12:00.836 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:12:00.836 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:12:00.837 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=3 2026-02-18T08:12:00.837 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=54, tag_tid=2, entry_tid=55538' 2026-02-18T08:12:00.837 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:12:00.838 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:12:00.838 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=2 2026-02-18T08:12:00.839 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=57, tag_tid=3, entry_tid=1' 2026-02-18T08:12:00.839 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:12:00.840 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:12:00.841 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:12:00.842 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=54, tag_tid=2, entry_tid=55538' 2026-02-18T08:12:00.842 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:12:00.842 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:12:00.843 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=55538 2026-02-18T08:12:00.843 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 3 = 2 -a 1 -le 55538 2026-02-18T08:12:00.843 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:12:00.843 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:12:00.844 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:12:01.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:12:01.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:12:01.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:12:01.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:12:01.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:12:01.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:12:01.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:12:01.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:12:01.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:12:01.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:12:01.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:12:01.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:12:01.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:12:01.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:12:01.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:12:01.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:12:01.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:12:01.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:12:01.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:12:01.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:12:01.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:12:01.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:12:01.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:12:01.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:12:01.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:12:01.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:12:01.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:12:01.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:12:01.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:12:01.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:12:01.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:12:01.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:12:01.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:12:01.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:12:01.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:12:01.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:12:01.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:12:01.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:12:01.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:12:01.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:12:01.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:12:01.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:12:01.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:12:01.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:12:01.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:12:01.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:12:01.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:12:01.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:12:01.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:12:01.059 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:12:01.060 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:12:01.060 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:12:01.060 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:12:01.120 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:12:01.121 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:12:01.121 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:12:01.121 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:12:01.121 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:12:01.121 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:12:01.121 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:12:01.121 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:12:01.121 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:12:01.121 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:12:01.122 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:12:01.122 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:12:01.122 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:12:01.122 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:12:01.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:12:01.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:12:01.149 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 13 2026-02-18T08:12:01.149 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 14 2026-02-18T08:12:01.149 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:12:01.149 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=57, tag_tid=3, entry_tid=1], [object_number=56, tag_tid=3, entry_tid=0], [object_number=59, tag_tid=2, entry_tid=59619], [object_number=58, tag_tid=2, entry_tid=59618]]], state=connected] 2026-02-18T08:12:01.149 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=53, tag_tid=2, entry_tid=55777], [object_number=52, tag_tid=2, entry_tid=55776], [object_number=55, tag_tid=2, entry_tid=55775], [object_number=54, tag_tid=2, entry_tid=55774]]], state=connected] 2026-02-18T08:12:01.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:12:01.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=57, tag_tid=3, entry_tid=1' 2026-02-18T08:12:01.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:12:01.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:12:01.155 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:12:01.155 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:12:01.155 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:12:01.155 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:12:01.155 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:12:01.155 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:12:01.155 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:12:01.155 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:12:01.156 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:12:01.156 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:12:01.156 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:12:01.157 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:12:01.157 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:12:01.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 13 2026-02-18T08:12:01.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 14 2026-02-18T08:12:01.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:12:01.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=57, tag_tid=3, entry_tid=1], [object_number=56, tag_tid=3, entry_tid=0], [object_number=59, tag_tid=2, entry_tid=59619], [object_number=58, tag_tid=2, entry_tid=59618]]], state=connected] 2026-02-18T08:12:01.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=53, tag_tid=2, entry_tid=55777], [object_number=52, tag_tid=2, entry_tid=55776], [object_number=55, tag_tid=2, entry_tid=55775], [object_number=54, tag_tid=2, entry_tid=55774]]], state=connected] 2026-02-18T08:12:01.196 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:12:01.198 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=53, tag_tid=2, entry_tid=55777' 2026-02-18T08:12:01.198 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=57, tag_tid=3, entry_tid=1' -a 'object_number=57, tag_tid=3, entry_tid=1' = 'object_number=53, tag_tid=2, entry_tid=55777' 2026-02-18T08:12:01.199 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=53, tag_tid=2, entry_tid=55777' '!=' 'object_number=54, tag_tid=2, entry_tid=55538' 2026-02-18T08:12:01.199 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:12:01.199 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=53, tag_tid=2, entry_tid=55777' = 'object_number=54, tag_tid=2, entry_tid=55538' 2026-02-18T08:12:01.199 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=53, tag_tid=2, entry_tid=55777' 2026-02-18T08:12:01.199 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=57, tag_tid=3, entry_tid=1' 2026-02-18T08:12:01.200 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:12:01.200 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:12:01.200 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=3 2026-02-18T08:12:01.201 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=53, tag_tid=2, entry_tid=55777' 2026-02-18T08:12:01.201 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:12:01.201 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:12:01.203 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=2 2026-02-18T08:12:01.203 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=57, tag_tid=3, entry_tid=1' 2026-02-18T08:12:01.204 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:12:01.204 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:12:01.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:12:01.206 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=53, tag_tid=2, entry_tid=55777' 2026-02-18T08:12:01.206 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:12:01.206 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:12:01.208 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=55777 2026-02-18T08:12:01.208 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 3 = 2 -a 1 -le 55777 2026-02-18T08:12:01.208 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:12:01.208 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:12:01.208 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:12:01.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:12:01.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:12:01.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:12:01.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:12:01.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:12:01.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:12:01.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:12:01.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:12:01.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:12:01.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:12:01.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:12:01.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:12:01.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:12:01.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:12:01.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:12:01.412 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:12:01.414 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:12:01.414 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:12:01.414 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:12:01.414 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:12:01.415 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:12:01.415 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:12:01.415 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:12:01.415 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:12:01.415 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:12:01.415 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:12:01.415 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:12:01.415 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:12:01.415 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:12:01.415 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:12:01.415 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:12:01.415 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:12:01.416 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:12:01.416 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:12:01.416 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:12:01.416 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:12:01.416 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:12:01.416 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:12:01.416 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:12:01.416 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:12:01.416 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:12:01.416 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:12:01.416 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:12:01.416 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:12:01.417 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:12:01.417 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:12:01.417 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:12:01.417 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:12:01.417 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:12:01.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:12:01.425 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:12:01.425 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:12:01.425 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:12:01.485 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:12:01.485 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:12:01.485 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:12:01.485 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:12:01.486 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:12:01.486 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:12:01.486 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:12:01.486 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:12:01.486 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:12:01.486 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:12:01.486 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:12:01.486 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:12:01.487 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:12:01.487 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:12:01.487 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:12:01.488 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:12:01.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 13 2026-02-18T08:12:01.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 14 2026-02-18T08:12:01.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:12:01.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=57, tag_tid=3, entry_tid=1], [object_number=56, tag_tid=3, entry_tid=0], [object_number=59, tag_tid=2, entry_tid=59619], [object_number=58, tag_tid=2, entry_tid=59618]]], state=connected] 2026-02-18T08:12:01.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=54, tag_tid=2, entry_tid=56022], [object_number=53, tag_tid=2, entry_tid=56021], [object_number=52, tag_tid=2, entry_tid=56020], [object_number=55, tag_tid=2, entry_tid=56019]]], state=connected] 2026-02-18T08:12:01.525 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:12:01.527 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=57, tag_tid=3, entry_tid=1' 2026-02-18T08:12:01.527 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:12:01.528 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:12:01.528 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:12:01.528 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:12:01.528 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:12:01.528 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:12:01.528 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:12:01.528 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:12:01.528 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:12:01.528 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:12:01.529 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:12:01.529 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:12:01.530 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:12:01.530 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:12:01.530 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:12:01.559 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 13 2026-02-18T08:12:01.560 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 14 2026-02-18T08:12:01.560 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:12:01.560 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=57, tag_tid=3, entry_tid=1], [object_number=56, tag_tid=3, entry_tid=0], [object_number=59, tag_tid=2, entry_tid=59619], [object_number=58, tag_tid=2, entry_tid=59618]]], state=connected] 2026-02-18T08:12:01.560 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=54, tag_tid=2, entry_tid=56022], [object_number=53, tag_tid=2, entry_tid=56021], [object_number=52, tag_tid=2, entry_tid=56020], [object_number=55, tag_tid=2, entry_tid=56019]]], state=connected] 2026-02-18T08:12:01.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:12:01.565 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=54, tag_tid=2, entry_tid=56022' 2026-02-18T08:12:01.566 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=57, tag_tid=3, entry_tid=1' -a 'object_number=57, tag_tid=3, entry_tid=1' = 'object_number=54, tag_tid=2, entry_tid=56022' 2026-02-18T08:12:01.566 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=54, tag_tid=2, entry_tid=56022' '!=' 'object_number=53, tag_tid=2, entry_tid=55777' 2026-02-18T08:12:01.566 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:12:01.566 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=54, tag_tid=2, entry_tid=56022' = 'object_number=53, tag_tid=2, entry_tid=55777' 2026-02-18T08:12:01.566 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=54, tag_tid=2, entry_tid=56022' 2026-02-18T08:12:01.567 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=57, tag_tid=3, entry_tid=1' 2026-02-18T08:12:01.567 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:12:01.567 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:12:01.568 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=3 2026-02-18T08:12:01.568 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=54, tag_tid=2, entry_tid=56022' 2026-02-18T08:12:01.569 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:12:01.569 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:12:01.570 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=2 2026-02-18T08:12:01.571 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=57, tag_tid=3, entry_tid=1' 2026-02-18T08:12:01.571 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:12:01.571 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:12:01.572 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:12:01.573 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=54, tag_tid=2, entry_tid=56022' 2026-02-18T08:12:01.573 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:12:01.573 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:12:01.574 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=56022 2026-02-18T08:12:01.575 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 3 = 2 -a 1 -le 56022 2026-02-18T08:12:01.575 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:12:01.575 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:12:01.575 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:12:01.776 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:12:01.776 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:12:01.777 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:12:01.777 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:12:01.777 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:12:01.777 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:12:01.777 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:12:01.777 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:12:01.777 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:12:01.777 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:12:01.777 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:12:01.777 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:12:01.777 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:12:01.777 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:12:01.778 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:12:01.778 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:12:01.780 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:12:01.780 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:12:01.781 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:12:01.781 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:12:01.781 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:12:01.781 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:12:01.781 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:12:01.781 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:12:01.781 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:12:01.781 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:12:01.781 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:12:01.781 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:12:01.781 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:12:01.782 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:12:01.782 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:12:01.782 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:12:01.782 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:12:01.782 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:12:01.782 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:12:01.782 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:12:01.782 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:12:01.782 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:12:01.782 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:12:01.782 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:12:01.782 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:12:01.783 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:12:01.783 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:12:01.783 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:12:01.783 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:12:01.783 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:12:01.783 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:12:01.783 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:12:01.783 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:12:01.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:12:01.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:12:01.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:12:01.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:12:01.852 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:12:01.853 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:12:01.853 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:12:01.853 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:12:01.853 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:12:01.853 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:12:01.854 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:12:01.854 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:12:01.854 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:12:01.854 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:12:01.854 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:12:01.854 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:12:01.854 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:12:01.855 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:12:01.855 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:12:01.855 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:12:01.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 13 2026-02-18T08:12:01.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 14 2026-02-18T08:12:01.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:12:01.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=57, tag_tid=3, entry_tid=1], [object_number=56, tag_tid=3, entry_tid=0], [object_number=59, tag_tid=2, entry_tid=59619], [object_number=58, tag_tid=2, entry_tid=59618]]], state=connected] 2026-02-18T08:12:01.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=52, tag_tid=2, entry_tid=56292], [object_number=55, tag_tid=2, entry_tid=56291], [object_number=54, tag_tid=2, entry_tid=56290], [object_number=53, tag_tid=2, entry_tid=56289]]], state=connected] 2026-02-18T08:12:01.895 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:12:01.897 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=57, tag_tid=3, entry_tid=1' 2026-02-18T08:12:01.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:12:01.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:12:01.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:12:01.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:12:01.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:12:01.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:12:01.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:12:01.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:12:01.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:12:01.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:12:01.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:12:01.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:12:01.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:12:01.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:12:01.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:12:01.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 13 2026-02-18T08:12:01.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 14 2026-02-18T08:12:01.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:12:01.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=57, tag_tid=3, entry_tid=1], [object_number=56, tag_tid=3, entry_tid=0], [object_number=59, tag_tid=2, entry_tid=59619], [object_number=58, tag_tid=2, entry_tid=59618]]], state=connected] 2026-02-18T08:12:01.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=52, tag_tid=2, entry_tid=56292], [object_number=55, tag_tid=2, entry_tid=56291], [object_number=54, tag_tid=2, entry_tid=56290], [object_number=53, tag_tid=2, entry_tid=56289]]], state=connected] 2026-02-18T08:12:01.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:12:01.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=52, tag_tid=2, entry_tid=56292' 2026-02-18T08:12:01.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=57, tag_tid=3, entry_tid=1' -a 'object_number=57, tag_tid=3, entry_tid=1' = 'object_number=52, tag_tid=2, entry_tid=56292' 2026-02-18T08:12:01.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=52, tag_tid=2, entry_tid=56292' '!=' 'object_number=54, tag_tid=2, entry_tid=56022' 2026-02-18T08:12:01.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:12:01.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=52, tag_tid=2, entry_tid=56292' = 'object_number=54, tag_tid=2, entry_tid=56022' 2026-02-18T08:12:01.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=52, tag_tid=2, entry_tid=56292' 2026-02-18T08:12:01.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=57, tag_tid=3, entry_tid=1' 2026-02-18T08:12:01.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:12:01.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:12:01.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=3 2026-02-18T08:12:01.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=52, tag_tid=2, entry_tid=56292' 2026-02-18T08:12:01.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:12:01.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:12:01.944 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=2 2026-02-18T08:12:01.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=57, tag_tid=3, entry_tid=1' 2026-02-18T08:12:01.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:12:01.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:12:01.946 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:12:01.947 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=52, tag_tid=2, entry_tid=56292' 2026-02-18T08:12:01.947 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:12:01.947 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:12:01.949 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=56292 2026-02-18T08:12:01.949 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 3 = 2 -a 1 -le 56292 2026-02-18T08:12:01.949 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:12:01.949 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:12:01.949 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:12:02.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:12:02.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:12:02.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:12:02.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:12:02.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:12:02.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:12:02.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:12:02.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:12:02.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:12:02.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:12:02.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:12:02.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:12:02.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:12:02.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:12:02.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:12:02.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:12:02.155 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:12:02.155 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:12:02.155 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:12:02.155 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:12:02.155 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:12:02.155 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:12:02.155 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:12:02.155 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:12:02.155 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:12:02.156 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:12:02.156 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:12:02.156 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:12:02.156 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:12:02.156 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:12:02.156 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:12:02.156 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:12:02.156 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:12:02.156 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:12:02.156 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:12:02.156 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:12:02.156 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:12:02.157 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:12:02.157 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:12:02.157 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:12:02.157 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:12:02.157 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:12:02.157 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:12:02.157 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:12:02.157 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:12:02.157 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:12:02.157 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:12:02.157 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:12:02.157 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:12:02.165 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:12:02.166 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:12:02.166 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:12:02.166 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:12:02.223 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:12:02.224 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:12:02.224 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:12:02.224 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:12:02.225 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:12:02.225 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:12:02.225 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:12:02.225 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:12:02.225 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:12:02.225 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:12:02.225 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:12:02.225 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:12:02.225 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:12:02.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:12:02.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:12:02.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:12:02.253 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 13 2026-02-18T08:12:02.253 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 14 2026-02-18T08:12:02.253 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:12:02.254 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=57, tag_tid=3, entry_tid=1], [object_number=56, tag_tid=3, entry_tid=0], [object_number=59, tag_tid=2, entry_tid=59619], [object_number=58, tag_tid=2, entry_tid=59618]]], state=connected] 2026-02-18T08:12:02.254 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=54, tag_tid=2, entry_tid=56646], [object_number=53, tag_tid=2, entry_tid=56645], [object_number=52, tag_tid=2, entry_tid=56644], [object_number=55, tag_tid=2, entry_tid=56643]]], state=connected] 2026-02-18T08:12:02.256 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:12:02.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=57, tag_tid=3, entry_tid=1' 2026-02-18T08:12:02.259 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:12:02.259 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:12:02.259 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:12:02.259 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:12:02.259 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:12:02.260 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:12:02.260 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:12:02.260 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:12:02.260 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:12:02.260 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:12:02.260 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:12:02.260 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:12:02.261 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:12:02.261 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:12:02.261 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:12:02.287 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 13 2026-02-18T08:12:02.287 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 14 2026-02-18T08:12:02.287 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:12:02.287 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=57, tag_tid=3, entry_tid=1], [object_number=56, tag_tid=3, entry_tid=0], [object_number=59, tag_tid=2, entry_tid=59619], [object_number=58, tag_tid=2, entry_tid=59618]]], state=connected] 2026-02-18T08:12:02.287 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=54, tag_tid=2, entry_tid=56646], [object_number=53, tag_tid=2, entry_tid=56645], [object_number=52, tag_tid=2, entry_tid=56644], [object_number=55, tag_tid=2, entry_tid=56643]]], state=connected] 2026-02-18T08:12:02.289 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:12:02.291 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=54, tag_tid=2, entry_tid=56646' 2026-02-18T08:12:02.291 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=57, tag_tid=3, entry_tid=1' -a 'object_number=57, tag_tid=3, entry_tid=1' = 'object_number=54, tag_tid=2, entry_tid=56646' 2026-02-18T08:12:02.291 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=54, tag_tid=2, entry_tid=56646' '!=' 'object_number=52, tag_tid=2, entry_tid=56292' 2026-02-18T08:12:02.291 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:12:02.291 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=54, tag_tid=2, entry_tid=56646' = 'object_number=52, tag_tid=2, entry_tid=56292' 2026-02-18T08:12:02.291 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=54, tag_tid=2, entry_tid=56646' 2026-02-18T08:12:02.292 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=57, tag_tid=3, entry_tid=1' 2026-02-18T08:12:02.292 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:12:02.292 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:12:02.293 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=3 2026-02-18T08:12:02.293 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=54, tag_tid=2, entry_tid=56646' 2026-02-18T08:12:02.294 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:12:02.294 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:12:02.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=2 2026-02-18T08:12:02.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=57, tag_tid=3, entry_tid=1' 2026-02-18T08:12:02.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:12:02.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:12:02.297 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:12:02.298 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=54, tag_tid=2, entry_tid=56646' 2026-02-18T08:12:02.298 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:12:02.299 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:12:02.300 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=56646 2026-02-18T08:12:02.300 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 3 = 2 -a 1 -le 56646 2026-02-18T08:12:02.300 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:12:02.300 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:12:02.300 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:12:02.502 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:12:02.502 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:12:02.502 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:12:02.502 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:12:02.502 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:12:02.502 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:12:02.502 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:12:02.503 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:12:02.503 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:12:02.503 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:12:02.503 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:12:02.503 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:12:02.503 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:12:02.503 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:12:02.503 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:12:02.503 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:12:02.506 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:12:02.506 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:12:02.506 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:12:02.506 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:12:02.506 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:12:02.506 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:12:02.506 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:12:02.506 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:12:02.506 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:12:02.507 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:12:02.507 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:12:02.507 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:12:02.507 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:12:02.507 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:12:02.507 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:12:02.507 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:12:02.507 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:12:02.507 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:12:02.507 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:12:02.507 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:12:02.507 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:12:02.508 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:12:02.508 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:12:02.508 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:12:02.508 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:12:02.508 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:12:02.508 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:12:02.508 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:12:02.508 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:12:02.508 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:12:02.508 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:12:02.508 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:12:02.508 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:12:02.516 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:12:02.516 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:12:02.516 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:12:02.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:12:02.577 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:12:02.578 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:12:02.578 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:12:02.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:12:02.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:12:02.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:12:02.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:12:02.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:12:02.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:12:02.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:12:02.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:12:02.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:12:02.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:12:02.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:12:02.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:12:02.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:12:02.618 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 13 2026-02-18T08:12:02.618 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 14 2026-02-18T08:12:02.618 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:12:02.618 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=57, tag_tid=3, entry_tid=1], [object_number=56, tag_tid=3, entry_tid=0], [object_number=59, tag_tid=2, entry_tid=59619], [object_number=58, tag_tid=2, entry_tid=59618]]], state=connected] 2026-02-18T08:12:02.619 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=54, tag_tid=2, entry_tid=56922], [object_number=53, tag_tid=2, entry_tid=56921], [object_number=52, tag_tid=2, entry_tid=56920], [object_number=55, tag_tid=2, entry_tid=56919]]], state=connected] 2026-02-18T08:12:02.622 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:12:02.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=57, tag_tid=3, entry_tid=1' 2026-02-18T08:12:02.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:12:02.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:12:02.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:12:02.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:12:02.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:12:02.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:12:02.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:12:02.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:12:02.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:12:02.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:12:02.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:12:02.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:12:02.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:12:02.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:12:02.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:12:02.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 13 2026-02-18T08:12:02.653 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 14 2026-02-18T08:12:02.653 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:12:02.653 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=57, tag_tid=3, entry_tid=1], [object_number=56, tag_tid=3, entry_tid=0], [object_number=59, tag_tid=2, entry_tid=59619], [object_number=58, tag_tid=2, entry_tid=59618]]], state=connected] 2026-02-18T08:12:02.653 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=54, tag_tid=2, entry_tid=56922], [object_number=53, tag_tid=2, entry_tid=56921], [object_number=52, tag_tid=2, entry_tid=56920], [object_number=55, tag_tid=2, entry_tid=56919]]], state=connected] 2026-02-18T08:12:02.655 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:12:02.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=54, tag_tid=2, entry_tid=56922' 2026-02-18T08:12:02.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=57, tag_tid=3, entry_tid=1' -a 'object_number=57, tag_tid=3, entry_tid=1' = 'object_number=54, tag_tid=2, entry_tid=56922' 2026-02-18T08:12:02.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=54, tag_tid=2, entry_tid=56922' '!=' 'object_number=54, tag_tid=2, entry_tid=56646' 2026-02-18T08:12:02.658 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:12:02.658 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=54, tag_tid=2, entry_tid=56922' = 'object_number=54, tag_tid=2, entry_tid=56646' 2026-02-18T08:12:02.658 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=54, tag_tid=2, entry_tid=56922' 2026-02-18T08:12:02.658 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=57, tag_tid=3, entry_tid=1' 2026-02-18T08:12:02.658 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:12:02.659 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:12:02.659 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=3 2026-02-18T08:12:02.660 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=54, tag_tid=2, entry_tid=56922' 2026-02-18T08:12:02.660 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:12:02.660 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:12:02.661 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=2 2026-02-18T08:12:02.662 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=57, tag_tid=3, entry_tid=1' 2026-02-18T08:12:02.662 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:12:02.663 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:12:02.663 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:12:02.664 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=54, tag_tid=2, entry_tid=56922' 2026-02-18T08:12:02.664 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:12:02.664 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:12:02.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=56922 2026-02-18T08:12:02.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 3 = 2 -a 1 -le 56922 2026-02-18T08:12:02.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:12:02.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:12:02.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:12:02.868 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:12:02.868 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:12:02.868 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:12:02.868 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:12:02.869 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:12:02.869 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:12:02.869 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:12:02.869 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:12:02.869 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:12:02.869 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:12:02.869 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:12:02.869 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:12:02.869 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:12:02.870 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:12:02.870 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:12:02.870 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:12:02.870 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:12:02.875 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:12:02.875 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:12:02.875 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:12:02.875 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:12:02.875 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:12:02.876 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:12:02.876 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:12:02.876 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:12:02.876 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:12:02.876 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:12:02.876 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:12:02.876 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:12:02.877 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:12:02.877 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:12:02.877 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:12:02.877 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:12:02.877 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:12:02.877 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:12:02.877 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:12:02.877 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:12:02.878 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:12:02.878 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:12:02.878 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:12:02.878 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:12:02.878 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:12:02.878 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:12:02.878 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:12:02.879 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:12:02.879 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:12:02.879 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:12:02.879 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:12:02.879 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:12:02.881 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:12:02.882 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:12:02.882 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:12:02.882 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:12:02.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:12:02.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:12:02.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:12:02.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:12:02.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:12:02.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:12:02.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:12:02.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:12:02.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:12:02.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:12:02.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:12:02.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:12:02.944 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:12:02.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:12:02.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:12:02.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:12:02.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 13 2026-02-18T08:12:02.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 14 2026-02-18T08:12:02.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:12:02.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=57, tag_tid=3, entry_tid=1], [object_number=56, tag_tid=3, entry_tid=0], [object_number=59, tag_tid=2, entry_tid=59619], [object_number=58, tag_tid=2, entry_tid=59618]]], state=connected] 2026-02-18T08:12:02.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=54, tag_tid=2, entry_tid=57182], [object_number=53, tag_tid=2, entry_tid=57181], [object_number=52, tag_tid=2, entry_tid=57180], [object_number=55, tag_tid=2, entry_tid=57179]]], state=connected] 2026-02-18T08:12:02.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:12:02.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=57, tag_tid=3, entry_tid=1' 2026-02-18T08:12:02.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:12:02.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:12:02.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:12:02.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:12:02.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:12:02.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:12:02.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:12:02.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:12:02.985 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:12:02.985 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:12:02.985 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:12:02.985 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:12:02.986 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:12:02.986 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:12:02.986 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:12:03.012 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 13 2026-02-18T08:12:03.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 14 2026-02-18T08:12:03.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:12:03.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=57, tag_tid=3, entry_tid=1], [object_number=56, tag_tid=3, entry_tid=0], [object_number=59, tag_tid=2, entry_tid=59619], [object_number=58, tag_tid=2, entry_tid=59618]]], state=connected] 2026-02-18T08:12:03.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=54, tag_tid=2, entry_tid=57182], [object_number=53, tag_tid=2, entry_tid=57181], [object_number=52, tag_tid=2, entry_tid=57180], [object_number=55, tag_tid=2, entry_tid=57179]]], state=connected] 2026-02-18T08:12:03.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:12:03.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=54, tag_tid=2, entry_tid=57182' 2026-02-18T08:12:03.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=57, tag_tid=3, entry_tid=1' -a 'object_number=57, tag_tid=3, entry_tid=1' = 'object_number=54, tag_tid=2, entry_tid=57182' 2026-02-18T08:12:03.018 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=54, tag_tid=2, entry_tid=57182' '!=' 'object_number=54, tag_tid=2, entry_tid=56922' 2026-02-18T08:12:03.018 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:12:03.018 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=54, tag_tid=2, entry_tid=57182' = 'object_number=54, tag_tid=2, entry_tid=56922' 2026-02-18T08:12:03.018 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=54, tag_tid=2, entry_tid=57182' 2026-02-18T08:12:03.018 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=57, tag_tid=3, entry_tid=1' 2026-02-18T08:12:03.019 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:12:03.019 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:12:03.019 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=3 2026-02-18T08:12:03.020 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=54, tag_tid=2, entry_tid=57182' 2026-02-18T08:12:03.020 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:12:03.020 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:12:03.021 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=2 2026-02-18T08:12:03.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=57, tag_tid=3, entry_tid=1' 2026-02-18T08:12:03.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:12:03.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:12:03.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:12:03.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=54, tag_tid=2, entry_tid=57182' 2026-02-18T08:12:03.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:12:03.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:12:03.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=57182 2026-02-18T08:12:03.027 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 3 = 2 -a 1 -le 57182 2026-02-18T08:12:03.027 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:12:03.027 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:12:03.027 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:12:03.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:12:03.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:12:03.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:12:03.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:12:03.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:12:03.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:12:03.229 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:12:03.229 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:12:03.229 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:12:03.229 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:12:03.229 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:12:03.229 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:12:03.229 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:12:03.229 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:12:03.229 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:12:03.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:12:03.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:12:03.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:12:03.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:12:03.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:12:03.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:12:03.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:12:03.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:12:03.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:12:03.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:12:03.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:12:03.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:12:03.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:12:03.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:12:03.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:12:03.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:12:03.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:12:03.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:12:03.234 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:12:03.234 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:12:03.234 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:12:03.234 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:12:03.234 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:12:03.234 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:12:03.234 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:12:03.234 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:12:03.234 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:12:03.234 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:12:03.234 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:12:03.234 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:12:03.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:12:03.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:12:03.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:12:03.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:12:03.244 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:12:03.244 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:12:03.244 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:12:03.244 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:12:03.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:12:03.312 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:12:03.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:12:03.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:12:03.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:12:03.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:12:03.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:12:03.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:12:03.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:12:03.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:12:03.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:12:03.314 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:12:03.314 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:12:03.314 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:12:03.315 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:12:03.315 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:12:03.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 13 2026-02-18T08:12:03.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 14 2026-02-18T08:12:03.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:12:03.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=57, tag_tid=3, entry_tid=1], [object_number=56, tag_tid=3, entry_tid=0], [object_number=59, tag_tid=2, entry_tid=59619], [object_number=58, tag_tid=2, entry_tid=59618]]], state=connected] 2026-02-18T08:12:03.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=53, tag_tid=2, entry_tid=57449], [object_number=52, tag_tid=2, entry_tid=57448], [object_number=55, tag_tid=2, entry_tid=57447], [object_number=54, tag_tid=2, entry_tid=57446]]], state=connected] 2026-02-18T08:12:03.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:12:03.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=57, tag_tid=3, entry_tid=1' 2026-02-18T08:12:03.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:12:03.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:12:03.348 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:12:03.348 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:12:03.348 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:12:03.348 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:12:03.348 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:12:03.348 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:12:03.348 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:12:03.348 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:12:03.348 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:12:03.348 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:12:03.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:12:03.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:12:03.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:12:03.383 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 13 2026-02-18T08:12:03.383 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 14 2026-02-18T08:12:03.383 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:12:03.383 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=57, tag_tid=3, entry_tid=1], [object_number=56, tag_tid=3, entry_tid=0], [object_number=59, tag_tid=2, entry_tid=59619], [object_number=58, tag_tid=2, entry_tid=59618]]], state=connected] 2026-02-18T08:12:03.383 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=53, tag_tid=2, entry_tid=57449], [object_number=52, tag_tid=2, entry_tid=57448], [object_number=55, tag_tid=2, entry_tid=57447], [object_number=54, tag_tid=2, entry_tid=57446]]], state=connected] 2026-02-18T08:12:03.387 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:12:03.388 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=53, tag_tid=2, entry_tid=57449' 2026-02-18T08:12:03.389 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=57, tag_tid=3, entry_tid=1' -a 'object_number=57, tag_tid=3, entry_tid=1' = 'object_number=53, tag_tid=2, entry_tid=57449' 2026-02-18T08:12:03.389 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=53, tag_tid=2, entry_tid=57449' '!=' 'object_number=54, tag_tid=2, entry_tid=57182' 2026-02-18T08:12:03.389 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:12:03.389 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=53, tag_tid=2, entry_tid=57449' = 'object_number=54, tag_tid=2, entry_tid=57182' 2026-02-18T08:12:03.389 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=53, tag_tid=2, entry_tid=57449' 2026-02-18T08:12:03.389 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=57, tag_tid=3, entry_tid=1' 2026-02-18T08:12:03.389 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:12:03.389 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:12:03.390 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=3 2026-02-18T08:12:03.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=53, tag_tid=2, entry_tid=57449' 2026-02-18T08:12:03.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:12:03.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:12:03.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=2 2026-02-18T08:12:03.393 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=57, tag_tid=3, entry_tid=1' 2026-02-18T08:12:03.393 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:12:03.393 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:12:03.394 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:12:03.395 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=53, tag_tid=2, entry_tid=57449' 2026-02-18T08:12:03.395 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:12:03.395 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:12:03.397 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=57449 2026-02-18T08:12:03.397 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 3 = 2 -a 1 -le 57449 2026-02-18T08:12:03.397 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:12:03.397 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:12:03.397 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:12:03.599 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:12:03.599 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:12:03.599 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:12:03.599 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:12:03.599 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:12:03.600 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:12:03.600 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:12:03.600 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:12:03.600 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:12:03.600 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:12:03.600 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:12:03.600 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:12:03.600 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:12:03.600 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:12:03.601 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:12:03.601 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:12:03.601 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:12:03.604 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:12:03.604 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:12:03.604 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:12:03.604 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:12:03.604 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:12:03.604 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:12:03.604 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:12:03.604 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:12:03.604 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:12:03.605 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:12:03.605 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:12:03.605 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:12:03.605 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:12:03.605 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:12:03.605 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:12:03.605 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:12:03.605 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:12:03.605 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:12:03.605 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:12:03.605 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:12:03.606 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:12:03.606 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:12:03.606 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:12:03.606 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:12:03.606 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:12:03.606 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:12:03.606 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:12:03.606 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:12:03.606 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:12:03.606 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:12:03.606 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:12:03.606 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:12:03.612 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:12:03.613 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:12:03.613 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:12:03.613 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:12:03.671 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:12:03.672 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:12:03.672 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:12:03.673 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:12:03.673 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:12:03.673 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:12:03.673 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:12:03.673 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:12:03.673 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:12:03.673 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:12:03.673 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:12:03.673 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:12:03.673 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:12:03.674 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:12:03.674 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:12:03.674 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:12:03.701 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 13 2026-02-18T08:12:03.701 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 14 2026-02-18T08:12:03.701 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:12:03.701 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=57, tag_tid=3, entry_tid=1], [object_number=56, tag_tid=3, entry_tid=0], [object_number=59, tag_tid=2, entry_tid=59619], [object_number=58, tag_tid=2, entry_tid=59618]]], state=connected] 2026-02-18T08:12:03.701 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=54, tag_tid=2, entry_tid=57714], [object_number=53, tag_tid=2, entry_tid=57713], [object_number=52, tag_tid=2, entry_tid=57712], [object_number=55, tag_tid=2, entry_tid=57711]]], state=connected] 2026-02-18T08:12:03.703 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:12:03.705 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=57, tag_tid=3, entry_tid=1' 2026-02-18T08:12:03.706 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:12:03.706 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:12:03.706 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:12:03.706 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:12:03.706 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:12:03.707 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:12:03.707 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:12:03.707 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:12:03.707 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:12:03.707 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:12:03.707 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:12:03.708 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:12:03.708 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:12:03.708 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:12:03.708 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:12:03.734 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 13 2026-02-18T08:12:03.734 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 14 2026-02-18T08:12:03.734 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:12:03.734 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=57, tag_tid=3, entry_tid=1], [object_number=56, tag_tid=3, entry_tid=0], [object_number=59, tag_tid=2, entry_tid=59619], [object_number=58, tag_tid=2, entry_tid=59618]]], state=connected] 2026-02-18T08:12:03.734 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=54, tag_tid=2, entry_tid=57714], [object_number=53, tag_tid=2, entry_tid=57713], [object_number=52, tag_tid=2, entry_tid=57712], [object_number=55, tag_tid=2, entry_tid=57711]]], state=connected] 2026-02-18T08:12:03.736 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:12:03.738 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=54, tag_tid=2, entry_tid=57714' 2026-02-18T08:12:03.738 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=57, tag_tid=3, entry_tid=1' -a 'object_number=57, tag_tid=3, entry_tid=1' = 'object_number=54, tag_tid=2, entry_tid=57714' 2026-02-18T08:12:03.738 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=54, tag_tid=2, entry_tid=57714' '!=' 'object_number=53, tag_tid=2, entry_tid=57449' 2026-02-18T08:12:03.738 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:12:03.738 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=54, tag_tid=2, entry_tid=57714' = 'object_number=53, tag_tid=2, entry_tid=57449' 2026-02-18T08:12:03.738 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=54, tag_tid=2, entry_tid=57714' 2026-02-18T08:12:03.739 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=57, tag_tid=3, entry_tid=1' 2026-02-18T08:12:03.739 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:12:03.739 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:12:03.740 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=3 2026-02-18T08:12:03.741 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=54, tag_tid=2, entry_tid=57714' 2026-02-18T08:12:03.741 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:12:03.741 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:12:03.742 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=2 2026-02-18T08:12:03.743 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=57, tag_tid=3, entry_tid=1' 2026-02-18T08:12:03.743 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:12:03.743 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:12:03.744 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:12:03.745 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=54, tag_tid=2, entry_tid=57714' 2026-02-18T08:12:03.745 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:12:03.745 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:12:03.747 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=57714 2026-02-18T08:12:03.747 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 3 = 2 -a 1 -le 57714 2026-02-18T08:12:03.747 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:12:03.747 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:12:03.747 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:12:03.948 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:12:03.949 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:12:03.949 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:12:03.949 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:12:03.949 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:12:03.949 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:12:03.949 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:12:03.949 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:12:03.949 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:12:03.949 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:12:03.949 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:12:03.950 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:12:03.950 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:12:03.950 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:12:03.950 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:12:03.953 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:12:03.953 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:12:03.953 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:12:03.953 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:12:03.953 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:12:03.953 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:12:03.953 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:12:03.953 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:12:03.953 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:12:03.953 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:12:03.954 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:12:03.954 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:12:03.954 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:12:03.954 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:12:03.954 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:12:03.954 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:12:03.954 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:12:03.954 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:12:03.954 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:12:03.954 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:12:03.954 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:12:03.954 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:12:03.955 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:12:03.955 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:12:03.955 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:12:03.955 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:12:03.955 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:12:03.955 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:12:03.955 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:12:03.955 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:12:03.955 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:12:03.955 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:12:03.955 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:12:03.956 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:12:03.963 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:12:03.963 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:12:03.964 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:12:03.964 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:12:04.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:12:04.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:12:04.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:12:04.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:12:04.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:12:04.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:12:04.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:12:04.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:12:04.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:12:04.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:12:04.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:12:04.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:12:04.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:12:04.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:12:04.027 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:12:04.027 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:12:04.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 13 2026-02-18T08:12:04.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 14 2026-02-18T08:12:04.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:12:04.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=57, tag_tid=3, entry_tid=1], [object_number=56, tag_tid=3, entry_tid=0], [object_number=59, tag_tid=2, entry_tid=59619], [object_number=58, tag_tid=2, entry_tid=59618]]], state=connected] 2026-02-18T08:12:04.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=55, tag_tid=2, entry_tid=57963], [object_number=54, tag_tid=2, entry_tid=57962], [object_number=53, tag_tid=2, entry_tid=57961], [object_number=52, tag_tid=2, entry_tid=57960]]], state=connected] 2026-02-18T08:12:04.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:12:04.058 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=57, tag_tid=3, entry_tid=1' 2026-02-18T08:12:04.060 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:12:04.060 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:12:04.060 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:12:04.060 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:12:04.060 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:12:04.060 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:12:04.060 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:12:04.060 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:12:04.060 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:12:04.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:12:04.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:12:04.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:12:04.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:12:04.062 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:12:04.062 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:12:04.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 13 2026-02-18T08:12:04.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 14 2026-02-18T08:12:04.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:12:04.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=57, tag_tid=3, entry_tid=1], [object_number=56, tag_tid=3, entry_tid=0], [object_number=59, tag_tid=2, entry_tid=59619], [object_number=58, tag_tid=2, entry_tid=59618]]], state=connected] 2026-02-18T08:12:04.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=55, tag_tid=2, entry_tid=57963], [object_number=54, tag_tid=2, entry_tid=57962], [object_number=53, tag_tid=2, entry_tid=57961], [object_number=52, tag_tid=2, entry_tid=57960]]], state=connected] 2026-02-18T08:12:04.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:12:04.092 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=55, tag_tid=2, entry_tid=57963' 2026-02-18T08:12:04.092 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=57, tag_tid=3, entry_tid=1' -a 'object_number=57, tag_tid=3, entry_tid=1' = 'object_number=55, tag_tid=2, entry_tid=57963' 2026-02-18T08:12:04.093 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=55, tag_tid=2, entry_tid=57963' '!=' 'object_number=54, tag_tid=2, entry_tid=57714' 2026-02-18T08:12:04.093 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:12:04.093 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=55, tag_tid=2, entry_tid=57963' = 'object_number=54, tag_tid=2, entry_tid=57714' 2026-02-18T08:12:04.093 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=55, tag_tid=2, entry_tid=57963' 2026-02-18T08:12:04.093 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=57, tag_tid=3, entry_tid=1' 2026-02-18T08:12:04.094 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:12:04.094 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:12:04.094 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=3 2026-02-18T08:12:04.095 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=55, tag_tid=2, entry_tid=57963' 2026-02-18T08:12:04.095 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:12:04.096 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:12:04.097 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=2 2026-02-18T08:12:04.097 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=57, tag_tid=3, entry_tid=1' 2026-02-18T08:12:04.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:12:04.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:12:04.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:12:04.100 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=55, tag_tid=2, entry_tid=57963' 2026-02-18T08:12:04.100 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:12:04.100 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:12:04.102 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=57963 2026-02-18T08:12:04.102 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 3 = 2 -a 1 -le 57963 2026-02-18T08:12:04.102 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:12:04.102 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:12:04.102 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:12:04.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:12:04.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:12:04.304 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:12:04.304 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:12:04.304 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:12:04.304 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:12:04.304 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:12:04.304 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:12:04.304 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:12:04.304 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:12:04.304 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:12:04.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:12:04.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:12:04.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:12:04.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:12:04.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:12:04.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:12:04.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:12:04.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:12:04.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:12:04.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:12:04.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:12:04.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:12:04.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:12:04.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:12:04.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:12:04.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:12:04.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:12:04.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:12:04.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:12:04.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:12:04.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:12:04.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:12:04.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:12:04.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:12:04.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:12:04.310 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:12:04.310 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:12:04.310 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:12:04.310 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:12:04.310 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:12:04.310 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:12:04.310 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:12:04.310 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:12:04.310 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:12:04.310 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:12:04.310 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:12:04.310 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:12:04.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:12:04.317 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:12:04.317 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:12:04.317 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:12:04.317 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:12:04.377 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:12:04.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:12:04.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:12:04.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:12:04.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:12:04.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:12:04.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:12:04.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:12:04.379 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:12:04.379 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:12:04.379 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:12:04.379 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:12:04.379 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:12:04.379 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:12:04.380 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:12:04.380 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:12:04.407 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 13 2026-02-18T08:12:04.407 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 14 2026-02-18T08:12:04.407 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:12:04.407 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=57, tag_tid=3, entry_tid=1], [object_number=56, tag_tid=3, entry_tid=0], [object_number=59, tag_tid=2, entry_tid=59619], [object_number=58, tag_tid=2, entry_tid=59618]]], state=connected] 2026-02-18T08:12:04.408 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=52, tag_tid=2, entry_tid=58220], [object_number=55, tag_tid=2, entry_tid=58219], [object_number=54, tag_tid=2, entry_tid=58218], [object_number=53, tag_tid=2, entry_tid=58217]]], state=connected] 2026-02-18T08:12:04.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:12:04.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=57, tag_tid=3, entry_tid=1' 2026-02-18T08:12:04.413 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:12:04.413 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:12:04.413 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:12:04.413 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:12:04.413 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:12:04.413 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:12:04.413 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:12:04.413 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:12:04.413 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:12:04.414 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:12:04.414 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:12:04.414 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:12:04.414 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:12:04.415 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:12:04.415 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:12:04.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 13 2026-02-18T08:12:04.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 14 2026-02-18T08:12:04.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:12:04.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=57, tag_tid=3, entry_tid=1], [object_number=56, tag_tid=3, entry_tid=0], [object_number=59, tag_tid=2, entry_tid=59619], [object_number=58, tag_tid=2, entry_tid=59618]]], state=connected] 2026-02-18T08:12:04.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=52, tag_tid=2, entry_tid=58220], [object_number=55, tag_tid=2, entry_tid=58219], [object_number=54, tag_tid=2, entry_tid=58218], [object_number=53, tag_tid=2, entry_tid=58217]]], state=connected] 2026-02-18T08:12:04.443 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:12:04.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=52, tag_tid=2, entry_tid=58220' 2026-02-18T08:12:04.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=57, tag_tid=3, entry_tid=1' -a 'object_number=57, tag_tid=3, entry_tid=1' = 'object_number=52, tag_tid=2, entry_tid=58220' 2026-02-18T08:12:04.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=52, tag_tid=2, entry_tid=58220' '!=' 'object_number=55, tag_tid=2, entry_tid=57963' 2026-02-18T08:12:04.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:12:04.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=52, tag_tid=2, entry_tid=58220' = 'object_number=55, tag_tid=2, entry_tid=57963' 2026-02-18T08:12:04.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=52, tag_tid=2, entry_tid=58220' 2026-02-18T08:12:04.446 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=57, tag_tid=3, entry_tid=1' 2026-02-18T08:12:04.446 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:12:04.446 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:12:04.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=3 2026-02-18T08:12:04.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=52, tag_tid=2, entry_tid=58220' 2026-02-18T08:12:04.448 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:12:04.448 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:12:04.449 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=2 2026-02-18T08:12:04.450 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=57, tag_tid=3, entry_tid=1' 2026-02-18T08:12:04.450 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:12:04.450 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:12:04.451 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:12:04.452 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=52, tag_tid=2, entry_tid=58220' 2026-02-18T08:12:04.452 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:12:04.452 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:12:04.454 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=58220 2026-02-18T08:12:04.454 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 3 = 2 -a 1 -le 58220 2026-02-18T08:12:04.454 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:12:04.454 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:12:04.454 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:12:04.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:12:04.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:12:04.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:12:04.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:12:04.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:12:04.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:12:04.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:12:04.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:12:04.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:12:04.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:12:04.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:12:04.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:12:04.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:12:04.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:12:04.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:12:04.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:12:04.660 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:12:04.660 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:12:04.660 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:12:04.660 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:12:04.661 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:12:04.661 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:12:04.661 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:12:04.661 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:12:04.661 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:12:04.661 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:12:04.661 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:12:04.661 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:12:04.661 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:12:04.661 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:12:04.661 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:12:04.662 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:12:04.662 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:12:04.662 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:12:04.662 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:12:04.662 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:12:04.662 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:12:04.662 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:12:04.662 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:12:04.662 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:12:04.662 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:12:04.662 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:12:04.662 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:12:04.663 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:12:04.663 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:12:04.663 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:12:04.663 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:12:04.663 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:12:04.663 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:12:04.672 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:12:04.672 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:12:04.672 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:12:04.672 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:12:04.731 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:12:04.732 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:12:04.732 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:12:04.732 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:12:04.732 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:12:04.732 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:12:04.733 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:12:04.733 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:12:04.733 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:12:04.733 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:12:04.733 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:12:04.733 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:12:04.734 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:12:04.734 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:12:04.734 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:12:04.735 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:12:04.761 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 14 2026-02-18T08:12:04.762 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 14 2026-02-18T08:12:04.762 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:12:04.762 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=57, tag_tid=3, entry_tid=1], [object_number=56, tag_tid=3, entry_tid=0], [object_number=59, tag_tid=2, entry_tid=59619], [object_number=58, tag_tid=2, entry_tid=59618]]], state=connected] 2026-02-18T08:12:04.762 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=58, tag_tid=2, entry_tid=58450], [object_number=57, tag_tid=2, entry_tid=58449], [object_number=56, tag_tid=2, entry_tid=58448], [object_number=59, tag_tid=2, entry_tid=58447]]], state=connected] 2026-02-18T08:12:04.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:12:04.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=57, tag_tid=3, entry_tid=1' 2026-02-18T08:12:04.766 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:12:04.766 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:12:04.766 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:12:04.766 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:12:04.766 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:12:04.766 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:12:04.766 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:12:04.767 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:12:04.767 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:12:04.767 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:12:04.767 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:12:04.768 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:12:04.768 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:12:04.768 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:12:04.768 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:12:04.801 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 14 2026-02-18T08:12:04.801 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 14 2026-02-18T08:12:04.802 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:12:04.802 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=57, tag_tid=3, entry_tid=1], [object_number=56, tag_tid=3, entry_tid=0], [object_number=59, tag_tid=2, entry_tid=59619], [object_number=58, tag_tid=2, entry_tid=59618]]], state=connected] 2026-02-18T08:12:04.802 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=58, tag_tid=2, entry_tid=58450], [object_number=57, tag_tid=2, entry_tid=58449], [object_number=56, tag_tid=2, entry_tid=58448], [object_number=59, tag_tid=2, entry_tid=58447]]], state=connected] 2026-02-18T08:12:04.805 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:12:04.807 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=58, tag_tid=2, entry_tid=58450' 2026-02-18T08:12:04.807 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=57, tag_tid=3, entry_tid=1' -a 'object_number=57, tag_tid=3, entry_tid=1' = 'object_number=58, tag_tid=2, entry_tid=58450' 2026-02-18T08:12:04.807 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=58, tag_tid=2, entry_tid=58450' '!=' 'object_number=52, tag_tid=2, entry_tid=58220' 2026-02-18T08:12:04.807 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:12:04.807 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=58, tag_tid=2, entry_tid=58450' = 'object_number=52, tag_tid=2, entry_tid=58220' 2026-02-18T08:12:04.807 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=58, tag_tid=2, entry_tid=58450' 2026-02-18T08:12:04.808 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=57, tag_tid=3, entry_tid=1' 2026-02-18T08:12:04.808 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:12:04.808 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:12:04.809 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=3 2026-02-18T08:12:04.809 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=58, tag_tid=2, entry_tid=58450' 2026-02-18T08:12:04.810 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:12:04.810 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:12:04.811 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=2 2026-02-18T08:12:04.812 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=57, tag_tid=3, entry_tid=1' 2026-02-18T08:12:04.812 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:12:04.812 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:12:04.813 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:12:04.814 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=58, tag_tid=2, entry_tid=58450' 2026-02-18T08:12:04.814 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:12:04.814 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:12:04.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=58450 2026-02-18T08:12:04.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 3 = 2 -a 1 -le 58450 2026-02-18T08:12:04.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:12:04.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:12:04.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:12:05.018 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:12:05.018 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:12:05.018 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:12:05.018 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:12:05.018 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:12:05.018 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:12:05.018 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:12:05.018 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:12:05.018 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:12:05.019 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:12:05.019 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:12:05.019 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:12:05.019 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:12:05.019 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:12:05.019 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:12:05.021 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:12:05.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:12:05.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:12:05.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:12:05.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:12:05.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:12:05.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:12:05.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:12:05.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:12:05.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:12:05.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:12:05.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:12:05.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:12:05.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:12:05.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:12:05.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:12:05.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:12:05.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:12:05.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:12:05.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:12:05.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:12:05.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:12:05.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:12:05.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:12:05.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:12:05.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:12:05.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:12:05.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:12:05.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:12:05.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:12:05.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:12:05.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:12:05.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:12:05.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:12:05.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:12:05.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:12:05.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:12:05.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:12:05.100 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:12:05.101 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:12:05.101 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:12:05.101 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:12:05.101 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:12:05.101 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:12:05.101 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:12:05.101 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:12:05.101 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:12:05.101 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:12:05.101 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:12:05.102 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:12:05.102 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:12:05.102 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:12:05.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:12:05.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:12:05.129 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 14 2026-02-18T08:12:05.129 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 14 2026-02-18T08:12:05.129 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:12:05.129 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=57, tag_tid=3, entry_tid=1], [object_number=56, tag_tid=3, entry_tid=0], [object_number=59, tag_tid=2, entry_tid=59619], [object_number=58, tag_tid=2, entry_tid=59618]]], state=connected] 2026-02-18T08:12:05.130 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=59, tag_tid=2, entry_tid=58771], [object_number=58, tag_tid=2, entry_tid=58770], [object_number=57, tag_tid=2, entry_tid=58769], [object_number=56, tag_tid=2, entry_tid=58768]]], state=connected] 2026-02-18T08:12:05.132 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:12:05.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=57, tag_tid=3, entry_tid=1' 2026-02-18T08:12:05.134 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:12:05.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:12:05.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:12:05.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:12:05.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:12:05.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:12:05.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:12:05.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:12:05.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:12:05.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:12:05.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:12:05.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:12:05.136 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:12:05.137 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:12:05.137 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:12:05.173 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 14 2026-02-18T08:12:05.173 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 14 2026-02-18T08:12:05.173 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:12:05.173 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=57, tag_tid=3, entry_tid=1], [object_number=56, tag_tid=3, entry_tid=0], [object_number=59, tag_tid=2, entry_tid=59619], [object_number=58, tag_tid=2, entry_tid=59618]]], state=connected] 2026-02-18T08:12:05.173 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=59, tag_tid=2, entry_tid=58771], [object_number=58, tag_tid=2, entry_tid=58770], [object_number=57, tag_tid=2, entry_tid=58769], [object_number=56, tag_tid=2, entry_tid=58768]]], state=connected] 2026-02-18T08:12:05.177 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:12:05.178 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=59, tag_tid=2, entry_tid=58771' 2026-02-18T08:12:05.178 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=57, tag_tid=3, entry_tid=1' -a 'object_number=57, tag_tid=3, entry_tid=1' = 'object_number=59, tag_tid=2, entry_tid=58771' 2026-02-18T08:12:05.179 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=59, tag_tid=2, entry_tid=58771' '!=' 'object_number=58, tag_tid=2, entry_tid=58450' 2026-02-18T08:12:05.179 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:12:05.179 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=59, tag_tid=2, entry_tid=58771' = 'object_number=58, tag_tid=2, entry_tid=58450' 2026-02-18T08:12:05.179 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=59, tag_tid=2, entry_tid=58771' 2026-02-18T08:12:05.179 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=57, tag_tid=3, entry_tid=1' 2026-02-18T08:12:05.180 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:12:05.180 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:12:05.180 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=3 2026-02-18T08:12:05.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=59, tag_tid=2, entry_tid=58771' 2026-02-18T08:12:05.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:12:05.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:12:05.183 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=2 2026-02-18T08:12:05.183 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=57, tag_tid=3, entry_tid=1' 2026-02-18T08:12:05.183 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:12:05.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:12:05.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:12:05.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=59, tag_tid=2, entry_tid=58771' 2026-02-18T08:12:05.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:12:05.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:12:05.187 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=58771 2026-02-18T08:12:05.187 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 3 = 2 -a 1 -le 58771 2026-02-18T08:12:05.187 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:12:05.187 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:12:05.187 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:12:05.389 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:12:05.389 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:12:05.389 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:12:05.389 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:12:05.389 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:12:05.389 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:12:05.389 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:12:05.390 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:12:05.390 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:12:05.390 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:12:05.390 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:12:05.390 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:12:05.390 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:12:05.390 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:12:05.390 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:12:05.390 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:12:05.393 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:12:05.393 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:12:05.393 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:12:05.393 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:12:05.393 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:12:05.393 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:12:05.393 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:12:05.393 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:12:05.393 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:12:05.394 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:12:05.394 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:12:05.394 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:12:05.394 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:12:05.394 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:12:05.394 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:12:05.394 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:12:05.394 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:12:05.394 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:12:05.394 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:12:05.394 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:12:05.394 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:12:05.395 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:12:05.395 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:12:05.395 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:12:05.395 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:12:05.395 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:12:05.395 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:12:05.395 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:12:05.395 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:12:05.395 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:12:05.395 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:12:05.395 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:12:05.395 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:12:05.403 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:12:05.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:12:05.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:12:05.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:12:05.462 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:12:05.463 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:12:05.463 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:12:05.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:12:05.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:12:05.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:12:05.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:12:05.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:12:05.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:12:05.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:12:05.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:12:05.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:12:05.465 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:12:05.465 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:12:05.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:12:05.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:12:05.491 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 14 2026-02-18T08:12:05.492 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 14 2026-02-18T08:12:05.492 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:12:05.492 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=57, tag_tid=3, entry_tid=1], [object_number=56, tag_tid=3, entry_tid=0], [object_number=59, tag_tid=2, entry_tid=59619], [object_number=58, tag_tid=2, entry_tid=59618]]], state=connected] 2026-02-18T08:12:05.492 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=58, tag_tid=2, entry_tid=59070], [object_number=57, tag_tid=2, entry_tid=59069], [object_number=56, tag_tid=2, entry_tid=59068], [object_number=59, tag_tid=2, entry_tid=59067]]], state=connected] 2026-02-18T08:12:05.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:12:05.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=57, tag_tid=3, entry_tid=1' 2026-02-18T08:12:05.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:12:05.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:12:05.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:12:05.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:12:05.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:12:05.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:12:05.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:12:05.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:12:05.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:12:05.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:12:05.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:12:05.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:12:05.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:12:05.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:12:05.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:12:05.525 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 14 2026-02-18T08:12:05.525 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 14 2026-02-18T08:12:05.525 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:12:05.525 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=57, tag_tid=3, entry_tid=1], [object_number=56, tag_tid=3, entry_tid=0], [object_number=59, tag_tid=2, entry_tid=59619], [object_number=58, tag_tid=2, entry_tid=59618]]], state=connected] 2026-02-18T08:12:05.525 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=58, tag_tid=2, entry_tid=59070], [object_number=57, tag_tid=2, entry_tid=59069], [object_number=56, tag_tid=2, entry_tid=59068], [object_number=59, tag_tid=2, entry_tid=59067]]], state=connected] 2026-02-18T08:12:05.528 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:12:05.529 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=58, tag_tid=2, entry_tid=59070' 2026-02-18T08:12:05.530 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=57, tag_tid=3, entry_tid=1' -a 'object_number=57, tag_tid=3, entry_tid=1' = 'object_number=58, tag_tid=2, entry_tid=59070' 2026-02-18T08:12:05.530 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=58, tag_tid=2, entry_tid=59070' '!=' 'object_number=59, tag_tid=2, entry_tid=58771' 2026-02-18T08:12:05.530 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:12:05.530 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=58, tag_tid=2, entry_tid=59070' = 'object_number=59, tag_tid=2, entry_tid=58771' 2026-02-18T08:12:05.530 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=58, tag_tid=2, entry_tid=59070' 2026-02-18T08:12:05.531 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=57, tag_tid=3, entry_tid=1' 2026-02-18T08:12:05.531 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:12:05.531 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:12:05.531 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=3 2026-02-18T08:12:05.532 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=58, tag_tid=2, entry_tid=59070' 2026-02-18T08:12:05.532 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:12:05.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:12:05.534 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=2 2026-02-18T08:12:05.535 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=57, tag_tid=3, entry_tid=1' 2026-02-18T08:12:05.535 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:12:05.535 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:12:05.536 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:12:05.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=58, tag_tid=2, entry_tid=59070' 2026-02-18T08:12:05.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:12:05.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:12:05.539 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=59070 2026-02-18T08:12:05.539 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 3 = 2 -a 1 -le 59070 2026-02-18T08:12:05.539 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:12:05.539 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:12:05.539 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:12:05.741 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:12:05.741 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:12:05.741 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:12:05.741 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:12:05.741 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:12:05.741 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:12:05.741 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:12:05.741 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:12:05.741 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:12:05.742 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:12:05.742 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:12:05.742 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:12:05.742 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:12:05.742 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:12:05.742 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:12:05.745 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:12:05.745 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:12:05.745 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:12:05.745 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:12:05.746 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:12:05.746 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:12:05.746 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:12:05.746 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:12:05.746 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:12:05.746 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:12:05.746 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:12:05.746 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:12:05.746 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:12:05.746 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:12:05.746 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:12:05.746 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:12:05.747 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:12:05.747 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:12:05.747 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:12:05.747 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:12:05.747 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:12:05.747 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:12:05.747 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:12:05.747 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:12:05.747 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:12:05.747 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:12:05.747 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:12:05.747 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:12:05.748 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:12:05.748 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:12:05.748 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:12:05.748 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:12:05.748 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:12:05.748 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:12:05.756 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:12:05.756 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:12:05.756 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:12:05.756 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:12:05.822 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:12:05.823 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:12:05.823 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:12:05.823 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:12:05.823 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:12:05.823 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:12:05.823 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:12:05.824 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:12:05.824 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:12:05.824 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:12:05.824 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:12:05.824 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:12:05.824 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:12:05.825 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:12:05.825 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:12:05.825 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:12:05.861 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 14 2026-02-18T08:12:05.861 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 14 2026-02-18T08:12:05.861 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:12:05.861 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=57, tag_tid=3, entry_tid=1], [object_number=56, tag_tid=3, entry_tid=0], [object_number=59, tag_tid=2, entry_tid=59619], [object_number=58, tag_tid=2, entry_tid=59618]]], state=connected] 2026-02-18T08:12:05.861 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=59, tag_tid=2, entry_tid=59359], [object_number=58, tag_tid=2, entry_tid=59358], [object_number=57, tag_tid=2, entry_tid=59357], [object_number=56, tag_tid=2, entry_tid=59356]]], state=connected] 2026-02-18T08:12:05.864 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:12:05.866 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=57, tag_tid=3, entry_tid=1' 2026-02-18T08:12:05.867 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:12:05.867 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:12:05.867 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:12:05.867 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:12:05.867 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:12:05.867 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:12:05.867 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:12:05.868 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:12:05.868 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:12:05.868 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:12:05.868 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:12:05.868 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:12:05.869 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:12:05.869 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:12:05.869 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:12:05.895 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 14 2026-02-18T08:12:05.895 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 14 2026-02-18T08:12:05.895 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:12:05.895 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=57, tag_tid=3, entry_tid=1], [object_number=56, tag_tid=3, entry_tid=0], [object_number=59, tag_tid=2, entry_tid=59619], [object_number=58, tag_tid=2, entry_tid=59618]]], state=connected] 2026-02-18T08:12:05.895 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=59, tag_tid=2, entry_tid=59359], [object_number=58, tag_tid=2, entry_tid=59358], [object_number=57, tag_tid=2, entry_tid=59357], [object_number=56, tag_tid=2, entry_tid=59356]]], state=connected] 2026-02-18T08:12:05.897 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:12:05.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=59, tag_tid=2, entry_tid=59359' 2026-02-18T08:12:05.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=57, tag_tid=3, entry_tid=1' -a 'object_number=57, tag_tid=3, entry_tid=1' = 'object_number=59, tag_tid=2, entry_tid=59359' 2026-02-18T08:12:05.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=59, tag_tid=2, entry_tid=59359' '!=' 'object_number=58, tag_tid=2, entry_tid=59070' 2026-02-18T08:12:05.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:12:05.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=59, tag_tid=2, entry_tid=59359' = 'object_number=58, tag_tid=2, entry_tid=59070' 2026-02-18T08:12:05.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=59, tag_tid=2, entry_tid=59359' 2026-02-18T08:12:05.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=57, tag_tid=3, entry_tid=1' 2026-02-18T08:12:05.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:12:05.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:12:05.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=3 2026-02-18T08:12:05.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=59, tag_tid=2, entry_tid=59359' 2026-02-18T08:12:05.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:12:05.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:12:05.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=2 2026-02-18T08:12:05.904 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=57, tag_tid=3, entry_tid=1' 2026-02-18T08:12:05.904 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:12:05.904 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:12:05.905 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:12:05.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=59, tag_tid=2, entry_tid=59359' 2026-02-18T08:12:05.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:12:05.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:12:05.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=59359 2026-02-18T08:12:05.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 3 = 2 -a 1 -le 59359 2026-02-18T08:12:05.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:12:05.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:12:05.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:12:06.110 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:12:06.110 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:12:06.110 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:12:06.110 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:12:06.110 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:12:06.110 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:12:06.110 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:12:06.110 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:12:06.111 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:12:06.111 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:12:06.111 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:12:06.111 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:12:06.111 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:12:06.111 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:12:06.111 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:12:06.111 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:12:06.114 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:12:06.114 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:12:06.114 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:12:06.114 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:12:06.114 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:12:06.114 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:12:06.114 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:12:06.114 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:12:06.114 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:12:06.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:12:06.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:12:06.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:12:06.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:12:06.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:12:06.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:12:06.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:12:06.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:12:06.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:12:06.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:12:06.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:12:06.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:12:06.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:12:06.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:12:06.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:12:06.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:12:06.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:12:06.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:12:06.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:12:06.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:12:06.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:12:06.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:12:06.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:12:06.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:12:06.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:12:06.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:12:06.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:12:06.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:12:06.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:12:06.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:12:06.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:12:06.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:12:06.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:12:06.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:12:06.183 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:12:06.183 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:12:06.183 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:12:06.183 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:12:06.183 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:12:06.183 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:12:06.183 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:12:06.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:12:06.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:12:06.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:12:06.221 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 14 2026-02-18T08:12:06.221 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 14 2026-02-18T08:12:06.221 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:12:06.221 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=57, tag_tid=3, entry_tid=1], [object_number=56, tag_tid=3, entry_tid=0], [object_number=59, tag_tid=2, entry_tid=59619], [object_number=58, tag_tid=2, entry_tid=59618]]], state=connected] 2026-02-18T08:12:06.221 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=56, tag_tid=2, entry_tid=59620], [object_number=59, tag_tid=2, entry_tid=59619], [object_number=58, tag_tid=2, entry_tid=59618], [object_number=57, tag_tid=2, entry_tid=59617]]], state=connected] 2026-02-18T08:12:06.224 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:12:06.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=57, tag_tid=3, entry_tid=1' 2026-02-18T08:12:06.227 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:12:06.227 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:12:06.227 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:12:06.227 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:12:06.227 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:12:06.227 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:12:06.227 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:12:06.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:12:06.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:12:06.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:12:06.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:12:06.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:12:06.229 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:12:06.229 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:12:06.229 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:12:06.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 14 2026-02-18T08:12:06.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 14 2026-02-18T08:12:06.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:12:06.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=57, tag_tid=3, entry_tid=1], [object_number=56, tag_tid=3, entry_tid=0], [object_number=59, tag_tid=2, entry_tid=59619], [object_number=58, tag_tid=2, entry_tid=59618]]], state=connected] 2026-02-18T08:12:06.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=56, tag_tid=2, entry_tid=59620], [object_number=59, tag_tid=2, entry_tid=59619], [object_number=58, tag_tid=2, entry_tid=59618], [object_number=57, tag_tid=2, entry_tid=59617]]], state=connected] 2026-02-18T08:12:06.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:12:06.270 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=56, tag_tid=2, entry_tid=59620' 2026-02-18T08:12:06.270 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=57, tag_tid=3, entry_tid=1' -a 'object_number=57, tag_tid=3, entry_tid=1' = 'object_number=56, tag_tid=2, entry_tid=59620' 2026-02-18T08:12:06.270 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=56, tag_tid=2, entry_tid=59620' '!=' 'object_number=59, tag_tid=2, entry_tid=59359' 2026-02-18T08:12:06.270 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:12:06.270 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=56, tag_tid=2, entry_tid=59620' = 'object_number=59, tag_tid=2, entry_tid=59359' 2026-02-18T08:12:06.270 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=56, tag_tid=2, entry_tid=59620' 2026-02-18T08:12:06.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=57, tag_tid=3, entry_tid=1' 2026-02-18T08:12:06.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:12:06.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:12:06.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=3 2026-02-18T08:12:06.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=56, tag_tid=2, entry_tid=59620' 2026-02-18T08:12:06.273 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:12:06.273 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:12:06.274 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=2 2026-02-18T08:12:06.275 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=57, tag_tid=3, entry_tid=1' 2026-02-18T08:12:06.275 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:12:06.275 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:12:06.276 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:12:06.277 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=56, tag_tid=2, entry_tid=59620' 2026-02-18T08:12:06.277 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:12:06.277 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:12:06.279 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=59620 2026-02-18T08:12:06.279 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 3 = 2 -a 1 -le 59620 2026-02-18T08:12:06.279 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:12:06.279 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:12:06.279 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:12:06.480 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:12:06.481 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:12:06.481 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:12:06.481 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:12:06.481 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:12:06.481 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:12:06.481 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:12:06.481 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:12:06.481 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:12:06.481 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:12:06.481 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:12:06.481 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:12:06.482 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:12:06.482 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:12:06.482 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:12:06.484 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:12:06.485 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:12:06.485 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:12:06.485 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:12:06.485 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:12:06.485 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:12:06.485 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:12:06.485 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:12:06.485 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:12:06.485 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:12:06.485 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:12:06.486 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:12:06.486 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:12:06.486 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:12:06.486 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:12:06.486 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:12:06.486 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:12:06.486 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:12:06.486 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:12:06.486 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:12:06.486 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:12:06.486 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:12:06.486 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:12:06.486 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:12:06.487 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:12:06.487 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:12:06.487 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:12:06.487 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:12:06.487 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:12:06.487 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:12:06.487 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:12:06.487 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:12:06.487 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:12:06.487 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:12:06.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:12:06.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:12:06.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:12:06.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:12:06.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:12:06.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:12:06.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:12:06.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:12:06.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:12:06.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:12:06.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:12:06.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:12:06.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:12:06.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:12:06.565 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:12:06.565 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:12:06.565 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:12:06.565 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:12:06.566 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:12:06.566 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:12:06.601 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 14 2026-02-18T08:12:06.601 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 14 2026-02-18T08:12:06.602 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:12:06.602 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=57, tag_tid=3, entry_tid=1], [object_number=56, tag_tid=3, entry_tid=0], [object_number=59, tag_tid=2, entry_tid=59619], [object_number=58, tag_tid=2, entry_tid=59618]]], state=connected] 2026-02-18T08:12:06.602 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=57, tag_tid=3, entry_tid=1], [object_number=56, tag_tid=3, entry_tid=0], [object_number=59, tag_tid=2, entry_tid=59619], [object_number=58, tag_tid=2, entry_tid=59618]]], state=connected] 2026-02-18T08:12:06.605 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:12:06.606 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=57, tag_tid=3, entry_tid=1' 2026-02-18T08:12:06.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:12:06.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:12:06.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:12:06.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:12:06.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:12:06.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:12:06.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:12:06.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:12:06.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:12:06.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:12:06.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:12:06.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:12:06.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:12:06.610 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:12:06.610 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:12:06.637 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 14 2026-02-18T08:12:06.637 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 14 2026-02-18T08:12:06.637 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:12:06.637 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=57, tag_tid=3, entry_tid=1], [object_number=56, tag_tid=3, entry_tid=0], [object_number=59, tag_tid=2, entry_tid=59619], [object_number=58, tag_tid=2, entry_tid=59618]]], state=connected] 2026-02-18T08:12:06.637 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=57, tag_tid=3, entry_tid=1], [object_number=56, tag_tid=3, entry_tid=0], [object_number=59, tag_tid=2, entry_tid=59619], [object_number=58, tag_tid=2, entry_tid=59618]]], state=connected] 2026-02-18T08:12:06.639 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:12:06.641 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=57, tag_tid=3, entry_tid=1' 2026-02-18T08:12:06.641 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=57, tag_tid=3, entry_tid=1' -a 'object_number=57, tag_tid=3, entry_tid=1' = 'object_number=57, tag_tid=3, entry_tid=1' 2026-02-18T08:12:06.641 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:12:06.642 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_snap_present cluster1 mirror test snap1 2026-02-18T08:12:06.642 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:12:06.642 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:12:06.642 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:12:06.642 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap1 2026-02-18T08:12:06.642 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-18T08:12:06.642 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-18T08:12:06.642 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1 2026-02-18T08:12:07.642 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster1 info mirror/test@snap1 2026-02-18T08:12:07.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:rbd image 'test': 2026-02-18T08:12:07.670 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: size 512 MiB in 128 objects 2026-02-18T08:12:07.670 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: order 22 (4 MiB objects) 2026-02-18T08:12:07.670 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: snapshot_count: 1 2026-02-18T08:12:07.670 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: id: 10a74aa8ebc0 2026-02-18T08:12:07.670 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: block_name_prefix: rbd_data.10a74aa8ebc0 2026-02-18T08:12:07.670 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: format: 2 2026-02-18T08:12:07.670 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: features: layering, exclusive-lock, journaling 2026-02-18T08:12:07.670 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: op_features: 2026-02-18T08:12:07.670 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: flags: 2026-02-18T08:12:07.671 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: create_timestamp: Wed Feb 18 08:10:48 2026 2026-02-18T08:12:07.671 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: access_timestamp: Wed Feb 18 08:10:48 2026 2026-02-18T08:12:07.671 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: modify_timestamp: Wed Feb 18 08:11:49 2026 2026-02-18T08:12:07.671 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: protected: False 2026-02-18T08:12:07.671 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: journal: 10a74aa8ebc0 2026-02-18T08:12:07.671 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring state: enabled 2026-02-18T08:12:07.671 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring mode: journal 2026-02-18T08:12:07.671 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring global id: 0b548823-d751-43df-a076-4efac2da4d3d 2026-02-18T08:12:07.671 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring primary: false 2026-02-18T08:12:07.673 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:12:07.673 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n '' ']' 2026-02-18T08:12:07.673 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ compare_image_snaps mirror test snap1 2026-02-18T08:12:07.673 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:12:07.673 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:12:07.673 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap1 2026-02-18T08:12:07.673 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local ret=0 2026-02-18T08:12:07.673 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local rmt_export=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.export 2026-02-18T08:12:07.673 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loc_export=/tmp/tmp.vybh1m9yJF/cluster1-mirror-test.export 2026-02-18T08:12:07.674 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.export /tmp/tmp.vybh1m9yJF/cluster1-mirror-test.export 2026-02-18T08:12:07.674 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 -p mirror export test@snap1 /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.export 2026-02-18T08:12:08.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-18T08:12:08.138 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster1 -p mirror export test@snap1 /tmp/tmp.vybh1m9yJF/cluster1-mirror-test.export 2026-02-18T08:12:08.693 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-18T08:12:08.706 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmp /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.export /tmp/tmp.vybh1m9yJF/cluster1-mirror-test.export 2026-02-18T08:12:08.894 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.export /tmp/tmp.vybh1m9yJF/cluster1-mirror-test.export 2026-02-18T08:12:09.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:12:09.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ clean_snap_name=snap1-clean 2026-02-18T08:12:09.035 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_snap cluster2 mirror test snap1-clean 2026-02-18T08:12:09.035 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:12:09.035 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:12:09.035 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:12:09.035 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap1-clean 2026-02-18T08:12:09.035 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 -p mirror snap create test@snap1-clean --debug-rbd=20 --debug-journaler=20 2026-02-18T08:12:09.582 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 10` 2026-02-18T08:12:09.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ stress_write_image cluster2 mirror test 2026-02-18T08:12:09.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:12:09.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:12:09.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:12:09.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-18T08:12:09.584 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local duration=8 2026-02-18T08:12:09.585 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set +e 2026-02-18T08:12:09.585 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ timeout 8s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror test --debug-rbd=20 --debug-journaler=20 2026-02-18T08:12:10.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:100 IOs, 59776000 bytes 2026-02-18T08:12:10.820 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:200 IOs, 82705408 bytes 2026-02-18T08:12:11.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:300 IOs, 171746304 bytes 2026-02-18T08:12:12.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:400 IOs, 217388032 bytes 2026-02-18T08:12:12.375 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:500 IOs, 229228544 bytes 2026-02-18T08:12:12.588 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.0 with skip (0.7263448834163865) > weight (0.25) 2026-02-18T08:12:12.588 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.1 with skip (0.6779175608404326) > weight (0.25) 2026-02-18T08:12:12.588 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.2 with skip (0.7754165392913518) > weight (0.25) 2026-02-18T08:12:12.588 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.3 with skip (0.33139622547704517) > weight (0.25) 2026-02-18T08:12:12.588 INFO:tasks.rbd_mirror_thrash:waiting for 104.3330711418213 secs before thrashing 2026-02-18T08:12:13.007 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:600 IOs, 278771712 bytes 2026-02-18T08:12:13.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:700 IOs, 301147136 bytes 2026-02-18T08:12:13.876 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:800 IOs, 347835392 bytes 2026-02-18T08:12:14.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:900 IOs, 358068224 bytes 2026-02-18T08:12:14.429 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1000 IOs, 398317568 bytes 2026-02-18T08:12:14.746 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1100 IOs, 423009280 bytes 2026-02-18T08:12:15.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1200 IOs, 448438272 bytes 2026-02-18T08:12:15.396 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1300 IOs, 504691712 bytes 2026-02-18T08:12:16.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1400 IOs, 540350464 bytes 2026-02-18T08:12:16.663 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1500 IOs, 612680704 bytes 2026-02-18T08:12:17.450 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1600 IOs, 670315520 bytes 2026-02-18T08:12:17.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ error_code=124 2026-02-18T08:12:17.594 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set -e 2026-02-18T08:12:17.594 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 124 -eq 124 ']' 2026-02-18T08:12:17.594 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:12:17.594 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+replaying 2026-02-18T08:12:17.594 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:12:17.594 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:12:17.594 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:12:17.594 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local state_pattern=up+replaying 2026-02-18T08:12:17.595 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local description_pattern= 2026-02-18T08:12:17.595 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local service_pattern= 2026-02-18T08:12:17.595 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:12:17.595 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1 2026-02-18T08:12:18.595 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test_status_in_pool_dir cluster1 mirror test up+replaying '' '' 2026-02-18T08:12:18.595 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:12:18.595 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:12:18.595 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:12:18.595 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local state_pattern=up+replaying 2026-02-18T08:12:18.595 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local description_pattern= 2026-02-18T08:12:18.595 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local service_pattern= 2026-02-18T08:12:18.596 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local status 2026-02-18T08:12:18.596 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ CEPH_ARGS= 2026-02-18T08:12:18.596 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-18T08:12:18.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ status='test: 2026-02-18T08:12:18.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: global_id: 0b548823-d751-43df-a076-4efac2da4d3d 2026-02-18T08:12:18.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: state: up+replaying 2026-02-18T08:12:18.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: description: replaying, {"bytes_per_second":11287585.3,"entries_behind_primary":1,"entries_per_second":703.27,"non_primary_position":{"entry_tid":1,"object_number":57,"tag_tid":3},"primary_position":{"entry_tid":1,"object_number":57,"tag_tid":4},"seconds_until_synced":0} 2026-02-18T08:12:18.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: service: mirror.0 on trial163 2026-02-18T08:12:18.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: last_update: 2026-02-18 08:12:16 2026-02-18T08:12:18.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: peer_sites: 2026-02-18T08:12:18.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: name: cluster2 2026-02-18T08:12:18.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: state: up+stopped 2026-02-18T08:12:18.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: description: local image is primary 2026-02-18T08:12:18.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: last_update: 2026-02-18 08:12:16' 2026-02-18T08:12:18.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ grep '^ state: .*up+replaying' 2026-02-18T08:12:18.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: state: up+replaying 2026-02-18T08:12:18.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ grep '^ description: .*' 2026-02-18T08:12:18.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: description: replaying, {"bytes_per_second":11287585.3,"entries_behind_primary":1,"entries_per_second":703.27,"non_primary_position":{"entry_tid":1,"object_number":57,"tag_tid":3},"primary_position":{"entry_tid":1,"object_number":57,"tag_tid":4},"seconds_until_synced":0} 2026-02-18T08:12:18.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n '' ']' 2026-02-18T08:12:18.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ echo up+replaying 2026-02-18T08:12:18.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ grep '^up+' 2026-02-18T08:12:18.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:up+replaying 2026-02-18T08:12:18.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ grep 'service: *mirror..* on ' 2026-02-18T08:12:18.635 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: service: mirror.0 on trial163 2026-02-18T08:12:18.636 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local last_update 2026-02-18T08:12:18.636 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-18T08:12:18.638 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_update='2026-02-18 08:12:16' 2026-02-18T08:12:18.639 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+replaying '2026-02-18 08:12:16' 2026-02-18T08:12:18.639 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:12:18.639 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:12:18.639 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:12:18.639 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local state_pattern=up+replaying 2026-02-18T08:12:18.639 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'prev_last_update=2026-02-18 08:12:16' 2026-02-18T08:12:18.639 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local status 2026-02-18T08:12:18.639 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ CEPH_ARGS= 2026-02-18T08:12:18.639 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-18T08:12:18.684 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ status='OKOKOK142074263mirror.0trial16320.2.0-670-gadc19233falseOK42114265mirror.1trial16320.2.0-670-gadc19233trueOK42154297mirror.2trial16320.2.0-670-gadc19233falseOK42194268mirror.3trial16320.2.0-670-gadc19233falseOKtest0b548823-d751-43df-a076-4efac2da4d3dup+replayingreplaying, {"bytes_per_second":11287585.3,"entries_behind_primary":1,"entries_per_second":703.27,"non_primary_position":{"entry_tid":1,"object_number":57,"tag_tid":3},"primary_position":{"entry_tid":1,"object_number":57,"tag_tid":4},"seconds_until_synced":0}42074263mirror.0trial1632026-02-18 08:12:16cluster2c5e4f85e-ec31-430b-a71d-56e000d0de0bup+stoppedlocal image is primary2026-02-18 08:12:16' 2026-02-18T08:12:18.684 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local last_update state 2026-02-18T08:12:18.685 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-18T08:12:18.687 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_update='2026-02-18 08:12:16' 2026-02-18T08:12:18.687 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-18T08:12:18.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ state=up+replaying 2026-02-18T08:12:18.691 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ echo up+replaying 2026-02-18T08:12:18.691 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ grep up+replaying 2026-02-18T08:12:18.692 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:up+replaying 2026-02-18T08:12:18.693 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:12:18.693 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:12:18.693 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ snap_name=snap2 2026-02-18T08:12:18.693 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_snap cluster2 mirror test snap2 2026-02-18T08:12:18.693 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:12:18.693 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:12:18.693 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:12:18.694 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap2 2026-02-18T08:12:18.694 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 -p mirror snap create test@snap2 --debug-rbd=20 --debug-journaler=20 2026-02-18T08:12:49.613 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_replay_started cluster1 mirror test 2026-02-18T08:12:49.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:12:49.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:12:49.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:12:49.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_replay_state cluster1 mirror test started 2026-02-18T08:12:49.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:12:49.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:12:49.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:12:49.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local state=started 2026-02-18T08:12:49.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-18T08:12:49.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:12:49.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1 2026-02-18T08:12:50.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test_image_replay_state cluster1 mirror test started 2026-02-18T08:12:50.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:12:50.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:12:50.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:12:50.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local test_state=started 2026-02-18T08:12:50.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local status_result 2026-02-18T08:12:50.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_state=stopped 2026-02-18T08:12:50.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-18T08:12:50.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_instance=cluster1 2026-02-18T08:12:50.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-18T08:12:50.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -i state 2026-02-18T08:12:50.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:12:50.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance=cluster1 2026-02-18T08:12:50.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local loop_instance 2026-02-18T08:12:50.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:12:50.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sleep 0 2026-02-18T08:12:50.619 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:12:50.619 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ seq 0 3 2026-02-18T08:12:50.619 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:12:50.619 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-18T08:12:50.619 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:0 2026-02-18T08:12:50.620 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-18T08:12:50.620 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:12:50.620 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:12:50.620 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:12:50.620 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-18T08:12:50.620 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:12:50.620 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:12:50.620 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:12:50.620 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:12:50.620 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-18T08:12:50.620 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:12:50.620 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:12:50.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:12:50.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:12:50.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:0 2026-02-18T08:12:50.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-18T08:12:50.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-18T08:12:50.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:12:50.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:0 2026-02-18T08:12:50.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T08:12:50.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-18T08:12:50.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:12:50.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:12:50.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-18T08:12:50.622 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-18T08:12:50.622 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-18T08:12:50.622 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=0 2026-02-18T08:12:50.622 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=0 2026-02-18T08:12:50.622 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:12:50.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:12:50.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:12:50.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:12:50.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-18T08:12:50.684 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ return 0 2026-02-18T08:12:50.685 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ status_result=' "state": "Replaying"' 2026-02-18T08:12:50.685 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ echo ' "state": "Replaying"' 2026-02-18T08:12:50.685 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ grep -i Replaying 2026-02-18T08:12:50.686 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: "state": "Replaying" 2026-02-18T08:12:50.688 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ current_state=started 2026-02-18T08:12:50.688 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test started = started 2026-02-18T08:12:50.688 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:12:50.688 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror test 2026-02-18T08:12:50.688 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_cluster=cluster1 2026-02-18T08:12:50.688 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:12:50.688 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_pool=mirror 2026-02-18T08:12:50.688 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local remote_pool=mirror 2026-02-18T08:12:50.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:12:50.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' journal = journal ']' 2026-02-18T08:12:50.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror mirror test 2026-02-18T08:12:50.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_cluster=cluster1 2026-02-18T08:12:50.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:12:50.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_pool=mirror 2026-02-18T08:12:50.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local remote_pool=mirror 2026-02-18T08:12:50.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:12:50.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-02-18T08:12:50.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-02-18T08:12:50.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:12:50.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:12:50.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:12:50.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:12:50.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:12:50.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:12:50.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:12:50.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:12:50.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:12:50.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:12:50.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:12:50.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:12:50.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:12:50.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:12:50.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:12:50.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:12:50.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:12:50.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:12:50.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:12:50.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:12:50.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:12:50.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:12:50.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:12:50.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:12:50.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:12:50.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:12:50.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:12:50.894 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:12:50.894 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:12:50.894 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:12:50.894 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:12:50.894 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:12:50.894 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:12:50.894 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:12:50.894 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:12:50.894 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:12:50.894 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:12:50.894 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:12:50.894 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:12:50.895 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:12:50.895 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:12:50.895 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:12:50.895 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:12:50.895 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:12:50.895 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:12:50.895 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:12:50.895 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:12:50.895 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:12:50.895 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:12:50.895 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:12:50.895 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:12:50.896 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:12:50.904 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:12:50.904 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:12:50.904 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:12:50.905 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:12:50.971 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:12:50.971 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:12:50.971 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:12:50.971 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:12:50.971 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:12:50.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:12:50.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:12:50.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:12:50.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:12:50.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:12:50.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:12:50.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:12:50.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:12:50.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:12:50.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:12:50.974 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:12:51.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 21 2026-02-18T08:12:51.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 24 2026-02-18T08:12:51.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:12:51.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=97, tag_tid=6, entry_tid=1], [object_number=96, tag_tid=6, entry_tid=0], [object_number=99, tag_tid=5, entry_tid=42603], [object_number=98, tag_tid=5, entry_tid=42602]]], state=connected] 2026-02-18T08:12:51.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=85, tag_tid=5, entry_tid=28233], [object_number=84, tag_tid=5, entry_tid=28232], [object_number=87, tag_tid=5, entry_tid=28231], [object_number=86, tag_tid=5, entry_tid=28230]]], state=connected] 2026-02-18T08:12:51.004 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:12:51.006 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=97, tag_tid=6, entry_tid=1' 2026-02-18T08:12:51.007 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:12:51.008 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:12:51.008 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:12:51.008 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:12:51.008 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:12:51.008 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:12:51.008 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:12:51.008 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:12:51.008 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:12:51.008 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:12:51.008 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:12:51.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:12:51.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:12:51.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:12:51.010 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:12:51.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 21 2026-02-18T08:12:51.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 24 2026-02-18T08:12:51.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:12:51.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=97, tag_tid=6, entry_tid=1], [object_number=96, tag_tid=6, entry_tid=0], [object_number=99, tag_tid=5, entry_tid=42603], [object_number=98, tag_tid=5, entry_tid=42602]]], state=connected] 2026-02-18T08:12:51.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=85, tag_tid=5, entry_tid=28233], [object_number=84, tag_tid=5, entry_tid=28232], [object_number=87, tag_tid=5, entry_tid=28231], [object_number=86, tag_tid=5, entry_tid=28230]]], state=connected] 2026-02-18T08:12:51.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:12:51.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=85, tag_tid=5, entry_tid=28233' 2026-02-18T08:12:51.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=97, tag_tid=6, entry_tid=1' -a 'object_number=97, tag_tid=6, entry_tid=1' = 'object_number=85, tag_tid=5, entry_tid=28233' 2026-02-18T08:12:51.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=85, tag_tid=5, entry_tid=28233' '!=' '' 2026-02-18T08:12:51.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:12:51.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=85, tag_tid=5, entry_tid=28233' = '' 2026-02-18T08:12:51.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=85, tag_tid=5, entry_tid=28233' 2026-02-18T08:12:51.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=97, tag_tid=6, entry_tid=1' 2026-02-18T08:12:51.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:12:51.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:12:51.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=6 2026-02-18T08:12:51.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=85, tag_tid=5, entry_tid=28233' 2026-02-18T08:12:51.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:12:51.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:12:51.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=5 2026-02-18T08:12:51.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=97, tag_tid=6, entry_tid=1' 2026-02-18T08:12:51.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:12:51.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:12:51.056 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:12:51.056 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=85, tag_tid=5, entry_tid=28233' 2026-02-18T08:12:51.056 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:12:51.056 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:12:51.058 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=28233 2026-02-18T08:12:51.058 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 6 = 5 -a 1 -le 28233 2026-02-18T08:12:51.058 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:12:51.058 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:12:51.058 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:12:51.260 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:12:51.261 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:12:51.261 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:12:51.261 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:12:51.261 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:12:51.261 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:12:51.261 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:12:51.261 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:12:51.261 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:12:51.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:12:51.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:12:51.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:12:51.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:12:51.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:12:51.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:12:51.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:12:51.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:12:51.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:12:51.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:12:51.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:12:51.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:12:51.266 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:12:51.266 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:12:51.266 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:12:51.266 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:12:51.266 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:12:51.266 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:12:51.266 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:12:51.266 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:12:51.266 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:12:51.266 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:12:51.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:12:51.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:12:51.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:12:51.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:12:51.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:12:51.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:12:51.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:12:51.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:12:51.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:12:51.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:12:51.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:12:51.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:12:51.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:12:51.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:12:51.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:12:51.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:12:51.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:12:51.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:12:51.275 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:12:51.275 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:12:51.275 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:12:51.275 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:12:51.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:12:51.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:12:51.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:12:51.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:12:51.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:12:51.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:12:51.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:12:51.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:12:51.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:12:51.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:12:51.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:12:51.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:12:51.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:12:51.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:12:51.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:12:51.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:12:51.363 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 21 2026-02-18T08:12:51.363 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 24 2026-02-18T08:12:51.363 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:12:51.363 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=97, tag_tid=6, entry_tid=1], [object_number=96, tag_tid=6, entry_tid=0], [object_number=99, tag_tid=5, entry_tid=42603], [object_number=98, tag_tid=5, entry_tid=42602]]], state=connected] 2026-02-18T08:12:51.363 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=85, tag_tid=5, entry_tid=28469], [object_number=84, tag_tid=5, entry_tid=28468], [object_number=87, tag_tid=5, entry_tid=28467], [object_number=86, tag_tid=5, entry_tid=28466]]], state=connected] 2026-02-18T08:12:51.366 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:12:51.367 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=97, tag_tid=6, entry_tid=1' 2026-02-18T08:12:51.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:12:51.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:12:51.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:12:51.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:12:51.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:12:51.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:12:51.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:12:51.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:12:51.370 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:12:51.370 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:12:51.370 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:12:51.370 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:12:51.371 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:12:51.371 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:12:51.371 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:12:51.397 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 21 2026-02-18T08:12:51.397 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 24 2026-02-18T08:12:51.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:12:51.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=97, tag_tid=6, entry_tid=1], [object_number=96, tag_tid=6, entry_tid=0], [object_number=99, tag_tid=5, entry_tid=42603], [object_number=98, tag_tid=5, entry_tid=42602]]], state=connected] 2026-02-18T08:12:51.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=85, tag_tid=5, entry_tid=28469], [object_number=84, tag_tid=5, entry_tid=28468], [object_number=87, tag_tid=5, entry_tid=28467], [object_number=86, tag_tid=5, entry_tid=28466]]], state=connected] 2026-02-18T08:12:51.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:12:51.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=85, tag_tid=5, entry_tid=28469' 2026-02-18T08:12:51.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=97, tag_tid=6, entry_tid=1' -a 'object_number=97, tag_tid=6, entry_tid=1' = 'object_number=85, tag_tid=5, entry_tid=28469' 2026-02-18T08:12:51.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=85, tag_tid=5, entry_tid=28469' '!=' 'object_number=85, tag_tid=5, entry_tid=28233' 2026-02-18T08:12:51.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:12:51.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=85, tag_tid=5, entry_tid=28469' = 'object_number=85, tag_tid=5, entry_tid=28233' 2026-02-18T08:12:51.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=85, tag_tid=5, entry_tid=28469' 2026-02-18T08:12:51.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=97, tag_tid=6, entry_tid=1' 2026-02-18T08:12:51.403 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:12:51.403 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:12:51.403 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=6 2026-02-18T08:12:51.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=85, tag_tid=5, entry_tid=28469' 2026-02-18T08:12:51.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:12:51.405 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:12:51.406 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=5 2026-02-18T08:12:51.406 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=97, tag_tid=6, entry_tid=1' 2026-02-18T08:12:51.407 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:12:51.407 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:12:51.408 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:12:51.409 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=85, tag_tid=5, entry_tid=28469' 2026-02-18T08:12:51.409 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:12:51.409 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:12:51.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=28469 2026-02-18T08:12:51.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 6 = 5 -a 1 -le 28469 2026-02-18T08:12:51.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:12:51.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:12:51.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:12:51.613 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:12:51.613 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:12:51.613 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:12:51.613 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:12:51.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:12:51.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:12:51.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:12:51.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:12:51.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:12:51.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:12:51.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:12:51.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:12:51.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:12:51.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:12:51.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:12:51.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:12:51.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:12:51.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:12:51.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:12:51.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:12:51.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:12:51.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:12:51.618 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:12:51.618 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:12:51.618 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:12:51.618 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:12:51.618 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:12:51.618 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:12:51.618 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:12:51.618 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:12:51.618 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:12:51.618 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:12:51.618 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:12:51.618 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:12:51.619 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:12:51.619 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:12:51.619 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:12:51.619 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:12:51.619 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:12:51.619 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:12:51.619 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:12:51.619 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:12:51.619 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:12:51.619 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:12:51.619 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:12:51.619 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:12:51.620 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:12:51.620 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:12:51.620 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:12:51.629 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:12:51.629 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:12:51.629 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:12:51.629 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:12:51.687 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:12:51.687 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:12:51.687 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:12:51.687 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:12:51.687 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:12:51.687 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:12:51.687 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:12:51.688 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:12:51.688 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:12:51.688 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:12:51.688 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:12:51.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:12:51.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:12:51.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:12:51.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:12:51.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:12:51.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 21 2026-02-18T08:12:51.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 24 2026-02-18T08:12:51.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:12:51.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=97, tag_tid=6, entry_tid=1], [object_number=96, tag_tid=6, entry_tid=0], [object_number=99, tag_tid=5, entry_tid=42603], [object_number=98, tag_tid=5, entry_tid=42602]]], state=connected] 2026-02-18T08:12:51.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=87, tag_tid=5, entry_tid=28703], [object_number=86, tag_tid=5, entry_tid=28702], [object_number=85, tag_tid=5, entry_tid=28701], [object_number=84, tag_tid=5, entry_tid=28700]]], state=connected] 2026-02-18T08:12:51.718 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:12:51.720 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=97, tag_tid=6, entry_tid=1' 2026-02-18T08:12:51.721 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:12:51.721 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:12:51.722 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:12:51.722 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:12:51.722 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:12:51.722 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:12:51.722 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:12:51.722 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:12:51.722 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:12:51.722 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:12:51.722 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:12:51.722 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:12:51.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:12:51.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:12:51.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:12:51.756 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 21 2026-02-18T08:12:51.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 24 2026-02-18T08:12:51.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:12:51.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=97, tag_tid=6, entry_tid=1], [object_number=96, tag_tid=6, entry_tid=0], [object_number=99, tag_tid=5, entry_tid=42603], [object_number=98, tag_tid=5, entry_tid=42602]]], state=connected] 2026-02-18T08:12:51.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=87, tag_tid=5, entry_tid=28703], [object_number=86, tag_tid=5, entry_tid=28702], [object_number=85, tag_tid=5, entry_tid=28701], [object_number=84, tag_tid=5, entry_tid=28700]]], state=connected] 2026-02-18T08:12:51.759 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:12:51.761 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=87, tag_tid=5, entry_tid=28703' 2026-02-18T08:12:51.761 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=97, tag_tid=6, entry_tid=1' -a 'object_number=97, tag_tid=6, entry_tid=1' = 'object_number=87, tag_tid=5, entry_tid=28703' 2026-02-18T08:12:51.761 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=87, tag_tid=5, entry_tid=28703' '!=' 'object_number=85, tag_tid=5, entry_tid=28469' 2026-02-18T08:12:51.761 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:12:51.762 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=87, tag_tid=5, entry_tid=28703' = 'object_number=85, tag_tid=5, entry_tid=28469' 2026-02-18T08:12:51.762 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=87, tag_tid=5, entry_tid=28703' 2026-02-18T08:12:51.762 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=97, tag_tid=6, entry_tid=1' 2026-02-18T08:12:51.762 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:12:51.762 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:12:51.762 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=6 2026-02-18T08:12:51.763 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=87, tag_tid=5, entry_tid=28703' 2026-02-18T08:12:51.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:12:51.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:12:51.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=5 2026-02-18T08:12:51.766 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=97, tag_tid=6, entry_tid=1' 2026-02-18T08:12:51.766 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:12:51.766 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:12:51.767 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:12:51.768 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=87, tag_tid=5, entry_tid=28703' 2026-02-18T08:12:51.768 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:12:51.768 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:12:51.770 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=28703 2026-02-18T08:12:51.770 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 6 = 5 -a 1 -le 28703 2026-02-18T08:12:51.770 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:12:51.770 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:12:51.770 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:12:51.873 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.0 with skip (0.7500775656962001) > weight (0.25) 2026-02-18T08:12:51.873 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.1 with skip (0.5070382853012013) > weight (0.25) 2026-02-18T08:12:51.873 INFO:tasks.rbd_mirror_thrash:kill cluster1.client.mirror.2 2026-02-18T08:12:51.874 INFO:tasks.rbd_mirror.cluster1.client.mirror.2:Sent signal 15 2026-02-18T08:12:51.874 INFO:tasks.rbd_mirror_thrash:waiting for 3.49182463116527 secs before reviving daemons 2026-02-18T08:12:51.874 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.trial163.stderr:2026-02-18T08:12:51.870+0000 7ff549540640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.2 (PID: 14926) UID: 1000 2026-02-18T08:12:51.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:12:51.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:12:51.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:12:51.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:12:51.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:12:51.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:12:51.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:12:51.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:12:51.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:12:51.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:12:51.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:12:51.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:12:51.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:12:51.974 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:12:51.974 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:12:51.974 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:12:51.974 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:12:51.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:12:51.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:12:51.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:12:51.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:12:51.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:12:51.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:12:51.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:12:51.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:12:51.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:12:51.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:12:51.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:12:51.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:12:51.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:12:51.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:12:51.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:12:51.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:12:51.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:12:51.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:12:51.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:12:51.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:12:51.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:12:51.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:12:51.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:12:51.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:12:51.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:12:51.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:12:51.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:12:51.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:12:51.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:12:51.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:12:51.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:12:51.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:12:51.987 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:12:51.988 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:12:51.988 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:12:51.988 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:12:52.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:12:52.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:12:52.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:12:52.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:12:52.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:12:52.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:12:52.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:12:52.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:12:52.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:12:52.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:12:52.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:12:52.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:12:52.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:12:52.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:12:52.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:12:52.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:12:52.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 21 2026-02-18T08:12:52.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 24 2026-02-18T08:12:52.086 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:12:52.086 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=97, tag_tid=6, entry_tid=1], [object_number=96, tag_tid=6, entry_tid=0], [object_number=99, tag_tid=5, entry_tid=42603], [object_number=98, tag_tid=5, entry_tid=42602]]], state=connected] 2026-02-18T08:12:52.086 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=85, tag_tid=5, entry_tid=28961], [object_number=84, tag_tid=5, entry_tid=28960], [object_number=87, tag_tid=5, entry_tid=28959], [object_number=86, tag_tid=5, entry_tid=28958]]], state=connected] 2026-02-18T08:12:52.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:12:52.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=97, tag_tid=6, entry_tid=1' 2026-02-18T08:12:52.093 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:12:52.093 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:12:52.093 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:12:52.093 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:12:52.093 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:12:52.094 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:12:52.094 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:12:52.094 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:12:52.094 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:12:52.094 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:12:52.094 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:12:52.094 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:12:52.095 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:12:52.095 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:12:52.095 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:12:52.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 21 2026-02-18T08:12:52.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 24 2026-02-18T08:12:52.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:12:52.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=97, tag_tid=6, entry_tid=1], [object_number=96, tag_tid=6, entry_tid=0], [object_number=99, tag_tid=5, entry_tid=42603], [object_number=98, tag_tid=5, entry_tid=42602]]], state=connected] 2026-02-18T08:12:52.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=85, tag_tid=5, entry_tid=28961], [object_number=84, tag_tid=5, entry_tid=28960], [object_number=87, tag_tid=5, entry_tid=28959], [object_number=86, tag_tid=5, entry_tid=28958]]], state=connected] 2026-02-18T08:12:52.130 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:12:52.132 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=85, tag_tid=5, entry_tid=28961' 2026-02-18T08:12:52.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=97, tag_tid=6, entry_tid=1' -a 'object_number=97, tag_tid=6, entry_tid=1' = 'object_number=85, tag_tid=5, entry_tid=28961' 2026-02-18T08:12:52.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=85, tag_tid=5, entry_tid=28961' '!=' 'object_number=87, tag_tid=5, entry_tid=28703' 2026-02-18T08:12:52.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:12:52.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=85, tag_tid=5, entry_tid=28961' = 'object_number=87, tag_tid=5, entry_tid=28703' 2026-02-18T08:12:52.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=85, tag_tid=5, entry_tid=28961' 2026-02-18T08:12:52.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=97, tag_tid=6, entry_tid=1' 2026-02-18T08:12:52.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:12:52.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:12:52.134 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=6 2026-02-18T08:12:52.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=85, tag_tid=5, entry_tid=28961' 2026-02-18T08:12:52.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:12:52.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:12:52.136 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=5 2026-02-18T08:12:52.137 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=97, tag_tid=6, entry_tid=1' 2026-02-18T08:12:52.137 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:12:52.137 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:12:52.138 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:12:52.139 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=85, tag_tid=5, entry_tid=28961' 2026-02-18T08:12:52.139 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:12:52.139 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:12:52.141 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=28961 2026-02-18T08:12:52.141 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 6 = 5 -a 1 -le 28961 2026-02-18T08:12:52.141 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:12:52.141 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:12:52.142 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:12:52.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:12:52.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:12:52.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:12:52.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:12:52.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:12:52.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:12:52.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:12:52.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:12:52.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:12:52.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:12:52.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:12:52.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:12:52.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:12:52.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:12:52.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:12:52.344 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:12:52.344 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:12:52.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:12:52.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:12:52.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:12:52.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:12:52.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:12:52.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:12:52.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:12:52.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:12:52.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:12:52.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:12:52.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:12:52.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:12:52.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:12:52.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:12:52.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:12:52.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:12:52.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:12:52.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:12:52.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:12:52.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:12:52.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:12:52.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:12:52.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:12:52.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:12:52.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:12:52.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:12:52.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:12:52.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:12:52.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:12:52.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:12:52.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:12:52.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:12:52.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:12:52.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:12:52.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:12:52.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:12:52.416 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:12:52.416 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:12:52.416 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:12:52.416 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:12:52.416 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:12:52.416 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:12:52.416 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:12:52.417 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:12:52.417 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:12:52.417 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:12:52.417 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:12:52.417 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:12:52.417 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:12:52.418 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:12:52.418 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:12:52.418 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:12:52.454 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 21 2026-02-18T08:12:52.454 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 24 2026-02-18T08:12:52.454 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:12:52.454 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=97, tag_tid=6, entry_tid=1], [object_number=96, tag_tid=6, entry_tid=0], [object_number=99, tag_tid=5, entry_tid=42603], [object_number=98, tag_tid=5, entry_tid=42602]]], state=connected] 2026-02-18T08:12:52.454 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=85, tag_tid=5, entry_tid=29229], [object_number=84, tag_tid=5, entry_tid=29228], [object_number=87, tag_tid=5, entry_tid=29227], [object_number=86, tag_tid=5, entry_tid=29226]]], state=connected] 2026-02-18T08:12:52.458 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:12:52.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=97, tag_tid=6, entry_tid=1' 2026-02-18T08:12:52.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:12:52.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:12:52.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:12:52.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:12:52.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:12:52.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:12:52.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:12:52.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:12:52.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:12:52.462 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:12:52.462 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:12:52.462 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:12:52.462 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:12:52.463 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:12:52.463 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:12:52.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 21 2026-02-18T08:12:52.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 24 2026-02-18T08:12:52.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:12:52.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=97, tag_tid=6, entry_tid=1], [object_number=96, tag_tid=6, entry_tid=0], [object_number=99, tag_tid=5, entry_tid=42603], [object_number=98, tag_tid=5, entry_tid=42602]]], state=connected] 2026-02-18T08:12:52.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=85, tag_tid=5, entry_tid=29229], [object_number=84, tag_tid=5, entry_tid=29228], [object_number=87, tag_tid=5, entry_tid=29227], [object_number=86, tag_tid=5, entry_tid=29226]]], state=connected] 2026-02-18T08:12:52.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:12:52.504 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=85, tag_tid=5, entry_tid=29229' 2026-02-18T08:12:52.504 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=97, tag_tid=6, entry_tid=1' -a 'object_number=97, tag_tid=6, entry_tid=1' = 'object_number=85, tag_tid=5, entry_tid=29229' 2026-02-18T08:12:52.504 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=85, tag_tid=5, entry_tid=29229' '!=' 'object_number=85, tag_tid=5, entry_tid=28961' 2026-02-18T08:12:52.504 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:12:52.505 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=85, tag_tid=5, entry_tid=29229' = 'object_number=85, tag_tid=5, entry_tid=28961' 2026-02-18T08:12:52.505 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=85, tag_tid=5, entry_tid=29229' 2026-02-18T08:12:52.505 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=97, tag_tid=6, entry_tid=1' 2026-02-18T08:12:52.505 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:12:52.505 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:12:52.506 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=6 2026-02-18T08:12:52.506 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=85, tag_tid=5, entry_tid=29229' 2026-02-18T08:12:52.506 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:12:52.506 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:12:52.508 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=5 2026-02-18T08:12:52.509 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=97, tag_tid=6, entry_tid=1' 2026-02-18T08:12:52.509 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:12:52.509 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:12:52.509 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:12:52.510 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=85, tag_tid=5, entry_tid=29229' 2026-02-18T08:12:52.510 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:12:52.510 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:12:52.511 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=29229 2026-02-18T08:12:52.511 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 6 = 5 -a 1 -le 29229 2026-02-18T08:12:52.512 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:12:52.512 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:12:52.512 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:12:52.712 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:12:52.712 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:12:52.712 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:12:52.712 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:12:52.712 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:12:52.712 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:12:52.712 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:12:52.713 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:12:52.713 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:12:52.713 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:12:52.713 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:12:52.713 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:12:52.713 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:12:52.713 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:12:52.713 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:12:52.713 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:12:52.713 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:12:52.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:12:52.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:12:52.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:12:52.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:12:52.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:12:52.717 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:12:52.717 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:12:52.717 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:12:52.717 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:12:52.717 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:12:52.717 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:12:52.717 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:12:52.717 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:12:52.717 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:12:52.717 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:12:52.717 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:12:52.717 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:12:52.718 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:12:52.718 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:12:52.718 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:12:52.718 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:12:52.718 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:12:52.718 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:12:52.718 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:12:52.718 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:12:52.718 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:12:52.718 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:12:52.718 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:12:52.719 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:12:52.719 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:12:52.719 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:12:52.719 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:12:52.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:12:52.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:12:52.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:12:52.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:12:52.783 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:12:52.786 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:12:52.786 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:12:52.786 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:12:52.786 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:12:52.786 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:12:52.786 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:12:52.787 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:12:52.787 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:12:52.787 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:12:52.787 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:12:52.787 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:12:52.787 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:12:52.787 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:12:52.788 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:12:52.788 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:12:52.814 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 21 2026-02-18T08:12:52.815 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 24 2026-02-18T08:12:52.815 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:12:52.815 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=97, tag_tid=6, entry_tid=1], [object_number=96, tag_tid=6, entry_tid=0], [object_number=99, tag_tid=5, entry_tid=42603], [object_number=98, tag_tid=5, entry_tid=42602]]], state=connected] 2026-02-18T08:12:52.815 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=85, tag_tid=5, entry_tid=29461], [object_number=84, tag_tid=5, entry_tid=29460], [object_number=87, tag_tid=5, entry_tid=29459], [object_number=86, tag_tid=5, entry_tid=29458]]], state=connected] 2026-02-18T08:12:52.817 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:12:52.820 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=97, tag_tid=6, entry_tid=1' 2026-02-18T08:12:52.821 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:12:52.821 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:12:52.821 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:12:52.821 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:12:52.821 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:12:52.822 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:12:52.822 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:12:52.822 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:12:52.822 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:12:52.822 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:12:52.823 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:12:52.823 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:12:52.824 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:12:52.824 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:12:52.824 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:12:52.850 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 21 2026-02-18T08:12:52.850 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 24 2026-02-18T08:12:52.850 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:12:52.850 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=97, tag_tid=6, entry_tid=1], [object_number=96, tag_tid=6, entry_tid=0], [object_number=99, tag_tid=5, entry_tid=42603], [object_number=98, tag_tid=5, entry_tid=42602]]], state=connected] 2026-02-18T08:12:52.850 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=85, tag_tid=5, entry_tid=29461], [object_number=84, tag_tid=5, entry_tid=29460], [object_number=87, tag_tid=5, entry_tid=29459], [object_number=86, tag_tid=5, entry_tid=29458]]], state=connected] 2026-02-18T08:12:52.852 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:12:52.854 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=85, tag_tid=5, entry_tid=29461' 2026-02-18T08:12:52.854 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=97, tag_tid=6, entry_tid=1' -a 'object_number=97, tag_tid=6, entry_tid=1' = 'object_number=85, tag_tid=5, entry_tid=29461' 2026-02-18T08:12:52.854 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=85, tag_tid=5, entry_tid=29461' '!=' 'object_number=85, tag_tid=5, entry_tid=29229' 2026-02-18T08:12:52.854 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:12:52.854 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=85, tag_tid=5, entry_tid=29461' = 'object_number=85, tag_tid=5, entry_tid=29229' 2026-02-18T08:12:52.854 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=85, tag_tid=5, entry_tid=29461' 2026-02-18T08:12:52.855 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=97, tag_tid=6, entry_tid=1' 2026-02-18T08:12:52.855 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:12:52.855 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:12:52.856 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=6 2026-02-18T08:12:52.857 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=85, tag_tid=5, entry_tid=29461' 2026-02-18T08:12:52.857 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:12:52.857 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:12:52.858 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=5 2026-02-18T08:12:52.859 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=97, tag_tid=6, entry_tid=1' 2026-02-18T08:12:52.859 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:12:52.859 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:12:52.860 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:12:52.861 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=85, tag_tid=5, entry_tid=29461' 2026-02-18T08:12:52.861 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:12:52.861 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:12:52.863 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=29461 2026-02-18T08:12:52.863 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 6 = 5 -a 1 -le 29461 2026-02-18T08:12:52.863 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:12:52.863 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:12:52.864 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:12:53.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:12:53.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:12:53.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:12:53.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:12:53.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:12:53.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:12:53.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:12:53.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:12:53.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:12:53.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:12:53.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:12:53.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:12:53.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:12:53.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:12:53.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:12:53.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:12:53.069 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:12:53.069 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:12:53.069 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:12:53.069 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:12:53.069 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:12:53.069 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:12:53.069 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:12:53.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:12:53.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:12:53.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:12:53.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:12:53.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:12:53.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:12:53.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:12:53.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:12:53.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:12:53.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:12:53.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:12:53.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:12:53.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:12:53.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:12:53.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:12:53.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:12:53.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:12:53.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:12:53.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:12:53.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:12:53.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:12:53.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:12:53.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:12:53.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:12:53.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:12:53.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:12:53.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:12:53.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:12:53.080 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:12:53.080 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:12:53.140 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:12:53.140 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:12:53.140 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:12:53.140 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:12:53.140 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:12:53.140 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:12:53.141 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:12:53.141 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:12:53.141 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:12:53.141 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:12:53.142 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:12:53.142 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:12:53.142 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:12:53.143 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:12:53.143 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:12:53.144 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:12:53.178 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 21 2026-02-18T08:12:53.178 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 24 2026-02-18T08:12:53.178 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:12:53.178 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=97, tag_tid=6, entry_tid=1], [object_number=96, tag_tid=6, entry_tid=0], [object_number=99, tag_tid=5, entry_tid=42603], [object_number=98, tag_tid=5, entry_tid=42602]]], state=connected] 2026-02-18T08:12:53.179 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=84, tag_tid=5, entry_tid=29676], [object_number=87, tag_tid=5, entry_tid=29675], [object_number=86, tag_tid=5, entry_tid=29674], [object_number=85, tag_tid=5, entry_tid=29673]]], state=connected] 2026-02-18T08:12:53.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:12:53.183 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=97, tag_tid=6, entry_tid=1' 2026-02-18T08:12:53.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:12:53.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:12:53.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:12:53.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:12:53.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:12:53.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:12:53.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:12:53.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:12:53.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:12:53.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:12:53.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:12:53.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:12:53.187 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:12:53.187 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:12:53.187 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:12:53.220 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 21 2026-02-18T08:12:53.220 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 24 2026-02-18T08:12:53.220 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:12:53.220 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=97, tag_tid=6, entry_tid=1], [object_number=96, tag_tid=6, entry_tid=0], [object_number=99, tag_tid=5, entry_tid=42603], [object_number=98, tag_tid=5, entry_tid=42602]]], state=connected] 2026-02-18T08:12:53.220 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=84, tag_tid=5, entry_tid=29676], [object_number=87, tag_tid=5, entry_tid=29675], [object_number=86, tag_tid=5, entry_tid=29674], [object_number=85, tag_tid=5, entry_tid=29673]]], state=connected] 2026-02-18T08:12:53.224 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:12:53.225 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=84, tag_tid=5, entry_tid=29676' 2026-02-18T08:12:53.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=97, tag_tid=6, entry_tid=1' -a 'object_number=97, tag_tid=6, entry_tid=1' = 'object_number=84, tag_tid=5, entry_tid=29676' 2026-02-18T08:12:53.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=84, tag_tid=5, entry_tid=29676' '!=' 'object_number=85, tag_tid=5, entry_tid=29461' 2026-02-18T08:12:53.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:12:53.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=84, tag_tid=5, entry_tid=29676' = 'object_number=85, tag_tid=5, entry_tid=29461' 2026-02-18T08:12:53.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=84, tag_tid=5, entry_tid=29676' 2026-02-18T08:12:53.227 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=97, tag_tid=6, entry_tid=1' 2026-02-18T08:12:53.227 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:12:53.227 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:12:53.227 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=6 2026-02-18T08:12:53.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=84, tag_tid=5, entry_tid=29676' 2026-02-18T08:12:53.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:12:53.229 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:12:53.230 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=5 2026-02-18T08:12:53.230 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=97, tag_tid=6, entry_tid=1' 2026-02-18T08:12:53.231 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:12:53.231 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:12:53.231 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:12:53.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=84, tag_tid=5, entry_tid=29676' 2026-02-18T08:12:53.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:12:53.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:12:53.234 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=29676 2026-02-18T08:12:53.234 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 6 = 5 -a 1 -le 29676 2026-02-18T08:12:53.234 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:12:53.234 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:12:53.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:12:53.436 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:12:53.436 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:12:53.436 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:12:53.436 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:12:53.437 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:12:53.437 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:12:53.437 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:12:53.437 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:12:53.437 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:12:53.437 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:12:53.437 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:12:53.437 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:12:53.437 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:12:53.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:12:53.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:12:53.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:12:53.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:12:53.440 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:12:53.440 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:12:53.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:12:53.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:12:53.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:12:53.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:12:53.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:12:53.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:12:53.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:12:53.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:12:53.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:12:53.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:12:53.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:12:53.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:12:53.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:12:53.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:12:53.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:12:53.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:12:53.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:12:53.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:12:53.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:12:53.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:12:53.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:12:53.443 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:12:53.443 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:12:53.443 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:12:53.443 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:12:53.443 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:12:53.443 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:12:53.443 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:12:53.443 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:12:53.443 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:12:53.450 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:12:53.451 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:12:53.451 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:12:53.451 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:12:53.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:12:53.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:12:53.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:12:53.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:12:53.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:12:53.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:12:53.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:12:53.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:12:53.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:12:53.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:12:53.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:12:53.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:12:53.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:12:53.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:12:53.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:12:53.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:12:53.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 21 2026-02-18T08:12:53.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 24 2026-02-18T08:12:53.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:12:53.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=97, tag_tid=6, entry_tid=1], [object_number=96, tag_tid=6, entry_tid=0], [object_number=99, tag_tid=5, entry_tid=42603], [object_number=98, tag_tid=5, entry_tid=42602]]], state=connected] 2026-02-18T08:12:53.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=86, tag_tid=5, entry_tid=29954], [object_number=85, tag_tid=5, entry_tid=29953], [object_number=84, tag_tid=5, entry_tid=29952], [object_number=87, tag_tid=5, entry_tid=29951]]], state=connected] 2026-02-18T08:12:53.559 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:12:53.561 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=97, tag_tid=6, entry_tid=1' 2026-02-18T08:12:53.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:12:53.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:12:53.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:12:53.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:12:53.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:12:53.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:12:53.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:12:53.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:12:53.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:12:53.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:12:53.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:12:53.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:12:53.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:12:53.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:12:53.565 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:12:53.600 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 21 2026-02-18T08:12:53.600 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 24 2026-02-18T08:12:53.600 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:12:53.600 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=97, tag_tid=6, entry_tid=1], [object_number=96, tag_tid=6, entry_tid=0], [object_number=99, tag_tid=5, entry_tid=42603], [object_number=98, tag_tid=5, entry_tid=42602]]], state=connected] 2026-02-18T08:12:53.600 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=86, tag_tid=5, entry_tid=29954], [object_number=85, tag_tid=5, entry_tid=29953], [object_number=84, tag_tid=5, entry_tid=29952], [object_number=87, tag_tid=5, entry_tid=29951]]], state=connected] 2026-02-18T08:12:53.604 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:12:53.605 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=86, tag_tid=5, entry_tid=29954' 2026-02-18T08:12:53.605 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=97, tag_tid=6, entry_tid=1' -a 'object_number=97, tag_tid=6, entry_tid=1' = 'object_number=86, tag_tid=5, entry_tid=29954' 2026-02-18T08:12:53.606 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=86, tag_tid=5, entry_tid=29954' '!=' 'object_number=84, tag_tid=5, entry_tid=29676' 2026-02-18T08:12:53.606 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:12:53.606 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=86, tag_tid=5, entry_tid=29954' = 'object_number=84, tag_tid=5, entry_tid=29676' 2026-02-18T08:12:53.606 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=86, tag_tid=5, entry_tid=29954' 2026-02-18T08:12:53.606 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=97, tag_tid=6, entry_tid=1' 2026-02-18T08:12:53.606 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:12:53.606 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:12:53.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=6 2026-02-18T08:12:53.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=86, tag_tid=5, entry_tid=29954' 2026-02-18T08:12:53.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:12:53.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:12:53.610 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=5 2026-02-18T08:12:53.611 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=97, tag_tid=6, entry_tid=1' 2026-02-18T08:12:53.611 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:12:53.611 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:12:53.612 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:12:53.613 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=86, tag_tid=5, entry_tid=29954' 2026-02-18T08:12:53.613 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:12:53.613 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:12:53.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=29954 2026-02-18T08:12:53.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 6 = 5 -a 1 -le 29954 2026-02-18T08:12:53.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:12:53.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:12:53.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:12:53.817 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:12:53.818 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:12:53.818 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:12:53.818 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:12:53.818 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:12:53.818 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:12:53.818 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:12:53.819 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:12:53.819 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:12:53.819 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:12:53.819 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:12:53.819 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:12:53.819 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:12:53.819 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:12:53.819 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:12:53.820 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:12:53.820 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:12:53.823 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:12:53.823 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:12:53.823 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:12:53.823 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:12:53.823 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:12:53.824 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:12:53.824 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:12:53.824 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:12:53.824 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:12:53.824 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:12:53.824 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:12:53.824 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:12:53.824 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:12:53.824 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:12:53.825 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:12:53.825 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:12:53.825 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:12:53.825 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:12:53.825 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:12:53.825 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:12:53.825 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:12:53.825 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:12:53.825 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:12:53.826 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:12:53.826 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:12:53.826 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:12:53.826 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:12:53.826 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:12:53.826 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:12:53.826 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:12:53.826 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:12:53.826 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:12:53.832 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:12:53.832 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:12:53.832 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:12:53.832 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:12:53.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:12:53.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:12:53.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:12:53.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:12:53.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:12:53.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:12:53.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:12:53.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:12:53.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:12:53.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:12:53.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:12:53.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:12:53.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:12:53.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:12:53.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:12:53.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:12:53.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 21 2026-02-18T08:12:53.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 24 2026-02-18T08:12:53.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:12:53.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=97, tag_tid=6, entry_tid=1], [object_number=96, tag_tid=6, entry_tid=0], [object_number=99, tag_tid=5, entry_tid=42603], [object_number=98, tag_tid=5, entry_tid=42602]]], state=connected] 2026-02-18T08:12:53.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=85, tag_tid=5, entry_tid=30253], [object_number=84, tag_tid=5, entry_tid=30252], [object_number=87, tag_tid=5, entry_tid=30251], [object_number=86, tag_tid=5, entry_tid=30250]]], state=connected] 2026-02-18T08:12:53.923 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:12:53.924 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=97, tag_tid=6, entry_tid=1' 2026-02-18T08:12:53.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:12:53.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:12:53.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:12:53.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:12:53.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:12:53.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:12:53.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:12:53.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:12:53.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:12:53.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:12:53.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:12:53.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:12:53.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:12:53.928 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:12:53.928 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:12:53.963 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 21 2026-02-18T08:12:53.963 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 24 2026-02-18T08:12:53.963 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:12:53.963 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=97, tag_tid=6, entry_tid=1], [object_number=96, tag_tid=6, entry_tid=0], [object_number=99, tag_tid=5, entry_tid=42603], [object_number=98, tag_tid=5, entry_tid=42602]]], state=connected] 2026-02-18T08:12:53.964 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=85, tag_tid=5, entry_tid=30253], [object_number=84, tag_tid=5, entry_tid=30252], [object_number=87, tag_tid=5, entry_tid=30251], [object_number=86, tag_tid=5, entry_tid=30250]]], state=connected] 2026-02-18T08:12:53.967 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:12:53.969 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=85, tag_tid=5, entry_tid=30253' 2026-02-18T08:12:53.969 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=97, tag_tid=6, entry_tid=1' -a 'object_number=97, tag_tid=6, entry_tid=1' = 'object_number=85, tag_tid=5, entry_tid=30253' 2026-02-18T08:12:53.969 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=85, tag_tid=5, entry_tid=30253' '!=' 'object_number=86, tag_tid=5, entry_tid=29954' 2026-02-18T08:12:53.969 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:12:53.969 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=85, tag_tid=5, entry_tid=30253' = 'object_number=86, tag_tid=5, entry_tid=29954' 2026-02-18T08:12:53.969 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=85, tag_tid=5, entry_tid=30253' 2026-02-18T08:12:53.970 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=97, tag_tid=6, entry_tid=1' 2026-02-18T08:12:53.970 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:12:53.970 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:12:53.971 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=6 2026-02-18T08:12:53.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=85, tag_tid=5, entry_tid=30253' 2026-02-18T08:12:53.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:12:53.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:12:53.974 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=5 2026-02-18T08:12:53.974 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=97, tag_tid=6, entry_tid=1' 2026-02-18T08:12:53.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:12:53.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:12:53.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:12:53.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=85, tag_tid=5, entry_tid=30253' 2026-02-18T08:12:53.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:12:53.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:12:53.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=30253 2026-02-18T08:12:53.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 6 = 5 -a 1 -le 30253 2026-02-18T08:12:53.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:12:53.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:12:53.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:12:54.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:12:54.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:12:54.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:12:54.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:12:54.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:12:54.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:12:54.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:12:54.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:12:54.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:12:54.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:12:54.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:12:54.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:12:54.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:12:54.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:12:54.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:12:54.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:12:54.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:12:54.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:12:54.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:12:54.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:12:54.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:12:54.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:12:54.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:12:54.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:12:54.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:12:54.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:12:54.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:12:54.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:12:54.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:12:54.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:12:54.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:12:54.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:12:54.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:12:54.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:12:54.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:12:54.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:12:54.187 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:12:54.187 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:12:54.187 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:12:54.187 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:12:54.187 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:12:54.187 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:12:54.187 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:12:54.187 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:12:54.187 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:12:54.187 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:12:54.187 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:12:54.187 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:12:54.187 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:12:54.195 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:12:54.195 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:12:54.195 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:12:54.195 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:12:54.255 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:12:54.255 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:12:54.255 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:12:54.256 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:12:54.256 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:12:54.256 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:12:54.256 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:12:54.256 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:12:54.256 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:12:54.256 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:12:54.256 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:12:54.257 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:12:54.257 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:12:54.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:12:54.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:12:54.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:12:54.285 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 21 2026-02-18T08:12:54.286 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 24 2026-02-18T08:12:54.286 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:12:54.286 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=97, tag_tid=6, entry_tid=1], [object_number=96, tag_tid=6, entry_tid=0], [object_number=99, tag_tid=5, entry_tid=42603], [object_number=98, tag_tid=5, entry_tid=42602]]], state=connected] 2026-02-18T08:12:54.286 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=87, tag_tid=5, entry_tid=30511], [object_number=86, tag_tid=5, entry_tid=30510], [object_number=85, tag_tid=5, entry_tid=30509], [object_number=84, tag_tid=5, entry_tid=30508]]], state=connected] 2026-02-18T08:12:54.288 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:12:54.290 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=97, tag_tid=6, entry_tid=1' 2026-02-18T08:12:54.291 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:12:54.291 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:12:54.291 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:12:54.291 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:12:54.291 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:12:54.291 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:12:54.292 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:12:54.292 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:12:54.292 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:12:54.292 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:12:54.292 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:12:54.293 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:12:54.293 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:12:54.293 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:12:54.293 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:12:54.319 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 21 2026-02-18T08:12:54.320 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 24 2026-02-18T08:12:54.320 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:12:54.320 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=97, tag_tid=6, entry_tid=1], [object_number=96, tag_tid=6, entry_tid=0], [object_number=99, tag_tid=5, entry_tid=42603], [object_number=98, tag_tid=5, entry_tid=42602]]], state=connected] 2026-02-18T08:12:54.320 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=87, tag_tid=5, entry_tid=30511], [object_number=86, tag_tid=5, entry_tid=30510], [object_number=85, tag_tid=5, entry_tid=30509], [object_number=84, tag_tid=5, entry_tid=30508]]], state=connected] 2026-02-18T08:12:54.322 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:12:54.324 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=87, tag_tid=5, entry_tid=30511' 2026-02-18T08:12:54.324 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=97, tag_tid=6, entry_tid=1' -a 'object_number=97, tag_tid=6, entry_tid=1' = 'object_number=87, tag_tid=5, entry_tid=30511' 2026-02-18T08:12:54.324 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=87, tag_tid=5, entry_tid=30511' '!=' 'object_number=85, tag_tid=5, entry_tid=30253' 2026-02-18T08:12:54.324 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:12:54.325 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=87, tag_tid=5, entry_tid=30511' = 'object_number=85, tag_tid=5, entry_tid=30253' 2026-02-18T08:12:54.325 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=87, tag_tid=5, entry_tid=30511' 2026-02-18T08:12:54.325 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=97, tag_tid=6, entry_tid=1' 2026-02-18T08:12:54.325 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:12:54.326 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:12:54.326 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=6 2026-02-18T08:12:54.327 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=87, tag_tid=5, entry_tid=30511' 2026-02-18T08:12:54.327 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:12:54.327 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:12:54.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=5 2026-02-18T08:12:54.329 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=97, tag_tid=6, entry_tid=1' 2026-02-18T08:12:54.329 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:12:54.329 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:12:54.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:12:54.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=87, tag_tid=5, entry_tid=30511' 2026-02-18T08:12:54.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:12:54.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:12:54.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=30511 2026-02-18T08:12:54.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 6 = 5 -a 1 -le 30511 2026-02-18T08:12:54.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:12:54.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:12:54.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:12:54.535 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:12:54.536 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:12:54.536 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:12:54.536 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:12:54.536 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:12:54.536 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:12:54.536 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:12:54.536 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:12:54.536 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:12:54.536 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:12:54.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:12:54.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:12:54.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:12:54.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:12:54.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:12:54.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:12:54.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:12:54.539 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:12:54.540 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:12:54.540 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:12:54.540 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:12:54.540 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:12:54.540 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:12:54.540 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:12:54.540 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:12:54.540 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:12:54.540 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:12:54.540 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:12:54.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:12:54.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:12:54.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:12:54.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:12:54.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:12:54.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:12:54.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:12:54.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:12:54.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:12:54.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:12:54.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:12:54.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:12:54.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:12:54.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:12:54.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:12:54.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:12:54.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:12:54.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:12:54.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:12:54.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:12:54.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:12:54.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:12:54.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:12:54.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:12:54.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:12:54.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:12:54.610 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:12:54.610 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:12:54.610 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:12:54.610 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:12:54.610 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:12:54.611 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:12:54.611 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:12:54.611 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:12:54.611 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:12:54.611 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:12:54.611 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:12:54.611 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:12:54.612 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:12:54.612 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:12:54.612 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:12:54.647 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 21 2026-02-18T08:12:54.647 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 24 2026-02-18T08:12:54.647 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:12:54.647 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=97, tag_tid=6, entry_tid=1], [object_number=96, tag_tid=6, entry_tid=0], [object_number=99, tag_tid=5, entry_tid=42603], [object_number=98, tag_tid=5, entry_tid=42602]]], state=connected] 2026-02-18T08:12:54.647 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=86, tag_tid=5, entry_tid=30794], [object_number=85, tag_tid=5, entry_tid=30793], [object_number=84, tag_tid=5, entry_tid=30792], [object_number=87, tag_tid=5, entry_tid=30791]]], state=connected] 2026-02-18T08:12:54.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:12:54.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=97, tag_tid=6, entry_tid=1' 2026-02-18T08:12:54.653 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:12:54.653 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:12:54.653 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:12:54.653 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:12:54.653 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:12:54.653 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:12:54.654 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:12:54.654 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:12:54.654 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:12:54.654 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:12:54.654 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:12:54.654 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:12:54.655 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:12:54.655 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:12:54.655 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:12:54.681 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 21 2026-02-18T08:12:54.682 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 24 2026-02-18T08:12:54.682 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:12:54.682 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=97, tag_tid=6, entry_tid=1], [object_number=96, tag_tid=6, entry_tid=0], [object_number=99, tag_tid=5, entry_tid=42603], [object_number=98, tag_tid=5, entry_tid=42602]]], state=connected] 2026-02-18T08:12:54.682 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=86, tag_tid=5, entry_tid=30794], [object_number=85, tag_tid=5, entry_tid=30793], [object_number=84, tag_tid=5, entry_tid=30792], [object_number=87, tag_tid=5, entry_tid=30791]]], state=connected] 2026-02-18T08:12:54.684 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:12:54.686 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=86, tag_tid=5, entry_tid=30794' 2026-02-18T08:12:54.686 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=97, tag_tid=6, entry_tid=1' -a 'object_number=97, tag_tid=6, entry_tid=1' = 'object_number=86, tag_tid=5, entry_tid=30794' 2026-02-18T08:12:54.686 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=86, tag_tid=5, entry_tid=30794' '!=' 'object_number=87, tag_tid=5, entry_tid=30511' 2026-02-18T08:12:54.686 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:12:54.686 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=86, tag_tid=5, entry_tid=30794' = 'object_number=87, tag_tid=5, entry_tid=30511' 2026-02-18T08:12:54.686 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=86, tag_tid=5, entry_tid=30794' 2026-02-18T08:12:54.686 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=97, tag_tid=6, entry_tid=1' 2026-02-18T08:12:54.686 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:12:54.687 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:12:54.687 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=6 2026-02-18T08:12:54.688 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=86, tag_tid=5, entry_tid=30794' 2026-02-18T08:12:54.688 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:12:54.688 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:12:54.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=5 2026-02-18T08:12:54.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=97, tag_tid=6, entry_tid=1' 2026-02-18T08:12:54.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:12:54.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:12:54.691 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:12:54.692 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=86, tag_tid=5, entry_tid=30794' 2026-02-18T08:12:54.692 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:12:54.693 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:12:54.694 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=30794 2026-02-18T08:12:54.694 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 6 = 5 -a 1 -le 30794 2026-02-18T08:12:54.694 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:12:54.695 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:12:54.695 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:12:54.896 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:12:54.897 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:12:54.897 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:12:54.897 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:12:54.897 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:12:54.897 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:12:54.897 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:12:54.897 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:12:54.897 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:12:54.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:12:54.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:12:54.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:12:54.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:12:54.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:12:54.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:12:54.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:12:54.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:12:54.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:12:54.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:12:54.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:12:54.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:12:54.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:12:54.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:12:54.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:12:54.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:12:54.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:12:54.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:12:54.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:12:54.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:12:54.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:12:54.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:12:54.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:12:54.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:12:54.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:12:54.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:12:54.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:12:54.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:12:54.904 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:12:54.904 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:12:54.904 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:12:54.904 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:12:54.904 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:12:54.904 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:12:54.904 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:12:54.904 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:12:54.904 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:12:54.904 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:12:54.904 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:12:54.904 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:12:54.911 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:12:54.911 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:12:54.911 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:12:54.911 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:12:54.971 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:12:54.971 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:12:54.971 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:12:54.971 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:12:54.971 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:12:54.971 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:12:54.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:12:54.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:12:54.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:12:54.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:12:54.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:12:54.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:12:54.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:12:54.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:12:54.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:12:54.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:12:55.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 21 2026-02-18T08:12:55.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 24 2026-02-18T08:12:55.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:12:55.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=97, tag_tid=6, entry_tid=1], [object_number=96, tag_tid=6, entry_tid=0], [object_number=99, tag_tid=5, entry_tid=42603], [object_number=98, tag_tid=5, entry_tid=42602]]], state=connected] 2026-02-18T08:12:55.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=87, tag_tid=5, entry_tid=31079], [object_number=86, tag_tid=5, entry_tid=31078], [object_number=85, tag_tid=5, entry_tid=31077], [object_number=84, tag_tid=5, entry_tid=31076]]], state=connected] 2026-02-18T08:12:55.003 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:12:55.004 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=97, tag_tid=6, entry_tid=1' 2026-02-18T08:12:55.005 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:12:55.005 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:12:55.005 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:12:55.006 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:12:55.006 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:12:55.006 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:12:55.006 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:12:55.006 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:12:55.006 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:12:55.006 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:12:55.007 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:12:55.007 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:12:55.007 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:12:55.008 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:12:55.008 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:12:55.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 21 2026-02-18T08:12:55.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 24 2026-02-18T08:12:55.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:12:55.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=97, tag_tid=6, entry_tid=1], [object_number=96, tag_tid=6, entry_tid=0], [object_number=99, tag_tid=5, entry_tid=42603], [object_number=98, tag_tid=5, entry_tid=42602]]], state=connected] 2026-02-18T08:12:55.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=87, tag_tid=5, entry_tid=31079], [object_number=86, tag_tid=5, entry_tid=31078], [object_number=85, tag_tid=5, entry_tid=31077], [object_number=84, tag_tid=5, entry_tid=31076]]], state=connected] 2026-02-18T08:12:55.036 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:12:55.037 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=87, tag_tid=5, entry_tid=31079' 2026-02-18T08:12:55.037 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=97, tag_tid=6, entry_tid=1' -a 'object_number=97, tag_tid=6, entry_tid=1' = 'object_number=87, tag_tid=5, entry_tid=31079' 2026-02-18T08:12:55.037 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=87, tag_tid=5, entry_tid=31079' '!=' 'object_number=86, tag_tid=5, entry_tid=30794' 2026-02-18T08:12:55.037 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:12:55.038 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=87, tag_tid=5, entry_tid=31079' = 'object_number=86, tag_tid=5, entry_tid=30794' 2026-02-18T08:12:55.038 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=87, tag_tid=5, entry_tid=31079' 2026-02-18T08:12:55.038 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=97, tag_tid=6, entry_tid=1' 2026-02-18T08:12:55.039 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:12:55.039 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:12:55.039 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=6 2026-02-18T08:12:55.040 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=87, tag_tid=5, entry_tid=31079' 2026-02-18T08:12:55.040 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:12:55.040 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:12:55.041 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=5 2026-02-18T08:12:55.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=97, tag_tid=6, entry_tid=1' 2026-02-18T08:12:55.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:12:55.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:12:55.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:12:55.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=87, tag_tid=5, entry_tid=31079' 2026-02-18T08:12:55.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:12:55.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:12:55.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=31079 2026-02-18T08:12:55.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 6 = 5 -a 1 -le 31079 2026-02-18T08:12:55.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:12:55.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:12:55.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:12:55.249 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:12:55.249 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:12:55.249 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:12:55.249 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:12:55.249 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:12:55.250 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:12:55.250 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:12:55.250 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:12:55.250 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:12:55.250 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:12:55.250 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:12:55.250 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:12:55.250 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:12:55.250 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:12:55.250 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:12:55.251 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:12:55.251 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:12:55.253 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:12:55.254 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:12:55.254 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:12:55.254 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:12:55.254 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:12:55.254 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:12:55.254 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:12:55.254 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:12:55.254 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:12:55.254 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:12:55.255 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:12:55.255 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:12:55.255 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:12:55.255 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:12:55.255 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:12:55.255 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:12:55.255 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:12:55.255 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:12:55.255 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:12:55.255 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:12:55.255 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:12:55.255 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:12:55.256 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:12:55.256 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:12:55.256 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:12:55.256 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:12:55.256 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:12:55.256 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:12:55.256 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:12:55.256 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:12:55.256 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:12:55.256 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:12:55.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:12:55.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:12:55.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:12:55.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:12:55.321 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:12:55.323 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:12:55.323 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:12:55.323 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:12:55.323 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:12:55.323 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:12:55.324 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:12:55.324 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:12:55.324 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:12:55.324 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:12:55.324 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:12:55.324 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:12:55.324 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:12:55.325 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:12:55.325 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:12:55.325 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:12:55.360 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 21 2026-02-18T08:12:55.361 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 24 2026-02-18T08:12:55.361 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:12:55.361 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=97, tag_tid=6, entry_tid=1], [object_number=96, tag_tid=6, entry_tid=0], [object_number=99, tag_tid=5, entry_tid=42603], [object_number=98, tag_tid=5, entry_tid=42602]]], state=connected] 2026-02-18T08:12:55.361 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=84, tag_tid=5, entry_tid=31372], [object_number=87, tag_tid=5, entry_tid=31371], [object_number=86, tag_tid=5, entry_tid=31370], [object_number=85, tag_tid=5, entry_tid=31369]]], state=connected] 2026-02-18T08:12:55.364 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:12:55.366 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=97, tag_tid=6, entry_tid=1' 2026-02-18T08:12:55.367 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:12:55.367 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:12:55.367 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:12:55.367 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:12:55.367 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:12:55.367 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:12:55.367 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:12:55.367 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:12:55.367 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:12:55.368 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:12:55.368 INFO:tasks.rbd_mirror_thrash:waiting for cluster1.client.mirror.2 2026-02-18T08:12:55.368 INFO:teuthology.orchestra.run:waiting for 600 2026-02-18T08:12:55.368 INFO:tasks.rbd_mirror_thrash:reviving cluster1.client.mirror.2 2026-02-18T08:12:55.368 INFO:tasks.rbd_mirror.cluster1.client.mirror.2:Restarting daemon 2026-02-18T08:12:55.369 DEBUG:teuthology.orchestra.run.trial163:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.2 2026-02-18T08:12:55.370 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:12:55.370 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:12:55.370 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:12:55.370 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:12:55.370 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:12:55.372 INFO:tasks.rbd_mirror.cluster1.client.mirror.2:Started 2026-02-18T08:12:55.372 INFO:tasks.rbd_mirror_thrash:waiting for 89.68872910683902 secs before thrashing 2026-02-18T08:12:55.403 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 21 2026-02-18T08:12:55.403 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 24 2026-02-18T08:12:55.403 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:12:55.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=97, tag_tid=6, entry_tid=1], [object_number=96, tag_tid=6, entry_tid=0], [object_number=99, tag_tid=5, entry_tid=42603], [object_number=98, tag_tid=5, entry_tid=42602]]], state=connected] 2026-02-18T08:12:55.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=84, tag_tid=5, entry_tid=31372], [object_number=87, tag_tid=5, entry_tid=31371], [object_number=86, tag_tid=5, entry_tid=31370], [object_number=85, tag_tid=5, entry_tid=31369]]], state=connected] 2026-02-18T08:12:55.405 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:12:55.408 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=84, tag_tid=5, entry_tid=31372' 2026-02-18T08:12:55.408 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=97, tag_tid=6, entry_tid=1' -a 'object_number=97, tag_tid=6, entry_tid=1' = 'object_number=84, tag_tid=5, entry_tid=31372' 2026-02-18T08:12:55.408 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=84, tag_tid=5, entry_tid=31372' '!=' 'object_number=87, tag_tid=5, entry_tid=31079' 2026-02-18T08:12:55.408 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:12:55.408 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=84, tag_tid=5, entry_tid=31372' = 'object_number=87, tag_tid=5, entry_tid=31079' 2026-02-18T08:12:55.408 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=84, tag_tid=5, entry_tid=31372' 2026-02-18T08:12:55.409 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=97, tag_tid=6, entry_tid=1' 2026-02-18T08:12:55.409 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:12:55.409 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:12:55.409 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=6 2026-02-18T08:12:55.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=84, tag_tid=5, entry_tid=31372' 2026-02-18T08:12:55.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:12:55.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:12:55.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=5 2026-02-18T08:12:55.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=97, tag_tid=6, entry_tid=1' 2026-02-18T08:12:55.412 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:12:55.412 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:12:55.413 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:12:55.414 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=84, tag_tid=5, entry_tid=31372' 2026-02-18T08:12:55.414 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:12:55.414 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:12:55.416 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=31372 2026-02-18T08:12:55.416 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 6 = 5 -a 1 -le 31372 2026-02-18T08:12:55.416 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:12:55.416 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:12:55.416 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:12:55.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:12:55.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:12:55.618 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:12:55.618 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:12:55.618 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:12:55.618 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:12:55.618 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:12:55.618 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:12:55.618 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:12:55.618 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:12:55.618 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:12:55.619 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:12:55.619 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:12:55.619 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:12:55.619 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:12:55.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:12:55.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:12:55.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:12:55.622 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:12:55.622 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:12:55.622 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:12:55.622 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:12:55.622 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:12:55.622 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:12:55.622 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:12:55.622 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:12:55.622 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:12:55.622 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:12:55.622 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:12:55.623 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:12:55.623 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:12:55.623 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:12:55.623 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:12:55.623 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:12:55.623 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:12:55.623 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:12:55.623 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:12:55.623 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:12:55.623 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:12:55.623 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:12:55.623 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:12:55.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:12:55.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:12:55.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:12:55.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:12:55.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:12:55.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:12:55.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:12:55.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:12:55.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:12:55.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:12:55.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:12:55.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:12:55.702 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:12:55.703 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:12:55.703 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:12:55.703 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:12:55.703 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:12:55.703 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:12:55.703 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:12:55.703 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:12:55.703 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:12:55.704 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:12:55.704 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:12:55.704 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:12:55.704 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:12:55.705 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:12:55.705 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:12:55.705 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:12:55.742 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 21 2026-02-18T08:12:55.742 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 24 2026-02-18T08:12:55.742 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:12:55.742 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=97, tag_tid=6, entry_tid=1], [object_number=96, tag_tid=6, entry_tid=0], [object_number=99, tag_tid=5, entry_tid=42603], [object_number=98, tag_tid=5, entry_tid=42602]]], state=connected] 2026-02-18T08:12:55.742 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=85, tag_tid=5, entry_tid=31641], [object_number=84, tag_tid=5, entry_tid=31640], [object_number=87, tag_tid=5, entry_tid=31639], [object_number=86, tag_tid=5, entry_tid=31638]]], state=connected] 2026-02-18T08:12:55.745 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:12:55.747 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=97, tag_tid=6, entry_tid=1' 2026-02-18T08:12:55.748 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:12:55.749 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:12:55.749 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:12:55.749 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:12:55.749 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:12:55.749 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:12:55.749 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:12:55.749 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:12:55.749 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:12:55.749 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:12:55.750 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:12:55.750 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:12:55.750 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:12:55.751 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:12:55.751 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:12:55.785 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 21 2026-02-18T08:12:55.785 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 24 2026-02-18T08:12:55.785 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:12:55.785 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=97, tag_tid=6, entry_tid=1], [object_number=96, tag_tid=6, entry_tid=0], [object_number=99, tag_tid=5, entry_tid=42603], [object_number=98, tag_tid=5, entry_tid=42602]]], state=connected] 2026-02-18T08:12:55.785 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=85, tag_tid=5, entry_tid=31641], [object_number=84, tag_tid=5, entry_tid=31640], [object_number=87, tag_tid=5, entry_tid=31639], [object_number=86, tag_tid=5, entry_tid=31638]]], state=connected] 2026-02-18T08:12:55.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:12:55.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=85, tag_tid=5, entry_tid=31641' 2026-02-18T08:12:55.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=97, tag_tid=6, entry_tid=1' -a 'object_number=97, tag_tid=6, entry_tid=1' = 'object_number=85, tag_tid=5, entry_tid=31641' 2026-02-18T08:12:55.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=85, tag_tid=5, entry_tid=31641' '!=' 'object_number=84, tag_tid=5, entry_tid=31372' 2026-02-18T08:12:55.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:12:55.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=85, tag_tid=5, entry_tid=31641' = 'object_number=84, tag_tid=5, entry_tid=31372' 2026-02-18T08:12:55.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=85, tag_tid=5, entry_tid=31641' 2026-02-18T08:12:55.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=97, tag_tid=6, entry_tid=1' 2026-02-18T08:12:55.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:12:55.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:12:55.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=6 2026-02-18T08:12:55.793 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=85, tag_tid=5, entry_tid=31641' 2026-02-18T08:12:55.793 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:12:55.793 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:12:55.795 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=5 2026-02-18T08:12:55.795 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=97, tag_tid=6, entry_tid=1' 2026-02-18T08:12:55.795 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:12:55.796 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:12:55.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:12:55.798 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=85, tag_tid=5, entry_tid=31641' 2026-02-18T08:12:55.798 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:12:55.798 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:12:55.800 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=31641 2026-02-18T08:12:55.800 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 6 = 5 -a 1 -le 31641 2026-02-18T08:12:55.800 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:12:55.800 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:12:55.800 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:12:56.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:12:56.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:12:56.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:12:56.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:12:56.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:12:56.003 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:12:56.003 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:12:56.003 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:12:56.003 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:12:56.003 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:12:56.003 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:12:56.003 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:12:56.003 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:12:56.003 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:12:56.003 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:12:56.004 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:12:56.004 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:12:56.006 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:12:56.006 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:12:56.007 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:12:56.007 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:12:56.007 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:12:56.007 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:12:56.007 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:12:56.007 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:12:56.007 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:12:56.007 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:12:56.007 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:12:56.007 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:12:56.008 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:12:56.008 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:12:56.008 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:12:56.008 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:12:56.008 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:12:56.008 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:12:56.008 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:12:56.008 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:12:56.008 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:12:56.008 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:12:56.008 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:12:56.008 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:12:56.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:12:56.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:12:56.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:12:56.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:12:56.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:12:56.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:12:56.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:12:56.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:12:56.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:12:56.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:12:56.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:12:56.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:12:56.075 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:12:56.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:12:56.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:12:56.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:12:56.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:12:56.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:12:56.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:12:56.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:12:56.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:12:56.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:12:56.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:12:56.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:12:56.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:12:56.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:12:56.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:12:56.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:12:56.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 21 2026-02-18T08:12:56.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 24 2026-02-18T08:12:56.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:12:56.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=97, tag_tid=6, entry_tid=1], [object_number=96, tag_tid=6, entry_tid=0], [object_number=99, tag_tid=5, entry_tid=42603], [object_number=98, tag_tid=5, entry_tid=42602]]], state=connected] 2026-02-18T08:12:56.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=84, tag_tid=5, entry_tid=31884], [object_number=87, tag_tid=5, entry_tid=31883], [object_number=86, tag_tid=5, entry_tid=31882], [object_number=85, tag_tid=5, entry_tid=31881]]], state=connected] 2026-02-18T08:12:56.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:12:56.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=97, tag_tid=6, entry_tid=1' 2026-02-18T08:12:56.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:12:56.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:12:56.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:12:56.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:12:56.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:12:56.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:12:56.110 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:12:56.110 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:12:56.110 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:12:56.110 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:12:56.110 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:12:56.110 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:12:56.111 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:12:56.111 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:12:56.111 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:12:56.137 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 21 2026-02-18T08:12:56.137 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 24 2026-02-18T08:12:56.137 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:12:56.137 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=97, tag_tid=6, entry_tid=1], [object_number=96, tag_tid=6, entry_tid=0], [object_number=99, tag_tid=5, entry_tid=42603], [object_number=98, tag_tid=5, entry_tid=42602]]], state=connected] 2026-02-18T08:12:56.137 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=84, tag_tid=5, entry_tid=31884], [object_number=87, tag_tid=5, entry_tid=31883], [object_number=86, tag_tid=5, entry_tid=31882], [object_number=85, tag_tid=5, entry_tid=31881]]], state=connected] 2026-02-18T08:12:56.140 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:12:56.141 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=84, tag_tid=5, entry_tid=31884' 2026-02-18T08:12:56.141 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=97, tag_tid=6, entry_tid=1' -a 'object_number=97, tag_tid=6, entry_tid=1' = 'object_number=84, tag_tid=5, entry_tid=31884' 2026-02-18T08:12:56.141 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=84, tag_tid=5, entry_tid=31884' '!=' 'object_number=85, tag_tid=5, entry_tid=31641' 2026-02-18T08:12:56.142 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:12:56.142 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=84, tag_tid=5, entry_tid=31884' = 'object_number=85, tag_tid=5, entry_tid=31641' 2026-02-18T08:12:56.142 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=84, tag_tid=5, entry_tid=31884' 2026-02-18T08:12:56.143 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=97, tag_tid=6, entry_tid=1' 2026-02-18T08:12:56.143 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:12:56.143 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:12:56.143 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=6 2026-02-18T08:12:56.144 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=84, tag_tid=5, entry_tid=31884' 2026-02-18T08:12:56.144 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:12:56.144 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:12:56.145 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=5 2026-02-18T08:12:56.146 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=97, tag_tid=6, entry_tid=1' 2026-02-18T08:12:56.146 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:12:56.147 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:12:56.148 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:12:56.149 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=84, tag_tid=5, entry_tid=31884' 2026-02-18T08:12:56.149 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:12:56.149 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:12:56.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=31884 2026-02-18T08:12:56.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 6 = 5 -a 1 -le 31884 2026-02-18T08:12:56.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:12:56.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:12:56.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:12:56.353 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:12:56.353 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:12:56.353 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:12:56.353 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:12:56.353 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:12:56.353 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:12:56.353 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:12:56.354 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:12:56.354 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:12:56.354 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:12:56.354 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:12:56.354 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:12:56.354 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:12:56.354 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:12:56.354 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:12:56.354 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:12:56.354 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:12:56.357 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:12:56.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:12:56.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:12:56.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:12:56.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:12:56.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:12:56.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:12:56.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:12:56.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:12:56.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:12:56.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:12:56.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:12:56.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:12:56.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:12:56.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:12:56.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:12:56.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:12:56.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:12:56.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:12:56.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:12:56.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:12:56.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:12:56.360 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:12:56.360 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:12:56.360 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:12:56.360 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:12:56.360 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:12:56.360 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:12:56.360 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:12:56.360 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:12:56.360 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:12:56.360 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:12:56.368 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:12:56.368 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:12:56.368 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:12:56.368 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:12:56.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:12:56.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:12:56.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:12:56.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:12:56.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:12:56.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:12:56.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:12:56.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:12:56.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:12:56.427 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:12:56.427 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:12:56.427 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:12:56.427 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:12:56.428 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:12:56.428 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:12:56.428 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:12:56.454 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 21 2026-02-18T08:12:56.454 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 24 2026-02-18T08:12:56.454 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:12:56.454 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=97, tag_tid=6, entry_tid=1], [object_number=96, tag_tid=6, entry_tid=0], [object_number=99, tag_tid=5, entry_tid=42603], [object_number=98, tag_tid=5, entry_tid=42602]]], state=connected] 2026-02-18T08:12:56.455 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=86, tag_tid=5, entry_tid=32106], [object_number=85, tag_tid=5, entry_tid=32105], [object_number=84, tag_tid=5, entry_tid=32104], [object_number=87, tag_tid=5, entry_tid=32103]]], state=connected] 2026-02-18T08:12:56.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:12:56.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=97, tag_tid=6, entry_tid=1' 2026-02-18T08:12:56.460 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:12:56.460 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:12:56.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:12:56.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:12:56.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:12:56.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:12:56.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:12:56.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:12:56.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:12:56.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:12:56.462 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:12:56.462 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:12:56.462 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:12:56.463 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:12:56.463 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:12:56.490 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 21 2026-02-18T08:12:56.490 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 24 2026-02-18T08:12:56.490 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:12:56.490 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=97, tag_tid=6, entry_tid=1], [object_number=96, tag_tid=6, entry_tid=0], [object_number=99, tag_tid=5, entry_tid=42603], [object_number=98, tag_tid=5, entry_tid=42602]]], state=connected] 2026-02-18T08:12:56.490 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=86, tag_tid=5, entry_tid=32106], [object_number=85, tag_tid=5, entry_tid=32105], [object_number=84, tag_tid=5, entry_tid=32104], [object_number=87, tag_tid=5, entry_tid=32103]]], state=connected] 2026-02-18T08:12:56.492 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:12:56.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=86, tag_tid=5, entry_tid=32106' 2026-02-18T08:12:56.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=97, tag_tid=6, entry_tid=1' -a 'object_number=97, tag_tid=6, entry_tid=1' = 'object_number=86, tag_tid=5, entry_tid=32106' 2026-02-18T08:12:56.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=86, tag_tid=5, entry_tid=32106' '!=' 'object_number=84, tag_tid=5, entry_tid=31884' 2026-02-18T08:12:56.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:12:56.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=86, tag_tid=5, entry_tid=32106' = 'object_number=84, tag_tid=5, entry_tid=31884' 2026-02-18T08:12:56.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=86, tag_tid=5, entry_tid=32106' 2026-02-18T08:12:56.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=97, tag_tid=6, entry_tid=1' 2026-02-18T08:12:56.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:12:56.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:12:56.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=6 2026-02-18T08:12:56.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=86, tag_tid=5, entry_tid=32106' 2026-02-18T08:12:56.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:12:56.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:12:56.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=5 2026-02-18T08:12:56.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=97, tag_tid=6, entry_tid=1' 2026-02-18T08:12:56.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:12:56.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:12:56.500 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:12:56.501 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=86, tag_tid=5, entry_tid=32106' 2026-02-18T08:12:56.501 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:12:56.501 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:12:56.502 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=32106 2026-02-18T08:12:56.502 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 6 = 5 -a 1 -le 32106 2026-02-18T08:12:56.503 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:12:56.503 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:12:56.503 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:12:56.704 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:12:56.705 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:12:56.705 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:12:56.705 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:12:56.705 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:12:56.705 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:12:56.705 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:12:56.705 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:12:56.705 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:12:56.705 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:12:56.706 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:12:56.706 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:12:56.706 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:12:56.706 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:12:56.706 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:12:56.706 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:12:56.709 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:12:56.709 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:12:56.709 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:12:56.709 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:12:56.709 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:12:56.709 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:12:56.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:12:56.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:12:56.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:12:56.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:12:56.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:12:56.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:12:56.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:12:56.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:12:56.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:12:56.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:12:56.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:12:56.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:12:56.711 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:12:56.711 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:12:56.711 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:12:56.711 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:12:56.711 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:12:56.711 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:12:56.711 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:12:56.711 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:12:56.711 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:12:56.711 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:12:56.711 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:12:56.712 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:12:56.712 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:12:56.712 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:12:56.712 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:12:56.720 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:12:56.720 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:12:56.720 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:12:56.720 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:12:56.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:12:56.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:12:56.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:12:56.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:12:56.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:12:56.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:12:56.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:12:56.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:12:56.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:12:56.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:12:56.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:12:56.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:12:56.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:12:56.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:12:56.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:12:56.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:12:56.820 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 22 2026-02-18T08:12:56.820 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 24 2026-02-18T08:12:56.820 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:12:56.820 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=97, tag_tid=6, entry_tid=1], [object_number=96, tag_tid=6, entry_tid=0], [object_number=99, tag_tid=5, entry_tid=42603], [object_number=98, tag_tid=5, entry_tid=42602]]], state=connected] 2026-02-18T08:12:56.820 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=90, tag_tid=5, entry_tid=32246], [object_number=89, tag_tid=5, entry_tid=32245], [object_number=88, tag_tid=5, entry_tid=32244], [object_number=91, tag_tid=5, entry_tid=32243]]], state=connected] 2026-02-18T08:12:56.822 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:12:56.824 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=97, tag_tid=6, entry_tid=1' 2026-02-18T08:12:56.825 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:12:56.825 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:12:56.825 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:12:56.825 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:12:56.825 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:12:56.825 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:12:56.826 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:12:56.826 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:12:56.826 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:12:56.826 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:12:56.826 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:12:56.826 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:12:56.827 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:12:56.827 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:12:56.827 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:12:56.854 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 22 2026-02-18T08:12:56.854 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 24 2026-02-18T08:12:56.854 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:12:56.855 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=97, tag_tid=6, entry_tid=1], [object_number=96, tag_tid=6, entry_tid=0], [object_number=99, tag_tid=5, entry_tid=42603], [object_number=98, tag_tid=5, entry_tid=42602]]], state=connected] 2026-02-18T08:12:56.855 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=90, tag_tid=5, entry_tid=32246], [object_number=89, tag_tid=5, entry_tid=32245], [object_number=88, tag_tid=5, entry_tid=32244], [object_number=91, tag_tid=5, entry_tid=32243]]], state=connected] 2026-02-18T08:12:56.857 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:12:56.859 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=90, tag_tid=5, entry_tid=32246' 2026-02-18T08:12:56.859 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=97, tag_tid=6, entry_tid=1' -a 'object_number=97, tag_tid=6, entry_tid=1' = 'object_number=90, tag_tid=5, entry_tid=32246' 2026-02-18T08:12:56.859 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=90, tag_tid=5, entry_tid=32246' '!=' 'object_number=86, tag_tid=5, entry_tid=32106' 2026-02-18T08:12:56.859 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:12:56.859 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=90, tag_tid=5, entry_tid=32246' = 'object_number=86, tag_tid=5, entry_tid=32106' 2026-02-18T08:12:56.859 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=90, tag_tid=5, entry_tid=32246' 2026-02-18T08:12:56.860 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=97, tag_tid=6, entry_tid=1' 2026-02-18T08:12:56.860 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:12:56.860 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:12:56.861 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=6 2026-02-18T08:12:56.862 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=90, tag_tid=5, entry_tid=32246' 2026-02-18T08:12:56.862 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:12:56.862 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:12:56.863 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=5 2026-02-18T08:12:56.864 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=97, tag_tid=6, entry_tid=1' 2026-02-18T08:12:56.864 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:12:56.865 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:12:56.866 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:12:56.866 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=90, tag_tid=5, entry_tid=32246' 2026-02-18T08:12:56.867 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:12:56.867 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:12:56.868 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=32246 2026-02-18T08:12:56.868 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 6 = 5 -a 1 -le 32246 2026-02-18T08:12:56.868 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:12:56.869 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:12:56.869 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:12:57.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:12:57.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:12:57.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:12:57.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:12:57.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:12:57.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:12:57.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:12:57.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:12:57.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:12:57.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:12:57.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:12:57.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:12:57.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:12:57.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:12:57.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:12:57.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:12:57.074 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:12:57.074 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:12:57.074 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:12:57.074 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:12:57.074 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:12:57.074 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:12:57.074 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:12:57.074 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:12:57.074 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:12:57.075 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:12:57.075 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:12:57.075 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:12:57.075 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:12:57.075 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:12:57.075 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:12:57.075 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:12:57.075 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:12:57.075 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:12:57.075 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:12:57.075 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:12:57.076 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:12:57.076 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:12:57.076 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:12:57.076 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:12:57.076 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:12:57.076 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:12:57.076 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:12:57.076 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:12:57.076 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:12:57.076 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:12:57.076 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:12:57.076 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:12:57.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:12:57.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:12:57.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:12:57.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:12:57.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:12:57.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:12:57.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:12:57.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:12:57.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:12:57.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:12:57.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:12:57.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:12:57.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:12:57.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:12:57.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:12:57.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:12:57.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:12:57.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:12:57.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:12:57.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:12:57.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:12:57.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 22 2026-02-18T08:12:57.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 24 2026-02-18T08:12:57.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:12:57.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=97, tag_tid=6, entry_tid=1], [object_number=96, tag_tid=6, entry_tid=0], [object_number=99, tag_tid=5, entry_tid=42603], [object_number=98, tag_tid=5, entry_tid=42602]]], state=connected] 2026-02-18T08:12:57.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=91, tag_tid=5, entry_tid=32479], [object_number=90, tag_tid=5, entry_tid=32478], [object_number=89, tag_tid=5, entry_tid=32477], [object_number=88, tag_tid=5, entry_tid=32476]]], state=connected] 2026-02-18T08:12:57.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:12:57.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=97, tag_tid=6, entry_tid=1' 2026-02-18T08:12:57.187 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:12:57.187 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:12:57.187 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:12:57.187 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:12:57.187 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:12:57.187 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:12:57.187 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:12:57.188 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:12:57.188 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:12:57.188 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:12:57.188 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:12:57.188 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:12:57.189 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:12:57.189 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:12:57.189 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:12:57.224 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 22 2026-02-18T08:12:57.225 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 24 2026-02-18T08:12:57.225 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:12:57.225 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=97, tag_tid=6, entry_tid=1], [object_number=96, tag_tid=6, entry_tid=0], [object_number=99, tag_tid=5, entry_tid=42603], [object_number=98, tag_tid=5, entry_tid=42602]]], state=connected] 2026-02-18T08:12:57.225 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=91, tag_tid=5, entry_tid=32479], [object_number=90, tag_tid=5, entry_tid=32478], [object_number=89, tag_tid=5, entry_tid=32477], [object_number=88, tag_tid=5, entry_tid=32476]]], state=connected] 2026-02-18T08:12:57.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:12:57.230 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=91, tag_tid=5, entry_tid=32479' 2026-02-18T08:12:57.230 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=97, tag_tid=6, entry_tid=1' -a 'object_number=97, tag_tid=6, entry_tid=1' = 'object_number=91, tag_tid=5, entry_tid=32479' 2026-02-18T08:12:57.230 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=91, tag_tid=5, entry_tid=32479' '!=' 'object_number=90, tag_tid=5, entry_tid=32246' 2026-02-18T08:12:57.230 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:12:57.230 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=91, tag_tid=5, entry_tid=32479' = 'object_number=90, tag_tid=5, entry_tid=32246' 2026-02-18T08:12:57.230 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=91, tag_tid=5, entry_tid=32479' 2026-02-18T08:12:57.231 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=97, tag_tid=6, entry_tid=1' 2026-02-18T08:12:57.231 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:12:57.231 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:12:57.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=6 2026-02-18T08:12:57.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=91, tag_tid=5, entry_tid=32479' 2026-02-18T08:12:57.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:12:57.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:12:57.234 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=5 2026-02-18T08:12:57.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=97, tag_tid=6, entry_tid=1' 2026-02-18T08:12:57.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:12:57.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:12:57.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:12:57.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=91, tag_tid=5, entry_tid=32479' 2026-02-18T08:12:57.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:12:57.238 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:12:57.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=32479 2026-02-18T08:12:57.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 6 = 5 -a 1 -le 32479 2026-02-18T08:12:57.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:12:57.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:12:57.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:12:57.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:12:57.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:12:57.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:12:57.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:12:57.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:12:57.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:12:57.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:12:57.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:12:57.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:12:57.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:12:57.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:12:57.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:12:57.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:12:57.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:12:57.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:12:57.443 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:12:57.443 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:12:57.443 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:12:57.443 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:12:57.443 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:12:57.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:12:57.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:12:57.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:12:57.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:12:57.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:12:57.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:12:57.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:12:57.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:12:57.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:12:57.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:12:57.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:12:57.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:12:57.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:12:57.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:12:57.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:12:57.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:12:57.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:12:57.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:12:57.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:12:57.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:12:57.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:12:57.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:12:57.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:12:57.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:12:57.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:12:57.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:12:57.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:12:57.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:12:57.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:12:57.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:12:57.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:12:57.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:12:57.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:12:57.514 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:12:57.516 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:12:57.516 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:12:57.516 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:12:57.516 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:12:57.516 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:12:57.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:12:57.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:12:57.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:12:57.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:12:57.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:12:57.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:12:57.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:12:57.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:12:57.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:12:57.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:12:57.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 22 2026-02-18T08:12:57.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 24 2026-02-18T08:12:57.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:12:57.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=97, tag_tid=6, entry_tid=1], [object_number=96, tag_tid=6, entry_tid=0], [object_number=99, tag_tid=5, entry_tid=42603], [object_number=98, tag_tid=5, entry_tid=42602]]], state=connected] 2026-02-18T08:12:57.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=89, tag_tid=5, entry_tid=32713], [object_number=88, tag_tid=5, entry_tid=32712], [object_number=91, tag_tid=5, entry_tid=32711], [object_number=90, tag_tid=5, entry_tid=32710]]], state=connected] 2026-02-18T08:12:57.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:12:57.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=97, tag_tid=6, entry_tid=1' 2026-02-18T08:12:57.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:12:57.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:12:57.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:12:57.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:12:57.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:12:57.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:12:57.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:12:57.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:12:57.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:12:57.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:12:57.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:12:57.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:12:57.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:12:57.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:12:57.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:12:57.584 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 22 2026-02-18T08:12:57.584 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 24 2026-02-18T08:12:57.584 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:12:57.584 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=97, tag_tid=6, entry_tid=1], [object_number=96, tag_tid=6, entry_tid=0], [object_number=99, tag_tid=5, entry_tid=42603], [object_number=98, tag_tid=5, entry_tid=42602]]], state=connected] 2026-02-18T08:12:57.584 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=89, tag_tid=5, entry_tid=32713], [object_number=88, tag_tid=5, entry_tid=32712], [object_number=91, tag_tid=5, entry_tid=32711], [object_number=90, tag_tid=5, entry_tid=32710]]], state=connected] 2026-02-18T08:12:57.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:12:57.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=89, tag_tid=5, entry_tid=32713' 2026-02-18T08:12:57.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=97, tag_tid=6, entry_tid=1' -a 'object_number=97, tag_tid=6, entry_tid=1' = 'object_number=89, tag_tid=5, entry_tid=32713' 2026-02-18T08:12:57.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=89, tag_tid=5, entry_tid=32713' '!=' 'object_number=91, tag_tid=5, entry_tid=32479' 2026-02-18T08:12:57.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:12:57.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=89, tag_tid=5, entry_tid=32713' = 'object_number=91, tag_tid=5, entry_tid=32479' 2026-02-18T08:12:57.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=89, tag_tid=5, entry_tid=32713' 2026-02-18T08:12:57.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=97, tag_tid=6, entry_tid=1' 2026-02-18T08:12:57.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:12:57.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:12:57.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=6 2026-02-18T08:12:57.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=89, tag_tid=5, entry_tid=32713' 2026-02-18T08:12:57.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:12:57.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:12:57.594 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=5 2026-02-18T08:12:57.594 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=97, tag_tid=6, entry_tid=1' 2026-02-18T08:12:57.594 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:12:57.595 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:12:57.596 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:12:57.597 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=89, tag_tid=5, entry_tid=32713' 2026-02-18T08:12:57.597 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:12:57.597 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:12:57.598 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=32713 2026-02-18T08:12:57.599 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 6 = 5 -a 1 -le 32713 2026-02-18T08:12:57.599 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:12:57.599 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:12:57.599 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:12:57.800 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:12:57.801 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:12:57.801 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:12:57.801 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:12:57.801 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:12:57.801 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:12:57.801 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:12:57.801 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:12:57.801 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:12:57.801 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:12:57.801 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:12:57.801 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:12:57.801 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:12:57.802 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:12:57.802 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:12:57.804 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:12:57.804 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:12:57.804 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:12:57.804 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:12:57.804 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:12:57.804 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:12:57.804 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:12:57.804 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:12:57.805 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:12:57.805 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:12:57.805 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:12:57.805 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:12:57.805 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:12:57.805 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:12:57.805 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:12:57.805 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:12:57.805 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:12:57.806 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:12:57.806 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:12:57.806 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:12:57.806 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:12:57.806 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:12:57.806 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:12:57.806 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:12:57.806 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:12:57.806 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:12:57.807 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:12:57.807 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:12:57.807 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:12:57.807 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:12:57.807 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:12:57.807 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:12:57.807 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:12:57.807 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:12:57.815 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:12:57.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:12:57.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:12:57.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:12:57.882 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:12:57.883 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:12:57.883 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:12:57.883 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:12:57.883 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:12:57.883 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:12:57.883 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:12:57.883 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:12:57.883 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:12:57.884 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:12:57.884 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:12:57.884 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:12:57.884 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:12:57.885 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:12:57.885 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:12:57.885 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:12:57.922 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 22 2026-02-18T08:12:57.922 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 24 2026-02-18T08:12:57.922 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:12:57.922 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=97, tag_tid=6, entry_tid=1], [object_number=96, tag_tid=6, entry_tid=0], [object_number=99, tag_tid=5, entry_tid=42603], [object_number=98, tag_tid=5, entry_tid=42602]]], state=connected] 2026-02-18T08:12:57.922 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=90, tag_tid=5, entry_tid=32946], [object_number=89, tag_tid=5, entry_tid=32945], [object_number=88, tag_tid=5, entry_tid=32944], [object_number=91, tag_tid=5, entry_tid=32943]]], state=connected] 2026-02-18T08:12:57.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:12:57.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=97, tag_tid=6, entry_tid=1' 2026-02-18T08:12:57.928 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:12:57.928 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:12:57.928 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:12:57.928 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:12:57.928 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:12:57.928 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:12:57.928 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:12:57.929 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:12:57.929 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:12:57.929 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:12:57.929 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:12:57.929 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:12:57.930 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:12:57.930 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:12:57.930 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:12:57.964 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 22 2026-02-18T08:12:57.964 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 24 2026-02-18T08:12:57.964 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:12:57.964 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=97, tag_tid=6, entry_tid=1], [object_number=96, tag_tid=6, entry_tid=0], [object_number=99, tag_tid=5, entry_tid=42603], [object_number=98, tag_tid=5, entry_tid=42602]]], state=connected] 2026-02-18T08:12:57.964 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=90, tag_tid=5, entry_tid=32946], [object_number=89, tag_tid=5, entry_tid=32945], [object_number=88, tag_tid=5, entry_tid=32944], [object_number=91, tag_tid=5, entry_tid=32943]]], state=connected] 2026-02-18T08:12:57.967 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:12:57.969 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=90, tag_tid=5, entry_tid=32946' 2026-02-18T08:12:57.969 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=97, tag_tid=6, entry_tid=1' -a 'object_number=97, tag_tid=6, entry_tid=1' = 'object_number=90, tag_tid=5, entry_tid=32946' 2026-02-18T08:12:57.969 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=90, tag_tid=5, entry_tid=32946' '!=' 'object_number=89, tag_tid=5, entry_tid=32713' 2026-02-18T08:12:57.969 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:12:57.969 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=90, tag_tid=5, entry_tid=32946' = 'object_number=89, tag_tid=5, entry_tid=32713' 2026-02-18T08:12:57.969 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=90, tag_tid=5, entry_tid=32946' 2026-02-18T08:12:57.969 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=97, tag_tid=6, entry_tid=1' 2026-02-18T08:12:57.969 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:12:57.969 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:12:57.971 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=6 2026-02-18T08:12:57.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=90, tag_tid=5, entry_tid=32946' 2026-02-18T08:12:57.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:12:57.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:12:57.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=5 2026-02-18T08:12:57.974 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=97, tag_tid=6, entry_tid=1' 2026-02-18T08:12:57.974 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:12:57.974 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:12:57.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:12:57.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=90, tag_tid=5, entry_tid=32946' 2026-02-18T08:12:57.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:12:57.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:12:57.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=32946 2026-02-18T08:12:57.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 6 = 5 -a 1 -le 32946 2026-02-18T08:12:57.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:12:57.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:12:57.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:12:58.179 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:12:58.179 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:12:58.179 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:12:58.179 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:12:58.179 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:12:58.180 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:12:58.180 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:12:58.180 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:12:58.180 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:12:58.180 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:12:58.180 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:12:58.180 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:12:58.180 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:12:58.180 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:12:58.180 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:12:58.180 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:12:58.180 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:12:58.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:12:58.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:12:58.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:12:58.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:12:58.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:12:58.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:12:58.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:12:58.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:12:58.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:12:58.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:12:58.183 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:12:58.183 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:12:58.183 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:12:58.183 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:12:58.183 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:12:58.183 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:12:58.183 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:12:58.183 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:12:58.183 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:12:58.183 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:12:58.183 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:12:58.183 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:12:58.183 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:12:58.183 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:12:58.183 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:12:58.183 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:12:58.183 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:12:58.183 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:12:58.183 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:12:58.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:12:58.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:12:58.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:12:58.195 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:12:58.195 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:12:58.195 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:12:58.195 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:12:58.257 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:12:58.257 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:12:58.257 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:12:58.257 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:12:58.257 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:12:58.257 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:12:58.257 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:12:58.257 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:12:58.257 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:12:58.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:12:58.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:12:58.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:12:58.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:12:58.259 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:12:58.259 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:12:58.259 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:12:58.285 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 22 2026-02-18T08:12:58.286 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 24 2026-02-18T08:12:58.286 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:12:58.286 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=97, tag_tid=6, entry_tid=1], [object_number=96, tag_tid=6, entry_tid=0], [object_number=99, tag_tid=5, entry_tid=42603], [object_number=98, tag_tid=5, entry_tid=42602]]], state=connected] 2026-02-18T08:12:58.286 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=91, tag_tid=5, entry_tid=33183], [object_number=90, tag_tid=5, entry_tid=33182], [object_number=89, tag_tid=5, entry_tid=33181], [object_number=88, tag_tid=5, entry_tid=33180]]], state=connected] 2026-02-18T08:12:58.288 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:12:58.290 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=97, tag_tid=6, entry_tid=1' 2026-02-18T08:12:58.291 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:12:58.291 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:12:58.291 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:12:58.291 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:12:58.291 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:12:58.292 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:12:58.292 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:12:58.292 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:12:58.292 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:12:58.292 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:12:58.292 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:12:58.292 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:12:58.293 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:12:58.293 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:12:58.293 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:12:58.319 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 22 2026-02-18T08:12:58.319 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 24 2026-02-18T08:12:58.319 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:12:58.319 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=97, tag_tid=6, entry_tid=1], [object_number=96, tag_tid=6, entry_tid=0], [object_number=99, tag_tid=5, entry_tid=42603], [object_number=98, tag_tid=5, entry_tid=42602]]], state=connected] 2026-02-18T08:12:58.319 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=91, tag_tid=5, entry_tid=33183], [object_number=90, tag_tid=5, entry_tid=33182], [object_number=89, tag_tid=5, entry_tid=33181], [object_number=88, tag_tid=5, entry_tid=33180]]], state=connected] 2026-02-18T08:12:58.322 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:12:58.323 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=91, tag_tid=5, entry_tid=33183' 2026-02-18T08:12:58.323 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=97, tag_tid=6, entry_tid=1' -a 'object_number=97, tag_tid=6, entry_tid=1' = 'object_number=91, tag_tid=5, entry_tid=33183' 2026-02-18T08:12:58.323 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=91, tag_tid=5, entry_tid=33183' '!=' 'object_number=90, tag_tid=5, entry_tid=32946' 2026-02-18T08:12:58.323 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:12:58.323 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=91, tag_tid=5, entry_tid=33183' = 'object_number=90, tag_tid=5, entry_tid=32946' 2026-02-18T08:12:58.323 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=91, tag_tid=5, entry_tid=33183' 2026-02-18T08:12:58.324 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=97, tag_tid=6, entry_tid=1' 2026-02-18T08:12:58.324 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:12:58.324 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:12:58.326 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=6 2026-02-18T08:12:58.326 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=91, tag_tid=5, entry_tid=33183' 2026-02-18T08:12:58.326 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:12:58.326 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:12:58.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=5 2026-02-18T08:12:58.329 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=97, tag_tid=6, entry_tid=1' 2026-02-18T08:12:58.329 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:12:58.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:12:58.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:12:58.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=91, tag_tid=5, entry_tid=33183' 2026-02-18T08:12:58.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:12:58.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:12:58.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=33183 2026-02-18T08:12:58.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 6 = 5 -a 1 -le 33183 2026-02-18T08:12:58.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:12:58.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:12:58.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:12:58.535 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:12:58.535 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:12:58.535 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:12:58.535 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:12:58.535 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:12:58.535 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:12:58.535 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:12:58.535 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:12:58.536 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:12:58.536 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:12:58.536 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:12:58.536 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:12:58.536 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:12:58.536 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:12:58.536 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:12:58.536 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:12:58.539 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:12:58.539 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:12:58.539 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:12:58.539 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:12:58.539 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:12:58.539 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:12:58.539 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:12:58.539 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:12:58.539 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:12:58.540 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:12:58.540 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:12:58.540 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:12:58.540 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:12:58.540 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:12:58.540 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:12:58.540 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:12:58.540 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:12:58.540 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:12:58.540 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:12:58.540 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:12:58.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:12:58.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:12:58.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:12:58.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:12:58.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:12:58.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:12:58.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:12:58.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:12:58.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:12:58.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:12:58.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:12:58.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:12:58.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:12:58.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:12:58.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:12:58.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:12:58.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:12:58.607 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:12:58.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:12:58.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:12:58.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:12:58.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:12:58.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:12:58.610 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:12:58.610 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:12:58.610 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:12:58.610 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:12:58.610 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:12:58.610 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:12:58.610 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:12:58.611 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:12:58.611 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:12:58.611 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:12:58.638 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 22 2026-02-18T08:12:58.638 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 24 2026-02-18T08:12:58.638 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:12:58.638 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=97, tag_tid=6, entry_tid=1], [object_number=96, tag_tid=6, entry_tid=0], [object_number=99, tag_tid=5, entry_tid=42603], [object_number=98, tag_tid=5, entry_tid=42602]]], state=connected] 2026-02-18T08:12:58.639 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=91, tag_tid=5, entry_tid=33423], [object_number=90, tag_tid=5, entry_tid=33422], [object_number=89, tag_tid=5, entry_tid=33421], [object_number=88, tag_tid=5, entry_tid=33420]]], state=connected] 2026-02-18T08:12:58.641 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:12:58.642 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=97, tag_tid=6, entry_tid=1' 2026-02-18T08:12:58.643 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:12:58.643 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:12:58.643 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:12:58.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:12:58.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:12:58.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:12:58.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:12:58.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:12:58.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:12:58.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:12:58.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:12:58.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:12:58.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:12:58.646 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:12:58.646 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:12:58.673 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 22 2026-02-18T08:12:58.673 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 24 2026-02-18T08:12:58.673 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:12:58.673 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=97, tag_tid=6, entry_tid=1], [object_number=96, tag_tid=6, entry_tid=0], [object_number=99, tag_tid=5, entry_tid=42603], [object_number=98, tag_tid=5, entry_tid=42602]]], state=connected] 2026-02-18T08:12:58.673 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=91, tag_tid=5, entry_tid=33423], [object_number=90, tag_tid=5, entry_tid=33422], [object_number=89, tag_tid=5, entry_tid=33421], [object_number=88, tag_tid=5, entry_tid=33420]]], state=connected] 2026-02-18T08:12:58.675 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:12:58.677 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=91, tag_tid=5, entry_tid=33423' 2026-02-18T08:12:58.677 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=97, tag_tid=6, entry_tid=1' -a 'object_number=97, tag_tid=6, entry_tid=1' = 'object_number=91, tag_tid=5, entry_tid=33423' 2026-02-18T08:12:58.677 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=91, tag_tid=5, entry_tid=33423' '!=' 'object_number=91, tag_tid=5, entry_tid=33183' 2026-02-18T08:12:58.677 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:12:58.677 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=91, tag_tid=5, entry_tid=33423' = 'object_number=91, tag_tid=5, entry_tid=33183' 2026-02-18T08:12:58.677 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=91, tag_tid=5, entry_tid=33423' 2026-02-18T08:12:58.678 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=97, tag_tid=6, entry_tid=1' 2026-02-18T08:12:58.678 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:12:58.678 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:12:58.679 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=6 2026-02-18T08:12:58.679 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=91, tag_tid=5, entry_tid=33423' 2026-02-18T08:12:58.679 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:12:58.680 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:12:58.681 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=5 2026-02-18T08:12:58.682 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=97, tag_tid=6, entry_tid=1' 2026-02-18T08:12:58.682 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:12:58.682 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:12:58.683 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:12:58.684 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=91, tag_tid=5, entry_tid=33423' 2026-02-18T08:12:58.684 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:12:58.684 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:12:58.685 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=33423 2026-02-18T08:12:58.686 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 6 = 5 -a 1 -le 33423 2026-02-18T08:12:58.686 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:12:58.686 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:12:58.686 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:12:58.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:12:58.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:12:58.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:12:58.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:12:58.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:12:58.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:12:58.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:12:58.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:12:58.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:12:58.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:12:58.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:12:58.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:12:58.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:12:58.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:12:58.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:12:58.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:12:58.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:12:58.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:12:58.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:12:58.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:12:58.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:12:58.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:12:58.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:12:58.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:12:58.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:12:58.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:12:58.894 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:12:58.894 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:12:58.894 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:12:58.894 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:12:58.894 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:12:58.894 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:12:58.895 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:12:58.895 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:12:58.895 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:12:58.895 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:12:58.895 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:12:58.895 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:12:58.895 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:12:58.895 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:12:58.895 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:12:58.895 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:12:58.896 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:12:58.896 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:12:58.896 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:12:58.896 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:12:58.896 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:12:58.896 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:12:58.896 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:12:58.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:12:58.904 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:12:58.904 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:12:58.904 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:12:58.961 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:12:58.962 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:12:58.962 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:12:58.962 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:12:58.962 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:12:58.963 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:12:58.963 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:12:58.963 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:12:58.963 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:12:58.963 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:12:58.963 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:12:58.963 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:12:58.963 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:12:58.964 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:12:58.964 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:12:58.964 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:12:59.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 22 2026-02-18T08:12:59.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 24 2026-02-18T08:12:59.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:12:59.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=97, tag_tid=6, entry_tid=1], [object_number=96, tag_tid=6, entry_tid=0], [object_number=99, tag_tid=5, entry_tid=42603], [object_number=98, tag_tid=5, entry_tid=42602]]], state=connected] 2026-02-18T08:12:59.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=88, tag_tid=5, entry_tid=33672], [object_number=91, tag_tid=5, entry_tid=33671], [object_number=90, tag_tid=5, entry_tid=33670], [object_number=89, tag_tid=5, entry_tid=33669]]], state=connected] 2026-02-18T08:12:59.004 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:12:59.005 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=97, tag_tid=6, entry_tid=1' 2026-02-18T08:12:59.007 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:12:59.007 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:12:59.007 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:12:59.007 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:12:59.007 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:12:59.007 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:12:59.007 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:12:59.007 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:12:59.007 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:12:59.007 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:12:59.008 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:12:59.008 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:12:59.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:12:59.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:12:59.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:12:59.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 22 2026-02-18T08:12:59.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 24 2026-02-18T08:12:59.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:12:59.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=97, tag_tid=6, entry_tid=1], [object_number=96, tag_tid=6, entry_tid=0], [object_number=99, tag_tid=5, entry_tid=42603], [object_number=98, tag_tid=5, entry_tid=42602]]], state=connected] 2026-02-18T08:12:59.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=88, tag_tid=5, entry_tid=33672], [object_number=91, tag_tid=5, entry_tid=33671], [object_number=90, tag_tid=5, entry_tid=33670], [object_number=89, tag_tid=5, entry_tid=33669]]], state=connected] 2026-02-18T08:12:59.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:12:59.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=88, tag_tid=5, entry_tid=33672' 2026-02-18T08:12:59.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=97, tag_tid=6, entry_tid=1' -a 'object_number=97, tag_tid=6, entry_tid=1' = 'object_number=88, tag_tid=5, entry_tid=33672' 2026-02-18T08:12:59.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=88, tag_tid=5, entry_tid=33672' '!=' 'object_number=91, tag_tid=5, entry_tid=33423' 2026-02-18T08:12:59.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:12:59.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=88, tag_tid=5, entry_tid=33672' = 'object_number=91, tag_tid=5, entry_tid=33423' 2026-02-18T08:12:59.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=88, tag_tid=5, entry_tid=33672' 2026-02-18T08:12:59.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=97, tag_tid=6, entry_tid=1' 2026-02-18T08:12:59.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:12:59.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:12:59.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=6 2026-02-18T08:12:59.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=88, tag_tid=5, entry_tid=33672' 2026-02-18T08:12:59.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:12:59.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:12:59.055 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=5 2026-02-18T08:12:59.056 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=97, tag_tid=6, entry_tid=1' 2026-02-18T08:12:59.056 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:12:59.056 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:12:59.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:12:59.058 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=88, tag_tid=5, entry_tid=33672' 2026-02-18T08:12:59.058 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:12:59.058 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:12:59.059 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=33672 2026-02-18T08:12:59.060 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 6 = 5 -a 1 -le 33672 2026-02-18T08:12:59.060 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:12:59.060 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:12:59.060 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:12:59.261 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:12:59.261 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:12:59.261 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:12:59.261 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:12:59.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:12:59.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:12:59.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:12:59.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:12:59.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:12:59.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:12:59.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:12:59.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:12:59.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:12:59.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:12:59.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:12:59.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:12:59.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:12:59.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:12:59.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:12:59.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:12:59.266 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:12:59.266 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:12:59.266 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:12:59.266 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:12:59.266 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:12:59.266 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:12:59.266 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:12:59.266 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:12:59.266 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:12:59.266 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:12:59.266 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:12:59.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:12:59.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:12:59.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:12:59.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:12:59.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:12:59.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:12:59.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:12:59.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:12:59.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:12:59.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:12:59.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:12:59.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:12:59.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:12:59.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:12:59.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:12:59.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:12:59.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:12:59.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:12:59.275 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:12:59.276 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:12:59.276 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:12:59.276 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:12:59.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:12:59.344 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:12:59.344 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:12:59.344 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:12:59.344 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:12:59.344 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:12:59.344 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:12:59.344 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:12:59.344 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:12:59.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:12:59.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:12:59.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:12:59.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:12:59.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:12:59.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:12:59.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:12:59.373 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 22 2026-02-18T08:12:59.373 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 24 2026-02-18T08:12:59.374 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:12:59.374 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=97, tag_tid=6, entry_tid=1], [object_number=96, tag_tid=6, entry_tid=0], [object_number=99, tag_tid=5, entry_tid=42603], [object_number=98, tag_tid=5, entry_tid=42602]]], state=connected] 2026-02-18T08:12:59.374 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=90, tag_tid=5, entry_tid=33970], [object_number=89, tag_tid=5, entry_tid=33969], [object_number=88, tag_tid=5, entry_tid=33968], [object_number=91, tag_tid=5, entry_tid=33967]]], state=connected] 2026-02-18T08:12:59.376 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:12:59.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=97, tag_tid=6, entry_tid=1' 2026-02-18T08:12:59.379 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:12:59.379 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:12:59.379 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:12:59.379 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:12:59.379 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:12:59.379 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:12:59.379 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:12:59.379 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:12:59.379 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:12:59.379 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:12:59.380 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:12:59.380 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:12:59.380 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:12:59.381 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:12:59.381 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:12:59.415 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 22 2026-02-18T08:12:59.415 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 24 2026-02-18T08:12:59.415 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:12:59.416 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=97, tag_tid=6, entry_tid=1], [object_number=96, tag_tid=6, entry_tid=0], [object_number=99, tag_tid=5, entry_tid=42603], [object_number=98, tag_tid=5, entry_tid=42602]]], state=connected] 2026-02-18T08:12:59.416 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=90, tag_tid=5, entry_tid=33970], [object_number=89, tag_tid=5, entry_tid=33969], [object_number=88, tag_tid=5, entry_tid=33968], [object_number=91, tag_tid=5, entry_tid=33967]]], state=connected] 2026-02-18T08:12:59.419 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:12:59.420 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=90, tag_tid=5, entry_tid=33970' 2026-02-18T08:12:59.420 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=97, tag_tid=6, entry_tid=1' -a 'object_number=97, tag_tid=6, entry_tid=1' = 'object_number=90, tag_tid=5, entry_tid=33970' 2026-02-18T08:12:59.421 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=90, tag_tid=5, entry_tid=33970' '!=' 'object_number=88, tag_tid=5, entry_tid=33672' 2026-02-18T08:12:59.421 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:12:59.421 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=90, tag_tid=5, entry_tid=33970' = 'object_number=88, tag_tid=5, entry_tid=33672' 2026-02-18T08:12:59.421 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=90, tag_tid=5, entry_tid=33970' 2026-02-18T08:12:59.422 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=97, tag_tid=6, entry_tid=1' 2026-02-18T08:12:59.422 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:12:59.422 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:12:59.422 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=6 2026-02-18T08:12:59.423 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=90, tag_tid=5, entry_tid=33970' 2026-02-18T08:12:59.423 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:12:59.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:12:59.425 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=5 2026-02-18T08:12:59.425 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=97, tag_tid=6, entry_tid=1' 2026-02-18T08:12:59.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:12:59.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:12:59.427 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:12:59.428 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=90, tag_tid=5, entry_tid=33970' 2026-02-18T08:12:59.428 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:12:59.428 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:12:59.430 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=33970 2026-02-18T08:12:59.430 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 6 = 5 -a 1 -le 33970 2026-02-18T08:12:59.430 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:12:59.430 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:12:59.430 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:12:59.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:12:59.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:12:59.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:12:59.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:12:59.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:12:59.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:12:59.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:12:59.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:12:59.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:12:59.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:12:59.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:12:59.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:12:59.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:12:59.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:12:59.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:12:59.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:12:59.635 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:12:59.636 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:12:59.636 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:12:59.636 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:12:59.636 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:12:59.636 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:12:59.636 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:12:59.636 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:12:59.636 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:12:59.636 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:12:59.636 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:12:59.636 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:12:59.636 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:12:59.637 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:12:59.637 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:12:59.637 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:12:59.637 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:12:59.637 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:12:59.637 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:12:59.637 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:12:59.637 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:12:59.637 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:12:59.637 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:12:59.637 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:12:59.638 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:12:59.638 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:12:59.638 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:12:59.638 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:12:59.638 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:12:59.638 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:12:59.638 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:12:59.638 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:12:59.638 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:12:59.646 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:12:59.647 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:12:59.647 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:12:59.647 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:12:59.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:12:59.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:12:59.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:12:59.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:12:59.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:12:59.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:12:59.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:12:59.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:12:59.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:12:59.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:12:59.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:12:59.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:12:59.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:12:59.717 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:12:59.717 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:12:59.717 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:12:59.745 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 22 2026-02-18T08:12:59.745 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 24 2026-02-18T08:12:59.745 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:12:59.745 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=97, tag_tid=6, entry_tid=1], [object_number=96, tag_tid=6, entry_tid=0], [object_number=99, tag_tid=5, entry_tid=42603], [object_number=98, tag_tid=5, entry_tid=42602]]], state=connected] 2026-02-18T08:12:59.745 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=89, tag_tid=5, entry_tid=34233], [object_number=88, tag_tid=5, entry_tid=34232], [object_number=91, tag_tid=5, entry_tid=34231], [object_number=90, tag_tid=5, entry_tid=34230]]], state=connected] 2026-02-18T08:12:59.748 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:12:59.749 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=97, tag_tid=6, entry_tid=1' 2026-02-18T08:12:59.749 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:12:59.749 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:12:59.749 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:12:59.749 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:12:59.749 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:12:59.750 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:12:59.750 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:12:59.750 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:12:59.750 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:12:59.750 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:12:59.750 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:12:59.750 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:12:59.752 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:12:59.752 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:12:59.752 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:12:59.778 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 22 2026-02-18T08:12:59.778 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 24 2026-02-18T08:12:59.778 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:12:59.778 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=97, tag_tid=6, entry_tid=1], [object_number=96, tag_tid=6, entry_tid=0], [object_number=99, tag_tid=5, entry_tid=42603], [object_number=98, tag_tid=5, entry_tid=42602]]], state=connected] 2026-02-18T08:12:59.778 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=89, tag_tid=5, entry_tid=34233], [object_number=88, tag_tid=5, entry_tid=34232], [object_number=91, tag_tid=5, entry_tid=34231], [object_number=90, tag_tid=5, entry_tid=34230]]], state=connected] 2026-02-18T08:12:59.781 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:12:59.782 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=89, tag_tid=5, entry_tid=34233' 2026-02-18T08:12:59.782 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=97, tag_tid=6, entry_tid=1' -a 'object_number=97, tag_tid=6, entry_tid=1' = 'object_number=89, tag_tid=5, entry_tid=34233' 2026-02-18T08:12:59.782 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=89, tag_tid=5, entry_tid=34233' '!=' 'object_number=90, tag_tid=5, entry_tid=33970' 2026-02-18T08:12:59.782 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:12:59.782 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=89, tag_tid=5, entry_tid=34233' = 'object_number=90, tag_tid=5, entry_tid=33970' 2026-02-18T08:12:59.782 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=89, tag_tid=5, entry_tid=34233' 2026-02-18T08:12:59.783 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=97, tag_tid=6, entry_tid=1' 2026-02-18T08:12:59.783 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:12:59.783 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:12:59.784 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=6 2026-02-18T08:12:59.785 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=89, tag_tid=5, entry_tid=34233' 2026-02-18T08:12:59.785 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:12:59.785 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:12:59.786 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=5 2026-02-18T08:12:59.787 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=97, tag_tid=6, entry_tid=1' 2026-02-18T08:12:59.787 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:12:59.787 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:12:59.788 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:12:59.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=89, tag_tid=5, entry_tid=34233' 2026-02-18T08:12:59.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:12:59.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:12:59.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=34233 2026-02-18T08:12:59.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 6 = 5 -a 1 -le 34233 2026-02-18T08:12:59.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:12:59.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:12:59.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:12:59.992 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:12:59.992 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:12:59.992 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:12:59.992 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:12:59.992 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:12:59.992 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:12:59.992 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:12:59.992 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:12:59.992 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:12:59.992 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:12:59.992 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:12:59.992 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:12:59.992 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:12:59.992 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:12:59.992 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:12:59.993 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:12:59.993 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:12:59.994 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:12:59.994 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:12:59.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:12:59.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:12:59.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:12:59.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:12:59.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:12:59.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:12:59.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:12:59.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:12:59.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:12:59.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:12:59.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:12:59.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:12:59.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:12:59.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:12:59.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:12:59.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:12:59.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:12:59.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:12:59.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:12:59.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:12:59.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:12:59.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:12:59.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:12:59.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:12:59.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:12:59.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:12:59.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:12:59.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:12:59.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:12:59.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:13:00.007 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:13:00.007 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:13:00.007 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:13:00.007 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:13:00.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:13:00.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:13:00.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:13:00.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:13:00.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:13:00.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:13:00.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:13:00.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:13:00.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:13:00.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:13:00.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:13:00.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:13:00.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:13:00.067 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:13:00.067 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:13:00.067 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:13:00.093 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 22 2026-02-18T08:13:00.093 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 24 2026-02-18T08:13:00.093 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:13:00.093 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=97, tag_tid=6, entry_tid=1], [object_number=96, tag_tid=6, entry_tid=0], [object_number=99, tag_tid=5, entry_tid=42603], [object_number=98, tag_tid=5, entry_tid=42602]]], state=connected] 2026-02-18T08:13:00.093 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=91, tag_tid=5, entry_tid=34443], [object_number=90, tag_tid=5, entry_tid=34442], [object_number=89, tag_tid=5, entry_tid=34441], [object_number=88, tag_tid=5, entry_tid=34440]]], state=connected] 2026-02-18T08:13:00.096 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:13:00.097 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=97, tag_tid=6, entry_tid=1' 2026-02-18T08:13:00.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:13:00.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:13:00.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:13:00.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:13:00.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:13:00.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:13:00.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:13:00.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:13:00.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:13:00.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:13:00.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:13:00.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:13:00.100 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:13:00.100 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:13:00.100 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:13:00.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 22 2026-02-18T08:13:00.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 24 2026-02-18T08:13:00.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:13:00.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=97, tag_tid=6, entry_tid=1], [object_number=96, tag_tid=6, entry_tid=0], [object_number=99, tag_tid=5, entry_tid=42603], [object_number=98, tag_tid=5, entry_tid=42602]]], state=connected] 2026-02-18T08:13:00.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=91, tag_tid=5, entry_tid=34443], [object_number=90, tag_tid=5, entry_tid=34442], [object_number=89, tag_tid=5, entry_tid=34441], [object_number=88, tag_tid=5, entry_tid=34440]]], state=connected] 2026-02-18T08:13:00.129 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:13:00.130 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=91, tag_tid=5, entry_tid=34443' 2026-02-18T08:13:00.130 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=97, tag_tid=6, entry_tid=1' -a 'object_number=97, tag_tid=6, entry_tid=1' = 'object_number=91, tag_tid=5, entry_tid=34443' 2026-02-18T08:13:00.130 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=91, tag_tid=5, entry_tid=34443' '!=' 'object_number=89, tag_tid=5, entry_tid=34233' 2026-02-18T08:13:00.131 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:13:00.131 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=91, tag_tid=5, entry_tid=34443' = 'object_number=89, tag_tid=5, entry_tid=34233' 2026-02-18T08:13:00.131 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=91, tag_tid=5, entry_tid=34443' 2026-02-18T08:13:00.131 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=97, tag_tid=6, entry_tid=1' 2026-02-18T08:13:00.131 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:13:00.131 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:13:00.132 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=6 2026-02-18T08:13:00.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=91, tag_tid=5, entry_tid=34443' 2026-02-18T08:13:00.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:13:00.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:13:00.134 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=5 2026-02-18T08:13:00.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=97, tag_tid=6, entry_tid=1' 2026-02-18T08:13:00.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:13:00.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:13:00.136 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:13:00.137 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=91, tag_tid=5, entry_tid=34443' 2026-02-18T08:13:00.137 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:13:00.137 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:13:00.139 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=34443 2026-02-18T08:13:00.139 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 6 = 5 -a 1 -le 34443 2026-02-18T08:13:00.139 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:13:00.139 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:13:00.139 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:13:00.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:13:00.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:13:00.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:13:00.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:13:00.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:13:00.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:13:00.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:13:00.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:13:00.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:13:00.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:13:00.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:13:00.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:13:00.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:13:00.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:13:00.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:13:00.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:13:00.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:13:00.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:13:00.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:13:00.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:13:00.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:13:00.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:13:00.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:13:00.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:13:00.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:13:00.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:13:00.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:13:00.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:13:00.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:13:00.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:13:00.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:13:00.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:13:00.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:13:00.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:13:00.344 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:13:00.344 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:13:00.344 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:13:00.344 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:13:00.344 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:13:00.344 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:13:00.344 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:13:00.344 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:13:00.344 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:13:00.344 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:13:00.344 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:13:00.344 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:13:00.344 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:13:00.344 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:13:00.344 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:13:00.356 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:13:00.356 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:13:00.356 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:13:00.356 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:13:00.415 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:13:00.416 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:13:00.416 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:13:00.416 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:13:00.416 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:13:00.416 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:13:00.416 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:13:00.416 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:13:00.416 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:13:00.416 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:13:00.416 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:13:00.416 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:13:00.417 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:13:00.418 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:13:00.418 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:13:00.418 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:13:00.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 22 2026-02-18T08:13:00.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 24 2026-02-18T08:13:00.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:13:00.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=97, tag_tid=6, entry_tid=1], [object_number=96, tag_tid=6, entry_tid=0], [object_number=99, tag_tid=5, entry_tid=42603], [object_number=98, tag_tid=5, entry_tid=42602]]], state=connected] 2026-02-18T08:13:00.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=90, tag_tid=5, entry_tid=34686], [object_number=89, tag_tid=5, entry_tid=34685], [object_number=88, tag_tid=5, entry_tid=34684], [object_number=91, tag_tid=5, entry_tid=34683]]], state=connected] 2026-02-18T08:13:00.450 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:13:00.451 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=97, tag_tid=6, entry_tid=1' 2026-02-18T08:13:00.452 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:13:00.452 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:13:00.452 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:13:00.452 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:13:00.452 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:13:00.452 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:13:00.452 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:13:00.452 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:13:00.452 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:13:00.453 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:13:00.453 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:13:00.453 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:13:00.454 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:13:00.455 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:13:00.455 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:13:00.481 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 22 2026-02-18T08:13:00.481 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 24 2026-02-18T08:13:00.481 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:13:00.481 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=97, tag_tid=6, entry_tid=1], [object_number=96, tag_tid=6, entry_tid=0], [object_number=99, tag_tid=5, entry_tid=42603], [object_number=98, tag_tid=5, entry_tid=42602]]], state=connected] 2026-02-18T08:13:00.481 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=90, tag_tid=5, entry_tid=34686], [object_number=89, tag_tid=5, entry_tid=34685], [object_number=88, tag_tid=5, entry_tid=34684], [object_number=91, tag_tid=5, entry_tid=34683]]], state=connected] 2026-02-18T08:13:00.484 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:13:00.485 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=90, tag_tid=5, entry_tid=34686' 2026-02-18T08:13:00.485 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=97, tag_tid=6, entry_tid=1' -a 'object_number=97, tag_tid=6, entry_tid=1' = 'object_number=90, tag_tid=5, entry_tid=34686' 2026-02-18T08:13:00.485 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=90, tag_tid=5, entry_tid=34686' '!=' 'object_number=91, tag_tid=5, entry_tid=34443' 2026-02-18T08:13:00.485 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:13:00.485 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=90, tag_tid=5, entry_tid=34686' = 'object_number=91, tag_tid=5, entry_tid=34443' 2026-02-18T08:13:00.485 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=90, tag_tid=5, entry_tid=34686' 2026-02-18T08:13:00.485 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=97, tag_tid=6, entry_tid=1' 2026-02-18T08:13:00.486 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:13:00.486 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:13:00.487 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=6 2026-02-18T08:13:00.487 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=90, tag_tid=5, entry_tid=34686' 2026-02-18T08:13:00.488 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:13:00.488 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:13:00.489 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=5 2026-02-18T08:13:00.490 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=97, tag_tid=6, entry_tid=1' 2026-02-18T08:13:00.490 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:13:00.490 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:13:00.492 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:13:00.492 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=90, tag_tid=5, entry_tid=34686' 2026-02-18T08:13:00.492 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:13:00.492 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:13:00.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=34686 2026-02-18T08:13:00.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 6 = 5 -a 1 -le 34686 2026-02-18T08:13:00.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:13:00.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:13:00.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:13:00.696 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:13:00.696 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:13:00.696 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:13:00.696 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:13:00.696 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:13:00.696 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:13:00.696 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:13:00.696 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:13:00.696 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:13:00.696 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:13:00.696 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:13:00.696 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:13:00.696 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:13:00.696 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:13:00.696 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:13:00.696 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:13:00.696 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:13:00.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:13:00.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:13:00.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:13:00.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:13:00.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:13:00.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:13:00.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:13:00.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:13:00.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:13:00.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:13:00.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:13:00.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:13:00.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:13:00.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:13:00.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:13:00.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:13:00.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:13:00.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:13:00.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:13:00.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:13:00.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:13:00.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:13:00.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:13:00.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:13:00.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:13:00.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:13:00.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:13:00.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:13:00.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:13:00.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:13:00.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:13:00.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:13:00.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:13:00.711 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:13:00.711 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:13:00.711 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:13:00.781 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:13:00.781 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:13:00.781 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:13:00.781 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:13:00.781 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:13:00.782 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:13:00.782 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:13:00.782 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:13:00.782 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:13:00.782 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:13:00.782 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:13:00.783 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:13:00.783 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:13:00.783 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:13:00.783 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:13:00.784 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:13:00.819 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 22 2026-02-18T08:13:00.819 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 24 2026-02-18T08:13:00.819 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:13:00.819 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=97, tag_tid=6, entry_tid=1], [object_number=96, tag_tid=6, entry_tid=0], [object_number=99, tag_tid=5, entry_tid=42603], [object_number=98, tag_tid=5, entry_tid=42602]]], state=connected] 2026-02-18T08:13:00.819 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=88, tag_tid=5, entry_tid=34932], [object_number=91, tag_tid=5, entry_tid=34931], [object_number=90, tag_tid=5, entry_tid=34930], [object_number=89, tag_tid=5, entry_tid=34929]]], state=connected] 2026-02-18T08:13:00.822 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:13:00.824 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=97, tag_tid=6, entry_tid=1' 2026-02-18T08:13:00.825 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:13:00.825 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:13:00.825 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:13:00.825 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:13:00.825 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:13:00.825 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:13:00.825 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:13:00.825 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:13:00.825 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:13:00.825 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:13:00.825 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:13:00.826 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:13:00.827 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:13:00.827 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:13:00.828 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:13:00.854 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 22 2026-02-18T08:13:00.855 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 24 2026-02-18T08:13:00.855 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:13:00.855 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=97, tag_tid=6, entry_tid=1], [object_number=96, tag_tid=6, entry_tid=0], [object_number=99, tag_tid=5, entry_tid=42603], [object_number=98, tag_tid=5, entry_tid=42602]]], state=connected] 2026-02-18T08:13:00.855 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=88, tag_tid=5, entry_tid=34932], [object_number=91, tag_tid=5, entry_tid=34931], [object_number=90, tag_tid=5, entry_tid=34930], [object_number=89, tag_tid=5, entry_tid=34929]]], state=connected] 2026-02-18T08:13:00.857 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:13:00.859 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=88, tag_tid=5, entry_tid=34932' 2026-02-18T08:13:00.859 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=97, tag_tid=6, entry_tid=1' -a 'object_number=97, tag_tid=6, entry_tid=1' = 'object_number=88, tag_tid=5, entry_tid=34932' 2026-02-18T08:13:00.859 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=88, tag_tid=5, entry_tid=34932' '!=' 'object_number=90, tag_tid=5, entry_tid=34686' 2026-02-18T08:13:00.859 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:13:00.859 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=88, tag_tid=5, entry_tid=34932' = 'object_number=90, tag_tid=5, entry_tid=34686' 2026-02-18T08:13:00.859 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=88, tag_tid=5, entry_tid=34932' 2026-02-18T08:13:00.860 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=97, tag_tid=6, entry_tid=1' 2026-02-18T08:13:00.860 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:13:00.860 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:13:00.862 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=6 2026-02-18T08:13:00.862 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=88, tag_tid=5, entry_tid=34932' 2026-02-18T08:13:00.862 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:13:00.862 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:13:00.864 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=5 2026-02-18T08:13:00.864 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=97, tag_tid=6, entry_tid=1' 2026-02-18T08:13:00.865 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:13:00.865 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:13:00.866 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:13:00.867 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=88, tag_tid=5, entry_tid=34932' 2026-02-18T08:13:00.867 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:13:00.867 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:13:00.868 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=34932 2026-02-18T08:13:00.869 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 6 = 5 -a 1 -le 34932 2026-02-18T08:13:00.869 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:13:00.869 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:13:00.869 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:13:01.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:13:01.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:13:01.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:13:01.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:13:01.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:13:01.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:13:01.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:13:01.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:13:01.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:13:01.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:13:01.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:13:01.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:13:01.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:13:01.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:13:01.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:13:01.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:13:01.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:13:01.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:13:01.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:13:01.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:13:01.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:13:01.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:13:01.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:13:01.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:13:01.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:13:01.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:13:01.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:13:01.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:13:01.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:13:01.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:13:01.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:13:01.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:13:01.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:13:01.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:13:01.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:13:01.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:13:01.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:13:01.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:13:01.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:13:01.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:13:01.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:13:01.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:13:01.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:13:01.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:13:01.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:13:01.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:13:01.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:13:01.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:13:01.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:13:01.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:13:01.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:13:01.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:13:01.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:13:01.142 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:13:01.142 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:13:01.142 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:13:01.142 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:13:01.143 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:13:01.143 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:13:01.143 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:13:01.143 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:13:01.143 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:13:01.143 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:13:01.143 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:13:01.143 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:13:01.143 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:13:01.145 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:13:01.145 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:13:01.145 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:13:01.179 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 22 2026-02-18T08:13:01.179 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 24 2026-02-18T08:13:01.179 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:13:01.179 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=97, tag_tid=6, entry_tid=1], [object_number=96, tag_tid=6, entry_tid=0], [object_number=99, tag_tid=5, entry_tid=42603], [object_number=98, tag_tid=5, entry_tid=42602]]], state=connected] 2026-02-18T08:13:01.179 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=89, tag_tid=5, entry_tid=35193], [object_number=88, tag_tid=5, entry_tid=35192], [object_number=91, tag_tid=5, entry_tid=35191], [object_number=90, tag_tid=5, entry_tid=35190]]], state=connected] 2026-02-18T08:13:01.183 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:13:01.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=97, tag_tid=6, entry_tid=1' 2026-02-18T08:13:01.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:13:01.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:13:01.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:13:01.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:13:01.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:13:01.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:13:01.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:13:01.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:13:01.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:13:01.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:13:01.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:13:01.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:13:01.187 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:13:01.187 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:13:01.187 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:13:01.221 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 22 2026-02-18T08:13:01.222 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 24 2026-02-18T08:13:01.222 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:13:01.222 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=97, tag_tid=6, entry_tid=1], [object_number=96, tag_tid=6, entry_tid=0], [object_number=99, tag_tid=5, entry_tid=42603], [object_number=98, tag_tid=5, entry_tid=42602]]], state=connected] 2026-02-18T08:13:01.222 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=89, tag_tid=5, entry_tid=35193], [object_number=88, tag_tid=5, entry_tid=35192], [object_number=91, tag_tid=5, entry_tid=35191], [object_number=90, tag_tid=5, entry_tid=35190]]], state=connected] 2026-02-18T08:13:01.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:13:01.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=89, tag_tid=5, entry_tid=35193' 2026-02-18T08:13:01.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=97, tag_tid=6, entry_tid=1' -a 'object_number=97, tag_tid=6, entry_tid=1' = 'object_number=89, tag_tid=5, entry_tid=35193' 2026-02-18T08:13:01.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=89, tag_tid=5, entry_tid=35193' '!=' 'object_number=88, tag_tid=5, entry_tid=34932' 2026-02-18T08:13:01.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:13:01.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=89, tag_tid=5, entry_tid=35193' = 'object_number=88, tag_tid=5, entry_tid=34932' 2026-02-18T08:13:01.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=89, tag_tid=5, entry_tid=35193' 2026-02-18T08:13:01.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=97, tag_tid=6, entry_tid=1' 2026-02-18T08:13:01.229 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:13:01.229 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:13:01.230 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=6 2026-02-18T08:13:01.231 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=89, tag_tid=5, entry_tid=35193' 2026-02-18T08:13:01.231 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:13:01.231 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:13:01.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=5 2026-02-18T08:13:01.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=97, tag_tid=6, entry_tid=1' 2026-02-18T08:13:01.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:13:01.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:13:01.234 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:13:01.234 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=89, tag_tid=5, entry_tid=35193' 2026-02-18T08:13:01.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:13:01.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:13:01.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=35193 2026-02-18T08:13:01.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 6 = 5 -a 1 -le 35193 2026-02-18T08:13:01.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:13:01.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:13:01.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:13:01.437 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:13:01.437 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:13:01.437 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:13:01.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:13:01.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:13:01.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:13:01.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:13:01.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:13:01.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:13:01.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:13:01.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:13:01.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:13:01.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:13:01.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:13:01.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:13:01.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:13:01.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:13:01.439 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:13:01.439 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:13:01.439 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:13:01.439 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:13:01.439 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:13:01.440 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:13:01.440 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:13:01.440 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:13:01.440 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:13:01.440 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:13:01.440 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:13:01.440 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:13:01.440 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:13:01.440 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:13:01.440 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:13:01.440 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:13:01.440 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:13:01.440 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:13:01.440 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:13:01.440 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:13:01.440 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:13:01.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:13:01.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:13:01.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:13:01.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:13:01.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:13:01.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:13:01.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:13:01.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:13:01.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:13:01.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:13:01.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:13:01.452 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:13:01.453 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:13:01.453 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:13:01.453 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:13:01.514 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:13:01.514 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:13:01.514 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:13:01.514 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:13:01.514 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:13:01.514 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:13:01.514 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:13:01.514 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:13:01.514 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:13:01.514 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:13:01.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:13:01.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:13:01.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:13:01.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:13:01.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:13:01.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:13:01.545 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 22 2026-02-18T08:13:01.545 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 24 2026-02-18T08:13:01.545 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:13:01.545 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=97, tag_tid=6, entry_tid=1], [object_number=96, tag_tid=6, entry_tid=0], [object_number=99, tag_tid=5, entry_tid=42603], [object_number=98, tag_tid=5, entry_tid=42602]]], state=connected] 2026-02-18T08:13:01.545 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=89, tag_tid=5, entry_tid=35437], [object_number=88, tag_tid=5, entry_tid=35436], [object_number=91, tag_tid=5, entry_tid=35435], [object_number=90, tag_tid=5, entry_tid=35434]]], state=connected] 2026-02-18T08:13:01.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:13:01.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=97, tag_tid=6, entry_tid=1' 2026-02-18T08:13:01.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:13:01.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:13:01.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:13:01.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:13:01.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:13:01.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:13:01.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:13:01.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:13:01.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:13:01.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:13:01.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:13:01.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:13:01.552 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:13:01.552 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:13:01.552 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:13:01.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 22 2026-02-18T08:13:01.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 24 2026-02-18T08:13:01.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:13:01.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=97, tag_tid=6, entry_tid=1], [object_number=96, tag_tid=6, entry_tid=0], [object_number=99, tag_tid=5, entry_tid=42603], [object_number=98, tag_tid=5, entry_tid=42602]]], state=connected] 2026-02-18T08:13:01.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=89, tag_tid=5, entry_tid=35437], [object_number=88, tag_tid=5, entry_tid=35436], [object_number=91, tag_tid=5, entry_tid=35435], [object_number=90, tag_tid=5, entry_tid=35434]]], state=connected] 2026-02-18T08:13:01.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:13:01.584 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=89, tag_tid=5, entry_tid=35437' 2026-02-18T08:13:01.584 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=97, tag_tid=6, entry_tid=1' -a 'object_number=97, tag_tid=6, entry_tid=1' = 'object_number=89, tag_tid=5, entry_tid=35437' 2026-02-18T08:13:01.584 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=89, tag_tid=5, entry_tid=35437' '!=' 'object_number=89, tag_tid=5, entry_tid=35193' 2026-02-18T08:13:01.584 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:13:01.584 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=89, tag_tid=5, entry_tid=35437' = 'object_number=89, tag_tid=5, entry_tid=35193' 2026-02-18T08:13:01.584 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=89, tag_tid=5, entry_tid=35437' 2026-02-18T08:13:01.585 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=97, tag_tid=6, entry_tid=1' 2026-02-18T08:13:01.585 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:13:01.585 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:13:01.586 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=6 2026-02-18T08:13:01.587 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=89, tag_tid=5, entry_tid=35437' 2026-02-18T08:13:01.587 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:13:01.587 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:13:01.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=5 2026-02-18T08:13:01.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=97, tag_tid=6, entry_tid=1' 2026-02-18T08:13:01.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:13:01.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:13:01.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:13:01.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=89, tag_tid=5, entry_tid=35437' 2026-02-18T08:13:01.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:13:01.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:13:01.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=35437 2026-02-18T08:13:01.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 6 = 5 -a 1 -le 35437 2026-02-18T08:13:01.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:13:01.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:13:01.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:13:01.794 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:13:01.794 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:13:01.794 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:13:01.794 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:13:01.795 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:13:01.795 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:13:01.795 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:13:01.795 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:13:01.795 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:13:01.795 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:13:01.795 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:13:01.795 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:13:01.795 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:13:01.795 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:13:01.795 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:13:01.795 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:13:01.796 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:13:01.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:13:01.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:13:01.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:13:01.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:13:01.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:13:01.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:13:01.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:13:01.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:13:01.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:13:01.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:13:01.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:13:01.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:13:01.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:13:01.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:13:01.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:13:01.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:13:01.798 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:13:01.798 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:13:01.798 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:13:01.798 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:13:01.798 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:13:01.798 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:13:01.798 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:13:01.798 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:13:01.798 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:13:01.798 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:13:01.798 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:13:01.798 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:13:01.798 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:13:01.798 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:13:01.798 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:13:01.798 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:13:01.811 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:13:01.811 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:13:01.811 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:13:01.811 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:13:01.870 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:13:01.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:13:01.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:13:01.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:13:01.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:13:01.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:13:01.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:13:01.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:13:01.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:13:01.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:13:01.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:13:01.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:13:01.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:13:01.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:13:01.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:13:01.874 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:13:01.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 22 2026-02-18T08:13:01.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 24 2026-02-18T08:13:01.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:13:01.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=97, tag_tid=6, entry_tid=1], [object_number=96, tag_tid=6, entry_tid=0], [object_number=99, tag_tid=5, entry_tid=42603], [object_number=98, tag_tid=5, entry_tid=42602]]], state=connected] 2026-02-18T08:13:01.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=88, tag_tid=5, entry_tid=35660], [object_number=91, tag_tid=5, entry_tid=35659], [object_number=90, tag_tid=5, entry_tid=35658], [object_number=89, tag_tid=5, entry_tid=35657]]], state=connected] 2026-02-18T08:13:01.905 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:13:01.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=97, tag_tid=6, entry_tid=1' 2026-02-18T08:13:01.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:13:01.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:13:01.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:13:01.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:13:01.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:13:01.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:13:01.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:13:01.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:13:01.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:13:01.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:13:01.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:13:01.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:13:01.910 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:13:01.910 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:13:01.910 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:13:01.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 22 2026-02-18T08:13:01.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 24 2026-02-18T08:13:01.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:13:01.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=97, tag_tid=6, entry_tid=1], [object_number=96, tag_tid=6, entry_tid=0], [object_number=99, tag_tid=5, entry_tid=42603], [object_number=98, tag_tid=5, entry_tid=42602]]], state=connected] 2026-02-18T08:13:01.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=88, tag_tid=5, entry_tid=35660], [object_number=91, tag_tid=5, entry_tid=35659], [object_number=90, tag_tid=5, entry_tid=35658], [object_number=89, tag_tid=5, entry_tid=35657]]], state=connected] 2026-02-18T08:13:01.948 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:13:01.949 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=88, tag_tid=5, entry_tid=35660' 2026-02-18T08:13:01.949 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=97, tag_tid=6, entry_tid=1' -a 'object_number=97, tag_tid=6, entry_tid=1' = 'object_number=88, tag_tid=5, entry_tid=35660' 2026-02-18T08:13:01.949 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=88, tag_tid=5, entry_tid=35660' '!=' 'object_number=89, tag_tid=5, entry_tid=35437' 2026-02-18T08:13:01.949 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:13:01.949 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=88, tag_tid=5, entry_tid=35660' = 'object_number=89, tag_tid=5, entry_tid=35437' 2026-02-18T08:13:01.949 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=88, tag_tid=5, entry_tid=35660' 2026-02-18T08:13:01.950 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=97, tag_tid=6, entry_tid=1' 2026-02-18T08:13:01.950 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:13:01.950 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:13:01.952 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=6 2026-02-18T08:13:01.952 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=88, tag_tid=5, entry_tid=35660' 2026-02-18T08:13:01.952 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:13:01.953 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:13:01.954 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=5 2026-02-18T08:13:01.955 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=97, tag_tid=6, entry_tid=1' 2026-02-18T08:13:01.955 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:13:01.955 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:13:01.956 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:13:01.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=88, tag_tid=5, entry_tid=35660' 2026-02-18T08:13:01.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:13:01.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:13:01.959 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=35660 2026-02-18T08:13:01.959 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 6 = 5 -a 1 -le 35660 2026-02-18T08:13:01.959 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:13:01.959 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:13:01.959 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:13:02.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:13:02.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:13:02.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:13:02.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:13:02.161 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:13:02.161 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:13:02.161 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:13:02.161 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:13:02.161 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:13:02.161 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:13:02.161 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:13:02.161 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:13:02.161 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:13:02.161 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:13:02.161 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:13:02.161 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:13:02.161 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:13:02.163 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:13:02.163 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:13:02.163 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:13:02.163 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:13:02.163 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:13:02.163 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:13:02.163 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:13:02.163 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:13:02.163 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:13:02.163 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:13:02.163 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:13:02.163 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:13:02.163 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:13:02.163 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:13:02.163 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:13:02.163 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:13:02.164 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:13:02.164 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:13:02.164 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:13:02.164 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:13:02.164 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:13:02.164 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:13:02.164 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:13:02.164 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:13:02.164 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:13:02.164 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:13:02.164 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:13:02.164 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:13:02.164 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:13:02.164 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:13:02.164 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:13:02.164 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:13:02.175 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:13:02.176 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:13:02.176 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:13:02.176 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:13:02.245 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:13:02.245 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:13:02.246 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:13:02.246 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:13:02.246 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:13:02.246 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:13:02.246 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:13:02.246 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:13:02.246 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:13:02.246 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:13:02.246 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:13:02.246 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:13:02.246 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:13:02.248 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:13:02.248 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:13:02.248 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:13:02.284 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 22 2026-02-18T08:13:02.284 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 24 2026-02-18T08:13:02.284 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:13:02.284 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=97, tag_tid=6, entry_tid=1], [object_number=96, tag_tid=6, entry_tid=0], [object_number=99, tag_tid=5, entry_tid=42603], [object_number=98, tag_tid=5, entry_tid=42602]]], state=connected] 2026-02-18T08:13:02.284 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=89, tag_tid=5, entry_tid=35917], [object_number=88, tag_tid=5, entry_tid=35916], [object_number=91, tag_tid=5, entry_tid=35915], [object_number=90, tag_tid=5, entry_tid=35914]]], state=connected] 2026-02-18T08:13:02.288 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:13:02.289 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=97, tag_tid=6, entry_tid=1' 2026-02-18T08:13:02.290 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:13:02.290 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:13:02.290 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:13:02.290 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:13:02.290 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:13:02.290 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:13:02.290 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:13:02.290 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:13:02.290 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:13:02.291 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:13:02.291 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:13:02.291 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:13:02.292 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:13:02.293 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:13:02.293 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:13:02.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 22 2026-02-18T08:13:02.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 24 2026-02-18T08:13:02.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:13:02.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=97, tag_tid=6, entry_tid=1], [object_number=96, tag_tid=6, entry_tid=0], [object_number=99, tag_tid=5, entry_tid=42603], [object_number=98, tag_tid=5, entry_tid=42602]]], state=connected] 2026-02-18T08:13:02.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=89, tag_tid=5, entry_tid=35917], [object_number=88, tag_tid=5, entry_tid=35916], [object_number=91, tag_tid=5, entry_tid=35915], [object_number=90, tag_tid=5, entry_tid=35914]]], state=connected] 2026-02-18T08:13:02.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:13:02.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=89, tag_tid=5, entry_tid=35917' 2026-02-18T08:13:02.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=97, tag_tid=6, entry_tid=1' -a 'object_number=97, tag_tid=6, entry_tid=1' = 'object_number=89, tag_tid=5, entry_tid=35917' 2026-02-18T08:13:02.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=89, tag_tid=5, entry_tid=35917' '!=' 'object_number=88, tag_tid=5, entry_tid=35660' 2026-02-18T08:13:02.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:13:02.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=89, tag_tid=5, entry_tid=35917' = 'object_number=88, tag_tid=5, entry_tid=35660' 2026-02-18T08:13:02.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=89, tag_tid=5, entry_tid=35917' 2026-02-18T08:13:02.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=97, tag_tid=6, entry_tid=1' 2026-02-18T08:13:02.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:13:02.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:13:02.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=6 2026-02-18T08:13:02.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=89, tag_tid=5, entry_tid=35917' 2026-02-18T08:13:02.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:13:02.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:13:02.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=5 2026-02-18T08:13:02.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=97, tag_tid=6, entry_tid=1' 2026-02-18T08:13:02.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:13:02.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:13:02.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:13:02.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=89, tag_tid=5, entry_tid=35917' 2026-02-18T08:13:02.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:13:02.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:13:02.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=35917 2026-02-18T08:13:02.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 6 = 5 -a 1 -le 35917 2026-02-18T08:13:02.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:13:02.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:13:02.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:13:02.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:13:02.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:13:02.545 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:13:02.545 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:13:02.545 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:13:02.545 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:13:02.545 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:13:02.545 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:13:02.545 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:13:02.545 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:13:02.545 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:13:02.545 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:13:02.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:13:02.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:13:02.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:13:02.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:13:02.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:13:02.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:13:02.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:13:02.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:13:02.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:13:02.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:13:02.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:13:02.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:13:02.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:13:02.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:13:02.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:13:02.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:13:02.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:13:02.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:13:02.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:13:02.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:13:02.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:13:02.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:13:02.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:13:02.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:13:02.551 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:13:02.551 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:13:02.551 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:13:02.551 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:13:02.551 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:13:02.551 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:13:02.551 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:13:02.551 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:13:02.551 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:13:02.551 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:13:02.551 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:13:02.551 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:13:02.551 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:13:02.559 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:13:02.559 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:13:02.559 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:13:02.560 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:13:02.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:13:02.618 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:13:02.619 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:13:02.619 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:13:02.619 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:13:02.619 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:13:02.619 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:13:02.619 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:13:02.619 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:13:02.619 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:13:02.619 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:13:02.620 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:13:02.620 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:13:02.620 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:13:02.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:13:02.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:13:02.648 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 22 2026-02-18T08:13:02.648 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 24 2026-02-18T08:13:02.648 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:13:02.649 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=97, tag_tid=6, entry_tid=1], [object_number=96, tag_tid=6, entry_tid=0], [object_number=99, tag_tid=5, entry_tid=42603], [object_number=98, tag_tid=5, entry_tid=42602]]], state=connected] 2026-02-18T08:13:02.649 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=88, tag_tid=5, entry_tid=36184], [object_number=91, tag_tid=5, entry_tid=36183], [object_number=90, tag_tid=5, entry_tid=36182], [object_number=89, tag_tid=5, entry_tid=36181]]], state=connected] 2026-02-18T08:13:02.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:13:02.653 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=97, tag_tid=6, entry_tid=1' 2026-02-18T08:13:02.654 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:13:02.654 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:13:02.654 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:13:02.654 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:13:02.654 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:13:02.654 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:13:02.654 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:13:02.654 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:13:02.655 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:13:02.655 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:13:02.655 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:13:02.655 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:13:02.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:13:02.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:13:02.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:13:02.683 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 22 2026-02-18T08:13:02.683 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 24 2026-02-18T08:13:02.683 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:13:02.683 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=97, tag_tid=6, entry_tid=1], [object_number=96, tag_tid=6, entry_tid=0], [object_number=99, tag_tid=5, entry_tid=42603], [object_number=98, tag_tid=5, entry_tid=42602]]], state=connected] 2026-02-18T08:13:02.683 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=88, tag_tid=5, entry_tid=36184], [object_number=91, tag_tid=5, entry_tid=36183], [object_number=90, tag_tid=5, entry_tid=36182], [object_number=89, tag_tid=5, entry_tid=36181]]], state=connected] 2026-02-18T08:13:02.685 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:13:02.687 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=88, tag_tid=5, entry_tid=36184' 2026-02-18T08:13:02.688 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=97, tag_tid=6, entry_tid=1' -a 'object_number=97, tag_tid=6, entry_tid=1' = 'object_number=88, tag_tid=5, entry_tid=36184' 2026-02-18T08:13:02.688 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=88, tag_tid=5, entry_tid=36184' '!=' 'object_number=89, tag_tid=5, entry_tid=35917' 2026-02-18T08:13:02.688 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:13:02.688 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=88, tag_tid=5, entry_tid=36184' = 'object_number=89, tag_tid=5, entry_tid=35917' 2026-02-18T08:13:02.688 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=88, tag_tid=5, entry_tid=36184' 2026-02-18T08:13:02.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=97, tag_tid=6, entry_tid=1' 2026-02-18T08:13:02.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:13:02.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:13:02.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=6 2026-02-18T08:13:02.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=88, tag_tid=5, entry_tid=36184' 2026-02-18T08:13:02.691 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:13:02.691 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:13:02.692 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=5 2026-02-18T08:13:02.693 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=97, tag_tid=6, entry_tid=1' 2026-02-18T08:13:02.693 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:13:02.693 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:13:02.694 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:13:02.695 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=88, tag_tid=5, entry_tid=36184' 2026-02-18T08:13:02.695 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:13:02.695 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:13:02.697 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=36184 2026-02-18T08:13:02.697 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 6 = 5 -a 1 -le 36184 2026-02-18T08:13:02.697 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:13:02.697 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:13:02.697 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:13:02.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:13:02.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:13:02.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:13:02.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:13:02.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:13:02.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:13:02.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:13:02.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:13:02.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:13:02.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:13:02.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:13:02.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:13:02.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:13:02.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:13:02.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:13:02.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:13:02.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:13:02.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:13:02.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:13:02.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:13:02.904 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:13:02.904 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:13:02.904 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:13:02.904 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:13:02.904 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:13:02.904 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:13:02.904 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:13:02.904 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:13:02.904 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:13:02.904 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:13:02.904 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:13:02.904 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:13:02.905 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:13:02.905 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:13:02.905 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:13:02.905 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:13:02.905 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:13:02.905 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:13:02.905 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:13:02.905 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:13:02.905 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:13:02.905 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:13:02.905 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:13:02.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:13:02.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:13:02.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:13:02.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:13:02.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:13:02.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:13:02.914 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:13:02.915 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:13:02.915 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:13:02.915 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:13:02.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:13:02.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:13:02.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:13:02.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:13:02.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:13:02.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:13:02.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:13:02.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:13:02.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:13:02.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:13:02.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:13:02.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:13:02.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:13:02.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:13:02.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:13:02.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:13:03.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 23 2026-02-18T08:13:03.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 24 2026-02-18T08:13:03.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:13:03.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=97, tag_tid=6, entry_tid=1], [object_number=96, tag_tid=6, entry_tid=0], [object_number=99, tag_tid=5, entry_tid=42603], [object_number=98, tag_tid=5, entry_tid=42602]]], state=connected] 2026-02-18T08:13:03.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=95, tag_tid=5, entry_tid=36367], [object_number=94, tag_tid=5, entry_tid=36366], [object_number=93, tag_tid=5, entry_tid=36365], [object_number=92, tag_tid=5, entry_tid=36364]]], state=connected] 2026-02-18T08:13:03.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:13:03.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=97, tag_tid=6, entry_tid=1' 2026-02-18T08:13:03.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:13:03.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:13:03.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:13:03.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:13:03.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:13:03.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:13:03.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:13:03.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:13:03.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:13:03.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:13:03.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:13:03.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:13:03.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:13:03.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:13:03.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:13:03.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 23 2026-02-18T08:13:03.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 24 2026-02-18T08:13:03.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:13:03.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=97, tag_tid=6, entry_tid=1], [object_number=96, tag_tid=6, entry_tid=0], [object_number=99, tag_tid=5, entry_tid=42603], [object_number=98, tag_tid=5, entry_tid=42602]]], state=connected] 2026-02-18T08:13:03.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=95, tag_tid=5, entry_tid=36367], [object_number=94, tag_tid=5, entry_tid=36366], [object_number=93, tag_tid=5, entry_tid=36365], [object_number=92, tag_tid=5, entry_tid=36364]]], state=connected] 2026-02-18T08:13:03.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:13:03.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=95, tag_tid=5, entry_tid=36367' 2026-02-18T08:13:03.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=97, tag_tid=6, entry_tid=1' -a 'object_number=97, tag_tid=6, entry_tid=1' = 'object_number=95, tag_tid=5, entry_tid=36367' 2026-02-18T08:13:03.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=95, tag_tid=5, entry_tid=36367' '!=' 'object_number=88, tag_tid=5, entry_tid=36184' 2026-02-18T08:13:03.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:13:03.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=95, tag_tid=5, entry_tid=36367' = 'object_number=88, tag_tid=5, entry_tid=36184' 2026-02-18T08:13:03.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=95, tag_tid=5, entry_tid=36367' 2026-02-18T08:13:03.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=97, tag_tid=6, entry_tid=1' 2026-02-18T08:13:03.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:13:03.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:13:03.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=6 2026-02-18T08:13:03.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=95, tag_tid=5, entry_tid=36367' 2026-02-18T08:13:03.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:13:03.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:13:03.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=5 2026-02-18T08:13:03.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=97, tag_tid=6, entry_tid=1' 2026-02-18T08:13:03.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:13:03.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:13:03.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:13:03.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=95, tag_tid=5, entry_tid=36367' 2026-02-18T08:13:03.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:13:03.055 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:13:03.056 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=36367 2026-02-18T08:13:03.056 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 6 = 5 -a 1 -le 36367 2026-02-18T08:13:03.056 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:13:03.056 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:13:03.056 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:13:03.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:13:03.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:13:03.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:13:03.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:13:03.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:13:03.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:13:03.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:13:03.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:13:03.259 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:13:03.259 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:13:03.259 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:13:03.259 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:13:03.259 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:13:03.259 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:13:03.259 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:13:03.259 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:13:03.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:13:03.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:13:03.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:13:03.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:13:03.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:13:03.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:13:03.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:13:03.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:13:03.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:13:03.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:13:03.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:13:03.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:13:03.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:13:03.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:13:03.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:13:03.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:13:03.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:13:03.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:13:03.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:13:03.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:13:03.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:13:03.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:13:03.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:13:03.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:13:03.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:13:03.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:13:03.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:13:03.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:13:03.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:13:03.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:13:03.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:13:03.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:13:03.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:13:03.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:13:03.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:13:03.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:13:03.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:13:03.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:13:03.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:13:03.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:13:03.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:13:03.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:13:03.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:13:03.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:13:03.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:13:03.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:13:03.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:13:03.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:13:03.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:13:03.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:13:03.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:13:03.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:13:03.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:13:03.362 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 23 2026-02-18T08:13:03.362 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 24 2026-02-18T08:13:03.362 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:13:03.362 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=97, tag_tid=6, entry_tid=1], [object_number=96, tag_tid=6, entry_tid=0], [object_number=99, tag_tid=5, entry_tid=42603], [object_number=98, tag_tid=5, entry_tid=42602]]], state=connected] 2026-02-18T08:13:03.362 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=92, tag_tid=5, entry_tid=36632], [object_number=95, tag_tid=5, entry_tid=36631], [object_number=94, tag_tid=5, entry_tid=36630], [object_number=93, tag_tid=5, entry_tid=36629]]], state=connected] 2026-02-18T08:13:03.365 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:13:03.367 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=97, tag_tid=6, entry_tid=1' 2026-02-18T08:13:03.368 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:13:03.368 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:13:03.368 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:13:03.368 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:13:03.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:13:03.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:13:03.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:13:03.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:13:03.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:13:03.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:13:03.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:13:03.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:13:03.370 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:13:03.370 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:13:03.370 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:13:03.395 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 23 2026-02-18T08:13:03.395 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 24 2026-02-18T08:13:03.395 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:13:03.396 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=97, tag_tid=6, entry_tid=1], [object_number=96, tag_tid=6, entry_tid=0], [object_number=99, tag_tid=5, entry_tid=42603], [object_number=98, tag_tid=5, entry_tid=42602]]], state=connected] 2026-02-18T08:13:03.396 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=92, tag_tid=5, entry_tid=36632], [object_number=95, tag_tid=5, entry_tid=36631], [object_number=94, tag_tid=5, entry_tid=36630], [object_number=93, tag_tid=5, entry_tid=36629]]], state=connected] 2026-02-18T08:13:03.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:13:03.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=92, tag_tid=5, entry_tid=36632' 2026-02-18T08:13:03.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=97, tag_tid=6, entry_tid=1' -a 'object_number=97, tag_tid=6, entry_tid=1' = 'object_number=92, tag_tid=5, entry_tid=36632' 2026-02-18T08:13:03.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=92, tag_tid=5, entry_tid=36632' '!=' 'object_number=95, tag_tid=5, entry_tid=36367' 2026-02-18T08:13:03.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:13:03.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=92, tag_tid=5, entry_tid=36632' = 'object_number=95, tag_tid=5, entry_tid=36367' 2026-02-18T08:13:03.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=92, tag_tid=5, entry_tid=36632' 2026-02-18T08:13:03.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=97, tag_tid=6, entry_tid=1' 2026-02-18T08:13:03.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:13:03.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:13:03.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=6 2026-02-18T08:13:03.403 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=92, tag_tid=5, entry_tid=36632' 2026-02-18T08:13:03.403 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:13:03.403 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:13:03.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=5 2026-02-18T08:13:03.405 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=97, tag_tid=6, entry_tid=1' 2026-02-18T08:13:03.405 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:13:03.405 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:13:03.407 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:13:03.407 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=92, tag_tid=5, entry_tid=36632' 2026-02-18T08:13:03.408 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:13:03.408 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:13:03.409 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=36632 2026-02-18T08:13:03.409 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 6 = 5 -a 1 -le 36632 2026-02-18T08:13:03.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:13:03.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:13:03.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:13:03.611 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:13:03.611 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:13:03.612 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:13:03.612 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:13:03.612 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:13:03.612 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:13:03.612 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:13:03.612 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:13:03.612 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:13:03.612 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:13:03.612 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:13:03.612 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:13:03.613 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:13:03.613 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:13:03.613 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:13:03.613 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:13:03.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:13:03.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:13:03.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:13:03.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:13:03.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:13:03.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:13:03.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:13:03.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:13:03.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:13:03.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:13:03.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:13:03.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:13:03.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:13:03.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:13:03.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:13:03.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:13:03.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:13:03.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:13:03.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:13:03.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:13:03.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:13:03.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:13:03.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:13:03.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:13:03.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:13:03.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:13:03.618 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:13:03.618 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:13:03.618 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:13:03.618 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:13:03.618 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:13:03.618 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:13:03.618 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:13:03.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:13:03.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:13:03.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:13:03.628 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:13:03.694 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:13:03.695 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:13:03.695 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:13:03.695 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:13:03.695 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:13:03.695 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:13:03.695 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:13:03.695 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:13:03.695 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:13:03.696 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:13:03.696 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:13:03.696 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:13:03.696 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:13:03.696 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:13:03.697 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:13:03.697 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:13:03.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 23 2026-02-18T08:13:03.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 24 2026-02-18T08:13:03.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:13:03.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=97, tag_tid=6, entry_tid=1], [object_number=96, tag_tid=6, entry_tid=0], [object_number=99, tag_tid=5, entry_tid=42603], [object_number=98, tag_tid=5, entry_tid=42602]]], state=connected] 2026-02-18T08:13:03.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=94, tag_tid=5, entry_tid=36878], [object_number=93, tag_tid=5, entry_tid=36877], [object_number=92, tag_tid=5, entry_tid=36876], [object_number=95, tag_tid=5, entry_tid=36875]]], state=connected] 2026-02-18T08:13:03.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:13:03.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=97, tag_tid=6, entry_tid=1' 2026-02-18T08:13:03.729 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:13:03.729 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:13:03.729 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:13:03.729 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:13:03.729 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:13:03.729 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:13:03.729 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:13:03.729 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:13:03.729 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:13:03.729 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:13:03.729 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:13:03.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:13:03.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:13:03.731 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:13:03.731 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:13:03.760 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 23 2026-02-18T08:13:03.760 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 24 2026-02-18T08:13:03.760 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:13:03.760 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=97, tag_tid=6, entry_tid=1], [object_number=96, tag_tid=6, entry_tid=0], [object_number=99, tag_tid=5, entry_tid=42603], [object_number=98, tag_tid=5, entry_tid=42602]]], state=connected] 2026-02-18T08:13:03.760 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=94, tag_tid=5, entry_tid=36878], [object_number=93, tag_tid=5, entry_tid=36877], [object_number=92, tag_tid=5, entry_tid=36876], [object_number=95, tag_tid=5, entry_tid=36875]]], state=connected] 2026-02-18T08:13:03.762 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:13:03.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=94, tag_tid=5, entry_tid=36878' 2026-02-18T08:13:03.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=97, tag_tid=6, entry_tid=1' -a 'object_number=97, tag_tid=6, entry_tid=1' = 'object_number=94, tag_tid=5, entry_tid=36878' 2026-02-18T08:13:03.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=94, tag_tid=5, entry_tid=36878' '!=' 'object_number=92, tag_tid=5, entry_tid=36632' 2026-02-18T08:13:03.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:13:03.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=94, tag_tid=5, entry_tid=36878' = 'object_number=92, tag_tid=5, entry_tid=36632' 2026-02-18T08:13:03.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=94, tag_tid=5, entry_tid=36878' 2026-02-18T08:13:03.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=97, tag_tid=6, entry_tid=1' 2026-02-18T08:13:03.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:13:03.766 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:13:03.766 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=6 2026-02-18T08:13:03.767 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=94, tag_tid=5, entry_tid=36878' 2026-02-18T08:13:03.767 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:13:03.767 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:13:03.768 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=5 2026-02-18T08:13:03.769 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=97, tag_tid=6, entry_tid=1' 2026-02-18T08:13:03.769 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:13:03.769 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:13:03.771 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:13:03.771 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=94, tag_tid=5, entry_tid=36878' 2026-02-18T08:13:03.771 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:13:03.772 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:13:03.773 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=36878 2026-02-18T08:13:03.773 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 6 = 5 -a 1 -le 36878 2026-02-18T08:13:03.774 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:13:03.774 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:13:03.774 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:13:03.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:13:03.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:13:03.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:13:03.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:13:03.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:13:03.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:13:03.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:13:03.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:13:03.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:13:03.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:13:03.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:13:03.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:13:03.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:13:03.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:13:03.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:13:03.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:13:03.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:13:03.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:13:03.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:13:03.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:13:03.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:13:03.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:13:03.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:13:03.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:13:03.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:13:03.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:13:03.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:13:03.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:13:03.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:13:03.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:13:03.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:13:03.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:13:03.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:13:03.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:13:03.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:13:03.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:13:03.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:13:03.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:13:03.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:13:03.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:13:03.982 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:13:03.982 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:13:03.982 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:13:03.982 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:13:03.982 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:13:03.982 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:13:03.982 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:13:03.982 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:13:03.982 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:13:03.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:13:03.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:13:03.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:13:03.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:13:04.058 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:13:04.059 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:13:04.059 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:13:04.060 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:13:04.060 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:13:04.060 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:13:04.060 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:13:04.060 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:13:04.060 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:13:04.060 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:13:04.060 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:13:04.060 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:13:04.060 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:13:04.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:13:04.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:13:04.062 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:13:04.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 23 2026-02-18T08:13:04.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 24 2026-02-18T08:13:04.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:13:04.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=97, tag_tid=6, entry_tid=1], [object_number=96, tag_tid=6, entry_tid=0], [object_number=99, tag_tid=5, entry_tid=42603], [object_number=98, tag_tid=5, entry_tid=42602]]], state=connected] 2026-02-18T08:13:04.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=95, tag_tid=5, entry_tid=37151], [object_number=94, tag_tid=5, entry_tid=37150], [object_number=93, tag_tid=5, entry_tid=37149], [object_number=92, tag_tid=5, entry_tid=37148]]], state=connected] 2026-02-18T08:13:04.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:13:04.092 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=97, tag_tid=6, entry_tid=1' 2026-02-18T08:13:04.093 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:13:04.093 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:13:04.093 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:13:04.093 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:13:04.094 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:13:04.094 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:13:04.094 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:13:04.094 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:13:04.094 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:13:04.094 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:13:04.094 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:13:04.094 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:13:04.095 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:13:04.095 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:13:04.095 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:13:04.121 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 23 2026-02-18T08:13:04.121 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 24 2026-02-18T08:13:04.121 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:13:04.122 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=97, tag_tid=6, entry_tid=1], [object_number=96, tag_tid=6, entry_tid=0], [object_number=99, tag_tid=5, entry_tid=42603], [object_number=98, tag_tid=5, entry_tid=42602]]], state=connected] 2026-02-18T08:13:04.122 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=95, tag_tid=5, entry_tid=37151], [object_number=94, tag_tid=5, entry_tid=37150], [object_number=93, tag_tid=5, entry_tid=37149], [object_number=92, tag_tid=5, entry_tid=37148]]], state=connected] 2026-02-18T08:13:04.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:13:04.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=95, tag_tid=5, entry_tid=37151' 2026-02-18T08:13:04.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=97, tag_tid=6, entry_tid=1' -a 'object_number=97, tag_tid=6, entry_tid=1' = 'object_number=95, tag_tid=5, entry_tid=37151' 2026-02-18T08:13:04.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=95, tag_tid=5, entry_tid=37151' '!=' 'object_number=94, tag_tid=5, entry_tid=36878' 2026-02-18T08:13:04.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:13:04.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=95, tag_tid=5, entry_tid=37151' = 'object_number=94, tag_tid=5, entry_tid=36878' 2026-02-18T08:13:04.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=95, tag_tid=5, entry_tid=37151' 2026-02-18T08:13:04.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=97, tag_tid=6, entry_tid=1' 2026-02-18T08:13:04.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:13:04.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:13:04.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=6 2026-02-18T08:13:04.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=95, tag_tid=5, entry_tid=37151' 2026-02-18T08:13:04.129 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:13:04.129 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:13:04.130 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=5 2026-02-18T08:13:04.131 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=97, tag_tid=6, entry_tid=1' 2026-02-18T08:13:04.131 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:13:04.131 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:13:04.132 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:13:04.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=95, tag_tid=5, entry_tid=37151' 2026-02-18T08:13:04.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:13:04.134 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:13:04.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=37151 2026-02-18T08:13:04.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 6 = 5 -a 1 -le 37151 2026-02-18T08:13:04.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:13:04.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:13:04.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:13:04.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:13:04.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:13:04.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:13:04.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:13:04.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:13:04.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:13:04.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:13:04.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:13:04.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:13:04.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:13:04.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:13:04.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:13:04.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:13:04.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:13:04.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:13:04.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:13:04.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:13:04.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:13:04.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:13:04.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:13:04.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:13:04.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:13:04.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:13:04.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:13:04.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:13:04.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:13:04.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:13:04.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:13:04.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:13:04.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:13:04.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:13:04.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:13:04.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:13:04.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:13:04.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:13:04.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:13:04.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:13:04.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:13:04.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:13:04.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:13:04.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:13:04.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:13:04.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:13:04.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:13:04.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:13:04.344 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:13:04.344 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:13:04.344 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:13:04.344 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:13:04.351 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:13:04.352 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:13:04.352 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:13:04.352 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:13:04.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:13:04.413 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:13:04.413 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:13:04.413 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:13:04.413 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:13:04.413 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:13:04.413 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:13:04.413 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:13:04.413 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:13:04.413 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:13:04.414 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:13:04.414 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:13:04.414 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:13:04.414 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:13:04.415 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:13:04.415 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:13:04.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 23 2026-02-18T08:13:04.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 24 2026-02-18T08:13:04.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:13:04.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=97, tag_tid=6, entry_tid=1], [object_number=96, tag_tid=6, entry_tid=0], [object_number=99, tag_tid=5, entry_tid=42603], [object_number=98, tag_tid=5, entry_tid=42602]]], state=connected] 2026-02-18T08:13:04.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=92, tag_tid=5, entry_tid=37404], [object_number=95, tag_tid=5, entry_tid=37403], [object_number=94, tag_tid=5, entry_tid=37402], [object_number=93, tag_tid=5, entry_tid=37401]]], state=connected] 2026-02-18T08:13:04.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:13:04.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=97, tag_tid=6, entry_tid=1' 2026-02-18T08:13:04.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:13:04.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:13:04.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:13:04.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:13:04.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:13:04.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:13:04.448 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:13:04.448 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:13:04.448 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:13:04.448 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:13:04.448 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:13:04.448 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:13:04.448 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:13:04.449 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:13:04.449 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:13:04.474 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 23 2026-02-18T08:13:04.474 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 24 2026-02-18T08:13:04.474 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:13:04.474 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=97, tag_tid=6, entry_tid=1], [object_number=96, tag_tid=6, entry_tid=0], [object_number=99, tag_tid=5, entry_tid=42603], [object_number=98, tag_tid=5, entry_tid=42602]]], state=connected] 2026-02-18T08:13:04.474 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=92, tag_tid=5, entry_tid=37404], [object_number=95, tag_tid=5, entry_tid=37403], [object_number=94, tag_tid=5, entry_tid=37402], [object_number=93, tag_tid=5, entry_tid=37401]]], state=connected] 2026-02-18T08:13:04.476 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:13:04.478 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=92, tag_tid=5, entry_tid=37404' 2026-02-18T08:13:04.478 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=97, tag_tid=6, entry_tid=1' -a 'object_number=97, tag_tid=6, entry_tid=1' = 'object_number=92, tag_tid=5, entry_tid=37404' 2026-02-18T08:13:04.478 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=92, tag_tid=5, entry_tid=37404' '!=' 'object_number=95, tag_tid=5, entry_tid=37151' 2026-02-18T08:13:04.478 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:13:04.478 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=92, tag_tid=5, entry_tid=37404' = 'object_number=95, tag_tid=5, entry_tid=37151' 2026-02-18T08:13:04.478 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=92, tag_tid=5, entry_tid=37404' 2026-02-18T08:13:04.479 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=97, tag_tid=6, entry_tid=1' 2026-02-18T08:13:04.479 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:13:04.479 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:13:04.480 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=6 2026-02-18T08:13:04.481 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=92, tag_tid=5, entry_tid=37404' 2026-02-18T08:13:04.481 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:13:04.481 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:13:04.482 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=5 2026-02-18T08:13:04.483 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=97, tag_tid=6, entry_tid=1' 2026-02-18T08:13:04.483 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:13:04.483 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:13:04.484 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:13:04.485 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=92, tag_tid=5, entry_tid=37404' 2026-02-18T08:13:04.485 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:13:04.485 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:13:04.487 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=37404 2026-02-18T08:13:04.487 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 6 = 5 -a 1 -le 37404 2026-02-18T08:13:04.487 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:13:04.487 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:13:04.487 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:13:04.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:13:04.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:13:04.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:13:04.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:13:04.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:13:04.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:13:04.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:13:04.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:13:04.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:13:04.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:13:04.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:13:04.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:13:04.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:13:04.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:13:04.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:13:04.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:13:04.693 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:13:04.693 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:13:04.693 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:13:04.693 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:13:04.693 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:13:04.693 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:13:04.693 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:13:04.694 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:13:04.694 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:13:04.694 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:13:04.694 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:13:04.694 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:13:04.694 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:13:04.694 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:13:04.694 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:13:04.694 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:13:04.694 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:13:04.694 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:13:04.695 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:13:04.695 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:13:04.695 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:13:04.695 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:13:04.695 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:13:04.695 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:13:04.695 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:13:04.695 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:13:04.695 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:13:04.695 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:13:04.695 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:13:04.695 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:13:04.695 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:13:04.696 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:13:04.696 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:13:04.704 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:13:04.704 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:13:04.704 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:13:04.704 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:13:04.771 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:13:04.772 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:13:04.772 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:13:04.772 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:13:04.772 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:13:04.772 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:13:04.772 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:13:04.772 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:13:04.773 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:13:04.773 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:13:04.773 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:13:04.773 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:13:04.773 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:13:04.774 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:13:04.774 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:13:04.774 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:13:04.801 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 23 2026-02-18T08:13:04.802 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 24 2026-02-18T08:13:04.802 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:13:04.802 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=97, tag_tid=6, entry_tid=1], [object_number=96, tag_tid=6, entry_tid=0], [object_number=99, tag_tid=5, entry_tid=42603], [object_number=98, tag_tid=5, entry_tid=42602]]], state=connected] 2026-02-18T08:13:04.802 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=94, tag_tid=5, entry_tid=37670], [object_number=93, tag_tid=5, entry_tid=37669], [object_number=92, tag_tid=5, entry_tid=37668], [object_number=95, tag_tid=5, entry_tid=37667]]], state=connected] 2026-02-18T08:13:04.804 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:13:04.806 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=97, tag_tid=6, entry_tid=1' 2026-02-18T08:13:04.807 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:13:04.807 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:13:04.807 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:13:04.807 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:13:04.808 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:13:04.808 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:13:04.808 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:13:04.808 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:13:04.808 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:13:04.808 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:13:04.809 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:13:04.809 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:13:04.809 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:13:04.809 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:13:04.810 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:13:04.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 23 2026-02-18T08:13:04.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 24 2026-02-18T08:13:04.835 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:13:04.835 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=97, tag_tid=6, entry_tid=1], [object_number=96, tag_tid=6, entry_tid=0], [object_number=99, tag_tid=5, entry_tid=42603], [object_number=98, tag_tid=5, entry_tid=42602]]], state=connected] 2026-02-18T08:13:04.835 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=94, tag_tid=5, entry_tid=37670], [object_number=93, tag_tid=5, entry_tid=37669], [object_number=92, tag_tid=5, entry_tid=37668], [object_number=95, tag_tid=5, entry_tid=37667]]], state=connected] 2026-02-18T08:13:04.837 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:13:04.838 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=94, tag_tid=5, entry_tid=37670' 2026-02-18T08:13:04.839 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=97, tag_tid=6, entry_tid=1' -a 'object_number=97, tag_tid=6, entry_tid=1' = 'object_number=94, tag_tid=5, entry_tid=37670' 2026-02-18T08:13:04.839 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=94, tag_tid=5, entry_tid=37670' '!=' 'object_number=92, tag_tid=5, entry_tid=37404' 2026-02-18T08:13:04.839 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:13:04.839 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=94, tag_tid=5, entry_tid=37670' = 'object_number=92, tag_tid=5, entry_tid=37404' 2026-02-18T08:13:04.839 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=94, tag_tid=5, entry_tid=37670' 2026-02-18T08:13:04.840 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=97, tag_tid=6, entry_tid=1' 2026-02-18T08:13:04.840 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:13:04.840 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:13:04.841 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=6 2026-02-18T08:13:04.842 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=94, tag_tid=5, entry_tid=37670' 2026-02-18T08:13:04.842 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:13:04.842 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:13:04.843 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=5 2026-02-18T08:13:04.844 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=97, tag_tid=6, entry_tid=1' 2026-02-18T08:13:04.844 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:13:04.844 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:13:04.845 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:13:04.846 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=94, tag_tid=5, entry_tid=37670' 2026-02-18T08:13:04.846 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:13:04.846 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:13:04.848 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=37670 2026-02-18T08:13:04.848 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 6 = 5 -a 1 -le 37670 2026-02-18T08:13:04.848 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:13:04.848 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:13:04.848 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:13:05.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:13:05.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:13:05.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:13:05.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:13:05.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:13:05.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:13:05.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:13:05.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:13:05.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:13:05.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:13:05.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:13:05.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:13:05.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:13:05.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:13:05.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:13:05.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:13:05.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:13:05.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:13:05.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:13:05.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:13:05.055 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:13:05.055 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:13:05.055 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:13:05.055 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:13:05.055 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:13:05.055 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:13:05.055 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:13:05.055 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:13:05.055 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:13:05.055 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:13:05.055 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:13:05.055 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:13:05.056 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:13:05.056 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:13:05.056 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:13:05.056 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:13:05.056 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:13:05.056 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:13:05.056 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:13:05.056 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:13:05.056 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:13:05.056 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:13:05.056 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:13:05.056 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:13:05.056 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:13:05.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:13:05.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:13:05.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:13:05.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:13:05.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:13:05.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:13:05.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:13:05.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:13:05.121 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:13:05.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:13:05.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:13:05.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:13:05.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:13:05.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:13:05.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:13:05.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:13:05.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:13:05.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:13:05.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:13:05.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:13:05.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:13:05.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:13:05.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:13:05.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:13:05.161 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 23 2026-02-18T08:13:05.161 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 24 2026-02-18T08:13:05.161 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:13:05.161 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=97, tag_tid=6, entry_tid=1], [object_number=96, tag_tid=6, entry_tid=0], [object_number=99, tag_tid=5, entry_tid=42603], [object_number=98, tag_tid=5, entry_tid=42602]]], state=connected] 2026-02-18T08:13:05.162 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=92, tag_tid=5, entry_tid=37936], [object_number=95, tag_tid=5, entry_tid=37935], [object_number=94, tag_tid=5, entry_tid=37934], [object_number=93, tag_tid=5, entry_tid=37933]]], state=connected] 2026-02-18T08:13:05.165 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:13:05.167 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=97, tag_tid=6, entry_tid=1' 2026-02-18T08:13:05.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:13:05.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:13:05.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:13:05.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:13:05.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:13:05.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:13:05.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:13:05.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:13:05.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:13:05.169 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:13:05.169 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:13:05.169 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:13:05.170 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:13:05.170 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:13:05.170 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:13:05.196 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 23 2026-02-18T08:13:05.197 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 24 2026-02-18T08:13:05.197 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:13:05.197 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=97, tag_tid=6, entry_tid=1], [object_number=96, tag_tid=6, entry_tid=0], [object_number=99, tag_tid=5, entry_tid=42603], [object_number=98, tag_tid=5, entry_tid=42602]]], state=connected] 2026-02-18T08:13:05.197 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=92, tag_tid=5, entry_tid=37936], [object_number=95, tag_tid=5, entry_tid=37935], [object_number=94, tag_tid=5, entry_tid=37934], [object_number=93, tag_tid=5, entry_tid=37933]]], state=connected] 2026-02-18T08:13:05.199 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:13:05.201 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=92, tag_tid=5, entry_tid=37936' 2026-02-18T08:13:05.201 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=97, tag_tid=6, entry_tid=1' -a 'object_number=97, tag_tid=6, entry_tid=1' = 'object_number=92, tag_tid=5, entry_tid=37936' 2026-02-18T08:13:05.201 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=92, tag_tid=5, entry_tid=37936' '!=' 'object_number=94, tag_tid=5, entry_tid=37670' 2026-02-18T08:13:05.201 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:13:05.201 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=92, tag_tid=5, entry_tid=37936' = 'object_number=94, tag_tid=5, entry_tid=37670' 2026-02-18T08:13:05.201 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=92, tag_tid=5, entry_tid=37936' 2026-02-18T08:13:05.201 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=97, tag_tid=6, entry_tid=1' 2026-02-18T08:13:05.202 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:13:05.202 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:13:05.202 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=6 2026-02-18T08:13:05.203 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=92, tag_tid=5, entry_tid=37936' 2026-02-18T08:13:05.203 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:13:05.203 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:13:05.204 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=5 2026-02-18T08:13:05.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=97, tag_tid=6, entry_tid=1' 2026-02-18T08:13:05.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:13:05.206 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:13:05.207 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:13:05.207 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=92, tag_tid=5, entry_tid=37936' 2026-02-18T08:13:05.207 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:13:05.208 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:13:05.209 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=37936 2026-02-18T08:13:05.209 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 6 = 5 -a 1 -le 37936 2026-02-18T08:13:05.210 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:13:05.210 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:13:05.210 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:13:05.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:13:05.412 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:13:05.412 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:13:05.412 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:13:05.412 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:13:05.412 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:13:05.412 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:13:05.412 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:13:05.412 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:13:05.412 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:13:05.413 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:13:05.413 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:13:05.413 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:13:05.413 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:13:05.413 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:13:05.413 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:13:05.416 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:13:05.416 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:13:05.416 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:13:05.416 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:13:05.416 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:13:05.416 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:13:05.417 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:13:05.417 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:13:05.417 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:13:05.417 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:13:05.417 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:13:05.417 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:13:05.417 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:13:05.417 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:13:05.417 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:13:05.417 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:13:05.417 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:13:05.417 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:13:05.418 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:13:05.418 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:13:05.418 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:13:05.418 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:13:05.418 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:13:05.418 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:13:05.418 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:13:05.418 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:13:05.418 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:13:05.418 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:13:05.418 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:13:05.418 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:13:05.419 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:13:05.419 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:13:05.419 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:13:05.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:13:05.427 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:13:05.427 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:13:05.427 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:13:05.485 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:13:05.486 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:13:05.486 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:13:05.486 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:13:05.486 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:13:05.486 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:13:05.486 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:13:05.486 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:13:05.486 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:13:05.487 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:13:05.487 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:13:05.487 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:13:05.487 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:13:05.487 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:13:05.488 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:13:05.488 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:13:05.516 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 23 2026-02-18T08:13:05.516 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 24 2026-02-18T08:13:05.516 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:13:05.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=97, tag_tid=6, entry_tid=1], [object_number=96, tag_tid=6, entry_tid=0], [object_number=99, tag_tid=5, entry_tid=42603], [object_number=98, tag_tid=5, entry_tid=42602]]], state=connected] 2026-02-18T08:13:05.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=94, tag_tid=5, entry_tid=38214], [object_number=93, tag_tid=5, entry_tid=38213], [object_number=92, tag_tid=5, entry_tid=38212], [object_number=95, tag_tid=5, entry_tid=38211]]], state=connected] 2026-02-18T08:13:05.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:13:05.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=97, tag_tid=6, entry_tid=1' 2026-02-18T08:13:05.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:13:05.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:13:05.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:13:05.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:13:05.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:13:05.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:13:05.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:13:05.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:13:05.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:13:05.523 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:13:05.523 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:13:05.523 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:13:05.523 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:13:05.524 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:13:05.524 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:13:05.559 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 23 2026-02-18T08:13:05.559 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 24 2026-02-18T08:13:05.559 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:13:05.559 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=97, tag_tid=6, entry_tid=1], [object_number=96, tag_tid=6, entry_tid=0], [object_number=99, tag_tid=5, entry_tid=42603], [object_number=98, tag_tid=5, entry_tid=42602]]], state=connected] 2026-02-18T08:13:05.559 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=94, tag_tid=5, entry_tid=38214], [object_number=93, tag_tid=5, entry_tid=38213], [object_number=92, tag_tid=5, entry_tid=38212], [object_number=95, tag_tid=5, entry_tid=38211]]], state=connected] 2026-02-18T08:13:05.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:13:05.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=94, tag_tid=5, entry_tid=38214' 2026-02-18T08:13:05.565 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=97, tag_tid=6, entry_tid=1' -a 'object_number=97, tag_tid=6, entry_tid=1' = 'object_number=94, tag_tid=5, entry_tid=38214' 2026-02-18T08:13:05.565 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=94, tag_tid=5, entry_tid=38214' '!=' 'object_number=92, tag_tid=5, entry_tid=37936' 2026-02-18T08:13:05.565 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:13:05.565 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=94, tag_tid=5, entry_tid=38214' = 'object_number=92, tag_tid=5, entry_tid=37936' 2026-02-18T08:13:05.565 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=94, tag_tid=5, entry_tid=38214' 2026-02-18T08:13:05.566 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=97, tag_tid=6, entry_tid=1' 2026-02-18T08:13:05.566 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:13:05.566 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:13:05.566 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=6 2026-02-18T08:13:05.567 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=94, tag_tid=5, entry_tid=38214' 2026-02-18T08:13:05.567 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:13:05.568 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:13:05.569 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=5 2026-02-18T08:13:05.569 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=97, tag_tid=6, entry_tid=1' 2026-02-18T08:13:05.570 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:13:05.570 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:13:05.571 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:13:05.571 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=94, tag_tid=5, entry_tid=38214' 2026-02-18T08:13:05.572 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:13:05.572 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:13:05.573 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=38214 2026-02-18T08:13:05.573 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 6 = 5 -a 1 -le 38214 2026-02-18T08:13:05.574 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:13:05.574 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:13:05.574 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:13:05.776 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:13:05.776 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:13:05.776 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:13:05.776 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:13:05.776 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:13:05.776 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:13:05.776 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:13:05.777 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:13:05.777 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:13:05.777 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:13:05.777 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:13:05.777 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:13:05.777 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:13:05.777 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:13:05.777 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:13:05.777 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:13:05.777 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:13:05.780 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:13:05.780 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:13:05.780 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:13:05.780 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:13:05.780 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:13:05.780 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:13:05.780 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:13:05.780 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:13:05.781 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:13:05.781 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:13:05.781 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:13:05.781 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:13:05.781 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:13:05.781 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:13:05.781 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:13:05.781 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:13:05.781 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:13:05.781 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:13:05.781 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:13:05.781 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:13:05.782 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:13:05.782 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:13:05.782 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:13:05.782 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:13:05.782 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:13:05.782 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:13:05.782 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:13:05.782 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:13:05.782 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:13:05.782 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:13:05.782 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:13:05.782 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:13:05.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:13:05.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:13:05.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:13:05.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:13:05.851 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:13:05.852 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:13:05.852 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:13:05.852 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:13:05.852 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:13:05.852 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:13:05.852 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:13:05.852 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:13:05.852 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:13:05.852 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:13:05.852 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:13:05.853 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:13:05.853 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:13:05.854 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:13:05.854 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:13:05.854 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:13:05.882 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 23 2026-02-18T08:13:05.883 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 24 2026-02-18T08:13:05.883 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:13:05.883 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=97, tag_tid=6, entry_tid=1], [object_number=96, tag_tid=6, entry_tid=0], [object_number=99, tag_tid=5, entry_tid=42603], [object_number=98, tag_tid=5, entry_tid=42602]]], state=connected] 2026-02-18T08:13:05.883 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=92, tag_tid=5, entry_tid=38488], [object_number=95, tag_tid=5, entry_tid=38487], [object_number=94, tag_tid=5, entry_tid=38486], [object_number=93, tag_tid=5, entry_tid=38485]]], state=connected] 2026-02-18T08:13:05.885 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:13:05.887 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=97, tag_tid=6, entry_tid=1' 2026-02-18T08:13:05.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:13:05.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:13:05.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:13:05.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:13:05.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:13:05.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:13:05.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:13:05.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:13:05.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:13:05.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:13:05.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:13:05.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:13:05.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:13:05.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:13:05.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:13:05.918 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 23 2026-02-18T08:13:05.918 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 24 2026-02-18T08:13:05.918 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:13:05.918 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=97, tag_tid=6, entry_tid=1], [object_number=96, tag_tid=6, entry_tid=0], [object_number=99, tag_tid=5, entry_tid=42603], [object_number=98, tag_tid=5, entry_tid=42602]]], state=connected] 2026-02-18T08:13:05.918 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=92, tag_tid=5, entry_tid=38488], [object_number=95, tag_tid=5, entry_tid=38487], [object_number=94, tag_tid=5, entry_tid=38486], [object_number=93, tag_tid=5, entry_tid=38485]]], state=connected] 2026-02-18T08:13:05.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:13:05.922 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=92, tag_tid=5, entry_tid=38488' 2026-02-18T08:13:05.922 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=97, tag_tid=6, entry_tid=1' -a 'object_number=97, tag_tid=6, entry_tid=1' = 'object_number=92, tag_tid=5, entry_tid=38488' 2026-02-18T08:13:05.923 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=92, tag_tid=5, entry_tid=38488' '!=' 'object_number=94, tag_tid=5, entry_tid=38214' 2026-02-18T08:13:05.923 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:13:05.923 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=92, tag_tid=5, entry_tid=38488' = 'object_number=94, tag_tid=5, entry_tid=38214' 2026-02-18T08:13:05.923 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=92, tag_tid=5, entry_tid=38488' 2026-02-18T08:13:05.923 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=97, tag_tid=6, entry_tid=1' 2026-02-18T08:13:05.924 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:13:05.924 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:13:05.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=6 2026-02-18T08:13:05.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=92, tag_tid=5, entry_tid=38488' 2026-02-18T08:13:05.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:13:05.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:13:05.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=5 2026-02-18T08:13:05.928 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=97, tag_tid=6, entry_tid=1' 2026-02-18T08:13:05.928 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:13:05.928 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:13:05.929 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:13:05.930 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=92, tag_tid=5, entry_tid=38488' 2026-02-18T08:13:05.930 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:13:05.930 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:13:05.932 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=38488 2026-02-18T08:13:05.932 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 6 = 5 -a 1 -le 38488 2026-02-18T08:13:05.932 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:13:05.932 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:13:05.932 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:13:06.134 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:13:06.134 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:13:06.134 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:13:06.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:13:06.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:13:06.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:13:06.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:13:06.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:13:06.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:13:06.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:13:06.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:13:06.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:13:06.136 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:13:06.136 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:13:06.136 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:13:06.136 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:13:06.136 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:13:06.138 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:13:06.138 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:13:06.139 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:13:06.139 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:13:06.139 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:13:06.139 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:13:06.139 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:13:06.139 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:13:06.139 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:13:06.139 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:13:06.139 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:13:06.139 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:13:06.139 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:13:06.140 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:13:06.140 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:13:06.140 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:13:06.140 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:13:06.140 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:13:06.140 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:13:06.140 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:13:06.140 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:13:06.140 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:13:06.140 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:13:06.140 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:13:06.140 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:13:06.141 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:13:06.141 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:13:06.141 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:13:06.141 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:13:06.141 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:13:06.141 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:13:06.141 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:13:06.148 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:13:06.149 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:13:06.149 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:13:06.149 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:13:06.217 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:13:06.217 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:13:06.218 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:13:06.218 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:13:06.218 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:13:06.218 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:13:06.218 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:13:06.218 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:13:06.218 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:13:06.218 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:13:06.218 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:13:06.219 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:13:06.219 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:13:06.219 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:13:06.220 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:13:06.220 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:13:06.247 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 23 2026-02-18T08:13:06.247 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 24 2026-02-18T08:13:06.247 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:13:06.247 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=97, tag_tid=6, entry_tid=1], [object_number=96, tag_tid=6, entry_tid=0], [object_number=99, tag_tid=5, entry_tid=42603], [object_number=98, tag_tid=5, entry_tid=42602]]], state=connected] 2026-02-18T08:13:06.247 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=93, tag_tid=5, entry_tid=38737], [object_number=92, tag_tid=5, entry_tid=38736], [object_number=95, tag_tid=5, entry_tid=38735], [object_number=94, tag_tid=5, entry_tid=38734]]], state=connected] 2026-02-18T08:13:06.249 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:13:06.251 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=97, tag_tid=6, entry_tid=1' 2026-02-18T08:13:06.252 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:13:06.252 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:13:06.252 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:13:06.253 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:13:06.253 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:13:06.253 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:13:06.253 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:13:06.253 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:13:06.253 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:13:06.253 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:13:06.253 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:13:06.253 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:13:06.254 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:13:06.254 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:13:06.255 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:13:06.289 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 23 2026-02-18T08:13:06.289 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 24 2026-02-18T08:13:06.289 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:13:06.289 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=97, tag_tid=6, entry_tid=1], [object_number=96, tag_tid=6, entry_tid=0], [object_number=99, tag_tid=5, entry_tid=42603], [object_number=98, tag_tid=5, entry_tid=42602]]], state=connected] 2026-02-18T08:13:06.289 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=93, tag_tid=5, entry_tid=38737], [object_number=92, tag_tid=5, entry_tid=38736], [object_number=95, tag_tid=5, entry_tid=38735], [object_number=94, tag_tid=5, entry_tid=38734]]], state=connected] 2026-02-18T08:13:06.293 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:13:06.294 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=93, tag_tid=5, entry_tid=38737' 2026-02-18T08:13:06.294 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=97, tag_tid=6, entry_tid=1' -a 'object_number=97, tag_tid=6, entry_tid=1' = 'object_number=93, tag_tid=5, entry_tid=38737' 2026-02-18T08:13:06.294 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=93, tag_tid=5, entry_tid=38737' '!=' 'object_number=92, tag_tid=5, entry_tid=38488' 2026-02-18T08:13:06.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:13:06.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=93, tag_tid=5, entry_tid=38737' = 'object_number=92, tag_tid=5, entry_tid=38488' 2026-02-18T08:13:06.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=93, tag_tid=5, entry_tid=38737' 2026-02-18T08:13:06.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=97, tag_tid=6, entry_tid=1' 2026-02-18T08:13:06.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:13:06.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:13:06.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=6 2026-02-18T08:13:06.297 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=93, tag_tid=5, entry_tid=38737' 2026-02-18T08:13:06.297 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:13:06.297 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:13:06.298 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=5 2026-02-18T08:13:06.299 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=97, tag_tid=6, entry_tid=1' 2026-02-18T08:13:06.299 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:13:06.299 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:13:06.300 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:13:06.301 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=93, tag_tid=5, entry_tid=38737' 2026-02-18T08:13:06.301 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:13:06.301 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:13:06.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=38737 2026-02-18T08:13:06.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 6 = 5 -a 1 -le 38737 2026-02-18T08:13:06.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:13:06.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:13:06.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:13:06.505 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:13:06.505 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:13:06.505 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:13:06.505 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:13:06.506 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:13:06.506 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:13:06.506 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:13:06.506 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:13:06.506 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:13:06.506 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:13:06.506 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:13:06.506 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:13:06.506 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:13:06.506 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:13:06.506 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:13:06.507 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:13:06.507 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:13:06.509 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:13:06.509 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:13:06.509 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:13:06.509 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:13:06.510 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:13:06.510 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:13:06.510 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:13:06.510 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:13:06.510 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:13:06.510 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:13:06.510 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:13:06.510 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:13:06.510 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:13:06.510 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:13:06.510 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:13:06.510 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:13:06.511 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:13:06.511 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:13:06.511 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:13:06.511 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:13:06.511 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:13:06.511 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:13:06.511 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:13:06.511 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:13:06.511 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:13:06.511 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:13:06.511 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:13:06.511 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:13:06.511 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:13:06.512 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:13:06.512 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:13:06.512 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:13:06.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:13:06.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:13:06.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:13:06.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:13:06.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:13:06.582 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:13:06.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:13:06.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:13:06.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:13:06.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:13:06.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:13:06.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:13:06.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:13:06.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:13:06.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:13:06.584 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:13:06.584 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:13:06.585 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:13:06.585 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:13:06.585 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:13:06.611 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 23 2026-02-18T08:13:06.611 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 24 2026-02-18T08:13:06.611 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:13:06.611 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=97, tag_tid=6, entry_tid=1], [object_number=96, tag_tid=6, entry_tid=0], [object_number=99, tag_tid=5, entry_tid=42603], [object_number=98, tag_tid=5, entry_tid=42602]]], state=connected] 2026-02-18T08:13:06.611 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=94, tag_tid=5, entry_tid=38986], [object_number=93, tag_tid=5, entry_tid=38985], [object_number=92, tag_tid=5, entry_tid=38984], [object_number=95, tag_tid=5, entry_tid=38983]]], state=connected] 2026-02-18T08:13:06.613 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:13:06.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=97, tag_tid=6, entry_tid=1' 2026-02-18T08:13:06.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:13:06.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:13:06.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:13:06.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:13:06.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:13:06.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:13:06.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:13:06.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:13:06.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:13:06.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:13:06.618 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:13:06.618 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:13:06.618 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:13:06.619 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:13:06.619 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:13:06.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 23 2026-02-18T08:13:06.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 24 2026-02-18T08:13:06.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:13:06.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=97, tag_tid=6, entry_tid=1], [object_number=96, tag_tid=6, entry_tid=0], [object_number=99, tag_tid=5, entry_tid=42603], [object_number=98, tag_tid=5, entry_tid=42602]]], state=connected] 2026-02-18T08:13:06.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=94, tag_tid=5, entry_tid=38986], [object_number=93, tag_tid=5, entry_tid=38985], [object_number=92, tag_tid=5, entry_tid=38984], [object_number=95, tag_tid=5, entry_tid=38983]]], state=connected] 2026-02-18T08:13:06.647 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:13:06.648 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=94, tag_tid=5, entry_tid=38986' 2026-02-18T08:13:06.648 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=97, tag_tid=6, entry_tid=1' -a 'object_number=97, tag_tid=6, entry_tid=1' = 'object_number=94, tag_tid=5, entry_tid=38986' 2026-02-18T08:13:06.649 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=94, tag_tid=5, entry_tid=38986' '!=' 'object_number=93, tag_tid=5, entry_tid=38737' 2026-02-18T08:13:06.649 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:13:06.649 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=94, tag_tid=5, entry_tid=38986' = 'object_number=93, tag_tid=5, entry_tid=38737' 2026-02-18T08:13:06.649 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=94, tag_tid=5, entry_tid=38986' 2026-02-18T08:13:06.649 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=97, tag_tid=6, entry_tid=1' 2026-02-18T08:13:06.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:13:06.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:13:06.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=6 2026-02-18T08:13:06.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=94, tag_tid=5, entry_tid=38986' 2026-02-18T08:13:06.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:13:06.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:13:06.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=5 2026-02-18T08:13:06.653 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=97, tag_tid=6, entry_tid=1' 2026-02-18T08:13:06.654 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:13:06.654 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:13:06.655 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:13:06.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=94, tag_tid=5, entry_tid=38986' 2026-02-18T08:13:06.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:13:06.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:13:06.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=38986 2026-02-18T08:13:06.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 6 = 5 -a 1 -le 38986 2026-02-18T08:13:06.658 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:13:06.658 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:13:06.658 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:13:06.859 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:13:06.860 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:13:06.860 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:13:06.860 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:13:06.860 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:13:06.860 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:13:06.860 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:13:06.860 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:13:06.860 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:13:06.861 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:13:06.861 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:13:06.861 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:13:06.861 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:13:06.861 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:13:06.861 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:13:06.861 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:13:06.864 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:13:06.864 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:13:06.864 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:13:06.864 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:13:06.864 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:13:06.864 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:13:06.864 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:13:06.864 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:13:06.864 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:13:06.864 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:13:06.865 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:13:06.865 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:13:06.865 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:13:06.865 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:13:06.865 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:13:06.865 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:13:06.865 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:13:06.865 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:13:06.865 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:13:06.865 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:13:06.865 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:13:06.865 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:13:06.866 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:13:06.866 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:13:06.866 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:13:06.866 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:13:06.866 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:13:06.866 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:13:06.866 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:13:06.866 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:13:06.866 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:13:06.866 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:13:06.866 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:13:06.875 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:13:06.875 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:13:06.875 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:13:06.876 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:13:06.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:13:06.936 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:13:06.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:13:06.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:13:06.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:13:06.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:13:06.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:13:06.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:13:06.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:13:06.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:13:06.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:13:06.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:13:06.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:13:06.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:13:06.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:13:06.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:13:06.965 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 23 2026-02-18T08:13:06.965 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 24 2026-02-18T08:13:06.965 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:13:06.966 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=97, tag_tid=6, entry_tid=1], [object_number=96, tag_tid=6, entry_tid=0], [object_number=99, tag_tid=5, entry_tid=42603], [object_number=98, tag_tid=5, entry_tid=42602]]], state=connected] 2026-02-18T08:13:06.966 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=95, tag_tid=5, entry_tid=39239], [object_number=94, tag_tid=5, entry_tid=39238], [object_number=93, tag_tid=5, entry_tid=39237], [object_number=92, tag_tid=5, entry_tid=39236]]], state=connected] 2026-02-18T08:13:06.968 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:13:06.970 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=97, tag_tid=6, entry_tid=1' 2026-02-18T08:13:06.971 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:13:06.971 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:13:06.971 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:13:06.971 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:13:06.971 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:13:06.971 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:13:06.971 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:13:06.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:13:06.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:13:06.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:13:06.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:13:06.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:13:06.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:13:06.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:13:06.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:13:06.999 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 23 2026-02-18T08:13:07.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 24 2026-02-18T08:13:07.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:13:07.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=97, tag_tid=6, entry_tid=1], [object_number=96, tag_tid=6, entry_tid=0], [object_number=99, tag_tid=5, entry_tid=42603], [object_number=98, tag_tid=5, entry_tid=42602]]], state=connected] 2026-02-18T08:13:07.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=95, tag_tid=5, entry_tid=39239], [object_number=94, tag_tid=5, entry_tid=39238], [object_number=93, tag_tid=5, entry_tid=39237], [object_number=92, tag_tid=5, entry_tid=39236]]], state=connected] 2026-02-18T08:13:07.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:13:07.004 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=95, tag_tid=5, entry_tid=39239' 2026-02-18T08:13:07.004 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=97, tag_tid=6, entry_tid=1' -a 'object_number=97, tag_tid=6, entry_tid=1' = 'object_number=95, tag_tid=5, entry_tid=39239' 2026-02-18T08:13:07.004 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=95, tag_tid=5, entry_tid=39239' '!=' 'object_number=94, tag_tid=5, entry_tid=38986' 2026-02-18T08:13:07.004 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:13:07.004 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=95, tag_tid=5, entry_tid=39239' = 'object_number=94, tag_tid=5, entry_tid=38986' 2026-02-18T08:13:07.004 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=95, tag_tid=5, entry_tid=39239' 2026-02-18T08:13:07.005 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=97, tag_tid=6, entry_tid=1' 2026-02-18T08:13:07.005 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:13:07.005 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:13:07.006 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=6 2026-02-18T08:13:07.007 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=95, tag_tid=5, entry_tid=39239' 2026-02-18T08:13:07.007 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:13:07.007 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:13:07.008 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=5 2026-02-18T08:13:07.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=97, tag_tid=6, entry_tid=1' 2026-02-18T08:13:07.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:13:07.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:13:07.010 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:13:07.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=95, tag_tid=5, entry_tid=39239' 2026-02-18T08:13:07.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:13:07.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:13:07.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=39239 2026-02-18T08:13:07.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 6 = 5 -a 1 -le 39239 2026-02-18T08:13:07.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:13:07.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:13:07.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:13:07.215 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:13:07.215 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:13:07.215 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:13:07.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:13:07.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:13:07.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:13:07.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:13:07.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:13:07.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:13:07.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:13:07.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:13:07.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:13:07.217 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:13:07.217 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:13:07.217 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:13:07.217 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:13:07.217 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:13:07.219 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:13:07.220 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:13:07.220 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:13:07.220 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:13:07.220 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:13:07.220 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:13:07.220 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:13:07.220 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:13:07.220 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:13:07.220 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:13:07.220 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:13:07.220 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:13:07.221 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:13:07.221 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:13:07.221 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:13:07.221 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:13:07.221 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:13:07.221 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:13:07.221 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:13:07.221 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:13:07.221 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:13:07.221 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:13:07.221 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:13:07.221 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:13:07.222 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:13:07.222 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:13:07.222 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:13:07.222 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:13:07.222 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:13:07.222 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:13:07.222 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:13:07.222 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:13:07.229 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:13:07.229 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:13:07.229 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:13:07.230 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:13:07.290 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:13:07.291 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:13:07.292 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:13:07.292 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:13:07.292 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:13:07.292 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:13:07.292 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:13:07.292 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:13:07.292 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:13:07.292 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:13:07.292 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:13:07.292 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:13:07.293 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:13:07.293 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:13:07.294 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:13:07.294 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:13:07.321 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 23 2026-02-18T08:13:07.321 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 24 2026-02-18T08:13:07.321 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:13:07.321 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=97, tag_tid=6, entry_tid=1], [object_number=96, tag_tid=6, entry_tid=0], [object_number=99, tag_tid=5, entry_tid=42603], [object_number=98, tag_tid=5, entry_tid=42602]]], state=connected] 2026-02-18T08:13:07.322 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=92, tag_tid=5, entry_tid=39496], [object_number=95, tag_tid=5, entry_tid=39495], [object_number=94, tag_tid=5, entry_tid=39494], [object_number=93, tag_tid=5, entry_tid=39493]]], state=connected] 2026-02-18T08:13:07.324 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:13:07.326 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=97, tag_tid=6, entry_tid=1' 2026-02-18T08:13:07.327 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:13:07.327 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:13:07.327 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:13:07.327 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:13:07.327 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:13:07.327 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:13:07.327 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:13:07.327 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:13:07.327 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:13:07.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:13:07.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:13:07.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:13:07.329 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:13:07.329 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:13:07.329 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:13:07.355 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 23 2026-02-18T08:13:07.356 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 24 2026-02-18T08:13:07.356 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:13:07.356 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=97, tag_tid=6, entry_tid=1], [object_number=96, tag_tid=6, entry_tid=0], [object_number=99, tag_tid=5, entry_tid=42603], [object_number=98, tag_tid=5, entry_tid=42602]]], state=connected] 2026-02-18T08:13:07.356 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=92, tag_tid=5, entry_tid=39496], [object_number=95, tag_tid=5, entry_tid=39495], [object_number=94, tag_tid=5, entry_tid=39494], [object_number=93, tag_tid=5, entry_tid=39493]]], state=connected] 2026-02-18T08:13:07.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:13:07.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=92, tag_tid=5, entry_tid=39496' 2026-02-18T08:13:07.360 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=97, tag_tid=6, entry_tid=1' -a 'object_number=97, tag_tid=6, entry_tid=1' = 'object_number=92, tag_tid=5, entry_tid=39496' 2026-02-18T08:13:07.360 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=92, tag_tid=5, entry_tid=39496' '!=' 'object_number=95, tag_tid=5, entry_tid=39239' 2026-02-18T08:13:07.360 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:13:07.360 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=92, tag_tid=5, entry_tid=39496' = 'object_number=95, tag_tid=5, entry_tid=39239' 2026-02-18T08:13:07.360 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=92, tag_tid=5, entry_tid=39496' 2026-02-18T08:13:07.361 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=97, tag_tid=6, entry_tid=1' 2026-02-18T08:13:07.361 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:13:07.361 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:13:07.362 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=6 2026-02-18T08:13:07.362 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=92, tag_tid=5, entry_tid=39496' 2026-02-18T08:13:07.363 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:13:07.363 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:13:07.364 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=5 2026-02-18T08:13:07.365 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=97, tag_tid=6, entry_tid=1' 2026-02-18T08:13:07.365 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:13:07.365 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:13:07.366 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:13:07.367 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=92, tag_tid=5, entry_tid=39496' 2026-02-18T08:13:07.367 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:13:07.367 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:13:07.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=39496 2026-02-18T08:13:07.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 6 = 5 -a 1 -le 39496 2026-02-18T08:13:07.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:13:07.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:13:07.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:13:07.570 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:13:07.571 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:13:07.571 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:13:07.571 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:13:07.571 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:13:07.571 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:13:07.571 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:13:07.571 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:13:07.571 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:13:07.571 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:13:07.571 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:13:07.571 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:13:07.572 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:13:07.572 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:13:07.572 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:13:07.574 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:13:07.575 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:13:07.575 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:13:07.575 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:13:07.575 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:13:07.575 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:13:07.575 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:13:07.575 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:13:07.575 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:13:07.575 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:13:07.575 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:13:07.575 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:13:07.576 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:13:07.576 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:13:07.576 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:13:07.576 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:13:07.576 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:13:07.576 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:13:07.576 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:13:07.576 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:13:07.576 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:13:07.576 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:13:07.576 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:13:07.576 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:13:07.577 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:13:07.577 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:13:07.577 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:13:07.577 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:13:07.577 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:13:07.577 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:13:07.577 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:13:07.577 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:13:07.577 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:13:07.577 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:13:07.584 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:13:07.585 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:13:07.585 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:13:07.585 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:13:07.642 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:13:07.643 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:13:07.643 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:13:07.643 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:13:07.643 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:13:07.643 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:13:07.643 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:13:07.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:13:07.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:13:07.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:13:07.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:13:07.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:13:07.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:13:07.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:13:07.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:13:07.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:13:07.672 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 23 2026-02-18T08:13:07.672 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 24 2026-02-18T08:13:07.672 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:13:07.672 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=97, tag_tid=6, entry_tid=1], [object_number=96, tag_tid=6, entry_tid=0], [object_number=99, tag_tid=5, entry_tid=42603], [object_number=98, tag_tid=5, entry_tid=42602]]], state=connected] 2026-02-18T08:13:07.672 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=92, tag_tid=5, entry_tid=39764], [object_number=95, tag_tid=5, entry_tid=39763], [object_number=94, tag_tid=5, entry_tid=39762], [object_number=93, tag_tid=5, entry_tid=39761]]], state=connected] 2026-02-18T08:13:07.674 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:13:07.676 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=97, tag_tid=6, entry_tid=1' 2026-02-18T08:13:07.677 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:13:07.677 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:13:07.677 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:13:07.677 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:13:07.677 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:13:07.677 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:13:07.677 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:13:07.677 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:13:07.677 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:13:07.678 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:13:07.678 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:13:07.678 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:13:07.678 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:13:07.679 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:13:07.679 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:13:07.705 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 23 2026-02-18T08:13:07.705 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 24 2026-02-18T08:13:07.705 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:13:07.705 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=97, tag_tid=6, entry_tid=1], [object_number=96, tag_tid=6, entry_tid=0], [object_number=99, tag_tid=5, entry_tid=42603], [object_number=98, tag_tid=5, entry_tid=42602]]], state=connected] 2026-02-18T08:13:07.705 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=92, tag_tid=5, entry_tid=39764], [object_number=95, tag_tid=5, entry_tid=39763], [object_number=94, tag_tid=5, entry_tid=39762], [object_number=93, tag_tid=5, entry_tid=39761]]], state=connected] 2026-02-18T08:13:07.707 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:13:07.709 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=92, tag_tid=5, entry_tid=39764' 2026-02-18T08:13:07.709 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=97, tag_tid=6, entry_tid=1' -a 'object_number=97, tag_tid=6, entry_tid=1' = 'object_number=92, tag_tid=5, entry_tid=39764' 2026-02-18T08:13:07.709 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=92, tag_tid=5, entry_tid=39764' '!=' 'object_number=92, tag_tid=5, entry_tid=39496' 2026-02-18T08:13:07.709 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:13:07.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=92, tag_tid=5, entry_tid=39764' = 'object_number=92, tag_tid=5, entry_tid=39496' 2026-02-18T08:13:07.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=92, tag_tid=5, entry_tid=39764' 2026-02-18T08:13:07.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=97, tag_tid=6, entry_tid=1' 2026-02-18T08:13:07.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:13:07.711 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:13:07.711 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=6 2026-02-18T08:13:07.712 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=92, tag_tid=5, entry_tid=39764' 2026-02-18T08:13:07.712 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:13:07.712 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:13:07.713 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=5 2026-02-18T08:13:07.714 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=97, tag_tid=6, entry_tid=1' 2026-02-18T08:13:07.714 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:13:07.714 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:13:07.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:13:07.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=92, tag_tid=5, entry_tid=39764' 2026-02-18T08:13:07.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:13:07.717 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:13:07.718 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=39764 2026-02-18T08:13:07.718 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 6 = 5 -a 1 -le 39764 2026-02-18T08:13:07.718 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:13:07.718 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:13:07.718 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:13:07.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:13:07.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:13:07.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:13:07.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:13:07.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:13:07.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:13:07.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:13:07.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:13:07.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:13:07.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:13:07.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:13:07.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:13:07.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:13:07.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:13:07.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:13:07.924 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:13:07.924 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:13:07.924 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:13:07.924 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:13:07.924 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:13:07.924 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:13:07.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:13:07.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:13:07.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:13:07.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:13:07.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:13:07.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:13:07.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:13:07.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:13:07.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:13:07.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:13:07.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:13:07.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:13:07.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:13:07.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:13:07.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:13:07.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:13:07.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:13:07.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:13:07.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:13:07.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:13:07.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:13:07.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:13:07.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:13:07.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:13:07.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:13:07.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:13:07.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:13:07.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:13:07.936 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:13:07.936 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:13:07.936 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:13:07.936 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:13:08.004 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:13:08.005 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:13:08.005 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:13:08.006 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:13:08.006 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:13:08.006 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:13:08.006 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:13:08.006 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:13:08.006 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:13:08.006 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:13:08.006 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:13:08.006 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:13:08.006 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:13:08.007 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:13:08.007 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:13:08.007 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:13:08.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 23 2026-02-18T08:13:08.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 24 2026-02-18T08:13:08.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:13:08.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=97, tag_tid=6, entry_tid=1], [object_number=96, tag_tid=6, entry_tid=0], [object_number=99, tag_tid=5, entry_tid=42603], [object_number=98, tag_tid=5, entry_tid=42602]]], state=connected] 2026-02-18T08:13:08.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=92, tag_tid=5, entry_tid=40008], [object_number=95, tag_tid=5, entry_tid=40007], [object_number=94, tag_tid=5, entry_tid=40006], [object_number=93, tag_tid=5, entry_tid=40005]]], state=connected] 2026-02-18T08:13:08.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:13:08.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=97, tag_tid=6, entry_tid=1' 2026-02-18T08:13:08.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:13:08.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:13:08.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:13:08.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:13:08.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:13:08.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:13:08.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:13:08.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:13:08.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:13:08.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:13:08.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:13:08.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:13:08.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:13:08.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:13:08.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:13:08.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 23 2026-02-18T08:13:08.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 24 2026-02-18T08:13:08.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:13:08.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=97, tag_tid=6, entry_tid=1], [object_number=96, tag_tid=6, entry_tid=0], [object_number=99, tag_tid=5, entry_tid=42603], [object_number=98, tag_tid=5, entry_tid=42602]]], state=connected] 2026-02-18T08:13:08.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=92, tag_tid=5, entry_tid=40008], [object_number=95, tag_tid=5, entry_tid=40007], [object_number=94, tag_tid=5, entry_tid=40006], [object_number=93, tag_tid=5, entry_tid=40005]]], state=connected] 2026-02-18T08:13:08.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:13:08.092 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=92, tag_tid=5, entry_tid=40008' 2026-02-18T08:13:08.092 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=97, tag_tid=6, entry_tid=1' -a 'object_number=97, tag_tid=6, entry_tid=1' = 'object_number=92, tag_tid=5, entry_tid=40008' 2026-02-18T08:13:08.093 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=92, tag_tid=5, entry_tid=40008' '!=' 'object_number=92, tag_tid=5, entry_tid=39764' 2026-02-18T08:13:08.093 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:13:08.093 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=92, tag_tid=5, entry_tid=40008' = 'object_number=92, tag_tid=5, entry_tid=39764' 2026-02-18T08:13:08.093 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=92, tag_tid=5, entry_tid=40008' 2026-02-18T08:13:08.094 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=97, tag_tid=6, entry_tid=1' 2026-02-18T08:13:08.094 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:13:08.094 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:13:08.094 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=6 2026-02-18T08:13:08.095 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=92, tag_tid=5, entry_tid=40008' 2026-02-18T08:13:08.095 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:13:08.095 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:13:08.097 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=5 2026-02-18T08:13:08.097 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=97, tag_tid=6, entry_tid=1' 2026-02-18T08:13:08.097 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:13:08.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:13:08.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:13:08.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=92, tag_tid=5, entry_tid=40008' 2026-02-18T08:13:08.100 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:13:08.100 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:13:08.101 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=40008 2026-02-18T08:13:08.101 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 6 = 5 -a 1 -le 40008 2026-02-18T08:13:08.101 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:13:08.102 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:13:08.102 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:13:08.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:13:08.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:13:08.304 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:13:08.304 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:13:08.304 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:13:08.304 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:13:08.304 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:13:08.304 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:13:08.304 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:13:08.304 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:13:08.304 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:13:08.304 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:13:08.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:13:08.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:13:08.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:13:08.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:13:08.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:13:08.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:13:08.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:13:08.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:13:08.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:13:08.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:13:08.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:13:08.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:13:08.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:13:08.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:13:08.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:13:08.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:13:08.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:13:08.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:13:08.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:13:08.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:13:08.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:13:08.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:13:08.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:13:08.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:13:08.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:13:08.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:13:08.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:13:08.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:13:08.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:13:08.310 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:13:08.310 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:13:08.310 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:13:08.310 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:13:08.310 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:13:08.310 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:13:08.310 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:13:08.310 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:13:08.319 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:13:08.319 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:13:08.319 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:13:08.319 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:13:08.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:13:08.379 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:13:08.379 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:13:08.379 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:13:08.379 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:13:08.379 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:13:08.379 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:13:08.379 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:13:08.379 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:13:08.379 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:13:08.380 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:13:08.380 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:13:08.380 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:13:08.381 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:13:08.381 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:13:08.381 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:13:08.417 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 23 2026-02-18T08:13:08.417 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 24 2026-02-18T08:13:08.417 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:13:08.417 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=97, tag_tid=6, entry_tid=1], [object_number=96, tag_tid=6, entry_tid=0], [object_number=99, tag_tid=5, entry_tid=42603], [object_number=98, tag_tid=5, entry_tid=42602]]], state=connected] 2026-02-18T08:13:08.417 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=93, tag_tid=5, entry_tid=40293], [object_number=92, tag_tid=5, entry_tid=40292], [object_number=95, tag_tid=5, entry_tid=40291], [object_number=94, tag_tid=5, entry_tid=40290]]], state=connected] 2026-02-18T08:13:08.420 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:13:08.422 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=97, tag_tid=6, entry_tid=1' 2026-02-18T08:13:08.423 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:13:08.423 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:13:08.423 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:13:08.423 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:13:08.423 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:13:08.423 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:13:08.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:13:08.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:13:08.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:13:08.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:13:08.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:13:08.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:13:08.425 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:13:08.425 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:13:08.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:13:08.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 23 2026-02-18T08:13:08.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 24 2026-02-18T08:13:08.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:13:08.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=97, tag_tid=6, entry_tid=1], [object_number=96, tag_tid=6, entry_tid=0], [object_number=99, tag_tid=5, entry_tid=42603], [object_number=98, tag_tid=5, entry_tid=42602]]], state=connected] 2026-02-18T08:13:08.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=93, tag_tid=5, entry_tid=40293], [object_number=92, tag_tid=5, entry_tid=40292], [object_number=95, tag_tid=5, entry_tid=40291], [object_number=94, tag_tid=5, entry_tid=40290]]], state=connected] 2026-02-18T08:13:08.465 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:13:08.467 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=93, tag_tid=5, entry_tid=40293' 2026-02-18T08:13:08.467 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=97, tag_tid=6, entry_tid=1' -a 'object_number=97, tag_tid=6, entry_tid=1' = 'object_number=93, tag_tid=5, entry_tid=40293' 2026-02-18T08:13:08.467 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=93, tag_tid=5, entry_tid=40293' '!=' 'object_number=92, tag_tid=5, entry_tid=40008' 2026-02-18T08:13:08.467 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:13:08.467 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=93, tag_tid=5, entry_tid=40293' = 'object_number=92, tag_tid=5, entry_tid=40008' 2026-02-18T08:13:08.467 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=93, tag_tid=5, entry_tid=40293' 2026-02-18T08:13:08.467 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=97, tag_tid=6, entry_tid=1' 2026-02-18T08:13:08.468 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:13:08.468 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:13:08.468 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=6 2026-02-18T08:13:08.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=93, tag_tid=5, entry_tid=40293' 2026-02-18T08:13:08.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:13:08.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:13:08.470 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=5 2026-02-18T08:13:08.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=97, tag_tid=6, entry_tid=1' 2026-02-18T08:13:08.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:13:08.472 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:13:08.473 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:13:08.474 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=93, tag_tid=5, entry_tid=40293' 2026-02-18T08:13:08.474 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:13:08.474 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:13:08.475 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=40293 2026-02-18T08:13:08.475 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 6 = 5 -a 1 -le 40293 2026-02-18T08:13:08.475 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:13:08.475 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:13:08.476 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:13:08.677 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:13:08.677 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:13:08.677 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:13:08.677 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:13:08.677 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:13:08.677 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:13:08.678 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:13:08.678 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:13:08.678 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:13:08.678 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:13:08.678 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:13:08.678 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:13:08.678 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:13:08.678 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:13:08.678 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:13:08.681 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:13:08.681 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:13:08.681 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:13:08.681 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:13:08.681 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:13:08.681 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:13:08.681 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:13:08.681 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:13:08.682 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:13:08.682 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:13:08.682 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:13:08.682 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:13:08.682 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:13:08.682 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:13:08.682 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:13:08.682 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:13:08.682 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:13:08.682 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:13:08.682 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:13:08.682 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:13:08.683 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:13:08.683 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:13:08.683 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:13:08.683 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:13:08.683 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:13:08.683 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:13:08.683 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:13:08.683 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:13:08.683 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:13:08.683 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:13:08.683 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:13:08.683 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:13:08.684 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:13:08.684 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:13:08.693 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:13:08.693 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:13:08.693 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:13:08.693 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:13:08.752 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:13:08.753 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:13:08.753 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:13:08.753 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:13:08.754 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:13:08.754 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:13:08.754 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:13:08.754 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:13:08.754 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:13:08.754 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:13:08.754 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:13:08.754 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:13:08.754 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:13:08.755 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:13:08.755 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:13:08.755 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:13:08.781 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 24 2026-02-18T08:13:08.781 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 24 2026-02-18T08:13:08.782 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:13:08.782 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=97, tag_tid=6, entry_tid=1], [object_number=96, tag_tid=6, entry_tid=0], [object_number=99, tag_tid=5, entry_tid=42603], [object_number=98, tag_tid=5, entry_tid=42602]]], state=connected] 2026-02-18T08:13:08.782 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=99, tag_tid=5, entry_tid=40507], [object_number=98, tag_tid=5, entry_tid=40506], [object_number=97, tag_tid=5, entry_tid=40505], [object_number=96, tag_tid=5, entry_tid=40504]]], state=connected] 2026-02-18T08:13:08.784 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:13:08.786 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=97, tag_tid=6, entry_tid=1' 2026-02-18T08:13:08.787 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:13:08.787 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:13:08.787 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:13:08.787 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:13:08.787 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:13:08.788 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:13:08.788 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:13:08.788 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:13:08.788 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:13:08.788 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:13:08.788 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:13:08.788 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:13:08.788 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:13:08.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:13:08.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:13:08.815 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 24 2026-02-18T08:13:08.815 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 24 2026-02-18T08:13:08.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:13:08.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=97, tag_tid=6, entry_tid=1], [object_number=96, tag_tid=6, entry_tid=0], [object_number=99, tag_tid=5, entry_tid=42603], [object_number=98, tag_tid=5, entry_tid=42602]]], state=connected] 2026-02-18T08:13:08.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=99, tag_tid=5, entry_tid=40507], [object_number=98, tag_tid=5, entry_tid=40506], [object_number=97, tag_tid=5, entry_tid=40505], [object_number=96, tag_tid=5, entry_tid=40504]]], state=connected] 2026-02-18T08:13:08.818 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:13:08.820 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=99, tag_tid=5, entry_tid=40507' 2026-02-18T08:13:08.820 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=97, tag_tid=6, entry_tid=1' -a 'object_number=97, tag_tid=6, entry_tid=1' = 'object_number=99, tag_tid=5, entry_tid=40507' 2026-02-18T08:13:08.820 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=99, tag_tid=5, entry_tid=40507' '!=' 'object_number=93, tag_tid=5, entry_tid=40293' 2026-02-18T08:13:08.820 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:13:08.821 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=99, tag_tid=5, entry_tid=40507' = 'object_number=93, tag_tid=5, entry_tid=40293' 2026-02-18T08:13:08.821 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=99, tag_tid=5, entry_tid=40507' 2026-02-18T08:13:08.821 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=97, tag_tid=6, entry_tid=1' 2026-02-18T08:13:08.821 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:13:08.822 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:13:08.822 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=6 2026-02-18T08:13:08.823 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=99, tag_tid=5, entry_tid=40507' 2026-02-18T08:13:08.823 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:13:08.823 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:13:08.824 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=5 2026-02-18T08:13:08.825 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=97, tag_tid=6, entry_tid=1' 2026-02-18T08:13:08.825 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:13:08.825 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:13:08.826 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:13:08.827 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=99, tag_tid=5, entry_tid=40507' 2026-02-18T08:13:08.827 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:13:08.827 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:13:08.829 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=40507 2026-02-18T08:13:08.829 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 6 = 5 -a 1 -le 40507 2026-02-18T08:13:08.829 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:13:08.829 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:13:08.829 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:13:09.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:13:09.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:13:09.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:13:09.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:13:09.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:13:09.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:13:09.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:13:09.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:13:09.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:13:09.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:13:09.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:13:09.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:13:09.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:13:09.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:13:09.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:13:09.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:13:09.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:13:09.035 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:13:09.036 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:13:09.036 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:13:09.036 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:13:09.036 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:13:09.036 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:13:09.036 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:13:09.036 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:13:09.036 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:13:09.036 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:13:09.036 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:13:09.036 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:13:09.037 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:13:09.037 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:13:09.037 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:13:09.037 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:13:09.037 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:13:09.037 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:13:09.037 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:13:09.037 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:13:09.037 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:13:09.037 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:13:09.037 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:13:09.037 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:13:09.037 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:13:09.038 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:13:09.038 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:13:09.038 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:13:09.038 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:13:09.038 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:13:09.038 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:13:09.038 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:13:09.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:13:09.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:13:09.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:13:09.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:13:09.114 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:13:09.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:13:09.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:13:09.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:13:09.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:13:09.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:13:09.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:13:09.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:13:09.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:13:09.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:13:09.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:13:09.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:13:09.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:13:09.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:13:09.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:13:09.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:13:09.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 24 2026-02-18T08:13:09.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 24 2026-02-18T08:13:09.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:13:09.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=97, tag_tid=6, entry_tid=1], [object_number=96, tag_tid=6, entry_tid=0], [object_number=99, tag_tid=5, entry_tid=42603], [object_number=98, tag_tid=5, entry_tid=42602]]], state=connected] 2026-02-18T08:13:09.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=99, tag_tid=5, entry_tid=40747], [object_number=98, tag_tid=5, entry_tid=40746], [object_number=97, tag_tid=5, entry_tid=40745], [object_number=96, tag_tid=5, entry_tid=40744]]], state=connected] 2026-02-18T08:13:09.156 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:13:09.157 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=97, tag_tid=6, entry_tid=1' 2026-02-18T08:13:09.158 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:13:09.158 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:13:09.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:13:09.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:13:09.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:13:09.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:13:09.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:13:09.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:13:09.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:13:09.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:13:09.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:13:09.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:13:09.161 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:13:09.161 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:13:09.161 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:13:09.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 24 2026-02-18T08:13:09.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 24 2026-02-18T08:13:09.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:13:09.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=97, tag_tid=6, entry_tid=1], [object_number=96, tag_tid=6, entry_tid=0], [object_number=99, tag_tid=5, entry_tid=42603], [object_number=98, tag_tid=5, entry_tid=42602]]], state=connected] 2026-02-18T08:13:09.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=99, tag_tid=5, entry_tid=40747], [object_number=98, tag_tid=5, entry_tid=40746], [object_number=97, tag_tid=5, entry_tid=40745], [object_number=96, tag_tid=5, entry_tid=40744]]], state=connected] 2026-02-18T08:13:09.189 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:13:09.190 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=99, tag_tid=5, entry_tid=40747' 2026-02-18T08:13:09.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=97, tag_tid=6, entry_tid=1' -a 'object_number=97, tag_tid=6, entry_tid=1' = 'object_number=99, tag_tid=5, entry_tid=40747' 2026-02-18T08:13:09.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=99, tag_tid=5, entry_tid=40747' '!=' 'object_number=99, tag_tid=5, entry_tid=40507' 2026-02-18T08:13:09.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:13:09.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=99, tag_tid=5, entry_tid=40747' = 'object_number=99, tag_tid=5, entry_tid=40507' 2026-02-18T08:13:09.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=99, tag_tid=5, entry_tid=40747' 2026-02-18T08:13:09.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=97, tag_tid=6, entry_tid=1' 2026-02-18T08:13:09.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:13:09.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:13:09.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=6 2026-02-18T08:13:09.194 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=99, tag_tid=5, entry_tid=40747' 2026-02-18T08:13:09.194 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:13:09.194 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:13:09.195 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=5 2026-02-18T08:13:09.196 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=97, tag_tid=6, entry_tid=1' 2026-02-18T08:13:09.196 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:13:09.196 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:13:09.197 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:13:09.198 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=99, tag_tid=5, entry_tid=40747' 2026-02-18T08:13:09.198 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:13:09.198 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:13:09.199 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=40747 2026-02-18T08:13:09.200 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 6 = 5 -a 1 -le 40747 2026-02-18T08:13:09.200 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:13:09.200 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:13:09.200 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:13:09.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:13:09.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:13:09.403 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:13:09.403 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:13:09.403 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:13:09.403 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:13:09.403 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:13:09.403 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:13:09.403 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:13:09.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:13:09.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:13:09.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:13:09.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:13:09.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:13:09.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:13:09.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:13:09.409 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:13:09.409 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:13:09.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:13:09.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:13:09.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:13:09.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:13:09.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:13:09.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:13:09.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:13:09.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:13:09.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:13:09.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:13:09.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:13:09.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:13:09.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:13:09.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:13:09.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:13:09.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:13:09.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:13:09.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:13:09.412 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:13:09.412 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:13:09.412 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:13:09.412 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:13:09.412 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:13:09.412 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:13:09.412 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:13:09.412 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:13:09.412 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:13:09.412 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:13:09.413 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:13:09.413 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:13:09.413 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:13:09.420 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:13:09.420 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:13:09.420 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:13:09.420 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:13:09.478 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:13:09.479 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:13:09.480 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:13:09.480 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:13:09.480 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:13:09.480 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:13:09.480 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:13:09.480 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:13:09.480 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:13:09.480 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:13:09.480 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:13:09.480 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:13:09.481 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:13:09.481 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:13:09.481 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:13:09.482 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:13:09.508 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 24 2026-02-18T08:13:09.509 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 24 2026-02-18T08:13:09.509 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:13:09.509 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=97, tag_tid=6, entry_tid=1], [object_number=96, tag_tid=6, entry_tid=0], [object_number=99, tag_tid=5, entry_tid=42603], [object_number=98, tag_tid=5, entry_tid=42602]]], state=connected] 2026-02-18T08:13:09.509 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=99, tag_tid=5, entry_tid=40991], [object_number=98, tag_tid=5, entry_tid=40990], [object_number=97, tag_tid=5, entry_tid=40989], [object_number=96, tag_tid=5, entry_tid=40988]]], state=connected] 2026-02-18T08:13:09.511 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:13:09.512 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=97, tag_tid=6, entry_tid=1' 2026-02-18T08:13:09.513 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:13:09.513 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:13:09.513 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:13:09.513 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:13:09.513 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:13:09.513 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:13:09.513 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:13:09.514 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:13:09.514 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:13:09.514 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:13:09.514 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:13:09.514 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:13:09.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:13:09.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:13:09.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:13:09.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 24 2026-02-18T08:13:09.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 24 2026-02-18T08:13:09.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:13:09.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=97, tag_tid=6, entry_tid=1], [object_number=96, tag_tid=6, entry_tid=0], [object_number=99, tag_tid=5, entry_tid=42603], [object_number=98, tag_tid=5, entry_tid=42602]]], state=connected] 2026-02-18T08:13:09.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=99, tag_tid=5, entry_tid=40991], [object_number=98, tag_tid=5, entry_tid=40990], [object_number=97, tag_tid=5, entry_tid=40989], [object_number=96, tag_tid=5, entry_tid=40988]]], state=connected] 2026-02-18T08:13:09.545 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:13:09.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=99, tag_tid=5, entry_tid=40991' 2026-02-18T08:13:09.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=97, tag_tid=6, entry_tid=1' -a 'object_number=97, tag_tid=6, entry_tid=1' = 'object_number=99, tag_tid=5, entry_tid=40991' 2026-02-18T08:13:09.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=99, tag_tid=5, entry_tid=40991' '!=' 'object_number=99, tag_tid=5, entry_tid=40747' 2026-02-18T08:13:09.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:13:09.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=99, tag_tid=5, entry_tid=40991' = 'object_number=99, tag_tid=5, entry_tid=40747' 2026-02-18T08:13:09.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=99, tag_tid=5, entry_tid=40991' 2026-02-18T08:13:09.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=97, tag_tid=6, entry_tid=1' 2026-02-18T08:13:09.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:13:09.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:13:09.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=6 2026-02-18T08:13:09.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=99, tag_tid=5, entry_tid=40991' 2026-02-18T08:13:09.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:13:09.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:13:09.551 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=5 2026-02-18T08:13:09.552 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=97, tag_tid=6, entry_tid=1' 2026-02-18T08:13:09.552 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:13:09.552 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:13:09.553 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:13:09.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=99, tag_tid=5, entry_tid=40991' 2026-02-18T08:13:09.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:13:09.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:13:09.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=40991 2026-02-18T08:13:09.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 6 = 5 -a 1 -le 40991 2026-02-18T08:13:09.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:13:09.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:13:09.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:13:09.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:13:09.758 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:13:09.758 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:13:09.758 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:13:09.758 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:13:09.758 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:13:09.758 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:13:09.758 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:13:09.758 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:13:09.759 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:13:09.759 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:13:09.759 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:13:09.759 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:13:09.759 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:13:09.759 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:13:09.759 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:13:09.762 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:13:09.762 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:13:09.762 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:13:09.762 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:13:09.762 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:13:09.762 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:13:09.762 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:13:09.762 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:13:09.763 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:13:09.763 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:13:09.763 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:13:09.763 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:13:09.763 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:13:09.763 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:13:09.763 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:13:09.763 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:13:09.763 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:13:09.763 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:13:09.763 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:13:09.763 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:13:09.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:13:09.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:13:09.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:13:09.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:13:09.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:13:09.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:13:09.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:13:09.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:13:09.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:13:09.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:13:09.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:13:09.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:13:09.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:13:09.771 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:13:09.772 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:13:09.772 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:13:09.772 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:13:09.838 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:13:09.839 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:13:09.839 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:13:09.839 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:13:09.839 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:13:09.840 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:13:09.840 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:13:09.840 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:13:09.840 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:13:09.840 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:13:09.840 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:13:09.840 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:13:09.840 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:13:09.841 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:13:09.842 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:13:09.842 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:13:09.870 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 24 2026-02-18T08:13:09.870 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 24 2026-02-18T08:13:09.870 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:13:09.870 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=97, tag_tid=6, entry_tid=1], [object_number=96, tag_tid=6, entry_tid=0], [object_number=99, tag_tid=5, entry_tid=42603], [object_number=98, tag_tid=5, entry_tid=42602]]], state=connected] 2026-02-18T08:13:09.870 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=98, tag_tid=5, entry_tid=41254], [object_number=97, tag_tid=5, entry_tid=41253], [object_number=96, tag_tid=5, entry_tid=41252], [object_number=99, tag_tid=5, entry_tid=41251]]], state=connected] 2026-02-18T08:13:09.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:13:09.874 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=97, tag_tid=6, entry_tid=1' 2026-02-18T08:13:09.876 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:13:09.876 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:13:09.876 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:13:09.876 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:13:09.876 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:13:09.876 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:13:09.876 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:13:09.876 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:13:09.876 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:13:09.876 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:13:09.877 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:13:09.877 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:13:09.878 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:13:09.878 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:13:09.878 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:13:09.905 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 24 2026-02-18T08:13:09.905 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 24 2026-02-18T08:13:09.905 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:13:09.905 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=97, tag_tid=6, entry_tid=1], [object_number=96, tag_tid=6, entry_tid=0], [object_number=99, tag_tid=5, entry_tid=42603], [object_number=98, tag_tid=5, entry_tid=42602]]], state=connected] 2026-02-18T08:13:09.905 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=98, tag_tid=5, entry_tid=41254], [object_number=97, tag_tid=5, entry_tid=41253], [object_number=96, tag_tid=5, entry_tid=41252], [object_number=99, tag_tid=5, entry_tid=41251]]], state=connected] 2026-02-18T08:13:09.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:13:09.909 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=98, tag_tid=5, entry_tid=41254' 2026-02-18T08:13:09.909 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=97, tag_tid=6, entry_tid=1' -a 'object_number=97, tag_tid=6, entry_tid=1' = 'object_number=98, tag_tid=5, entry_tid=41254' 2026-02-18T08:13:09.909 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=98, tag_tid=5, entry_tid=41254' '!=' 'object_number=99, tag_tid=5, entry_tid=40991' 2026-02-18T08:13:09.909 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:13:09.909 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=98, tag_tid=5, entry_tid=41254' = 'object_number=99, tag_tid=5, entry_tid=40991' 2026-02-18T08:13:09.910 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=98, tag_tid=5, entry_tid=41254' 2026-02-18T08:13:09.910 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=97, tag_tid=6, entry_tid=1' 2026-02-18T08:13:09.910 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:13:09.910 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:13:09.911 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=6 2026-02-18T08:13:09.912 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=98, tag_tid=5, entry_tid=41254' 2026-02-18T08:13:09.912 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:13:09.912 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:13:09.913 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=5 2026-02-18T08:13:09.914 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=97, tag_tid=6, entry_tid=1' 2026-02-18T08:13:09.914 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:13:09.914 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:13:09.916 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:13:09.916 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=98, tag_tid=5, entry_tid=41254' 2026-02-18T08:13:09.917 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:13:09.917 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:13:09.918 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=41254 2026-02-18T08:13:09.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 6 = 5 -a 1 -le 41254 2026-02-18T08:13:09.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:13:09.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:13:09.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:13:10.120 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:13:10.121 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:13:10.121 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:13:10.121 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:13:10.121 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:13:10.121 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:13:10.121 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:13:10.121 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:13:10.121 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:13:10.121 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:13:10.121 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:13:10.122 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:13:10.122 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:13:10.122 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:13:10.122 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:13:10.122 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:13:10.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:13:10.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:13:10.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:13:10.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:13:10.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:13:10.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:13:10.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:13:10.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:13:10.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:13:10.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:13:10.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:13:10.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:13:10.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:13:10.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:13:10.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:13:10.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:13:10.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:13:10.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:13:10.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:13:10.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:13:10.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:13:10.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:13:10.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:13:10.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:13:10.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:13:10.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:13:10.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:13:10.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:13:10.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:13:10.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:13:10.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:13:10.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:13:10.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:13:10.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:13:10.136 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:13:10.136 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:13:10.136 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:13:10.196 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:13:10.197 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:13:10.197 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:13:10.197 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:13:10.197 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:13:10.197 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:13:10.197 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:13:10.197 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:13:10.198 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:13:10.198 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:13:10.198 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:13:10.198 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:13:10.198 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:13:10.199 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:13:10.199 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:13:10.199 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:13:10.225 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 24 2026-02-18T08:13:10.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 24 2026-02-18T08:13:10.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:13:10.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=97, tag_tid=6, entry_tid=1], [object_number=96, tag_tid=6, entry_tid=0], [object_number=99, tag_tid=5, entry_tid=42603], [object_number=98, tag_tid=5, entry_tid=42602]]], state=connected] 2026-02-18T08:13:10.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=97, tag_tid=5, entry_tid=41521], [object_number=96, tag_tid=5, entry_tid=41520], [object_number=99, tag_tid=5, entry_tid=41519], [object_number=98, tag_tid=5, entry_tid=41518]]], state=connected] 2026-02-18T08:13:10.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:13:10.229 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=97, tag_tid=6, entry_tid=1' 2026-02-18T08:13:10.231 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:13:10.231 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:13:10.231 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:13:10.231 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:13:10.231 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:13:10.231 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:13:10.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:13:10.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:13:10.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:13:10.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:13:10.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:13:10.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:13:10.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:13:10.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:13:10.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:13:10.259 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 24 2026-02-18T08:13:10.260 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 24 2026-02-18T08:13:10.260 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:13:10.260 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=97, tag_tid=6, entry_tid=1], [object_number=96, tag_tid=6, entry_tid=0], [object_number=99, tag_tid=5, entry_tid=42603], [object_number=98, tag_tid=5, entry_tid=42602]]], state=connected] 2026-02-18T08:13:10.260 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=97, tag_tid=5, entry_tid=41521], [object_number=96, tag_tid=5, entry_tid=41520], [object_number=99, tag_tid=5, entry_tid=41519], [object_number=98, tag_tid=5, entry_tid=41518]]], state=connected] 2026-02-18T08:13:10.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:13:10.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=97, tag_tid=5, entry_tid=41521' 2026-02-18T08:13:10.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=97, tag_tid=6, entry_tid=1' -a 'object_number=97, tag_tid=6, entry_tid=1' = 'object_number=97, tag_tid=5, entry_tid=41521' 2026-02-18T08:13:10.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=97, tag_tid=5, entry_tid=41521' '!=' 'object_number=98, tag_tid=5, entry_tid=41254' 2026-02-18T08:13:10.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:13:10.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=97, tag_tid=5, entry_tid=41521' = 'object_number=98, tag_tid=5, entry_tid=41254' 2026-02-18T08:13:10.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=97, tag_tid=5, entry_tid=41521' 2026-02-18T08:13:10.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=97, tag_tid=6, entry_tid=1' 2026-02-18T08:13:10.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:13:10.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:13:10.266 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=6 2026-02-18T08:13:10.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=97, tag_tid=5, entry_tid=41521' 2026-02-18T08:13:10.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:13:10.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:13:10.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=5 2026-02-18T08:13:10.269 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=97, tag_tid=6, entry_tid=1' 2026-02-18T08:13:10.269 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:13:10.269 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:13:10.270 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:13:10.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=97, tag_tid=5, entry_tid=41521' 2026-02-18T08:13:10.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:13:10.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:13:10.273 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=41521 2026-02-18T08:13:10.273 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 6 = 5 -a 1 -le 41521 2026-02-18T08:13:10.273 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:13:10.273 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:13:10.273 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:13:10.474 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:13:10.475 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:13:10.475 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:13:10.475 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:13:10.475 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:13:10.475 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:13:10.475 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:13:10.475 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:13:10.475 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:13:10.476 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:13:10.476 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:13:10.476 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:13:10.476 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:13:10.476 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:13:10.476 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:13:10.476 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:13:10.479 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:13:10.479 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:13:10.479 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:13:10.479 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:13:10.479 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:13:10.479 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:13:10.479 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:13:10.479 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:13:10.479 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:13:10.479 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:13:10.480 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:13:10.480 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:13:10.480 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:13:10.480 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:13:10.480 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:13:10.480 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:13:10.480 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:13:10.480 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:13:10.480 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:13:10.480 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:13:10.480 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:13:10.480 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:13:10.480 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:13:10.481 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:13:10.481 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:13:10.481 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:13:10.481 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:13:10.481 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:13:10.481 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:13:10.481 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:13:10.481 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:13:10.481 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:13:10.481 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:13:10.489 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:13:10.489 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:13:10.489 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:13:10.490 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:13:10.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:13:10.551 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:13:10.551 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:13:10.551 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:13:10.551 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:13:10.551 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:13:10.551 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:13:10.551 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:13:10.551 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:13:10.552 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:13:10.552 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:13:10.552 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:13:10.552 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:13:10.553 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:13:10.553 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:13:10.553 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:13:10.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 24 2026-02-18T08:13:10.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 24 2026-02-18T08:13:10.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:13:10.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=97, tag_tid=6, entry_tid=1], [object_number=96, tag_tid=6, entry_tid=0], [object_number=99, tag_tid=5, entry_tid=42603], [object_number=98, tag_tid=5, entry_tid=42602]]], state=connected] 2026-02-18T08:13:10.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=96, tag_tid=5, entry_tid=41772], [object_number=99, tag_tid=5, entry_tid=41771], [object_number=98, tag_tid=5, entry_tid=41770], [object_number=97, tag_tid=5, entry_tid=41769]]], state=connected] 2026-02-18T08:13:10.582 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:13:10.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=97, tag_tid=6, entry_tid=1' 2026-02-18T08:13:10.584 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:13:10.584 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:13:10.584 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:13:10.584 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:13:10.585 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:13:10.585 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:13:10.585 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:13:10.585 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:13:10.585 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:13:10.585 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:13:10.586 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:13:10.586 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:13:10.586 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:13:10.587 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:13:10.587 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:13:10.612 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 24 2026-02-18T08:13:10.612 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 24 2026-02-18T08:13:10.612 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:13:10.612 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=97, tag_tid=6, entry_tid=1], [object_number=96, tag_tid=6, entry_tid=0], [object_number=99, tag_tid=5, entry_tid=42603], [object_number=98, tag_tid=5, entry_tid=42602]]], state=connected] 2026-02-18T08:13:10.612 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=96, tag_tid=5, entry_tid=41772], [object_number=99, tag_tid=5, entry_tid=41771], [object_number=98, tag_tid=5, entry_tid=41770], [object_number=97, tag_tid=5, entry_tid=41769]]], state=connected] 2026-02-18T08:13:10.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:13:10.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=96, tag_tid=5, entry_tid=41772' 2026-02-18T08:13:10.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=97, tag_tid=6, entry_tid=1' -a 'object_number=97, tag_tid=6, entry_tid=1' = 'object_number=96, tag_tid=5, entry_tid=41772' 2026-02-18T08:13:10.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=96, tag_tid=5, entry_tid=41772' '!=' 'object_number=97, tag_tid=5, entry_tid=41521' 2026-02-18T08:13:10.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:13:10.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=96, tag_tid=5, entry_tid=41772' = 'object_number=97, tag_tid=5, entry_tid=41521' 2026-02-18T08:13:10.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=96, tag_tid=5, entry_tid=41772' 2026-02-18T08:13:10.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=97, tag_tid=6, entry_tid=1' 2026-02-18T08:13:10.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:13:10.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:13:10.618 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=6 2026-02-18T08:13:10.619 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=96, tag_tid=5, entry_tid=41772' 2026-02-18T08:13:10.619 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:13:10.619 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:13:10.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=5 2026-02-18T08:13:10.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=97, tag_tid=6, entry_tid=1' 2026-02-18T08:13:10.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:13:10.622 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:13:10.623 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:13:10.623 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=96, tag_tid=5, entry_tid=41772' 2026-02-18T08:13:10.623 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:13:10.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:13:10.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=41772 2026-02-18T08:13:10.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 6 = 5 -a 1 -le 41772 2026-02-18T08:13:10.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:13:10.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:13:10.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:13:10.827 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:13:10.827 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:13:10.827 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:13:10.827 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:13:10.827 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:13:10.828 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:13:10.828 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:13:10.828 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:13:10.828 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:13:10.828 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:13:10.828 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:13:10.828 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:13:10.828 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:13:10.828 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:13:10.828 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:13:10.831 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:13:10.831 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:13:10.831 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:13:10.831 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:13:10.831 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:13:10.831 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:13:10.832 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:13:10.832 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:13:10.832 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:13:10.832 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:13:10.832 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:13:10.832 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:13:10.832 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:13:10.832 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:13:10.832 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:13:10.832 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:13:10.832 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:13:10.832 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:13:10.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:13:10.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:13:10.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:13:10.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:13:10.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:13:10.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:13:10.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:13:10.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:13:10.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:13:10.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:13:10.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:13:10.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:13:10.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:13:10.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:13:10.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:13:10.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:13:10.842 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:13:10.842 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:13:10.842 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:13:10.843 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:13:10.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:13:10.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:13:10.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:13:10.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:13:10.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:13:10.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:13:10.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:13:10.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:13:10.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:13:10.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:13:10.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:13:10.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:13:10.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:13:10.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:13:10.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:13:10.904 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:13:10.931 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 24 2026-02-18T08:13:10.931 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 24 2026-02-18T08:13:10.931 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:13:10.931 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=97, tag_tid=6, entry_tid=1], [object_number=96, tag_tid=6, entry_tid=0], [object_number=99, tag_tid=5, entry_tid=42603], [object_number=98, tag_tid=5, entry_tid=42602]]], state=connected] 2026-02-18T08:13:10.931 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=99, tag_tid=5, entry_tid=41991], [object_number=98, tag_tid=5, entry_tid=41990], [object_number=97, tag_tid=5, entry_tid=41989], [object_number=96, tag_tid=5, entry_tid=41988]]], state=connected] 2026-02-18T08:13:10.933 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:13:10.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=97, tag_tid=6, entry_tid=1' 2026-02-18T08:13:10.936 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:13:10.936 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:13:10.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:13:10.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:13:10.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:13:10.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:13:10.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:13:10.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:13:10.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:13:10.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:13:10.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:13:10.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:13:10.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:13:10.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:13:10.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:13:10.974 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 24 2026-02-18T08:13:10.974 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 24 2026-02-18T08:13:10.974 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:13:10.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=97, tag_tid=6, entry_tid=1], [object_number=96, tag_tid=6, entry_tid=0], [object_number=99, tag_tid=5, entry_tid=42603], [object_number=98, tag_tid=5, entry_tid=42602]]], state=connected] 2026-02-18T08:13:10.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=99, tag_tid=5, entry_tid=41991], [object_number=98, tag_tid=5, entry_tid=41990], [object_number=97, tag_tid=5, entry_tid=41989], [object_number=96, tag_tid=5, entry_tid=41988]]], state=connected] 2026-02-18T08:13:10.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:13:10.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=99, tag_tid=5, entry_tid=41991' 2026-02-18T08:13:10.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=97, tag_tid=6, entry_tid=1' -a 'object_number=97, tag_tid=6, entry_tid=1' = 'object_number=99, tag_tid=5, entry_tid=41991' 2026-02-18T08:13:10.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=99, tag_tid=5, entry_tid=41991' '!=' 'object_number=96, tag_tid=5, entry_tid=41772' 2026-02-18T08:13:10.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:13:10.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=99, tag_tid=5, entry_tid=41991' = 'object_number=96, tag_tid=5, entry_tid=41772' 2026-02-18T08:13:10.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=99, tag_tid=5, entry_tid=41991' 2026-02-18T08:13:10.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=97, tag_tid=6, entry_tid=1' 2026-02-18T08:13:10.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:13:10.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:13:10.982 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=6 2026-02-18T08:13:10.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=99, tag_tid=5, entry_tid=41991' 2026-02-18T08:13:10.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:13:10.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:13:10.985 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=5 2026-02-18T08:13:10.985 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=97, tag_tid=6, entry_tid=1' 2026-02-18T08:13:10.985 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:13:10.986 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:13:10.987 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:13:10.988 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=99, tag_tid=5, entry_tid=41991' 2026-02-18T08:13:10.988 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:13:10.988 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:13:10.990 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=41991 2026-02-18T08:13:10.990 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 6 = 5 -a 1 -le 41991 2026-02-18T08:13:10.990 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:13:10.990 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:13:10.990 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:13:11.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:13:11.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:13:11.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:13:11.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:13:11.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:13:11.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:13:11.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:13:11.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:13:11.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:13:11.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:13:11.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:13:11.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:13:11.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:13:11.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:13:11.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:13:11.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:13:11.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:13:11.195 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:13:11.196 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:13:11.196 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:13:11.196 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:13:11.196 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:13:11.196 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:13:11.196 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:13:11.196 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:13:11.196 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:13:11.196 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:13:11.196 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:13:11.197 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:13:11.197 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:13:11.197 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:13:11.197 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:13:11.197 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:13:11.197 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:13:11.197 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:13:11.197 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:13:11.197 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:13:11.197 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:13:11.197 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:13:11.197 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:13:11.197 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:13:11.198 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:13:11.198 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:13:11.198 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:13:11.198 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:13:11.198 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:13:11.198 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:13:11.198 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:13:11.198 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:13:11.206 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:13:11.207 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:13:11.207 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:13:11.207 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:13:11.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:13:11.266 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:13:11.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:13:11.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:13:11.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:13:11.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:13:11.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:13:11.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:13:11.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:13:11.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:13:11.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:13:11.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:13:11.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:13:11.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:13:11.269 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:13:11.269 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:13:11.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 24 2026-02-18T08:13:11.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 24 2026-02-18T08:13:11.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:13:11.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=97, tag_tid=6, entry_tid=1], [object_number=96, tag_tid=6, entry_tid=0], [object_number=99, tag_tid=5, entry_tid=42603], [object_number=98, tag_tid=5, entry_tid=42602]]], state=connected] 2026-02-18T08:13:11.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=99, tag_tid=5, entry_tid=42223], [object_number=98, tag_tid=5, entry_tid=42222], [object_number=97, tag_tid=5, entry_tid=42221], [object_number=96, tag_tid=5, entry_tid=42220]]], state=connected] 2026-02-18T08:13:11.298 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:13:11.299 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=97, tag_tid=6, entry_tid=1' 2026-02-18T08:13:11.301 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:13:11.301 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:13:11.301 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:13:11.301 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:13:11.301 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:13:11.301 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:13:11.301 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:13:11.301 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:13:11.302 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:13:11.302 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:13:11.302 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:13:11.302 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:13:11.302 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:13:11.302 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:13:11.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:13:11.329 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 24 2026-02-18T08:13:11.329 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 24 2026-02-18T08:13:11.329 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:13:11.329 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=97, tag_tid=6, entry_tid=1], [object_number=96, tag_tid=6, entry_tid=0], [object_number=99, tag_tid=5, entry_tid=42603], [object_number=98, tag_tid=5, entry_tid=42602]]], state=connected] 2026-02-18T08:13:11.329 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=99, tag_tid=5, entry_tid=42223], [object_number=98, tag_tid=5, entry_tid=42222], [object_number=97, tag_tid=5, entry_tid=42221], [object_number=96, tag_tid=5, entry_tid=42220]]], state=connected] 2026-02-18T08:13:11.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:13:11.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=99, tag_tid=5, entry_tid=42223' 2026-02-18T08:13:11.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=97, tag_tid=6, entry_tid=1' -a 'object_number=97, tag_tid=6, entry_tid=1' = 'object_number=99, tag_tid=5, entry_tid=42223' 2026-02-18T08:13:11.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=99, tag_tid=5, entry_tid=42223' '!=' 'object_number=99, tag_tid=5, entry_tid=41991' 2026-02-18T08:13:11.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:13:11.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=99, tag_tid=5, entry_tid=42223' = 'object_number=99, tag_tid=5, entry_tid=41991' 2026-02-18T08:13:11.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=99, tag_tid=5, entry_tid=42223' 2026-02-18T08:13:11.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=97, tag_tid=6, entry_tid=1' 2026-02-18T08:13:11.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:13:11.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:13:11.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=6 2026-02-18T08:13:11.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=99, tag_tid=5, entry_tid=42223' 2026-02-18T08:13:11.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:13:11.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:13:11.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=5 2026-02-18T08:13:11.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=97, tag_tid=6, entry_tid=1' 2026-02-18T08:13:11.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:13:11.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:13:11.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:13:11.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=99, tag_tid=5, entry_tid=42223' 2026-02-18T08:13:11.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:13:11.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:13:11.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=42223 2026-02-18T08:13:11.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 6 = 5 -a 1 -le 42223 2026-02-18T08:13:11.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:13:11.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:13:11.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:13:11.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:13:11.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:13:11.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:13:11.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:13:11.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:13:11.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:13:11.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:13:11.545 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:13:11.545 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:13:11.545 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:13:11.545 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:13:11.545 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:13:11.545 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:13:11.545 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:13:11.545 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:13:11.545 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:13:11.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:13:11.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:13:11.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:13:11.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:13:11.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:13:11.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:13:11.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:13:11.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:13:11.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:13:11.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:13:11.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:13:11.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:13:11.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:13:11.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:13:11.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:13:11.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:13:11.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:13:11.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:13:11.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:13:11.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:13:11.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:13:11.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:13:11.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:13:11.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:13:11.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:13:11.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:13:11.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:13:11.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:13:11.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:13:11.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:13:11.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:13:11.551 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:13:11.551 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:13:11.558 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:13:11.559 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:13:11.559 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:13:11.559 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:13:11.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:13:11.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:13:11.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:13:11.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:13:11.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:13:11.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:13:11.628 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:13:11.628 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:13:11.628 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:13:11.628 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:13:11.628 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:13:11.628 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:13:11.628 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:13:11.628 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:13:11.629 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:13:11.629 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:13:11.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 24 2026-02-18T08:13:11.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 24 2026-02-18T08:13:11.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:13:11.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=97, tag_tid=6, entry_tid=1], [object_number=96, tag_tid=6, entry_tid=0], [object_number=99, tag_tid=5, entry_tid=42603], [object_number=98, tag_tid=5, entry_tid=42602]]], state=connected] 2026-02-18T08:13:11.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=96, tag_tid=5, entry_tid=42496], [object_number=99, tag_tid=5, entry_tid=42495], [object_number=98, tag_tid=5, entry_tid=42494], [object_number=97, tag_tid=5, entry_tid=42493]]], state=connected] 2026-02-18T08:13:11.659 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:13:11.661 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=97, tag_tid=6, entry_tid=1' 2026-02-18T08:13:11.662 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:13:11.662 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:13:11.662 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:13:11.663 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:13:11.663 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:13:11.663 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:13:11.663 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:13:11.663 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:13:11.663 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:13:11.663 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:13:11.664 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:13:11.664 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:13:11.664 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:13:11.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:13:11.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:13:11.691 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 24 2026-02-18T08:13:11.692 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 24 2026-02-18T08:13:11.692 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:13:11.692 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=97, tag_tid=6, entry_tid=1], [object_number=96, tag_tid=6, entry_tid=0], [object_number=99, tag_tid=5, entry_tid=42603], [object_number=98, tag_tid=5, entry_tid=42602]]], state=connected] 2026-02-18T08:13:11.692 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=96, tag_tid=5, entry_tid=42496], [object_number=99, tag_tid=5, entry_tid=42495], [object_number=98, tag_tid=5, entry_tid=42494], [object_number=97, tag_tid=5, entry_tid=42493]]], state=connected] 2026-02-18T08:13:11.694 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:13:11.696 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=96, tag_tid=5, entry_tid=42496' 2026-02-18T08:13:11.696 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=97, tag_tid=6, entry_tid=1' -a 'object_number=97, tag_tid=6, entry_tid=1' = 'object_number=96, tag_tid=5, entry_tid=42496' 2026-02-18T08:13:11.696 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=96, tag_tid=5, entry_tid=42496' '!=' 'object_number=99, tag_tid=5, entry_tid=42223' 2026-02-18T08:13:11.696 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:13:11.696 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=96, tag_tid=5, entry_tid=42496' = 'object_number=99, tag_tid=5, entry_tid=42223' 2026-02-18T08:13:11.696 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=96, tag_tid=5, entry_tid=42496' 2026-02-18T08:13:11.697 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=97, tag_tid=6, entry_tid=1' 2026-02-18T08:13:11.697 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:13:11.697 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:13:11.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=6 2026-02-18T08:13:11.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=96, tag_tid=5, entry_tid=42496' 2026-02-18T08:13:11.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:13:11.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:13:11.700 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=5 2026-02-18T08:13:11.700 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=97, tag_tid=6, entry_tid=1' 2026-02-18T08:13:11.700 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:13:11.700 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:13:11.701 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:13:11.702 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=96, tag_tid=5, entry_tid=42496' 2026-02-18T08:13:11.703 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:13:11.703 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:13:11.704 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=42496 2026-02-18T08:13:11.704 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 6 = 5 -a 1 -le 42496 2026-02-18T08:13:11.705 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:13:11.705 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:13:11.705 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:13:11.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:13:11.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:13:11.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:13:11.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:13:11.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:13:11.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:13:11.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:13:11.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:13:11.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:13:11.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:13:11.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:13:11.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:13:11.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:13:11.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:13:11.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:13:11.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:13:11.910 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:13:11.911 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:13:11.911 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:13:11.911 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:13:11.911 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:13:11.911 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:13:11.911 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:13:11.911 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:13:11.911 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:13:11.911 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:13:11.911 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:13:11.911 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:13:11.911 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:13:11.912 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:13:11.912 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:13:11.912 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:13:11.912 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:13:11.912 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:13:11.912 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:13:11.912 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:13:11.912 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:13:11.912 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:13:11.912 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:13:11.912 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:13:11.912 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:13:11.913 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:13:11.913 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:13:11.913 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:13:11.913 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:13:11.913 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:13:11.913 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:13:11.913 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:13:11.913 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:13:11.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:13:11.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:13:11.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:13:11.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:13:11.989 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:13:11.990 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:13:11.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:13:11.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:13:11.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:13:11.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:13:11.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:13:11.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:13:11.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:13:11.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:13:11.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:13:11.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:13:11.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:13:11.992 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:13:11.993 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:13:11.993 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:13:12.019 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 24 2026-02-18T08:13:12.019 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 24 2026-02-18T08:13:12.019 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:13:12.019 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=97, tag_tid=6, entry_tid=1], [object_number=96, tag_tid=6, entry_tid=0], [object_number=99, tag_tid=5, entry_tid=42603], [object_number=98, tag_tid=5, entry_tid=42602]]], state=connected] 2026-02-18T08:13:12.019 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=97, tag_tid=6, entry_tid=1], [object_number=96, tag_tid=6, entry_tid=0], [object_number=99, tag_tid=5, entry_tid=42603], [object_number=98, tag_tid=5, entry_tid=42602]]], state=connected] 2026-02-18T08:13:12.021 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:13:12.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=97, tag_tid=6, entry_tid=1' 2026-02-18T08:13:12.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:13:12.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:13:12.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:13:12.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:13:12.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:13:12.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:13:12.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:13:12.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:13:12.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:13:12.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:13:12.027 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:13:12.027 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:13:12.027 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:13:12.027 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:13:12.028 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:13:12.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 24 2026-02-18T08:13:12.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 24 2026-02-18T08:13:12.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:13:12.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=97, tag_tid=6, entry_tid=1], [object_number=96, tag_tid=6, entry_tid=0], [object_number=99, tag_tid=5, entry_tid=42603], [object_number=98, tag_tid=5, entry_tid=42602]]], state=connected] 2026-02-18T08:13:12.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=97, tag_tid=6, entry_tid=1], [object_number=96, tag_tid=6, entry_tid=0], [object_number=99, tag_tid=5, entry_tid=42603], [object_number=98, tag_tid=5, entry_tid=42602]]], state=connected] 2026-02-18T08:13:12.056 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:13:12.059 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=97, tag_tid=6, entry_tid=1' 2026-02-18T08:13:12.059 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=97, tag_tid=6, entry_tid=1' -a 'object_number=97, tag_tid=6, entry_tid=1' = 'object_number=97, tag_tid=6, entry_tid=1' 2026-02-18T08:13:12.059 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:13:12.059 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_snap_present cluster1 mirror test snap2 2026-02-18T08:13:12.059 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:13:12.059 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:13:12.059 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:13:12.060 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap2 2026-02-18T08:13:12.060 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-18T08:13:12.060 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-18T08:13:12.060 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1 2026-02-18T08:13:13.060 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster1 info mirror/test@snap2 2026-02-18T08:13:13.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:rbd image 'test': 2026-02-18T08:13:13.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: size 512 MiB in 128 objects 2026-02-18T08:13:13.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: order 22 (4 MiB objects) 2026-02-18T08:13:13.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: snapshot_count: 3 2026-02-18T08:13:13.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: id: 10a74aa8ebc0 2026-02-18T08:13:13.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: block_name_prefix: rbd_data.10a74aa8ebc0 2026-02-18T08:13:13.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: format: 2 2026-02-18T08:13:13.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: features: layering, exclusive-lock, journaling 2026-02-18T08:13:13.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: op_features: 2026-02-18T08:13:13.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: flags: 2026-02-18T08:13:13.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: create_timestamp: Wed Feb 18 08:10:48 2026 2026-02-18T08:13:13.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: access_timestamp: Wed Feb 18 08:12:08 2026 2026-02-18T08:13:13.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: modify_timestamp: Wed Feb 18 08:12:50 2026 2026-02-18T08:13:13.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: protected: False 2026-02-18T08:13:13.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: journal: 10a74aa8ebc0 2026-02-18T08:13:13.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring state: enabled 2026-02-18T08:13:13.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring mode: journal 2026-02-18T08:13:13.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring global id: 0b548823-d751-43df-a076-4efac2da4d3d 2026-02-18T08:13:13.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring primary: false 2026-02-18T08:13:13.092 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:13:13.092 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n snap1-clean ']' 2026-02-18T08:13:13.092 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ compare_image_snaps mirror test snap1-clean 2026-02-18T08:13:13.092 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:13:13.092 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:13:13.093 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap1-clean 2026-02-18T08:13:13.093 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local ret=0 2026-02-18T08:13:13.093 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local rmt_export=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.export 2026-02-18T08:13:13.093 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loc_export=/tmp/tmp.vybh1m9yJF/cluster1-mirror-test.export 2026-02-18T08:13:13.093 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.export /tmp/tmp.vybh1m9yJF/cluster1-mirror-test.export 2026-02-18T08:13:13.093 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 -p mirror export test@snap1-clean /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.export 2026-02-18T08:13:13.561 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-18T08:13:13.574 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster1 -p mirror export test@snap1-clean /tmp/tmp.vybh1m9yJF/cluster1-mirror-test.export 2026-02-18T08:13:14.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-18T08:13:14.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmp /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.export /tmp/tmp.vybh1m9yJF/cluster1-mirror-test.export 2026-02-18T08:13:14.286 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.export /tmp/tmp.vybh1m9yJF/cluster1-mirror-test.export 2026-02-18T08:13:14.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:13:14.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ compare_image_snaps mirror test snap2 2026-02-18T08:13:14.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:13:14.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:13:14.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap2 2026-02-18T08:13:14.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local ret=0 2026-02-18T08:13:14.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local rmt_export=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.export 2026-02-18T08:13:14.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loc_export=/tmp/tmp.vybh1m9yJF/cluster1-mirror-test.export 2026-02-18T08:13:14.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.export /tmp/tmp.vybh1m9yJF/cluster1-mirror-test.export 2026-02-18T08:13:14.412 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 -p mirror export test@snap2 /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.export 2026-02-18T08:13:14.904 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-18T08:13:14.917 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster1 -p mirror export test@snap2 /tmp/tmp.vybh1m9yJF/cluster1-mirror-test.export 2026-02-18T08:13:15.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-18T08:13:15.511 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmp /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.export /tmp/tmp.vybh1m9yJF/cluster1-mirror-test.export 2026-02-18T08:13:15.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.export /tmp/tmp.vybh1m9yJF/cluster1-mirror-test.export 2026-02-18T08:13:15.840 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:13:15.840 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ clean_snap_name=snap2-clean 2026-02-18T08:13:15.841 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_snap cluster2 mirror test snap2-clean 2026-02-18T08:13:15.841 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:13:15.841 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:13:15.841 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:13:15.841 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap2-clean 2026-02-18T08:13:15.841 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 -p mirror snap create test@snap2-clean --debug-rbd=20 --debug-journaler=20 2026-02-18T08:13:16.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 10` 2026-02-18T08:13:16.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ stress_write_image cluster2 mirror test 2026-02-18T08:13:16.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:13:16.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:13:16.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:13:16.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-18T08:13:16.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local duration=13 2026-02-18T08:13:16.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set +e 2026-02-18T08:13:16.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ timeout 13s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror test --debug-rbd=20 --debug-journaler=20 2026-02-18T08:13:17.695 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:100 IOs, 65315840 bytes 2026-02-18T08:13:18.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:200 IOs, 86234112 bytes 2026-02-18T08:13:18.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:300 IOs, 103725056 bytes 2026-02-18T08:13:19.389 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:400 IOs, 162465792 bytes 2026-02-18T08:13:20.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:500 IOs, 205713408 bytes 2026-02-18T08:13:20.677 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:600 IOs, 255861760 bytes 2026-02-18T08:13:21.324 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:700 IOs, 289124352 bytes 2026-02-18T08:13:21.528 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:800 IOs, 316934144 bytes 2026-02-18T08:13:22.214 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:900 IOs, 342787072 bytes 2026-02-18T08:13:22.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1000 IOs, 377123840 bytes 2026-02-18T08:13:22.697 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1100 IOs, 397879296 bytes 2026-02-18T08:13:23.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1200 IOs, 415225856 bytes 2026-02-18T08:13:23.407 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1300 IOs, 447634432 bytes 2026-02-18T08:13:23.720 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1400 IOs, 470838272 bytes 2026-02-18T08:13:23.854 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1500 IOs, 478565376 bytes 2026-02-18T08:13:24.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1600 IOs, 537034752 bytes 2026-02-18T08:13:25.451 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1700 IOs, 628965376 bytes 2026-02-18T08:13:26.601 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1800 IOs, 696186880 bytes 2026-02-18T08:13:27.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1900 IOs, 728360960 bytes 2026-02-18T08:13:27.354 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:2000 IOs, 751555584 bytes 2026-02-18T08:13:27.918 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:2100 IOs, 806868992 bytes 2026-02-18T08:13:28.508 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:2200 IOs, 865680384 bytes 2026-02-18T08:13:28.839 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:2300 IOs, 898013184 bytes 2026-02-18T08:13:29.140 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:2400 IOs, 915876864 bytes 2026-02-18T08:13:29.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:2500 IOs, 926230528 bytes 2026-02-18T08:13:29.647 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ error_code=124 2026-02-18T08:13:29.647 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set -e 2026-02-18T08:13:29.647 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 124 -eq 124 ']' 2026-02-18T08:13:29.647 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:13:29.647 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+replaying 2026-02-18T08:13:29.647 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:13:29.647 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:13:29.648 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:13:29.648 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local state_pattern=up+replaying 2026-02-18T08:13:29.648 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local description_pattern= 2026-02-18T08:13:29.648 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local service_pattern= 2026-02-18T08:13:29.648 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:13:29.648 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1 2026-02-18T08:13:30.648 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test_status_in_pool_dir cluster1 mirror test up+replaying '' '' 2026-02-18T08:13:30.648 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:13:30.648 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:13:30.648 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:13:30.648 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local state_pattern=up+replaying 2026-02-18T08:13:30.649 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local description_pattern= 2026-02-18T08:13:30.649 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local service_pattern= 2026-02-18T08:13:30.649 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local status 2026-02-18T08:13:30.649 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ CEPH_ARGS= 2026-02-18T08:13:30.649 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-18T08:13:30.684 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ status='test: 2026-02-18T08:13:30.684 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: global_id: 0b548823-d751-43df-a076-4efac2da4d3d 2026-02-18T08:13:30.684 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: state: up+replaying 2026-02-18T08:13:30.684 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: description: replaying, {"bytes_per_second":11021382.13,"entries_behind_primary":0,"entries_per_second":690.9,"non_primary_position":{"entry_tid":1,"object_number":97,"tag_tid":6},"primary_position":{"entry_tid":1,"object_number":97,"tag_tid":6}} 2026-02-18T08:13:30.684 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: service: mirror.0 on trial163 2026-02-18T08:13:30.684 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: last_update: 2026-02-18 08:13:16 2026-02-18T08:13:30.685 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: peer_sites: 2026-02-18T08:13:30.685 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: name: cluster2 2026-02-18T08:13:30.685 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: state: up+stopped 2026-02-18T08:13:30.685 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: description: local image is primary 2026-02-18T08:13:30.685 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: last_update: 2026-02-18 08:13:16' 2026-02-18T08:13:30.685 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ grep '^ state: .*up+replaying' 2026-02-18T08:13:30.686 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: state: up+replaying 2026-02-18T08:13:30.686 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ grep '^ description: .*' 2026-02-18T08:13:30.687 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: description: replaying, {"bytes_per_second":11021382.13,"entries_behind_primary":0,"entries_per_second":690.9,"non_primary_position":{"entry_tid":1,"object_number":97,"tag_tid":6},"primary_position":{"entry_tid":1,"object_number":97,"tag_tid":6}} 2026-02-18T08:13:30.687 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n '' ']' 2026-02-18T08:13:30.687 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ echo up+replaying 2026-02-18T08:13:30.687 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ grep '^up+' 2026-02-18T08:13:30.688 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:up+replaying 2026-02-18T08:13:30.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ grep 'service: *mirror..* on ' 2026-02-18T08:13:30.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: service: mirror.0 on trial163 2026-02-18T08:13:30.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local last_update 2026-02-18T08:13:30.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-18T08:13:30.693 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_update='2026-02-18 08:13:16' 2026-02-18T08:13:30.693 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+replaying '2026-02-18 08:13:16' 2026-02-18T08:13:30.693 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:13:30.693 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:13:30.693 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:13:30.693 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local state_pattern=up+replaying 2026-02-18T08:13:30.693 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'prev_last_update=2026-02-18 08:13:16' 2026-02-18T08:13:30.693 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local status 2026-02-18T08:13:30.693 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ CEPH_ARGS= 2026-02-18T08:13:30.694 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-18T08:13:30.734 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ status='OKOKOK142074263mirror.0trial16320.2.0-670-gadc19233falseOK42114265mirror.1trial16320.2.0-670-gadc19233trueOK42194268mirror.3trial16320.2.0-670-gadc19233falseOK43304333mirror.2trial16320.2.0-670-gadc19233falseOKtest0b548823-d751-43df-a076-4efac2da4d3dup+replayingreplaying, {"bytes_per_second":11021382.13,"entries_behind_primary":0,"entries_per_second":690.9,"non_primary_position":{"entry_tid":1,"object_number":97,"tag_tid":6},"primary_position":{"entry_tid":1,"object_number":97,"tag_tid":6}}42074263mirror.0trial1632026-02-18 08:13:16cluster2c5e4f85e-ec31-430b-a71d-56e000d0de0bup+stoppedlocal image is primary2026-02-18 08:13:16' 2026-02-18T08:13:30.734 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local last_update state 2026-02-18T08:13:30.734 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-18T08:13:30.737 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_update='2026-02-18 08:13:16' 2026-02-18T08:13:30.737 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-18T08:13:30.740 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ state=up+replaying 2026-02-18T08:13:30.740 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ echo up+replaying 2026-02-18T08:13:30.741 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ grep up+replaying 2026-02-18T08:13:30.741 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:up+replaying 2026-02-18T08:13:30.742 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:13:30.742 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:13:30.742 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ snap_name=snap3 2026-02-18T08:13:30.743 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_snap cluster2 mirror test snap3 2026-02-18T08:13:30.743 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:13:30.743 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:13:30.743 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:13:30.743 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap3 2026-02-18T08:13:30.743 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 -p mirror snap create test@snap3 --debug-rbd=20 --debug-journaler=20 2026-02-18T08:13:56.922 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.0 with skip (0.2663162084743069) > weight (0.25) 2026-02-18T08:13:56.922 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.1 with skip (0.5805954728408658) > weight (0.25) 2026-02-18T08:13:56.922 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.2 with skip (0.8298811116545354) > weight (0.25) 2026-02-18T08:13:56.922 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.3 with skip (0.822514874218386) > weight (0.25) 2026-02-18T08:13:56.923 INFO:tasks.rbd_mirror_thrash:waiting for 103.45007638754774 secs before thrashing 2026-02-18T08:14:01.662 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_replay_started cluster1 mirror test 2026-02-18T08:14:01.662 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:14:01.663 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:14:01.663 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:14:01.663 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_replay_state cluster1 mirror test started 2026-02-18T08:14:01.663 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:14:01.663 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:14:01.663 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:14:01.663 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local state=started 2026-02-18T08:14:01.663 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-18T08:14:01.663 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:14:01.663 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1 2026-02-18T08:14:02.664 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test_image_replay_state cluster1 mirror test started 2026-02-18T08:14:02.664 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:14:02.664 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:14:02.664 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:14:02.664 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local test_state=started 2026-02-18T08:14:02.664 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local status_result 2026-02-18T08:14:02.664 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_state=stopped 2026-02-18T08:14:02.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-18T08:14:02.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -i state 2026-02-18T08:14:02.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_instance=cluster1 2026-02-18T08:14:02.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-18T08:14:02.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:14:02.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance=cluster1 2026-02-18T08:14:02.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local loop_instance 2026-02-18T08:14:02.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:14:02.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sleep 0 2026-02-18T08:14:02.667 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:14:02.667 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ seq 0 3 2026-02-18T08:14:02.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:14:02.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-18T08:14:02.670 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:0 2026-02-18T08:14:02.670 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-18T08:14:02.670 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:14:02.670 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:14:02.670 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:14:02.670 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-18T08:14:02.670 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:14:02.670 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:14:02.671 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:14:02.671 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:14:02.671 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-18T08:14:02.671 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:14:02.671 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:14:02.671 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:14:02.671 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:14:02.671 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:0 2026-02-18T08:14:02.671 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-18T08:14:02.671 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-18T08:14:02.671 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:14:02.671 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:0 2026-02-18T08:14:02.672 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T08:14:02.672 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-18T08:14:02.672 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:14:02.672 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:14:02.672 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-18T08:14:02.672 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-18T08:14:02.672 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-18T08:14:02.672 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=0 2026-02-18T08:14:02.672 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=0 2026-02-18T08:14:02.672 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:14:02.680 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:02.680 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:02.680 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:02.680 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-18T08:14:02.735 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ return 0 2026-02-18T08:14:02.735 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ status_result=' "state": "Replaying"' 2026-02-18T08:14:02.736 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ echo ' "state": "Replaying"' 2026-02-18T08:14:02.736 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ grep -i Replaying 2026-02-18T08:14:02.737 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: "state": "Replaying" 2026-02-18T08:14:02.738 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ current_state=started 2026-02-18T08:14:02.738 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test started = started 2026-02-18T08:14:02.739 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:14:02.739 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror test 2026-02-18T08:14:02.739 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_cluster=cluster1 2026-02-18T08:14:02.739 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:14:02.739 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_pool=mirror 2026-02-18T08:14:02.739 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local remote_pool=mirror 2026-02-18T08:14:02.739 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:14:02.739 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' journal = journal ']' 2026-02-18T08:14:02.739 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror mirror test 2026-02-18T08:14:02.739 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_cluster=cluster1 2026-02-18T08:14:02.739 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:14:02.739 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_pool=mirror 2026-02-18T08:14:02.739 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local remote_pool=mirror 2026-02-18T08:14:02.740 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:14:02.740 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-02-18T08:14:02.740 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-02-18T08:14:02.740 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:14:02.740 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:14:02.740 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:14:02.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:14:02.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:14:02.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:14:02.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:14:02.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:14:02.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:14:02.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:14:02.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:14:02.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:14:02.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:14:02.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:14:02.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:14:02.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:14:02.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:14:02.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:14:02.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:14:02.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:14:02.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:14:02.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:14:02.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:14:02.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:14:02.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:14:02.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:14:02.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:14:02.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:14:02.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:14:02.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:14:02.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:14:02.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:14:02.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:14:02.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:14:02.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:14:02.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:14:02.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:14:02.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:14:02.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:14:02.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:14:02.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:14:02.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:14:02.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:14:02.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:14:02.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:14:02.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:14:02.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:14:02.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:14:02.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:14:02.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:14:02.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:14:02.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:14:02.953 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:02.954 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:02.954 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:02.954 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:14:03.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:14:03.012 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:14:03.012 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:03.012 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:03.012 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:03.012 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:14:03.012 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:03.012 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:03.012 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:03.012 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:14:03.012 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:14:03.012 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:14:03.012 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:14:03.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:03.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:14:03.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:03.041 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 31 2026-02-18T08:14:03.041 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 38 2026-02-18T08:14:03.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:14:03.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=153, tag_tid=9, entry_tid=1], [object_number=152, tag_tid=9, entry_tid=0], [object_number=154, tag_tid=8, entry_tid=58722], [object_number=155, tag_tid=8, entry_tid=58719]]], state=connected] 2026-02-18T08:14:03.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=127, tag_tid=8, entry_tid=30319], [object_number=126, tag_tid=8, entry_tid=30318], [object_number=125, tag_tid=8, entry_tid=30317], [object_number=124, tag_tid=8, entry_tid=30316]]], state=connected] 2026-02-18T08:14:03.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:03.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:03.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:14:03.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:03.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:03.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:03.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:14:03.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:03.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:03.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:03.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:14:03.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:14:03.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:14:03.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:14:03.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:03.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:14:03.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:03.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 31 2026-02-18T08:14:03.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 38 2026-02-18T08:14:03.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:14:03.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=153, tag_tid=9, entry_tid=1], [object_number=152, tag_tid=9, entry_tid=0], [object_number=154, tag_tid=8, entry_tid=58722], [object_number=155, tag_tid=8, entry_tid=58719]]], state=connected] 2026-02-18T08:14:03.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=127, tag_tid=8, entry_tid=30319], [object_number=126, tag_tid=8, entry_tid=30318], [object_number=125, tag_tid=8, entry_tid=30317], [object_number=124, tag_tid=8, entry_tid=30316]]], state=connected] 2026-02-18T08:14:03.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:03.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=127, tag_tid=8, entry_tid=30319' 2026-02-18T08:14:03.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=153, tag_tid=9, entry_tid=1' -a 'object_number=153, tag_tid=9, entry_tid=1' = 'object_number=127, tag_tid=8, entry_tid=30319' 2026-02-18T08:14:03.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=127, tag_tid=8, entry_tid=30319' '!=' '' 2026-02-18T08:14:03.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:14:03.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=127, tag_tid=8, entry_tid=30319' = '' 2026-02-18T08:14:03.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=127, tag_tid=8, entry_tid=30319' 2026-02-18T08:14:03.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:03.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:14:03.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:03.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=9 2026-02-18T08:14:03.092 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=127, tag_tid=8, entry_tid=30319' 2026-02-18T08:14:03.092 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:14:03.093 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:03.094 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=8 2026-02-18T08:14:03.095 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:03.095 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:14:03.095 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:03.096 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:14:03.097 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=127, tag_tid=8, entry_tid=30319' 2026-02-18T08:14:03.097 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:14:03.097 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:03.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=30319 2026-02-18T08:14:03.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 9 = 8 -a 1 -le 30319 2026-02-18T08:14:03.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:14:03.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:14:03.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:14:03.300 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:14:03.300 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:14:03.300 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:14:03.300 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:14:03.300 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:14:03.300 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:14:03.300 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:14:03.300 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:14:03.300 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:14:03.300 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:14:03.300 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:14:03.300 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:14:03.300 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:14:03.300 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:14:03.300 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:14:03.300 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:14:03.301 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:14:03.302 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:14:03.302 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:14:03.302 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:14:03.302 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:14:03.302 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:14:03.302 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:14:03.302 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:14:03.302 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:14:03.302 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:14:03.302 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:14:03.302 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:14:03.302 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:14:03.302 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:14:03.302 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:14:03.302 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:14:03.302 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:14:03.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:14:03.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:14:03.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:14:03.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:14:03.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:14:03.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:14:03.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:14:03.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:14:03.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:14:03.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:14:03.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:14:03.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:14:03.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:14:03.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:14:03.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:14:03.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:14:03.314 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:03.314 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:03.315 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:03.315 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:14:03.375 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:14:03.376 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:14:03.376 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:03.376 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:03.377 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:03.377 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:14:03.377 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:03.377 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:03.377 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:03.377 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:14:03.377 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:14:03.377 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:14:03.377 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:14:03.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:03.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:14:03.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:03.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 31 2026-02-18T08:14:03.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 38 2026-02-18T08:14:03.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:14:03.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=153, tag_tid=9, entry_tid=1], [object_number=152, tag_tid=9, entry_tid=0], [object_number=154, tag_tid=8, entry_tid=58722], [object_number=155, tag_tid=8, entry_tid=58719]]], state=connected] 2026-02-18T08:14:03.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=127, tag_tid=8, entry_tid=30547], [object_number=126, tag_tid=8, entry_tid=30546], [object_number=125, tag_tid=8, entry_tid=30545], [object_number=124, tag_tid=8, entry_tid=30544]]], state=connected] 2026-02-18T08:14:03.612 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:03.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:03.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:14:03.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:03.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:03.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:03.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:14:03.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:03.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:03.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:03.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:14:03.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:14:03.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:14:03.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:14:03.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:03.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:14:03.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:03.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 31 2026-02-18T08:14:03.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 38 2026-02-18T08:14:03.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:14:03.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=153, tag_tid=9, entry_tid=1], [object_number=152, tag_tid=9, entry_tid=0], [object_number=154, tag_tid=8, entry_tid=58722], [object_number=155, tag_tid=8, entry_tid=58719]]], state=connected] 2026-02-18T08:14:03.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=127, tag_tid=8, entry_tid=30547], [object_number=126, tag_tid=8, entry_tid=30546], [object_number=125, tag_tid=8, entry_tid=30545], [object_number=124, tag_tid=8, entry_tid=30544]]], state=connected] 2026-02-18T08:14:03.647 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:03.649 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=127, tag_tid=8, entry_tid=30547' 2026-02-18T08:14:03.649 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=153, tag_tid=9, entry_tid=1' -a 'object_number=153, tag_tid=9, entry_tid=1' = 'object_number=127, tag_tid=8, entry_tid=30547' 2026-02-18T08:14:03.649 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=127, tag_tid=8, entry_tid=30547' '!=' 'object_number=127, tag_tid=8, entry_tid=30319' 2026-02-18T08:14:03.649 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:14:03.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=127, tag_tid=8, entry_tid=30547' = 'object_number=127, tag_tid=8, entry_tid=30319' 2026-02-18T08:14:03.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=127, tag_tid=8, entry_tid=30547' 2026-02-18T08:14:03.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:03.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:14:03.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:03.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=9 2026-02-18T08:14:03.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=127, tag_tid=8, entry_tid=30547' 2026-02-18T08:14:03.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:14:03.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:03.653 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=8 2026-02-18T08:14:03.654 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:03.654 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:14:03.654 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:03.655 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:14:03.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=127, tag_tid=8, entry_tid=30547' 2026-02-18T08:14:03.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:14:03.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:03.658 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=30547 2026-02-18T08:14:03.658 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 9 = 8 -a 1 -le 30547 2026-02-18T08:14:03.658 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:14:03.658 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:14:03.658 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:14:03.860 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:14:03.860 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:14:03.860 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:14:03.860 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:14:03.861 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:14:03.861 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:14:03.861 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:14:03.861 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:14:03.861 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:14:03.861 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:14:03.861 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:14:03.861 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:14:03.861 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:14:03.861 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:14:03.861 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:14:03.862 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:14:03.862 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:14:03.864 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:14:03.864 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:14:03.864 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:14:03.864 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:14:03.864 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:14:03.864 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:14:03.864 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:14:03.865 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:14:03.865 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:14:03.865 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:14:03.865 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:14:03.865 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:14:03.865 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:14:03.865 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:14:03.865 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:14:03.865 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:14:03.865 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:14:03.866 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:14:03.866 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:14:03.866 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:14:03.866 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:14:03.866 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:14:03.866 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:14:03.866 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:14:03.866 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:14:03.866 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:14:03.866 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:14:03.866 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:14:03.866 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:14:03.866 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:14:03.867 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:14:03.867 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:14:03.875 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:03.875 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:03.875 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:03.875 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:14:03.944 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:14:03.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:14:03.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:03.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:03.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:03.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:14:03.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:03.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:03.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:03.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:14:03.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:14:03.946 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:14:03.946 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:14:03.946 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:03.947 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:14:03.947 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:03.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 31 2026-02-18T08:14:03.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 38 2026-02-18T08:14:03.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:14:03.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=153, tag_tid=9, entry_tid=1], [object_number=152, tag_tid=9, entry_tid=0], [object_number=154, tag_tid=8, entry_tid=58722], [object_number=155, tag_tid=8, entry_tid=58719]]], state=connected] 2026-02-18T08:14:03.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=127, tag_tid=8, entry_tid=30927], [object_number=126, tag_tid=8, entry_tid=30926], [object_number=125, tag_tid=8, entry_tid=30925], [object_number=124, tag_tid=8, entry_tid=30924]]], state=connected] 2026-02-18T08:14:03.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:03.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:03.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:14:03.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:03.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:03.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:03.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:14:03.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:03.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:03.982 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:03.982 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:14:03.982 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:14:03.982 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:14:03.982 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:14:03.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:03.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:14:03.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:04.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 31 2026-02-18T08:14:04.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 38 2026-02-18T08:14:04.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:14:04.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=153, tag_tid=9, entry_tid=1], [object_number=152, tag_tid=9, entry_tid=0], [object_number=154, tag_tid=8, entry_tid=58722], [object_number=155, tag_tid=8, entry_tid=58719]]], state=connected] 2026-02-18T08:14:04.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=127, tag_tid=8, entry_tid=30927], [object_number=126, tag_tid=8, entry_tid=30926], [object_number=125, tag_tid=8, entry_tid=30925], [object_number=124, tag_tid=8, entry_tid=30924]]], state=connected] 2026-02-18T08:14:04.020 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:04.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=127, tag_tid=8, entry_tid=30927' 2026-02-18T08:14:04.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=153, tag_tid=9, entry_tid=1' -a 'object_number=153, tag_tid=9, entry_tid=1' = 'object_number=127, tag_tid=8, entry_tid=30927' 2026-02-18T08:14:04.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=127, tag_tid=8, entry_tid=30927' '!=' 'object_number=127, tag_tid=8, entry_tid=30547' 2026-02-18T08:14:04.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:14:04.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=127, tag_tid=8, entry_tid=30927' = 'object_number=127, tag_tid=8, entry_tid=30547' 2026-02-18T08:14:04.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=127, tag_tid=8, entry_tid=30927' 2026-02-18T08:14:04.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:04.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:14:04.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:04.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=9 2026-02-18T08:14:04.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=127, tag_tid=8, entry_tid=30927' 2026-02-18T08:14:04.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:14:04.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:04.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=8 2026-02-18T08:14:04.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:04.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:14:04.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:04.028 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:14:04.028 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=127, tag_tid=8, entry_tid=30927' 2026-02-18T08:14:04.028 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:14:04.029 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:04.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=30927 2026-02-18T08:14:04.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 9 = 8 -a 1 -le 30927 2026-02-18T08:14:04.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:14:04.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:14:04.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:14:04.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:14:04.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:14:04.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:14:04.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:14:04.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:14:04.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:14:04.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:14:04.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:14:04.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:14:04.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:14:04.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:14:04.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:14:04.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:14:04.234 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:14:04.234 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:14:04.234 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:14:04.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:14:04.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:14:04.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:14:04.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:14:04.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:14:04.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:14:04.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:14:04.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:14:04.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:14:04.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:14:04.238 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:14:04.238 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:14:04.238 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:14:04.238 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:14:04.238 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:14:04.238 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:14:04.238 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:14:04.238 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:14:04.238 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:14:04.238 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:14:04.238 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:14:04.238 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:14:04.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:14:04.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:14:04.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:14:04.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:14:04.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:14:04.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:14:04.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:14:04.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:14:04.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:14:04.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:14:04.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:14:04.246 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:04.247 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:04.247 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:04.247 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:14:04.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:14:04.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:14:04.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:04.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:04.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:04.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:14:04.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:04.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:04.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:04.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:14:04.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:14:04.310 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:14:04.310 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:14:04.310 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:04.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:14:04.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:04.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 31 2026-02-18T08:14:04.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 38 2026-02-18T08:14:04.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:14:04.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=153, tag_tid=9, entry_tid=1], [object_number=152, tag_tid=9, entry_tid=0], [object_number=154, tag_tid=8, entry_tid=58722], [object_number=155, tag_tid=8, entry_tid=58719]]], state=connected] 2026-02-18T08:14:04.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=124, tag_tid=8, entry_tid=31164], [object_number=127, tag_tid=8, entry_tid=31163], [object_number=126, tag_tid=8, entry_tid=31162], [object_number=125, tag_tid=8, entry_tid=31161]]], state=connected] 2026-02-18T08:14:04.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:04.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:04.344 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:14:04.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:04.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:04.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:04.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:14:04.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:04.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:04.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:04.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:14:04.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:14:04.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:14:04.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:14:04.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:04.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:14:04.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:04.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 31 2026-02-18T08:14:04.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 38 2026-02-18T08:14:04.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:14:04.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=153, tag_tid=9, entry_tid=1], [object_number=152, tag_tid=9, entry_tid=0], [object_number=154, tag_tid=8, entry_tid=58722], [object_number=155, tag_tid=8, entry_tid=58719]]], state=connected] 2026-02-18T08:14:04.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=124, tag_tid=8, entry_tid=31164], [object_number=127, tag_tid=8, entry_tid=31163], [object_number=126, tag_tid=8, entry_tid=31162], [object_number=125, tag_tid=8, entry_tid=31161]]], state=connected] 2026-02-18T08:14:04.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:04.585 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=124, tag_tid=8, entry_tid=31164' 2026-02-18T08:14:04.585 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=153, tag_tid=9, entry_tid=1' -a 'object_number=153, tag_tid=9, entry_tid=1' = 'object_number=124, tag_tid=8, entry_tid=31164' 2026-02-18T08:14:04.585 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=124, tag_tid=8, entry_tid=31164' '!=' 'object_number=127, tag_tid=8, entry_tid=30927' 2026-02-18T08:14:04.585 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:14:04.585 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=124, tag_tid=8, entry_tid=31164' = 'object_number=127, tag_tid=8, entry_tid=30927' 2026-02-18T08:14:04.585 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=124, tag_tid=8, entry_tid=31164' 2026-02-18T08:14:04.586 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:04.586 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:14:04.586 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:04.586 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=9 2026-02-18T08:14:04.587 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=124, tag_tid=8, entry_tid=31164' 2026-02-18T08:14:04.587 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:14:04.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:04.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=8 2026-02-18T08:14:04.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:04.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:14:04.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:04.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:14:04.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=124, tag_tid=8, entry_tid=31164' 2026-02-18T08:14:04.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:14:04.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:04.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=31164 2026-02-18T08:14:04.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 9 = 8 -a 1 -le 31164 2026-02-18T08:14:04.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:14:04.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:14:04.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:14:04.795 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:14:04.795 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:14:04.795 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:14:04.796 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:14:04.796 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:14:04.796 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:14:04.796 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:14:04.796 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:14:04.796 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:14:04.796 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:14:04.796 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:14:04.796 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:14:04.796 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:14:04.796 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:14:04.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:14:04.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:14:04.799 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:14:04.799 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:14:04.799 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:14:04.799 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:14:04.800 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:14:04.800 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:14:04.800 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:14:04.800 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:14:04.800 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:14:04.800 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:14:04.800 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:14:04.800 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:14:04.800 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:14:04.800 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:14:04.800 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:14:04.800 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:14:04.801 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:14:04.801 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:14:04.801 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:14:04.801 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:14:04.801 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:14:04.801 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:14:04.801 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:14:04.801 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:14:04.801 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:14:04.801 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:14:04.801 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:14:04.801 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:14:04.801 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:14:04.802 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:14:04.802 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:14:04.802 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:14:04.802 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:14:04.809 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:04.810 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:04.810 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:04.810 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:14:04.869 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:14:04.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:14:04.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:04.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:04.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:04.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:14:04.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:04.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:04.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:04.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:14:04.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:14:04.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:14:04.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:14:04.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:04.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:14:04.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:04.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 32 2026-02-18T08:14:04.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 38 2026-02-18T08:14:04.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:14:04.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=153, tag_tid=9, entry_tid=1], [object_number=152, tag_tid=9, entry_tid=0], [object_number=154, tag_tid=8, entry_tid=58722], [object_number=155, tag_tid=8, entry_tid=58719]]], state=connected] 2026-02-18T08:14:04.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=129, tag_tid=8, entry_tid=31429], [object_number=128, tag_tid=8, entry_tid=31428], [object_number=131, tag_tid=8, entry_tid=31427], [object_number=130, tag_tid=8, entry_tid=31426]]], state=connected] 2026-02-18T08:14:04.905 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:04.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:04.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:14:04.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:04.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:04.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:04.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:14:04.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:04.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:04.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:04.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:14:04.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:14:04.909 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:14:04.909 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:14:04.909 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:04.910 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:04.910 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:14:04.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 32 2026-02-18T08:14:04.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 38 2026-02-18T08:14:04.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:14:04.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=153, tag_tid=9, entry_tid=1], [object_number=152, tag_tid=9, entry_tid=0], [object_number=154, tag_tid=8, entry_tid=58722], [object_number=155, tag_tid=8, entry_tid=58719]]], state=connected] 2026-02-18T08:14:04.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=129, tag_tid=8, entry_tid=31429], [object_number=128, tag_tid=8, entry_tid=31428], [object_number=131, tag_tid=8, entry_tid=31427], [object_number=130, tag_tid=8, entry_tid=31426]]], state=connected] 2026-02-18T08:14:04.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:04.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=129, tag_tid=8, entry_tid=31429' 2026-02-18T08:14:04.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=153, tag_tid=9, entry_tid=1' -a 'object_number=153, tag_tid=9, entry_tid=1' = 'object_number=129, tag_tid=8, entry_tid=31429' 2026-02-18T08:14:04.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=129, tag_tid=8, entry_tid=31429' '!=' 'object_number=124, tag_tid=8, entry_tid=31164' 2026-02-18T08:14:04.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:14:04.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=129, tag_tid=8, entry_tid=31429' = 'object_number=124, tag_tid=8, entry_tid=31164' 2026-02-18T08:14:04.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=129, tag_tid=8, entry_tid=31429' 2026-02-18T08:14:04.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:04.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:14:04.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:04.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=9 2026-02-18T08:14:04.944 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=129, tag_tid=8, entry_tid=31429' 2026-02-18T08:14:04.944 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:14:04.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:04.946 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=8 2026-02-18T08:14:04.947 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:04.947 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:14:04.947 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:04.948 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:14:04.949 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=129, tag_tid=8, entry_tid=31429' 2026-02-18T08:14:04.949 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:14:04.949 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:04.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=31429 2026-02-18T08:14:04.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 9 = 8 -a 1 -le 31429 2026-02-18T08:14:04.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:14:04.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:14:04.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:14:05.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:14:05.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:14:05.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:14:05.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:14:05.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:14:05.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:14:05.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:14:05.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:14:05.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:14:05.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:14:05.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:14:05.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:14:05.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:14:05.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:14:05.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:14:05.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:14:05.156 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:14:05.156 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:14:05.156 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:14:05.157 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:14:05.157 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:14:05.157 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:14:05.157 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:14:05.157 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:14:05.157 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:14:05.157 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:14:05.157 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:14:05.157 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:14:05.157 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:14:05.157 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:14:05.157 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:14:05.158 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:14:05.158 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:14:05.158 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:14:05.158 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:14:05.158 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:14:05.158 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:14:05.158 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:14:05.158 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:14:05.158 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:14:05.158 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:14:05.158 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:14:05.158 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:14:05.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:14:05.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:14:05.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:14:05.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:14:05.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:14:05.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:14:05.167 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:05.167 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:05.167 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:05.167 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:14:05.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:14:05.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:14:05.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:05.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:05.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:05.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:14:05.227 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:05.227 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:05.227 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:05.227 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:14:05.227 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:14:05.227 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:14:05.227 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:14:05.229 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:05.229 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:14:05.229 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:05.257 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 32 2026-02-18T08:14:05.257 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 38 2026-02-18T08:14:05.257 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:14:05.257 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=153, tag_tid=9, entry_tid=1], [object_number=152, tag_tid=9, entry_tid=0], [object_number=154, tag_tid=8, entry_tid=58722], [object_number=155, tag_tid=8, entry_tid=58719]]], state=connected] 2026-02-18T08:14:05.257 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=128, tag_tid=8, entry_tid=31660], [object_number=131, tag_tid=8, entry_tid=31659], [object_number=130, tag_tid=8, entry_tid=31658], [object_number=129, tag_tid=8, entry_tid=31657]]], state=connected] 2026-02-18T08:14:05.260 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:05.261 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:05.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:14:05.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:05.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:05.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:05.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:14:05.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:05.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:05.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:05.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:14:05.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:14:05.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:14:05.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:14:05.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:05.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:14:05.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:05.291 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 32 2026-02-18T08:14:05.292 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 38 2026-02-18T08:14:05.292 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:14:05.292 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=153, tag_tid=9, entry_tid=1], [object_number=152, tag_tid=9, entry_tid=0], [object_number=154, tag_tid=8, entry_tid=58722], [object_number=155, tag_tid=8, entry_tid=58719]]], state=connected] 2026-02-18T08:14:05.292 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=128, tag_tid=8, entry_tid=31660], [object_number=131, tag_tid=8, entry_tid=31659], [object_number=130, tag_tid=8, entry_tid=31658], [object_number=129, tag_tid=8, entry_tid=31657]]], state=connected] 2026-02-18T08:14:05.294 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:05.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=128, tag_tid=8, entry_tid=31660' 2026-02-18T08:14:05.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=153, tag_tid=9, entry_tid=1' -a 'object_number=153, tag_tid=9, entry_tid=1' = 'object_number=128, tag_tid=8, entry_tid=31660' 2026-02-18T08:14:05.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=128, tag_tid=8, entry_tid=31660' '!=' 'object_number=129, tag_tid=8, entry_tid=31429' 2026-02-18T08:14:05.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:14:05.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=128, tag_tid=8, entry_tid=31660' = 'object_number=129, tag_tid=8, entry_tid=31429' 2026-02-18T08:14:05.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=128, tag_tid=8, entry_tid=31660' 2026-02-18T08:14:05.297 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:05.297 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:14:05.297 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:05.298 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=9 2026-02-18T08:14:05.299 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=128, tag_tid=8, entry_tid=31660' 2026-02-18T08:14:05.299 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:14:05.299 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:05.300 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=8 2026-02-18T08:14:05.301 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:05.301 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:14:05.301 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:05.302 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:14:05.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=128, tag_tid=8, entry_tid=31660' 2026-02-18T08:14:05.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:14:05.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:05.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=31660 2026-02-18T08:14:05.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 9 = 8 -a 1 -le 31660 2026-02-18T08:14:05.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:14:05.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:14:05.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:14:05.506 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:14:05.506 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:14:05.506 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:14:05.506 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:14:05.506 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:14:05.507 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:14:05.507 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:14:05.507 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:14:05.507 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:14:05.507 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:14:05.507 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:14:05.507 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:14:05.507 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:14:05.507 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:14:05.507 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:14:05.507 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:14:05.507 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:14:05.509 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:14:05.509 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:14:05.509 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:14:05.509 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:14:05.509 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:14:05.509 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:14:05.509 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:14:05.509 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:14:05.509 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:14:05.509 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:14:05.509 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:14:05.509 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:14:05.510 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:14:05.510 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:14:05.510 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:14:05.510 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:14:05.510 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:14:05.510 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:14:05.510 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:14:05.510 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:14:05.510 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:14:05.510 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:14:05.510 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:14:05.510 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:14:05.510 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:14:05.510 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:14:05.511 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:14:05.511 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:14:05.511 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:14:05.511 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:14:05.511 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:14:05.511 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:14:05.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:05.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:05.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:05.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:14:05.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:14:05.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:14:05.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:05.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:05.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:05.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:14:05.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:05.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:05.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:05.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:14:05.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:14:05.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:14:05.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:14:05.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:05.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:14:05.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:05.628 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 32 2026-02-18T08:14:05.628 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 38 2026-02-18T08:14:05.628 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:14:05.628 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=153, tag_tid=9, entry_tid=1], [object_number=152, tag_tid=9, entry_tid=0], [object_number=154, tag_tid=8, entry_tid=58722], [object_number=155, tag_tid=8, entry_tid=58719]]], state=connected] 2026-02-18T08:14:05.628 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=130, tag_tid=8, entry_tid=31898], [object_number=129, tag_tid=8, entry_tid=31897], [object_number=128, tag_tid=8, entry_tid=31896], [object_number=131, tag_tid=8, entry_tid=31895]]], state=connected] 2026-02-18T08:14:05.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:05.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:05.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:14:05.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:05.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:05.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:05.635 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:14:05.635 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:05.635 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:05.635 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:05.635 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:14:05.635 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:14:05.635 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:14:05.635 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:14:05.636 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:05.636 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:14:05.636 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:05.661 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 32 2026-02-18T08:14:05.662 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 38 2026-02-18T08:14:05.662 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:14:05.662 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=153, tag_tid=9, entry_tid=1], [object_number=152, tag_tid=9, entry_tid=0], [object_number=154, tag_tid=8, entry_tid=58722], [object_number=155, tag_tid=8, entry_tid=58719]]], state=connected] 2026-02-18T08:14:05.662 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=130, tag_tid=8, entry_tid=31898], [object_number=129, tag_tid=8, entry_tid=31897], [object_number=128, tag_tid=8, entry_tid=31896], [object_number=131, tag_tid=8, entry_tid=31895]]], state=connected] 2026-02-18T08:14:05.664 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:05.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=130, tag_tid=8, entry_tid=31898' 2026-02-18T08:14:05.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=153, tag_tid=9, entry_tid=1' -a 'object_number=153, tag_tid=9, entry_tid=1' = 'object_number=130, tag_tid=8, entry_tid=31898' 2026-02-18T08:14:05.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=130, tag_tid=8, entry_tid=31898' '!=' 'object_number=128, tag_tid=8, entry_tid=31660' 2026-02-18T08:14:05.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:14:05.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=130, tag_tid=8, entry_tid=31898' = 'object_number=128, tag_tid=8, entry_tid=31660' 2026-02-18T08:14:05.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=130, tag_tid=8, entry_tid=31898' 2026-02-18T08:14:05.667 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:05.667 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:14:05.667 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:05.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=9 2026-02-18T08:14:05.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=130, tag_tid=8, entry_tid=31898' 2026-02-18T08:14:05.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:14:05.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:05.670 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=8 2026-02-18T08:14:05.671 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:05.671 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:14:05.671 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:05.673 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:14:05.673 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=130, tag_tid=8, entry_tid=31898' 2026-02-18T08:14:05.674 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:14:05.674 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:05.675 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=31898 2026-02-18T08:14:05.675 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 9 = 8 -a 1 -le 31898 2026-02-18T08:14:05.675 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:14:05.675 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:14:05.675 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:14:05.877 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:14:05.877 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:14:05.877 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:14:05.877 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:14:05.877 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:14:05.877 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:14:05.877 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:14:05.877 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:14:05.877 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:14:05.877 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:14:05.877 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:14:05.877 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:14:05.877 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:14:05.877 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:14:05.877 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:14:05.878 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:14:05.878 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:14:05.880 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:14:05.880 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:14:05.880 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:14:05.880 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:14:05.881 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:14:05.881 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:14:05.881 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:14:05.881 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:14:05.881 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:14:05.881 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:14:05.881 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:14:05.881 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:14:05.881 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:14:05.881 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:14:05.881 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:14:05.881 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:14:05.881 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:14:05.881 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:14:05.881 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:14:05.882 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:14:05.882 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:14:05.882 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:14:05.882 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:14:05.882 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:14:05.882 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:14:05.882 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:14:05.882 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:14:05.882 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:14:05.882 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:14:05.882 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:14:05.882 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:14:05.882 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:14:05.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:05.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:05.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:05.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:14:05.959 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:14:05.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:14:05.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:05.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:05.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:05.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:14:05.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:05.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:05.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:05.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:14:05.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:14:05.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:14:05.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:14:05.962 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:05.962 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:14:05.962 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:05.990 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 32 2026-02-18T08:14:05.990 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 38 2026-02-18T08:14:05.990 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:14:05.990 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=153, tag_tid=9, entry_tid=1], [object_number=152, tag_tid=9, entry_tid=0], [object_number=154, tag_tid=8, entry_tid=58722], [object_number=155, tag_tid=8, entry_tid=58719]]], state=connected] 2026-02-18T08:14:05.990 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=131, tag_tid=8, entry_tid=32143], [object_number=130, tag_tid=8, entry_tid=32142], [object_number=129, tag_tid=8, entry_tid=32141], [object_number=128, tag_tid=8, entry_tid=32140]]], state=connected] 2026-02-18T08:14:05.993 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:05.994 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:05.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:14:05.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:05.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:05.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:05.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:14:05.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:05.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:05.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:05.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:14:05.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:14:05.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:14:05.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:14:05.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:05.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:14:05.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:06.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 32 2026-02-18T08:14:06.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 38 2026-02-18T08:14:06.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:14:06.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=153, tag_tid=9, entry_tid=1], [object_number=152, tag_tid=9, entry_tid=0], [object_number=154, tag_tid=8, entry_tid=58722], [object_number=155, tag_tid=8, entry_tid=58719]]], state=connected] 2026-02-18T08:14:06.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=131, tag_tid=8, entry_tid=32143], [object_number=130, tag_tid=8, entry_tid=32142], [object_number=129, tag_tid=8, entry_tid=32141], [object_number=128, tag_tid=8, entry_tid=32140]]], state=connected] 2026-02-18T08:14:06.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:06.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=131, tag_tid=8, entry_tid=32143' 2026-02-18T08:14:06.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=153, tag_tid=9, entry_tid=1' -a 'object_number=153, tag_tid=9, entry_tid=1' = 'object_number=131, tag_tid=8, entry_tid=32143' 2026-02-18T08:14:06.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=131, tag_tid=8, entry_tid=32143' '!=' 'object_number=130, tag_tid=8, entry_tid=31898' 2026-02-18T08:14:06.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:14:06.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=131, tag_tid=8, entry_tid=32143' = 'object_number=130, tag_tid=8, entry_tid=31898' 2026-02-18T08:14:06.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=131, tag_tid=8, entry_tid=32143' 2026-02-18T08:14:06.027 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:06.027 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:14:06.027 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:06.028 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=9 2026-02-18T08:14:06.029 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=131, tag_tid=8, entry_tid=32143' 2026-02-18T08:14:06.029 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:06.029 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:14:06.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=8 2026-02-18T08:14:06.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:06.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:14:06.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:06.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:14:06.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=131, tag_tid=8, entry_tid=32143' 2026-02-18T08:14:06.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:14:06.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:06.035 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=32143 2026-02-18T08:14:06.035 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 9 = 8 -a 1 -le 32143 2026-02-18T08:14:06.035 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:14:06.035 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:14:06.035 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:14:06.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:14:06.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:14:06.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:14:06.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:14:06.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:14:06.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:14:06.238 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:14:06.238 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:14:06.238 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:14:06.238 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:14:06.238 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:14:06.238 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:14:06.238 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:14:06.238 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:14:06.238 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:14:06.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:14:06.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:14:06.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:14:06.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:14:06.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:14:06.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:14:06.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:14:06.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:14:06.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:14:06.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:14:06.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:14:06.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:14:06.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:14:06.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:14:06.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:14:06.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:14:06.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:14:06.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:14:06.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:14:06.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:14:06.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:14:06.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:14:06.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:14:06.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:14:06.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:14:06.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:14:06.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:14:06.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:14:06.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:14:06.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:14:06.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:14:06.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:14:06.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:14:06.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:14:06.250 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:06.251 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:06.251 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:06.251 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:14:06.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:14:06.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:14:06.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:06.312 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:06.312 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:06.312 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:14:06.312 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:06.312 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:06.312 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:06.312 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:14:06.312 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:14:06.312 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:14:06.312 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:14:06.314 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:06.314 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:14:06.314 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:06.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 32 2026-02-18T08:14:06.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 38 2026-02-18T08:14:06.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:14:06.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=153, tag_tid=9, entry_tid=1], [object_number=152, tag_tid=9, entry_tid=0], [object_number=154, tag_tid=8, entry_tid=58722], [object_number=155, tag_tid=8, entry_tid=58719]]], state=connected] 2026-02-18T08:14:06.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=131, tag_tid=8, entry_tid=32439], [object_number=130, tag_tid=8, entry_tid=32438], [object_number=129, tag_tid=8, entry_tid=32437], [object_number=128, tag_tid=8, entry_tid=32436]]], state=connected] 2026-02-18T08:14:06.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:06.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:06.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:14:06.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:06.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:06.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:06.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:14:06.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:06.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:06.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:06.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:14:06.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:14:06.348 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:14:06.348 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:14:06.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:06.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:14:06.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:06.384 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 32 2026-02-18T08:14:06.384 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 38 2026-02-18T08:14:06.384 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:14:06.385 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=153, tag_tid=9, entry_tid=1], [object_number=152, tag_tid=9, entry_tid=0], [object_number=154, tag_tid=8, entry_tid=58722], [object_number=155, tag_tid=8, entry_tid=58719]]], state=connected] 2026-02-18T08:14:06.385 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=131, tag_tid=8, entry_tid=32439], [object_number=130, tag_tid=8, entry_tid=32438], [object_number=129, tag_tid=8, entry_tid=32437], [object_number=128, tag_tid=8, entry_tid=32436]]], state=connected] 2026-02-18T08:14:06.388 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:06.390 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=131, tag_tid=8, entry_tid=32439' 2026-02-18T08:14:06.390 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=153, tag_tid=9, entry_tid=1' -a 'object_number=153, tag_tid=9, entry_tid=1' = 'object_number=131, tag_tid=8, entry_tid=32439' 2026-02-18T08:14:06.390 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=131, tag_tid=8, entry_tid=32439' '!=' 'object_number=131, tag_tid=8, entry_tid=32143' 2026-02-18T08:14:06.390 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:14:06.390 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=131, tag_tid=8, entry_tid=32439' = 'object_number=131, tag_tid=8, entry_tid=32143' 2026-02-18T08:14:06.390 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=131, tag_tid=8, entry_tid=32439' 2026-02-18T08:14:06.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:06.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:14:06.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:06.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=9 2026-02-18T08:14:06.393 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=131, tag_tid=8, entry_tid=32439' 2026-02-18T08:14:06.393 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:14:06.393 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:06.394 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=8 2026-02-18T08:14:06.395 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:06.395 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:14:06.395 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:06.397 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:14:06.397 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=131, tag_tid=8, entry_tid=32439' 2026-02-18T08:14:06.397 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:14:06.397 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:06.399 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=32439 2026-02-18T08:14:06.399 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 9 = 8 -a 1 -le 32439 2026-02-18T08:14:06.399 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:14:06.399 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:14:06.399 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:14:06.601 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:14:06.601 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:14:06.601 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:14:06.601 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:14:06.601 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:14:06.601 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:14:06.601 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:14:06.602 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:14:06.602 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:14:06.602 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:14:06.602 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:14:06.602 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:14:06.602 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:14:06.602 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:14:06.602 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:14:06.602 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:14:06.602 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:14:06.604 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:14:06.604 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:14:06.604 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:14:06.604 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:14:06.604 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:14:06.604 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:14:06.604 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:14:06.604 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:14:06.604 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:14:06.605 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:14:06.605 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:14:06.605 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:14:06.605 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:14:06.605 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:14:06.605 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:14:06.605 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:14:06.605 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:14:06.605 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:14:06.605 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:14:06.605 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:14:06.605 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:14:06.605 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:14:06.605 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:14:06.605 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:14:06.605 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:14:06.605 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:14:06.605 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:14:06.605 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:14:06.605 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:14:06.605 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:14:06.606 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:14:06.606 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:14:06.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:06.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:06.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:06.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:14:06.691 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:14:06.692 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:14:06.692 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:06.692 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:06.692 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:06.692 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:14:06.692 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:06.692 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:06.692 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:06.692 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:14:06.692 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:14:06.693 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:14:06.693 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:14:06.694 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:06.694 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:14:06.695 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:06.724 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 32 2026-02-18T08:14:06.724 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 38 2026-02-18T08:14:06.724 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:14:06.724 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=153, tag_tid=9, entry_tid=1], [object_number=152, tag_tid=9, entry_tid=0], [object_number=154, tag_tid=8, entry_tid=58722], [object_number=155, tag_tid=8, entry_tid=58719]]], state=connected] 2026-02-18T08:14:06.724 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=131, tag_tid=8, entry_tid=32727], [object_number=130, tag_tid=8, entry_tid=32726], [object_number=129, tag_tid=8, entry_tid=32725], [object_number=128, tag_tid=8, entry_tid=32724]]], state=connected] 2026-02-18T08:14:06.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:06.729 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:06.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:14:06.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:06.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:06.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:06.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:14:06.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:06.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:06.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:06.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:14:06.731 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:14:06.731 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:14:06.731 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:14:06.732 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:06.732 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:14:06.732 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:06.758 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 32 2026-02-18T08:14:06.758 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 38 2026-02-18T08:14:06.758 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:14:06.758 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=153, tag_tid=9, entry_tid=1], [object_number=152, tag_tid=9, entry_tid=0], [object_number=154, tag_tid=8, entry_tid=58722], [object_number=155, tag_tid=8, entry_tid=58719]]], state=connected] 2026-02-18T08:14:06.758 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=131, tag_tid=8, entry_tid=32727], [object_number=130, tag_tid=8, entry_tid=32726], [object_number=129, tag_tid=8, entry_tid=32725], [object_number=128, tag_tid=8, entry_tid=32724]]], state=connected] 2026-02-18T08:14:06.760 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:06.763 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=131, tag_tid=8, entry_tid=32727' 2026-02-18T08:14:06.763 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=153, tag_tid=9, entry_tid=1' -a 'object_number=153, tag_tid=9, entry_tid=1' = 'object_number=131, tag_tid=8, entry_tid=32727' 2026-02-18T08:14:06.763 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=131, tag_tid=8, entry_tid=32727' '!=' 'object_number=131, tag_tid=8, entry_tid=32439' 2026-02-18T08:14:06.763 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:14:06.763 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=131, tag_tid=8, entry_tid=32727' = 'object_number=131, tag_tid=8, entry_tid=32439' 2026-02-18T08:14:06.763 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=131, tag_tid=8, entry_tid=32727' 2026-02-18T08:14:06.763 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:06.763 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:14:06.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:06.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=9 2026-02-18T08:14:06.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=131, tag_tid=8, entry_tid=32727' 2026-02-18T08:14:06.766 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:14:06.766 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:06.767 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=8 2026-02-18T08:14:06.768 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:06.768 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:14:06.768 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:06.769 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:14:06.770 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=131, tag_tid=8, entry_tid=32727' 2026-02-18T08:14:06.771 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:14:06.771 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:06.772 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=32727 2026-02-18T08:14:06.772 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 9 = 8 -a 1 -le 32727 2026-02-18T08:14:06.772 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:14:06.772 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:14:06.772 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:14:06.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:14:06.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:14:06.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:14:06.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:14:06.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:14:06.974 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:14:06.974 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:14:06.974 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:14:06.974 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:14:06.974 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:14:06.974 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:14:06.974 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:14:06.974 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:14:06.974 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:14:06.974 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:14:06.974 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:14:06.974 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:14:06.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:14:06.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:14:06.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:14:06.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:14:06.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:14:06.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:14:06.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:14:06.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:14:06.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:14:06.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:14:06.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:14:06.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:14:06.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:14:06.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:14:06.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:14:06.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:14:06.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:14:06.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:14:06.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:14:06.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:14:06.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:14:06.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:14:06.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:14:06.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:14:06.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:14:06.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:14:06.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:14:06.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:14:06.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:14:06.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:14:06.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:14:06.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:14:06.987 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:06.988 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:06.988 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:06.988 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:14:07.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:14:07.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:14:07.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:07.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:07.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:07.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:14:07.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:07.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:07.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:07.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:14:07.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:14:07.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:14:07.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:14:07.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:07.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:14:07.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:07.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 32 2026-02-18T08:14:07.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 38 2026-02-18T08:14:07.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:14:07.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=153, tag_tid=9, entry_tid=1], [object_number=152, tag_tid=9, entry_tid=0], [object_number=154, tag_tid=8, entry_tid=58722], [object_number=155, tag_tid=8, entry_tid=58719]]], state=connected] 2026-02-18T08:14:07.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=131, tag_tid=8, entry_tid=32951], [object_number=130, tag_tid=8, entry_tid=32950], [object_number=129, tag_tid=8, entry_tid=32949], [object_number=128, tag_tid=8, entry_tid=32948]]], state=connected] 2026-02-18T08:14:07.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:07.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:07.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:14:07.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:07.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:07.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:07.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:14:07.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:07.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:07.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:07.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:14:07.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:14:07.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:14:07.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:14:07.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:07.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:14:07.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:07.121 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 32 2026-02-18T08:14:07.122 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 38 2026-02-18T08:14:07.122 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:14:07.122 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=153, tag_tid=9, entry_tid=1], [object_number=152, tag_tid=9, entry_tid=0], [object_number=154, tag_tid=8, entry_tid=58722], [object_number=155, tag_tid=8, entry_tid=58719]]], state=connected] 2026-02-18T08:14:07.122 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=131, tag_tid=8, entry_tid=32951], [object_number=130, tag_tid=8, entry_tid=32950], [object_number=129, tag_tid=8, entry_tid=32949], [object_number=128, tag_tid=8, entry_tid=32948]]], state=connected] 2026-02-18T08:14:07.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:07.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=131, tag_tid=8, entry_tid=32951' 2026-02-18T08:14:07.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=153, tag_tid=9, entry_tid=1' -a 'object_number=153, tag_tid=9, entry_tid=1' = 'object_number=131, tag_tid=8, entry_tid=32951' 2026-02-18T08:14:07.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=131, tag_tid=8, entry_tid=32951' '!=' 'object_number=131, tag_tid=8, entry_tid=32727' 2026-02-18T08:14:07.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:14:07.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=131, tag_tid=8, entry_tid=32951' = 'object_number=131, tag_tid=8, entry_tid=32727' 2026-02-18T08:14:07.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=131, tag_tid=8, entry_tid=32951' 2026-02-18T08:14:07.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:07.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:14:07.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:07.130 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=9 2026-02-18T08:14:07.130 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=131, tag_tid=8, entry_tid=32951' 2026-02-18T08:14:07.131 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:14:07.131 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:07.132 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=8 2026-02-18T08:14:07.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:07.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:14:07.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:07.134 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:14:07.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=131, tag_tid=8, entry_tid=32951' 2026-02-18T08:14:07.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:14:07.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:07.137 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=32951 2026-02-18T08:14:07.137 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 9 = 8 -a 1 -le 32951 2026-02-18T08:14:07.137 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:14:07.137 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:14:07.137 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:14:07.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:14:07.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:14:07.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:14:07.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:14:07.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:14:07.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:14:07.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:14:07.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:14:07.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:14:07.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:14:07.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:14:07.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:14:07.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:14:07.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:14:07.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:14:07.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:14:07.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:14:07.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:14:07.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:14:07.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:14:07.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:14:07.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:14:07.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:14:07.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:14:07.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:14:07.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:14:07.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:14:07.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:14:07.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:14:07.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:14:07.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:14:07.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:14:07.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:14:07.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:14:07.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:14:07.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:14:07.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:14:07.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:14:07.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:14:07.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:14:07.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:14:07.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:14:07.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:14:07.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:14:07.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:14:07.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:14:07.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:14:07.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:14:07.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:14:07.354 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:07.354 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:07.354 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:07.354 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:14:07.419 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:14:07.420 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:14:07.420 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:07.420 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:07.420 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:07.420 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:14:07.420 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:07.420 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:07.420 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:07.420 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:14:07.420 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:14:07.421 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:14:07.421 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:14:07.422 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:07.422 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:14:07.422 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:07.450 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 32 2026-02-18T08:14:07.450 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 38 2026-02-18T08:14:07.450 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:14:07.450 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=153, tag_tid=9, entry_tid=1], [object_number=152, tag_tid=9, entry_tid=0], [object_number=154, tag_tid=8, entry_tid=58722], [object_number=155, tag_tid=8, entry_tid=58719]]], state=connected] 2026-02-18T08:14:07.450 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=130, tag_tid=8, entry_tid=33182], [object_number=129, tag_tid=8, entry_tid=33181], [object_number=128, tag_tid=8, entry_tid=33180], [object_number=131, tag_tid=8, entry_tid=33179]]], state=connected] 2026-02-18T08:14:07.452 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:07.454 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:07.454 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:14:07.454 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:07.454 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:07.454 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:07.454 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:14:07.454 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:07.454 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:07.454 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:07.454 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:14:07.455 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:14:07.455 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:14:07.455 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:14:07.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:07.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:14:07.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:07.491 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 32 2026-02-18T08:14:07.492 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 38 2026-02-18T08:14:07.492 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:14:07.492 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=153, tag_tid=9, entry_tid=1], [object_number=152, tag_tid=9, entry_tid=0], [object_number=154, tag_tid=8, entry_tid=58722], [object_number=155, tag_tid=8, entry_tid=58719]]], state=connected] 2026-02-18T08:14:07.492 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=130, tag_tid=8, entry_tid=33182], [object_number=129, tag_tid=8, entry_tid=33181], [object_number=128, tag_tid=8, entry_tid=33180], [object_number=131, tag_tid=8, entry_tid=33179]]], state=connected] 2026-02-18T08:14:07.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:07.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=130, tag_tid=8, entry_tid=33182' 2026-02-18T08:14:07.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=153, tag_tid=9, entry_tid=1' -a 'object_number=153, tag_tid=9, entry_tid=1' = 'object_number=130, tag_tid=8, entry_tid=33182' 2026-02-18T08:14:07.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=130, tag_tid=8, entry_tid=33182' '!=' 'object_number=131, tag_tid=8, entry_tid=32951' 2026-02-18T08:14:07.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:14:07.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=130, tag_tid=8, entry_tid=33182' = 'object_number=131, tag_tid=8, entry_tid=32951' 2026-02-18T08:14:07.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=130, tag_tid=8, entry_tid=33182' 2026-02-18T08:14:07.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:07.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:14:07.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:07.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=9 2026-02-18T08:14:07.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=130, tag_tid=8, entry_tid=33182' 2026-02-18T08:14:07.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:14:07.500 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:07.501 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=8 2026-02-18T08:14:07.502 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:07.502 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:14:07.502 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:07.503 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:14:07.504 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=130, tag_tid=8, entry_tid=33182' 2026-02-18T08:14:07.504 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:14:07.504 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:07.506 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=33182 2026-02-18T08:14:07.506 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 9 = 8 -a 1 -le 33182 2026-02-18T08:14:07.506 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:14:07.506 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:14:07.506 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:14:07.707 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:14:07.707 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:14:07.707 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:14:07.707 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:14:07.707 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:14:07.707 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:14:07.707 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:14:07.707 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:14:07.707 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:14:07.707 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:14:07.707 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:14:07.707 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:14:07.708 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:14:07.708 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:14:07.708 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:14:07.708 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:14:07.708 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:14:07.709 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:14:07.709 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:14:07.709 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:14:07.709 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:14:07.709 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:14:07.709 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:14:07.709 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:14:07.709 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:14:07.709 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:14:07.709 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:14:07.709 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:14:07.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:14:07.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:14:07.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:14:07.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:14:07.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:14:07.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:14:07.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:14:07.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:14:07.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:14:07.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:14:07.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:14:07.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:14:07.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:14:07.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:14:07.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:14:07.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:14:07.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:14:07.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:14:07.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:14:07.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:14:07.711 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:14:07.722 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:07.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:07.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:07.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:14:07.786 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:14:07.786 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:14:07.786 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:07.786 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:07.786 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:07.786 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:14:07.786 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:07.786 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:07.786 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:07.786 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:14:07.786 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:14:07.787 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:14:07.787 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:14:07.788 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:07.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:14:07.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:07.815 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 32 2026-02-18T08:14:07.815 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 38 2026-02-18T08:14:07.815 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:14:07.815 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=153, tag_tid=9, entry_tid=1], [object_number=152, tag_tid=9, entry_tid=0], [object_number=154, tag_tid=8, entry_tid=58722], [object_number=155, tag_tid=8, entry_tid=58719]]], state=connected] 2026-02-18T08:14:07.815 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=131, tag_tid=8, entry_tid=33427], [object_number=130, tag_tid=8, entry_tid=33426], [object_number=129, tag_tid=8, entry_tid=33425], [object_number=128, tag_tid=8, entry_tid=33424]]], state=connected] 2026-02-18T08:14:07.818 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:07.819 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:07.820 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:14:07.820 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:07.820 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:07.820 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:07.820 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:14:07.820 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:07.820 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:07.820 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:07.820 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:14:07.820 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:14:07.821 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:14:07.821 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:14:07.822 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:07.823 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:14:07.823 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:07.857 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 32 2026-02-18T08:14:07.857 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 38 2026-02-18T08:14:07.857 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:14:07.857 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=153, tag_tid=9, entry_tid=1], [object_number=152, tag_tid=9, entry_tid=0], [object_number=154, tag_tid=8, entry_tid=58722], [object_number=155, tag_tid=8, entry_tid=58719]]], state=connected] 2026-02-18T08:14:07.857 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=131, tag_tid=8, entry_tid=33427], [object_number=130, tag_tid=8, entry_tid=33426], [object_number=129, tag_tid=8, entry_tid=33425], [object_number=128, tag_tid=8, entry_tid=33424]]], state=connected] 2026-02-18T08:14:07.860 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:07.862 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=131, tag_tid=8, entry_tid=33427' 2026-02-18T08:14:07.862 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=153, tag_tid=9, entry_tid=1' -a 'object_number=153, tag_tid=9, entry_tid=1' = 'object_number=131, tag_tid=8, entry_tid=33427' 2026-02-18T08:14:07.862 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=131, tag_tid=8, entry_tid=33427' '!=' 'object_number=130, tag_tid=8, entry_tid=33182' 2026-02-18T08:14:07.862 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:14:07.862 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=131, tag_tid=8, entry_tid=33427' = 'object_number=130, tag_tid=8, entry_tid=33182' 2026-02-18T08:14:07.862 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=131, tag_tid=8, entry_tid=33427' 2026-02-18T08:14:07.862 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:07.862 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:14:07.862 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:07.864 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=9 2026-02-18T08:14:07.865 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=131, tag_tid=8, entry_tid=33427' 2026-02-18T08:14:07.865 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:14:07.865 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:07.866 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=8 2026-02-18T08:14:07.867 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:07.867 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:14:07.867 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:07.868 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:14:07.869 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=131, tag_tid=8, entry_tid=33427' 2026-02-18T08:14:07.869 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:14:07.869 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:07.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=33427 2026-02-18T08:14:07.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 9 = 8 -a 1 -le 33427 2026-02-18T08:14:07.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:14:07.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:14:07.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:14:08.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:14:08.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:14:08.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:14:08.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:14:08.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:14:08.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:14:08.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:14:08.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:14:08.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:14:08.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:14:08.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:14:08.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:14:08.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:14:08.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:14:08.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:14:08.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:14:08.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:14:08.074 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:14:08.074 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:14:08.074 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:14:08.074 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:14:08.074 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:14:08.074 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:14:08.075 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:14:08.075 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:14:08.075 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:14:08.075 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:14:08.075 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:14:08.075 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:14:08.075 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:14:08.075 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:14:08.075 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:14:08.075 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:14:08.075 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:14:08.075 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:14:08.075 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:14:08.075 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:14:08.075 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:14:08.075 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:14:08.075 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:14:08.075 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:14:08.075 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:14:08.075 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:14:08.075 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:14:08.076 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:14:08.076 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:14:08.076 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:14:08.076 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:14:08.076 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:14:08.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:08.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:08.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:08.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:14:08.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:14:08.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:14:08.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:08.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:08.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:08.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:14:08.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:08.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:08.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:08.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:14:08.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:14:08.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:14:08.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:14:08.162 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:08.162 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:14:08.162 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:08.196 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 32 2026-02-18T08:14:08.196 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 38 2026-02-18T08:14:08.196 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:14:08.196 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=153, tag_tid=9, entry_tid=1], [object_number=152, tag_tid=9, entry_tid=0], [object_number=154, tag_tid=8, entry_tid=58722], [object_number=155, tag_tid=8, entry_tid=58719]]], state=connected] 2026-02-18T08:14:08.196 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=130, tag_tid=8, entry_tid=33670], [object_number=129, tag_tid=8, entry_tid=33669], [object_number=128, tag_tid=8, entry_tid=33668], [object_number=131, tag_tid=8, entry_tid=33667]]], state=connected] 2026-02-18T08:14:08.199 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:08.201 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:08.201 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:14:08.201 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:08.201 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:08.201 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:08.201 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:14:08.201 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:08.201 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:08.201 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:08.201 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:14:08.202 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:14:08.202 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:14:08.202 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:14:08.204 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:08.204 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:14:08.204 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:08.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 32 2026-02-18T08:14:08.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 38 2026-02-18T08:14:08.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:14:08.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=153, tag_tid=9, entry_tid=1], [object_number=152, tag_tid=9, entry_tid=0], [object_number=154, tag_tid=8, entry_tid=58722], [object_number=155, tag_tid=8, entry_tid=58719]]], state=connected] 2026-02-18T08:14:08.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=130, tag_tid=8, entry_tid=33670], [object_number=129, tag_tid=8, entry_tid=33669], [object_number=128, tag_tid=8, entry_tid=33668], [object_number=131, tag_tid=8, entry_tid=33667]]], state=connected] 2026-02-18T08:14:08.244 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:08.245 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=130, tag_tid=8, entry_tid=33670' 2026-02-18T08:14:08.245 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=153, tag_tid=9, entry_tid=1' -a 'object_number=153, tag_tid=9, entry_tid=1' = 'object_number=130, tag_tid=8, entry_tid=33670' 2026-02-18T08:14:08.246 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=130, tag_tid=8, entry_tid=33670' '!=' 'object_number=131, tag_tid=8, entry_tid=33427' 2026-02-18T08:14:08.246 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:14:08.246 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=130, tag_tid=8, entry_tid=33670' = 'object_number=131, tag_tid=8, entry_tid=33427' 2026-02-18T08:14:08.246 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=130, tag_tid=8, entry_tid=33670' 2026-02-18T08:14:08.246 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:08.246 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:14:08.246 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:08.248 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=9 2026-02-18T08:14:08.248 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=130, tag_tid=8, entry_tid=33670' 2026-02-18T08:14:08.248 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:14:08.248 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:08.250 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=8 2026-02-18T08:14:08.250 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:08.251 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:14:08.251 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:08.252 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:14:08.253 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=130, tag_tid=8, entry_tid=33670' 2026-02-18T08:14:08.253 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:14:08.253 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:08.255 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=33670 2026-02-18T08:14:08.255 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 9 = 8 -a 1 -le 33670 2026-02-18T08:14:08.255 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:14:08.255 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:14:08.255 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:14:08.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:14:08.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:14:08.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:14:08.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:14:08.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:14:08.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:14:08.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:14:08.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:14:08.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:14:08.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:14:08.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:14:08.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:14:08.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:14:08.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:14:08.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:14:08.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:14:08.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:14:08.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:14:08.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:14:08.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:14:08.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:14:08.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:14:08.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:14:08.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:14:08.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:14:08.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:14:08.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:14:08.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:14:08.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:14:08.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:14:08.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:14:08.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:14:08.460 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:14:08.460 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:14:08.460 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:14:08.460 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:14:08.460 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:14:08.460 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:14:08.460 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:14:08.460 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:14:08.460 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:14:08.460 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:14:08.460 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:14:08.460 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:14:08.460 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:14:08.460 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:14:08.460 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:14:08.460 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:14:08.460 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:14:08.470 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:08.470 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:08.470 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:08.470 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:14:08.528 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:14:08.529 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:14:08.529 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:08.529 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:08.529 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:08.530 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:14:08.530 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:08.530 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:08.530 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:08.530 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:14:08.530 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:14:08.530 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:14:08.530 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:14:08.531 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:08.531 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:14:08.531 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:08.559 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 32 2026-02-18T08:14:08.559 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 38 2026-02-18T08:14:08.559 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:14:08.559 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=153, tag_tid=9, entry_tid=1], [object_number=152, tag_tid=9, entry_tid=0], [object_number=154, tag_tid=8, entry_tid=58722], [object_number=155, tag_tid=8, entry_tid=58719]]], state=connected] 2026-02-18T08:14:08.559 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=130, tag_tid=8, entry_tid=33930], [object_number=129, tag_tid=8, entry_tid=33929], [object_number=128, tag_tid=8, entry_tid=33928], [object_number=131, tag_tid=8, entry_tid=33927]]], state=connected] 2026-02-18T08:14:08.561 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:08.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:08.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:14:08.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:08.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:08.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:08.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:14:08.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:08.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:08.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:08.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:14:08.565 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:14:08.565 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:14:08.565 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:14:08.566 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:08.566 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:14:08.566 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:08.599 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 32 2026-02-18T08:14:08.599 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 38 2026-02-18T08:14:08.600 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:14:08.600 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=153, tag_tid=9, entry_tid=1], [object_number=152, tag_tid=9, entry_tid=0], [object_number=154, tag_tid=8, entry_tid=58722], [object_number=155, tag_tid=8, entry_tid=58719]]], state=connected] 2026-02-18T08:14:08.600 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=130, tag_tid=8, entry_tid=33930], [object_number=129, tag_tid=8, entry_tid=33929], [object_number=128, tag_tid=8, entry_tid=33928], [object_number=131, tag_tid=8, entry_tid=33927]]], state=connected] 2026-02-18T08:14:08.603 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:08.604 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=130, tag_tid=8, entry_tid=33930' 2026-02-18T08:14:08.605 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=153, tag_tid=9, entry_tid=1' -a 'object_number=153, tag_tid=9, entry_tid=1' = 'object_number=130, tag_tid=8, entry_tid=33930' 2026-02-18T08:14:08.605 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=130, tag_tid=8, entry_tid=33930' '!=' 'object_number=130, tag_tid=8, entry_tid=33670' 2026-02-18T08:14:08.605 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:14:08.605 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=130, tag_tid=8, entry_tid=33930' = 'object_number=130, tag_tid=8, entry_tid=33670' 2026-02-18T08:14:08.605 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=130, tag_tid=8, entry_tid=33930' 2026-02-18T08:14:08.605 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:08.605 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:14:08.605 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:08.607 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=9 2026-02-18T08:14:08.607 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=130, tag_tid=8, entry_tid=33930' 2026-02-18T08:14:08.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:14:08.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:08.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=8 2026-02-18T08:14:08.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:08.610 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:14:08.610 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:08.611 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:14:08.612 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=130, tag_tid=8, entry_tid=33930' 2026-02-18T08:14:08.612 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:14:08.612 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:08.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=33930 2026-02-18T08:14:08.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 9 = 8 -a 1 -le 33930 2026-02-18T08:14:08.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:14:08.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:14:08.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:14:08.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:14:08.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:14:08.817 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:14:08.817 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:14:08.817 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:14:08.817 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:14:08.817 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:14:08.817 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:14:08.817 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:14:08.817 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:14:08.817 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:14:08.817 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:14:08.817 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:14:08.818 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:14:08.818 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:14:08.818 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:14:08.818 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:14:08.820 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:14:08.820 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:14:08.820 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:14:08.821 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:14:08.821 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:14:08.821 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:14:08.821 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:14:08.821 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:14:08.821 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:14:08.821 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:14:08.821 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:14:08.821 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:14:08.821 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:14:08.822 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:14:08.822 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:14:08.822 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:14:08.822 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:14:08.822 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:14:08.822 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:14:08.822 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:14:08.822 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:14:08.822 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:14:08.822 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:14:08.822 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:14:08.822 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:14:08.823 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:14:08.823 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:14:08.823 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:14:08.823 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:14:08.823 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:14:08.823 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:14:08.823 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:14:08.830 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:08.831 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:08.831 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:08.831 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:14:08.904 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:14:08.905 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:14:08.905 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:08.905 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:08.905 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:08.905 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:14:08.905 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:08.905 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:08.905 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:08.905 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:14:08.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:14:08.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:14:08.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:14:08.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:08.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:14:08.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:08.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 32 2026-02-18T08:14:08.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 38 2026-02-18T08:14:08.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:14:08.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=153, tag_tid=9, entry_tid=1], [object_number=152, tag_tid=9, entry_tid=0], [object_number=154, tag_tid=8, entry_tid=58722], [object_number=155, tag_tid=8, entry_tid=58719]]], state=connected] 2026-02-18T08:14:08.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=130, tag_tid=8, entry_tid=34214], [object_number=129, tag_tid=8, entry_tid=34213], [object_number=128, tag_tid=8, entry_tid=34212], [object_number=131, tag_tid=8, entry_tid=34211]]], state=connected] 2026-02-18T08:14:08.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:08.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:08.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:14:08.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:08.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:08.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:08.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:14:08.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:08.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:08.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:08.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:14:08.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:14:08.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:14:08.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:14:08.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:08.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:14:08.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:08.967 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 32 2026-02-18T08:14:08.967 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 38 2026-02-18T08:14:08.967 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:14:08.967 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=153, tag_tid=9, entry_tid=1], [object_number=152, tag_tid=9, entry_tid=0], [object_number=154, tag_tid=8, entry_tid=58722], [object_number=155, tag_tid=8, entry_tid=58719]]], state=connected] 2026-02-18T08:14:08.967 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=130, tag_tid=8, entry_tid=34214], [object_number=129, tag_tid=8, entry_tid=34213], [object_number=128, tag_tid=8, entry_tid=34212], [object_number=131, tag_tid=8, entry_tid=34211]]], state=connected] 2026-02-18T08:14:08.970 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:08.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=130, tag_tid=8, entry_tid=34214' 2026-02-18T08:14:08.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=153, tag_tid=9, entry_tid=1' -a 'object_number=153, tag_tid=9, entry_tid=1' = 'object_number=130, tag_tid=8, entry_tid=34214' 2026-02-18T08:14:08.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=130, tag_tid=8, entry_tid=34214' '!=' 'object_number=130, tag_tid=8, entry_tid=33930' 2026-02-18T08:14:08.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:14:08.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=130, tag_tid=8, entry_tid=34214' = 'object_number=130, tag_tid=8, entry_tid=33930' 2026-02-18T08:14:08.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=130, tag_tid=8, entry_tid=34214' 2026-02-18T08:14:08.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:08.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:14:08.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:08.974 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=9 2026-02-18T08:14:08.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=130, tag_tid=8, entry_tid=34214' 2026-02-18T08:14:08.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:14:08.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:08.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=8 2026-02-18T08:14:08.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:08.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:14:08.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:08.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:14:08.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=130, tag_tid=8, entry_tid=34214' 2026-02-18T08:14:08.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:14:08.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:08.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=34214 2026-02-18T08:14:08.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 9 = 8 -a 1 -le 34214 2026-02-18T08:14:08.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:14:08.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:14:08.982 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:14:09.183 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:14:09.183 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:14:09.183 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:14:09.183 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:14:09.183 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:14:09.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:14:09.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:14:09.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:14:09.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:14:09.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:14:09.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:14:09.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:14:09.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:14:09.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:14:09.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:14:09.187 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:14:09.187 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:14:09.187 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:14:09.187 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:14:09.188 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:14:09.188 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:14:09.188 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:14:09.188 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:14:09.188 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:14:09.188 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:14:09.188 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:14:09.188 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:14:09.188 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:14:09.188 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:14:09.189 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:14:09.189 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:14:09.189 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:14:09.189 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:14:09.189 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:14:09.189 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:14:09.189 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:14:09.189 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:14:09.189 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:14:09.189 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:14:09.189 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:14:09.189 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:14:09.189 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:14:09.190 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:14:09.190 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:14:09.190 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:14:09.190 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:14:09.190 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:14:09.190 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:14:09.190 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:14:09.196 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:09.197 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:09.197 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:09.197 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:14:09.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:14:09.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:14:09.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:09.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:09.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:09.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:14:09.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:09.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:09.259 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:09.259 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:14:09.259 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:14:09.259 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:14:09.260 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:14:09.260 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:09.260 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:14:09.260 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:09.298 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 32 2026-02-18T08:14:09.298 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 38 2026-02-18T08:14:09.298 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:14:09.298 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=153, tag_tid=9, entry_tid=1], [object_number=152, tag_tid=9, entry_tid=0], [object_number=154, tag_tid=8, entry_tid=58722], [object_number=155, tag_tid=8, entry_tid=58719]]], state=connected] 2026-02-18T08:14:09.298 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=131, tag_tid=8, entry_tid=34463], [object_number=130, tag_tid=8, entry_tid=34462], [object_number=129, tag_tid=8, entry_tid=34461], [object_number=128, tag_tid=8, entry_tid=34460]]], state=connected] 2026-02-18T08:14:09.301 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:09.302 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:09.304 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:14:09.304 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:09.304 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:09.304 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:09.304 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:14:09.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:09.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:09.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:09.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:14:09.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:14:09.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:14:09.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:14:09.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:09.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:09.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:14:09.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 32 2026-02-18T08:14:09.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 38 2026-02-18T08:14:09.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:14:09.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=153, tag_tid=9, entry_tid=1], [object_number=152, tag_tid=9, entry_tid=0], [object_number=154, tag_tid=8, entry_tid=58722], [object_number=155, tag_tid=8, entry_tid=58719]]], state=connected] 2026-02-18T08:14:09.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=131, tag_tid=8, entry_tid=34463], [object_number=130, tag_tid=8, entry_tid=34462], [object_number=129, tag_tid=8, entry_tid=34461], [object_number=128, tag_tid=8, entry_tid=34460]]], state=connected] 2026-02-18T08:14:09.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:09.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=131, tag_tid=8, entry_tid=34463' 2026-02-18T08:14:09.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=153, tag_tid=9, entry_tid=1' -a 'object_number=153, tag_tid=9, entry_tid=1' = 'object_number=131, tag_tid=8, entry_tid=34463' 2026-02-18T08:14:09.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=131, tag_tid=8, entry_tid=34463' '!=' 'object_number=130, tag_tid=8, entry_tid=34214' 2026-02-18T08:14:09.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:14:09.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=131, tag_tid=8, entry_tid=34463' = 'object_number=130, tag_tid=8, entry_tid=34214' 2026-02-18T08:14:09.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=131, tag_tid=8, entry_tid=34463' 2026-02-18T08:14:09.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:09.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:14:09.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:09.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=9 2026-02-18T08:14:09.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=131, tag_tid=8, entry_tid=34463' 2026-02-18T08:14:09.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:14:09.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:09.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=8 2026-02-18T08:14:09.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:09.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:14:09.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:09.344 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:14:09.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=131, tag_tid=8, entry_tid=34463' 2026-02-18T08:14:09.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:14:09.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:09.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=34463 2026-02-18T08:14:09.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 9 = 8 -a 1 -le 34463 2026-02-18T08:14:09.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:14:09.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:14:09.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:14:09.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:14:09.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:14:09.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:14:09.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:14:09.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:14:09.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:14:09.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:14:09.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:14:09.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:14:09.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:14:09.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:14:09.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:14:09.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:14:09.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:14:09.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:14:09.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:14:09.553 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:14:09.553 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:14:09.553 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:14:09.553 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:14:09.553 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:14:09.553 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:14:09.553 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:14:09.553 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:14:09.553 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:14:09.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:14:09.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:14:09.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:14:09.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:14:09.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:14:09.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:14:09.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:14:09.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:14:09.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:14:09.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:14:09.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:14:09.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:14:09.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:14:09.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:14:09.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:14:09.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:14:09.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:14:09.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:14:09.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:14:09.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:14:09.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:14:09.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:14:09.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:14:09.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:14:09.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:09.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:09.565 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:09.565 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:14:09.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:14:09.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:14:09.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:09.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:09.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:09.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:14:09.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:09.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:09.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:09.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:14:09.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:14:09.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:14:09.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:14:09.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:09.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:14:09.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:09.654 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 32 2026-02-18T08:14:09.654 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 38 2026-02-18T08:14:09.654 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:14:09.655 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=153, tag_tid=9, entry_tid=1], [object_number=152, tag_tid=9, entry_tid=0], [object_number=154, tag_tid=8, entry_tid=58722], [object_number=155, tag_tid=8, entry_tid=58719]]], state=connected] 2026-02-18T08:14:09.655 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=129, tag_tid=8, entry_tid=34701], [object_number=128, tag_tid=8, entry_tid=34700], [object_number=131, tag_tid=8, entry_tid=34699], [object_number=130, tag_tid=8, entry_tid=34698]]], state=connected] 2026-02-18T08:14:09.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:09.659 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:09.660 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:14:09.660 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:09.660 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:09.660 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:09.660 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:14:09.660 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:09.660 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:09.660 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:09.660 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:14:09.660 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:14:09.661 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:14:09.661 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:14:09.661 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:09.662 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:14:09.662 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:09.688 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 32 2026-02-18T08:14:09.688 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 38 2026-02-18T08:14:09.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:14:09.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=153, tag_tid=9, entry_tid=1], [object_number=152, tag_tid=9, entry_tid=0], [object_number=154, tag_tid=8, entry_tid=58722], [object_number=155, tag_tid=8, entry_tid=58719]]], state=connected] 2026-02-18T08:14:09.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=129, tag_tid=8, entry_tid=34701], [object_number=128, tag_tid=8, entry_tid=34700], [object_number=131, tag_tid=8, entry_tid=34699], [object_number=130, tag_tid=8, entry_tid=34698]]], state=connected] 2026-02-18T08:14:09.691 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:09.693 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=129, tag_tid=8, entry_tid=34701' 2026-02-18T08:14:09.693 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=153, tag_tid=9, entry_tid=1' -a 'object_number=153, tag_tid=9, entry_tid=1' = 'object_number=129, tag_tid=8, entry_tid=34701' 2026-02-18T08:14:09.693 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=129, tag_tid=8, entry_tid=34701' '!=' 'object_number=131, tag_tid=8, entry_tid=34463' 2026-02-18T08:14:09.693 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:14:09.693 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=129, tag_tid=8, entry_tid=34701' = 'object_number=131, tag_tid=8, entry_tid=34463' 2026-02-18T08:14:09.693 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=129, tag_tid=8, entry_tid=34701' 2026-02-18T08:14:09.694 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:09.694 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:14:09.694 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:09.694 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=9 2026-02-18T08:14:09.695 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=129, tag_tid=8, entry_tid=34701' 2026-02-18T08:14:09.695 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:14:09.695 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:09.697 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=8 2026-02-18T08:14:09.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:09.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:14:09.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:09.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:14:09.700 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=129, tag_tid=8, entry_tid=34701' 2026-02-18T08:14:09.700 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:14:09.700 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:09.702 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=34701 2026-02-18T08:14:09.702 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 9 = 8 -a 1 -le 34701 2026-02-18T08:14:09.702 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:14:09.702 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:14:09.702 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:14:09.904 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:14:09.904 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:14:09.904 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:14:09.904 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:14:09.904 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:14:09.904 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:14:09.904 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:14:09.905 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:14:09.905 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:14:09.905 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:14:09.905 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:14:09.905 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:14:09.905 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:14:09.905 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:14:09.905 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:14:09.905 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:14:09.905 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:14:09.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:14:09.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:14:09.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:14:09.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:14:09.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:14:09.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:14:09.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:14:09.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:14:09.909 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:14:09.909 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:14:09.909 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:14:09.909 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:14:09.909 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:14:09.909 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:14:09.909 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:14:09.909 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:14:09.909 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:14:09.909 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:14:09.909 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:14:09.909 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:14:09.909 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:14:09.910 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:14:09.910 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:14:09.910 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:14:09.910 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:14:09.910 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:14:09.910 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:14:09.910 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:14:09.910 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:14:09.910 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:14:09.910 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:14:09.910 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:14:09.918 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:09.918 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:09.918 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:09.918 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:14:10.101 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:14:10.101 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:14:10.101 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:10.101 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:10.101 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:10.101 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:14:10.101 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:10.101 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:10.102 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:10.102 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:14:10.102 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:14:10.102 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:14:10.102 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:14:10.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:10.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:14:10.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:10.140 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 32 2026-02-18T08:14:10.140 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 38 2026-02-18T08:14:10.140 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:14:10.140 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=153, tag_tid=9, entry_tid=1], [object_number=152, tag_tid=9, entry_tid=0], [object_number=154, tag_tid=8, entry_tid=58722], [object_number=155, tag_tid=8, entry_tid=58719]]], state=connected] 2026-02-18T08:14:10.140 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=128, tag_tid=8, entry_tid=34928], [object_number=131, tag_tid=8, entry_tid=34927], [object_number=130, tag_tid=8, entry_tid=34926], [object_number=129, tag_tid=8, entry_tid=34925]]], state=connected] 2026-02-18T08:14:10.144 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:10.145 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:10.146 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:14:10.146 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:10.146 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:10.146 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:10.146 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:14:10.146 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:10.146 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:10.146 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:10.146 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:14:10.146 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:14:10.146 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:14:10.147 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:14:10.148 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:10.149 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:14:10.149 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:10.175 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 32 2026-02-18T08:14:10.175 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 38 2026-02-18T08:14:10.175 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:14:10.175 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=153, tag_tid=9, entry_tid=1], [object_number=152, tag_tid=9, entry_tid=0], [object_number=154, tag_tid=8, entry_tid=58722], [object_number=155, tag_tid=8, entry_tid=58719]]], state=connected] 2026-02-18T08:14:10.175 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=128, tag_tid=8, entry_tid=34928], [object_number=131, tag_tid=8, entry_tid=34927], [object_number=130, tag_tid=8, entry_tid=34926], [object_number=129, tag_tid=8, entry_tid=34925]]], state=connected] 2026-02-18T08:14:10.178 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:10.179 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=128, tag_tid=8, entry_tid=34928' 2026-02-18T08:14:10.179 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=153, tag_tid=9, entry_tid=1' -a 'object_number=153, tag_tid=9, entry_tid=1' = 'object_number=128, tag_tid=8, entry_tid=34928' 2026-02-18T08:14:10.179 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=128, tag_tid=8, entry_tid=34928' '!=' 'object_number=129, tag_tid=8, entry_tid=34701' 2026-02-18T08:14:10.179 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:14:10.180 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=128, tag_tid=8, entry_tid=34928' = 'object_number=129, tag_tid=8, entry_tid=34701' 2026-02-18T08:14:10.180 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=128, tag_tid=8, entry_tid=34928' 2026-02-18T08:14:10.180 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:10.180 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:14:10.180 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:10.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=9 2026-02-18T08:14:10.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=128, tag_tid=8, entry_tid=34928' 2026-02-18T08:14:10.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:14:10.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:10.183 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=8 2026-02-18T08:14:10.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:10.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:14:10.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:10.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:14:10.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=128, tag_tid=8, entry_tid=34928' 2026-02-18T08:14:10.187 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:14:10.187 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:10.188 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=34928 2026-02-18T08:14:10.189 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 9 = 8 -a 1 -le 34928 2026-02-18T08:14:10.189 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:14:10.189 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:14:10.189 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:14:10.390 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:14:10.390 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:14:10.390 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:14:10.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:14:10.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:14:10.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:14:10.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:14:10.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:14:10.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:14:10.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:14:10.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:14:10.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:14:10.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:14:10.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:14:10.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:14:10.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:14:10.394 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:14:10.394 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:14:10.394 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:14:10.394 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:14:10.394 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:14:10.395 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:14:10.395 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:14:10.395 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:14:10.395 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:14:10.395 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:14:10.395 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:14:10.395 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:14:10.395 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:14:10.395 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:14:10.395 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:14:10.395 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:14:10.395 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:14:10.396 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:14:10.396 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:14:10.396 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:14:10.396 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:14:10.396 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:14:10.396 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:14:10.396 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:14:10.396 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:14:10.396 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:14:10.396 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:14:10.396 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:14:10.396 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:14:10.397 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:14:10.397 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:14:10.397 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:14:10.397 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:14:10.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:10.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:10.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:10.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:14:10.474 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:14:10.474 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:14:10.475 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:10.475 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:10.475 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:10.475 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:14:10.475 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:10.475 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:10.475 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:10.475 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:14:10.475 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:14:10.476 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:14:10.476 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:14:10.476 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:10.477 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:14:10.477 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:10.503 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 32 2026-02-18T08:14:10.503 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 38 2026-02-18T08:14:10.503 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:14:10.503 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=153, tag_tid=9, entry_tid=1], [object_number=152, tag_tid=9, entry_tid=0], [object_number=154, tag_tid=8, entry_tid=58722], [object_number=155, tag_tid=8, entry_tid=58719]]], state=connected] 2026-02-18T08:14:10.503 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=129, tag_tid=8, entry_tid=35197], [object_number=128, tag_tid=8, entry_tid=35196], [object_number=131, tag_tid=8, entry_tid=35195], [object_number=130, tag_tid=8, entry_tid=35194]]], state=connected] 2026-02-18T08:14:10.506 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:10.507 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:10.508 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:14:10.508 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:10.508 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:10.508 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:10.508 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:14:10.508 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:10.508 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:10.509 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:10.509 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:14:10.509 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:14:10.509 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:14:10.509 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:14:10.510 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:10.510 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:14:10.510 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:10.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 32 2026-02-18T08:14:10.545 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 38 2026-02-18T08:14:10.545 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:14:10.545 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=153, tag_tid=9, entry_tid=1], [object_number=152, tag_tid=9, entry_tid=0], [object_number=154, tag_tid=8, entry_tid=58722], [object_number=155, tag_tid=8, entry_tid=58719]]], state=connected] 2026-02-18T08:14:10.545 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=128, tag_tid=8, entry_tid=35228], [object_number=131, tag_tid=8, entry_tid=35227], [object_number=130, tag_tid=8, entry_tid=35226], [object_number=129, tag_tid=8, entry_tid=35225]]], state=connected] 2026-02-18T08:14:10.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:10.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=128, tag_tid=8, entry_tid=35228' 2026-02-18T08:14:10.551 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=153, tag_tid=9, entry_tid=1' -a 'object_number=153, tag_tid=9, entry_tid=1' = 'object_number=128, tag_tid=8, entry_tid=35228' 2026-02-18T08:14:10.551 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=128, tag_tid=8, entry_tid=35228' '!=' 'object_number=128, tag_tid=8, entry_tid=34928' 2026-02-18T08:14:10.551 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:14:10.551 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=128, tag_tid=8, entry_tid=35228' = 'object_number=128, tag_tid=8, entry_tid=34928' 2026-02-18T08:14:10.551 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=128, tag_tid=8, entry_tid=35228' 2026-02-18T08:14:10.552 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:10.552 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:14:10.552 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:10.552 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=9 2026-02-18T08:14:10.553 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=128, tag_tid=8, entry_tid=35228' 2026-02-18T08:14:10.553 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:14:10.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:10.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=8 2026-02-18T08:14:10.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:10.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:14:10.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:10.557 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:14:10.558 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=128, tag_tid=8, entry_tid=35228' 2026-02-18T08:14:10.558 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:14:10.558 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:10.559 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=35228 2026-02-18T08:14:10.559 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 9 = 8 -a 1 -le 35228 2026-02-18T08:14:10.559 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:14:10.560 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:14:10.560 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:14:10.761 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:14:10.761 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:14:10.761 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:14:10.761 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:14:10.762 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:14:10.762 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:14:10.762 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:14:10.762 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:14:10.762 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:14:10.762 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:14:10.762 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:14:10.762 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:14:10.762 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:14:10.762 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:14:10.762 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:14:10.762 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:14:10.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:14:10.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:14:10.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:14:10.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:14:10.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:14:10.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:14:10.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:14:10.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:14:10.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:14:10.766 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:14:10.766 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:14:10.766 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:14:10.766 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:14:10.766 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:14:10.766 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:14:10.766 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:14:10.766 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:14:10.766 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:14:10.766 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:14:10.766 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:14:10.766 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:14:10.766 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:14:10.767 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:14:10.767 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:14:10.767 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:14:10.767 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:14:10.767 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:14:10.767 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:14:10.767 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:14:10.767 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:14:10.767 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:14:10.767 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:14:10.767 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:14:10.776 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:10.776 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:10.776 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:10.776 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:14:10.838 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:14:10.839 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:14:10.840 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:10.840 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:10.840 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:10.840 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:14:10.840 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:10.840 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:10.840 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:10.840 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:14:10.840 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:14:10.840 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:14:10.841 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:14:10.841 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:10.841 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:14:10.842 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:10.869 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 32 2026-02-18T08:14:10.869 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 38 2026-02-18T08:14:10.869 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:14:10.869 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=153, tag_tid=9, entry_tid=1], [object_number=152, tag_tid=9, entry_tid=0], [object_number=154, tag_tid=8, entry_tid=58722], [object_number=155, tag_tid=8, entry_tid=58719]]], state=connected] 2026-02-18T08:14:10.869 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=133, tag_tid=8, entry_tid=35353], [object_number=132, tag_tid=8, entry_tid=35352], [object_number=135, tag_tid=8, entry_tid=35351], [object_number=130, tag_tid=8, entry_tid=35350]]], state=connected] 2026-02-18T08:14:10.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:10.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:10.874 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:14:10.874 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:10.874 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:10.874 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:10.874 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:14:10.874 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:10.875 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:10.875 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:10.875 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:14:10.875 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:14:10.875 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:14:10.875 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:14:10.876 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:10.876 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:14:10.876 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:10.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 32 2026-02-18T08:14:10.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 38 2026-02-18T08:14:10.904 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:14:10.904 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=153, tag_tid=9, entry_tid=1], [object_number=152, tag_tid=9, entry_tid=0], [object_number=154, tag_tid=8, entry_tid=58722], [object_number=155, tag_tid=8, entry_tid=58719]]], state=connected] 2026-02-18T08:14:10.904 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=133, tag_tid=8, entry_tid=35353], [object_number=132, tag_tid=8, entry_tid=35352], [object_number=135, tag_tid=8, entry_tid=35351], [object_number=130, tag_tid=8, entry_tid=35350]]], state=connected] 2026-02-18T08:14:10.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:10.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=133, tag_tid=8, entry_tid=35353' 2026-02-18T08:14:10.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=153, tag_tid=9, entry_tid=1' -a 'object_number=153, tag_tid=9, entry_tid=1' = 'object_number=133, tag_tid=8, entry_tid=35353' 2026-02-18T08:14:10.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=133, tag_tid=8, entry_tid=35353' '!=' 'object_number=128, tag_tid=8, entry_tid=35228' 2026-02-18T08:14:10.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:14:10.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=133, tag_tid=8, entry_tid=35353' = 'object_number=128, tag_tid=8, entry_tid=35228' 2026-02-18T08:14:10.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=133, tag_tid=8, entry_tid=35353' 2026-02-18T08:14:10.909 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:10.909 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:14:10.909 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:10.910 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=9 2026-02-18T08:14:10.911 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=133, tag_tid=8, entry_tid=35353' 2026-02-18T08:14:10.911 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:14:10.911 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:10.912 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=8 2026-02-18T08:14:10.913 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:10.913 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:14:10.913 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:10.914 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:14:10.915 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=133, tag_tid=8, entry_tid=35353' 2026-02-18T08:14:10.915 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:14:10.915 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:10.917 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=35353 2026-02-18T08:14:10.917 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 9 = 8 -a 1 -le 35353 2026-02-18T08:14:10.917 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:14:10.917 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:14:10.918 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:14:11.119 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:14:11.119 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:14:11.119 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:14:11.119 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:14:11.120 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:14:11.120 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:14:11.120 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:14:11.120 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:14:11.120 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:14:11.120 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:14:11.120 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:14:11.120 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:14:11.120 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:14:11.120 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:14:11.120 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:14:11.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:14:11.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:14:11.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:14:11.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:14:11.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:14:11.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:14:11.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:14:11.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:14:11.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:14:11.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:14:11.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:14:11.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:14:11.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:14:11.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:14:11.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:14:11.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:14:11.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:14:11.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:14:11.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:14:11.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:14:11.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:14:11.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:14:11.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:14:11.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:14:11.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:14:11.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:14:11.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:14:11.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:14:11.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:14:11.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:14:11.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:14:11.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:14:11.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:14:11.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:14:11.134 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:11.134 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:11.134 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:11.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:14:11.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:14:11.194 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:14:11.194 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:11.194 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:11.194 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:11.194 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:14:11.194 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:11.194 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:11.195 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:11.195 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:14:11.195 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:14:11.195 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:14:11.195 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:14:11.196 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:11.196 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:14:11.196 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:11.223 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 33 2026-02-18T08:14:11.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 38 2026-02-18T08:14:11.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:14:11.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=153, tag_tid=9, entry_tid=1], [object_number=152, tag_tid=9, entry_tid=0], [object_number=154, tag_tid=8, entry_tid=58722], [object_number=155, tag_tid=8, entry_tid=58719]]], state=connected] 2026-02-18T08:14:11.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=133, tag_tid=8, entry_tid=35573], [object_number=132, tag_tid=8, entry_tid=35572], [object_number=135, tag_tid=8, entry_tid=35571], [object_number=134, tag_tid=8, entry_tid=35570]]], state=connected] 2026-02-18T08:14:11.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:11.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:11.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:14:11.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:11.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:11.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:11.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:14:11.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:11.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:11.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:11.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:14:11.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:14:11.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:14:11.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:14:11.244 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:11.244 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:14:11.244 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:11.257 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 33 2026-02-18T08:14:11.257 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 38 2026-02-18T08:14:11.257 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:14:11.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=153, tag_tid=9, entry_tid=1], [object_number=152, tag_tid=9, entry_tid=0], [object_number=154, tag_tid=8, entry_tid=58722], [object_number=155, tag_tid=8, entry_tid=58719]]], state=connected] 2026-02-18T08:14:11.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=133, tag_tid=8, entry_tid=35573], [object_number=132, tag_tid=8, entry_tid=35572], [object_number=135, tag_tid=8, entry_tid=35571], [object_number=134, tag_tid=8, entry_tid=35570]]], state=connected] 2026-02-18T08:14:11.260 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:11.261 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=133, tag_tid=8, entry_tid=35573' 2026-02-18T08:14:11.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=153, tag_tid=9, entry_tid=1' -a 'object_number=153, tag_tid=9, entry_tid=1' = 'object_number=133, tag_tid=8, entry_tid=35573' 2026-02-18T08:14:11.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=133, tag_tid=8, entry_tid=35573' '!=' 'object_number=133, tag_tid=8, entry_tid=35353' 2026-02-18T08:14:11.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:14:11.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=133, tag_tid=8, entry_tid=35573' = 'object_number=133, tag_tid=8, entry_tid=35353' 2026-02-18T08:14:11.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=133, tag_tid=8, entry_tid=35573' 2026-02-18T08:14:11.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:11.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:14:11.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:11.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=9 2026-02-18T08:14:11.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=133, tag_tid=8, entry_tid=35573' 2026-02-18T08:14:11.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:14:11.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:11.266 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=8 2026-02-18T08:14:11.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:11.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:14:11.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:11.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:14:11.269 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=133, tag_tid=8, entry_tid=35573' 2026-02-18T08:14:11.269 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:14:11.269 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:11.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=35573 2026-02-18T08:14:11.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 9 = 8 -a 1 -le 35573 2026-02-18T08:14:11.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:14:11.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:14:11.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:14:11.473 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:14:11.473 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:14:11.473 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:14:11.473 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:14:11.473 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:14:11.474 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:14:11.474 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:14:11.474 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:14:11.474 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:14:11.474 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:14:11.474 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:14:11.474 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:14:11.474 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:14:11.474 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:14:11.474 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:14:11.474 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:14:11.474 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:14:11.477 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:14:11.477 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:14:11.477 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:14:11.477 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:14:11.477 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:14:11.477 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:14:11.477 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:14:11.477 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:14:11.477 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:14:11.478 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:14:11.478 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:14:11.478 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:14:11.478 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:14:11.478 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:14:11.478 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:14:11.478 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:14:11.478 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:14:11.478 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:14:11.478 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:14:11.478 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:14:11.478 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:14:11.479 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:14:11.479 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:14:11.479 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:14:11.479 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:14:11.479 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:14:11.479 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:14:11.479 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:14:11.479 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:14:11.479 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:14:11.479 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:14:11.479 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:14:11.487 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:11.488 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:11.488 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:11.488 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:14:11.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:14:11.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:14:11.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:11.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:11.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:11.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:14:11.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:11.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:11.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:11.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:14:11.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:14:11.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:14:11.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:14:11.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:11.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:14:11.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:11.576 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 33 2026-02-18T08:14:11.576 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 38 2026-02-18T08:14:11.577 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:14:11.577 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=153, tag_tid=9, entry_tid=1], [object_number=152, tag_tid=9, entry_tid=0], [object_number=154, tag_tid=8, entry_tid=58722], [object_number=155, tag_tid=8, entry_tid=58719]]], state=connected] 2026-02-18T08:14:11.577 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=135, tag_tid=8, entry_tid=35803], [object_number=134, tag_tid=8, entry_tid=35802], [object_number=133, tag_tid=8, entry_tid=35801], [object_number=132, tag_tid=8, entry_tid=35800]]], state=connected] 2026-02-18T08:14:11.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:11.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:11.582 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:14:11.582 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:11.582 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:11.582 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:11.582 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:14:11.582 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:11.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:11.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:11.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:14:11.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:14:11.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:14:11.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:14:11.584 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:11.584 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:14:11.585 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:11.612 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 33 2026-02-18T08:14:11.612 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 38 2026-02-18T08:14:11.612 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:14:11.612 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=153, tag_tid=9, entry_tid=1], [object_number=152, tag_tid=9, entry_tid=0], [object_number=154, tag_tid=8, entry_tid=58722], [object_number=155, tag_tid=8, entry_tid=58719]]], state=connected] 2026-02-18T08:14:11.613 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=135, tag_tid=8, entry_tid=35803], [object_number=134, tag_tid=8, entry_tid=35802], [object_number=133, tag_tid=8, entry_tid=35801], [object_number=132, tag_tid=8, entry_tid=35800]]], state=connected] 2026-02-18T08:14:11.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:11.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=135, tag_tid=8, entry_tid=35803' 2026-02-18T08:14:11.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=153, tag_tid=9, entry_tid=1' -a 'object_number=153, tag_tid=9, entry_tid=1' = 'object_number=135, tag_tid=8, entry_tid=35803' 2026-02-18T08:14:11.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=135, tag_tid=8, entry_tid=35803' '!=' 'object_number=133, tag_tid=8, entry_tid=35573' 2026-02-18T08:14:11.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:14:11.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=135, tag_tid=8, entry_tid=35803' = 'object_number=133, tag_tid=8, entry_tid=35573' 2026-02-18T08:14:11.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=135, tag_tid=8, entry_tid=35803' 2026-02-18T08:14:11.618 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:11.618 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:14:11.618 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:11.619 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=9 2026-02-18T08:14:11.619 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=135, tag_tid=8, entry_tid=35803' 2026-02-18T08:14:11.620 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:14:11.620 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:11.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=8 2026-02-18T08:14:11.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:11.622 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:14:11.622 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:11.623 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:14:11.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=135, tag_tid=8, entry_tid=35803' 2026-02-18T08:14:11.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:14:11.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:11.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=35803 2026-02-18T08:14:11.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 9 = 8 -a 1 -le 35803 2026-02-18T08:14:11.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:14:11.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:14:11.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:14:11.827 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:14:11.827 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:14:11.827 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:14:11.828 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:14:11.828 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:14:11.828 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:14:11.828 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:14:11.828 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:14:11.828 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:14:11.828 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:14:11.828 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:14:11.828 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:14:11.828 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:14:11.829 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:14:11.829 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:14:11.829 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:14:11.831 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:14:11.832 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:14:11.832 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:14:11.832 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:14:11.832 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:14:11.832 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:14:11.832 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:14:11.832 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:14:11.832 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:14:11.832 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:14:11.832 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:14:11.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:14:11.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:14:11.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:14:11.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:14:11.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:14:11.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:14:11.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:14:11.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:14:11.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:14:11.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:14:11.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:14:11.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:14:11.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:14:11.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:14:11.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:14:11.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:14:11.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:14:11.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:14:11.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:14:11.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:14:11.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:14:11.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:14:11.841 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:11.841 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:11.841 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:11.841 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:14:11.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:14:11.904 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:14:11.904 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:11.904 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:11.904 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:11.905 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:14:11.905 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:11.905 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:11.905 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:11.905 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:14:11.905 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:14:11.905 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:14:11.905 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:14:11.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:11.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:14:11.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:11.936 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 33 2026-02-18T08:14:11.936 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 38 2026-02-18T08:14:11.936 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:14:11.936 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=153, tag_tid=9, entry_tid=1], [object_number=152, tag_tid=9, entry_tid=0], [object_number=154, tag_tid=8, entry_tid=58722], [object_number=155, tag_tid=8, entry_tid=58719]]], state=connected] 2026-02-18T08:14:11.936 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=135, tag_tid=8, entry_tid=36039], [object_number=134, tag_tid=8, entry_tid=36038], [object_number=133, tag_tid=8, entry_tid=36037], [object_number=132, tag_tid=8, entry_tid=36036]]], state=connected] 2026-02-18T08:14:11.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:11.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:11.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:14:11.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:11.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:11.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:11.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:14:11.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:11.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:11.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:11.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:14:11.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:14:11.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:14:11.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:14:11.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:11.944 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:14:11.944 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:11.971 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 33 2026-02-18T08:14:11.971 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 38 2026-02-18T08:14:11.971 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:14:11.971 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=153, tag_tid=9, entry_tid=1], [object_number=152, tag_tid=9, entry_tid=0], [object_number=154, tag_tid=8, entry_tid=58722], [object_number=155, tag_tid=8, entry_tid=58719]]], state=connected] 2026-02-18T08:14:11.971 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=135, tag_tid=8, entry_tid=36039], [object_number=134, tag_tid=8, entry_tid=36038], [object_number=133, tag_tid=8, entry_tid=36037], [object_number=132, tag_tid=8, entry_tid=36036]]], state=connected] 2026-02-18T08:14:11.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:11.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=135, tag_tid=8, entry_tid=36039' 2026-02-18T08:14:11.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=153, tag_tid=9, entry_tid=1' -a 'object_number=153, tag_tid=9, entry_tid=1' = 'object_number=135, tag_tid=8, entry_tid=36039' 2026-02-18T08:14:11.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=135, tag_tid=8, entry_tid=36039' '!=' 'object_number=135, tag_tid=8, entry_tid=35803' 2026-02-18T08:14:11.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:14:11.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=135, tag_tid=8, entry_tid=36039' = 'object_number=135, tag_tid=8, entry_tid=35803' 2026-02-18T08:14:11.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=135, tag_tid=8, entry_tid=36039' 2026-02-18T08:14:11.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:11.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:14:11.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:11.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=9 2026-02-18T08:14:11.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=135, tag_tid=8, entry_tid=36039' 2026-02-18T08:14:11.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:14:11.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:11.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=8 2026-02-18T08:14:11.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:11.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:14:11.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:11.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:14:11.982 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=135, tag_tid=8, entry_tid=36039' 2026-02-18T08:14:11.982 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:14:11.982 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:11.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=36039 2026-02-18T08:14:11.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 9 = 8 -a 1 -le 36039 2026-02-18T08:14:11.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:14:11.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:14:11.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:14:12.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:14:12.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:14:12.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:14:12.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:14:12.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:14:12.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:14:12.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:14:12.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:14:12.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:14:12.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:14:12.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:14:12.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:14:12.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:14:12.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:14:12.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:14:12.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:14:12.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:14:12.187 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:14:12.187 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:14:12.187 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:14:12.187 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:14:12.187 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:14:12.187 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:14:12.187 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:14:12.187 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:14:12.187 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:14:12.187 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:14:12.187 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:14:12.188 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:14:12.188 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:14:12.188 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:14:12.188 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:14:12.188 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:14:12.188 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:14:12.188 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:14:12.188 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:14:12.188 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:14:12.188 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:14:12.188 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:14:12.188 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:14:12.188 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:14:12.188 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:14:12.188 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:14:12.188 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:14:12.188 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:14:12.188 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:14:12.188 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:14:12.188 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:14:12.188 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:14:12.200 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:12.200 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:12.200 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:12.200 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:14:12.259 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:14:12.260 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:14:12.260 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:12.260 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:12.261 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:12.261 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:14:12.261 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:12.261 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:12.261 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:12.261 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:14:12.261 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:14:12.261 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:14:12.261 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:14:12.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:12.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:14:12.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:12.290 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 33 2026-02-18T08:14:12.290 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 38 2026-02-18T08:14:12.290 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:14:12.290 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=153, tag_tid=9, entry_tid=1], [object_number=152, tag_tid=9, entry_tid=0], [object_number=154, tag_tid=8, entry_tid=58722], [object_number=155, tag_tid=8, entry_tid=58719]]], state=connected] 2026-02-18T08:14:12.290 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=133, tag_tid=8, entry_tid=36273], [object_number=132, tag_tid=8, entry_tid=36272], [object_number=135, tag_tid=8, entry_tid=36271], [object_number=134, tag_tid=8, entry_tid=36270]]], state=connected] 2026-02-18T08:14:12.292 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:12.294 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:12.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:14:12.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:12.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:12.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:12.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:14:12.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:12.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:12.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:12.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:14:12.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:14:12.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:14:12.297 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:14:12.297 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:12.298 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:14:12.298 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:12.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 33 2026-02-18T08:14:12.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 38 2026-02-18T08:14:12.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:14:12.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=153, tag_tid=9, entry_tid=1], [object_number=152, tag_tid=9, entry_tid=0], [object_number=154, tag_tid=8, entry_tid=58722], [object_number=155, tag_tid=8, entry_tid=58719]]], state=connected] 2026-02-18T08:14:12.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=133, tag_tid=8, entry_tid=36273], [object_number=132, tag_tid=8, entry_tid=36272], [object_number=135, tag_tid=8, entry_tid=36271], [object_number=134, tag_tid=8, entry_tid=36270]]], state=connected] 2026-02-18T08:14:12.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:12.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=133, tag_tid=8, entry_tid=36273' 2026-02-18T08:14:12.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=153, tag_tid=9, entry_tid=1' -a 'object_number=153, tag_tid=9, entry_tid=1' = 'object_number=133, tag_tid=8, entry_tid=36273' 2026-02-18T08:14:12.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=133, tag_tid=8, entry_tid=36273' '!=' 'object_number=135, tag_tid=8, entry_tid=36039' 2026-02-18T08:14:12.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:14:12.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=133, tag_tid=8, entry_tid=36273' = 'object_number=135, tag_tid=8, entry_tid=36039' 2026-02-18T08:14:12.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=133, tag_tid=8, entry_tid=36273' 2026-02-18T08:14:12.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:12.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:14:12.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:12.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=9 2026-02-18T08:14:12.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=133, tag_tid=8, entry_tid=36273' 2026-02-18T08:14:12.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:14:12.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:12.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=8 2026-02-18T08:14:12.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:12.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:14:12.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:12.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:14:12.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=133, tag_tid=8, entry_tid=36273' 2026-02-18T08:14:12.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:14:12.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:12.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=36273 2026-02-18T08:14:12.348 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 9 = 8 -a 1 -le 36273 2026-02-18T08:14:12.348 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:14:12.348 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:14:12.348 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:14:12.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:14:12.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:14:12.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:14:12.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:14:12.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:14:12.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:14:12.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:14:12.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:14:12.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:14:12.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:14:12.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:14:12.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:14:12.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:14:12.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:14:12.551 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:14:12.551 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:14:12.553 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:14:12.553 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:14:12.553 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:14:12.553 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:14:12.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:14:12.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:14:12.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:14:12.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:14:12.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:14:12.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:14:12.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:14:12.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:14:12.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:14:12.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:14:12.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:14:12.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:14:12.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:14:12.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:14:12.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:14:12.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:14:12.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:14:12.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:14:12.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:14:12.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:14:12.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:14:12.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:14:12.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:14:12.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:14:12.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:14:12.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:14:12.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:14:12.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:14:12.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:14:12.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:12.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:12.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:12.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:14:12.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:14:12.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:14:12.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:12.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:12.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:12.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:14:12.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:12.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:12.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:12.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:14:12.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:14:12.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:14:12.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:14:12.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:12.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:14:12.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:12.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 33 2026-02-18T08:14:12.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 38 2026-02-18T08:14:12.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:14:12.653 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=153, tag_tid=9, entry_tid=1], [object_number=152, tag_tid=9, entry_tid=0], [object_number=154, tag_tid=8, entry_tid=58722], [object_number=155, tag_tid=8, entry_tid=58719]]], state=connected] 2026-02-18T08:14:12.653 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=132, tag_tid=8, entry_tid=36528], [object_number=135, tag_tid=8, entry_tid=36527], [object_number=134, tag_tid=8, entry_tid=36526], [object_number=133, tag_tid=8, entry_tid=36525]]], state=connected] 2026-02-18T08:14:12.655 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:12.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:12.658 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:14:12.658 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:12.658 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:12.658 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:12.658 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:14:12.658 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:12.658 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:12.659 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:12.659 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:14:12.659 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:14:12.659 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:14:12.660 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:14:12.660 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:12.660 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:14:12.660 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:12.688 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 33 2026-02-18T08:14:12.688 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 38 2026-02-18T08:14:12.688 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:14:12.688 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=153, tag_tid=9, entry_tid=1], [object_number=152, tag_tid=9, entry_tid=0], [object_number=154, tag_tid=8, entry_tid=58722], [object_number=155, tag_tid=8, entry_tid=58719]]], state=connected] 2026-02-18T08:14:12.688 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=132, tag_tid=8, entry_tid=36528], [object_number=135, tag_tid=8, entry_tid=36527], [object_number=134, tag_tid=8, entry_tid=36526], [object_number=133, tag_tid=8, entry_tid=36525]]], state=connected] 2026-02-18T08:14:12.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:12.691 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=132, tag_tid=8, entry_tid=36528' 2026-02-18T08:14:12.691 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=153, tag_tid=9, entry_tid=1' -a 'object_number=153, tag_tid=9, entry_tid=1' = 'object_number=132, tag_tid=8, entry_tid=36528' 2026-02-18T08:14:12.692 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=132, tag_tid=8, entry_tid=36528' '!=' 'object_number=133, tag_tid=8, entry_tid=36273' 2026-02-18T08:14:12.692 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:14:12.692 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=132, tag_tid=8, entry_tid=36528' = 'object_number=133, tag_tid=8, entry_tid=36273' 2026-02-18T08:14:12.692 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=132, tag_tid=8, entry_tid=36528' 2026-02-18T08:14:12.693 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:12.693 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:14:12.693 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:12.693 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=9 2026-02-18T08:14:12.694 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=132, tag_tid=8, entry_tid=36528' 2026-02-18T08:14:12.694 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:14:12.695 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:12.696 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=8 2026-02-18T08:14:12.697 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:12.697 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:14:12.697 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:12.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:14:12.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=132, tag_tid=8, entry_tid=36528' 2026-02-18T08:14:12.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:14:12.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:12.701 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=36528 2026-02-18T08:14:12.701 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 9 = 8 -a 1 -le 36528 2026-02-18T08:14:12.701 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:14:12.701 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:14:12.701 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:14:12.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:14:12.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:14:12.904 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:14:12.904 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:14:12.904 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:14:12.904 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:14:12.904 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:14:12.904 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:14:12.904 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:14:12.904 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:14:12.904 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:14:12.904 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:14:12.904 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:14:12.905 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:14:12.905 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:14:12.905 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:14:12.905 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:14:12.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:14:12.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:14:12.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:14:12.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:14:12.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:14:12.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:14:12.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:14:12.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:14:12.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:14:12.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:14:12.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:14:12.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:14:12.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:14:12.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:14:12.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:14:12.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:14:12.909 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:14:12.909 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:14:12.909 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:14:12.909 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:14:12.909 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:14:12.909 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:14:12.909 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:14:12.909 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:14:12.909 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:14:12.909 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:14:12.909 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:14:12.909 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:14:12.910 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:14:12.910 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:14:12.910 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:14:12.910 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:14:12.916 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:12.917 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:12.917 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:12.917 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:14:12.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:14:12.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:14:12.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:12.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:12.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:12.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:14:12.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:12.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:12.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:12.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:14:12.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:14:12.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:14:12.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:14:12.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:12.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:12.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:14:13.006 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 33 2026-02-18T08:14:13.006 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 38 2026-02-18T08:14:13.006 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:14:13.006 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=153, tag_tid=9, entry_tid=1], [object_number=152, tag_tid=9, entry_tid=0], [object_number=154, tag_tid=8, entry_tid=58722], [object_number=155, tag_tid=8, entry_tid=58719]]], state=connected] 2026-02-18T08:14:13.006 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=135, tag_tid=8, entry_tid=36755], [object_number=134, tag_tid=8, entry_tid=36754], [object_number=133, tag_tid=8, entry_tid=36753], [object_number=132, tag_tid=8, entry_tid=36752]]], state=connected] 2026-02-18T08:14:13.008 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:13.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:13.010 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:14:13.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:13.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:13.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:13.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:14:13.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:13.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:13.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:13.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:14:13.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:14:13.012 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:14:13.012 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:14:13.012 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:13.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:14:13.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:13.039 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 33 2026-02-18T08:14:13.039 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 38 2026-02-18T08:14:13.039 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:14:13.039 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=153, tag_tid=9, entry_tid=1], [object_number=152, tag_tid=9, entry_tid=0], [object_number=154, tag_tid=8, entry_tid=58722], [object_number=155, tag_tid=8, entry_tid=58719]]], state=connected] 2026-02-18T08:14:13.039 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=135, tag_tid=8, entry_tid=36755], [object_number=134, tag_tid=8, entry_tid=36754], [object_number=133, tag_tid=8, entry_tid=36753], [object_number=132, tag_tid=8, entry_tid=36752]]], state=connected] 2026-02-18T08:14:13.041 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:13.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=135, tag_tid=8, entry_tid=36755' 2026-02-18T08:14:13.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=153, tag_tid=9, entry_tid=1' -a 'object_number=153, tag_tid=9, entry_tid=1' = 'object_number=135, tag_tid=8, entry_tid=36755' 2026-02-18T08:14:13.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=135, tag_tid=8, entry_tid=36755' '!=' 'object_number=132, tag_tid=8, entry_tid=36528' 2026-02-18T08:14:13.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:14:13.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=135, tag_tid=8, entry_tid=36755' = 'object_number=132, tag_tid=8, entry_tid=36528' 2026-02-18T08:14:13.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=135, tag_tid=8, entry_tid=36755' 2026-02-18T08:14:13.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:13.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:14:13.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:13.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=9 2026-02-18T08:14:13.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=135, tag_tid=8, entry_tid=36755' 2026-02-18T08:14:13.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:14:13.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:13.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=8 2026-02-18T08:14:13.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:13.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:14:13.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:13.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:14:13.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=135, tag_tid=8, entry_tid=36755' 2026-02-18T08:14:13.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:14:13.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:13.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=36755 2026-02-18T08:14:13.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 9 = 8 -a 1 -le 36755 2026-02-18T08:14:13.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:14:13.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:14:13.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:14:13.254 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:14:13.254 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:14:13.254 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:14:13.254 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:14:13.254 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:14:13.255 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:14:13.255 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:14:13.255 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:14:13.255 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:14:13.255 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:14:13.255 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:14:13.255 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:14:13.255 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:14:13.255 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:14:13.255 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:14:13.255 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:14:13.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:14:13.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:14:13.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:14:13.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:14:13.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:14:13.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:14:13.259 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:14:13.259 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:14:13.259 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:14:13.259 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:14:13.259 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:14:13.259 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:14:13.259 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:14:13.259 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:14:13.259 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:14:13.259 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:14:13.259 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:14:13.259 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:14:13.260 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:14:13.260 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:14:13.260 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:14:13.260 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:14:13.260 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:14:13.260 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:14:13.260 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:14:13.260 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:14:13.260 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:14:13.260 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:14:13.260 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:14:13.260 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:14:13.260 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:14:13.261 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:14:13.261 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:14:13.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:13.269 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:13.269 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:13.269 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:14:13.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:14:13.329 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:14:13.329 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:13.329 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:13.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:13.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:14:13.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:13.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:13.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:13.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:14:13.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:14:13.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:14:13.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:14:13.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:13.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:14:13.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:13.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 33 2026-02-18T08:14:13.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 38 2026-02-18T08:14:13.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:14:13.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=153, tag_tid=9, entry_tid=1], [object_number=152, tag_tid=9, entry_tid=0], [object_number=154, tag_tid=8, entry_tid=58722], [object_number=155, tag_tid=8, entry_tid=58719]]], state=connected] 2026-02-18T08:14:13.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=133, tag_tid=8, entry_tid=36977], [object_number=132, tag_tid=8, entry_tid=36976], [object_number=135, tag_tid=8, entry_tid=36975], [object_number=134, tag_tid=8, entry_tid=36974]]], state=connected] 2026-02-18T08:14:13.361 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:13.362 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:13.363 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:14:13.364 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:13.364 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:13.364 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:13.364 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:14:13.364 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:13.364 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:13.364 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:13.364 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:14:13.364 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:14:13.365 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:14:13.365 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:14:13.365 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:13.366 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:14:13.366 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:13.399 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 33 2026-02-18T08:14:13.399 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 38 2026-02-18T08:14:13.399 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:14:13.399 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=153, tag_tid=9, entry_tid=1], [object_number=152, tag_tid=9, entry_tid=0], [object_number=154, tag_tid=8, entry_tid=58722], [object_number=155, tag_tid=8, entry_tid=58719]]], state=connected] 2026-02-18T08:14:13.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=133, tag_tid=8, entry_tid=36977], [object_number=132, tag_tid=8, entry_tid=36976], [object_number=135, tag_tid=8, entry_tid=36975], [object_number=134, tag_tid=8, entry_tid=36974]]], state=connected] 2026-02-18T08:14:13.403 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:13.405 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=133, tag_tid=8, entry_tid=36977' 2026-02-18T08:14:13.406 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=153, tag_tid=9, entry_tid=1' -a 'object_number=153, tag_tid=9, entry_tid=1' = 'object_number=133, tag_tid=8, entry_tid=36977' 2026-02-18T08:14:13.406 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=133, tag_tid=8, entry_tid=36977' '!=' 'object_number=135, tag_tid=8, entry_tid=36755' 2026-02-18T08:14:13.406 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:14:13.406 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=133, tag_tid=8, entry_tid=36977' = 'object_number=135, tag_tid=8, entry_tid=36755' 2026-02-18T08:14:13.406 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=133, tag_tid=8, entry_tid=36977' 2026-02-18T08:14:13.406 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:13.406 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:14:13.406 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:13.407 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=9 2026-02-18T08:14:13.407 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=133, tag_tid=8, entry_tid=36977' 2026-02-18T08:14:13.407 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:14:13.408 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:13.409 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=8 2026-02-18T08:14:13.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:13.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:14:13.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:13.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:14:13.412 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=133, tag_tid=8, entry_tid=36977' 2026-02-18T08:14:13.412 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:14:13.412 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:13.413 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=36977 2026-02-18T08:14:13.414 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 9 = 8 -a 1 -le 36977 2026-02-18T08:14:13.414 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:14:13.414 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:14:13.414 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:14:13.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:14:13.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:14:13.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:14:13.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:14:13.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:14:13.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:14:13.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:14:13.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:14:13.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:14:13.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:14:13.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:14:13.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:14:13.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:14:13.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:14:13.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:14:13.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:14:13.620 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:14:13.620 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:14:13.620 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:14:13.620 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:14:13.620 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:14:13.620 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:14:13.620 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:14:13.620 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:14:13.620 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:14:13.620 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:14:13.620 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:14:13.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:14:13.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:14:13.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:14:13.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:14:13.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:14:13.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:14:13.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:14:13.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:14:13.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:14:13.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:14:13.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:14:13.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:14:13.622 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:14:13.622 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:14:13.622 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:14:13.622 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:14:13.622 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:14:13.622 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:14:13.622 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:14:13.622 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:14:13.622 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:14:13.622 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:14:13.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:13.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:13.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:13.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:14:13.691 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:14:13.693 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:14:13.693 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:13.693 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:13.693 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:13.693 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:14:13.693 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:13.693 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:13.694 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:13.694 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:14:13.694 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:14:13.694 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:14:13.694 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:14:13.695 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:13.695 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:14:13.695 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:13.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 33 2026-02-18T08:14:13.724 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 38 2026-02-18T08:14:13.724 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:14:13.724 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=153, tag_tid=9, entry_tid=1], [object_number=152, tag_tid=9, entry_tid=0], [object_number=154, tag_tid=8, entry_tid=58722], [object_number=155, tag_tid=8, entry_tid=58719]]], state=connected] 2026-02-18T08:14:13.724 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=134, tag_tid=8, entry_tid=37186], [object_number=133, tag_tid=8, entry_tid=37185], [object_number=132, tag_tid=8, entry_tid=37184], [object_number=135, tag_tid=8, entry_tid=37183]]], state=connected] 2026-02-18T08:14:13.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:13.728 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:13.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:14:13.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:13.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:13.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:13.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:14:13.731 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:13.731 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:13.731 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:13.731 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:14:13.731 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:14:13.731 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:14:13.731 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:14:13.732 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:13.733 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:14:13.733 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:13.759 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 33 2026-02-18T08:14:13.760 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 38 2026-02-18T08:14:13.760 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:14:13.760 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=153, tag_tid=9, entry_tid=1], [object_number=152, tag_tid=9, entry_tid=0], [object_number=154, tag_tid=8, entry_tid=58722], [object_number=155, tag_tid=8, entry_tid=58719]]], state=connected] 2026-02-18T08:14:13.760 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=134, tag_tid=8, entry_tid=37186], [object_number=133, tag_tid=8, entry_tid=37185], [object_number=132, tag_tid=8, entry_tid=37184], [object_number=135, tag_tid=8, entry_tid=37183]]], state=connected] 2026-02-18T08:14:13.762 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:13.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=134, tag_tid=8, entry_tid=37186' 2026-02-18T08:14:13.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=153, tag_tid=9, entry_tid=1' -a 'object_number=153, tag_tid=9, entry_tid=1' = 'object_number=134, tag_tid=8, entry_tid=37186' 2026-02-18T08:14:13.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=134, tag_tid=8, entry_tid=37186' '!=' 'object_number=133, tag_tid=8, entry_tid=36977' 2026-02-18T08:14:13.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:14:13.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=134, tag_tid=8, entry_tid=37186' = 'object_number=133, tag_tid=8, entry_tid=36977' 2026-02-18T08:14:13.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=134, tag_tid=8, entry_tid=37186' 2026-02-18T08:14:13.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:13.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:14:13.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:13.766 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=9 2026-02-18T08:14:13.767 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=134, tag_tid=8, entry_tid=37186' 2026-02-18T08:14:13.767 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:14:13.767 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:13.768 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=8 2026-02-18T08:14:13.769 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:13.770 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:14:13.770 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:13.771 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:14:13.771 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=134, tag_tid=8, entry_tid=37186' 2026-02-18T08:14:13.772 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:14:13.772 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:13.773 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=37186 2026-02-18T08:14:13.773 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 9 = 8 -a 1 -le 37186 2026-02-18T08:14:13.773 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:14:13.774 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:14:13.774 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:14:13.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:14:13.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:14:13.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:14:13.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:14:13.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:14:13.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:14:13.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:14:13.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:14:13.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:14:13.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:14:13.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:14:13.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:14:13.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:14:13.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:14:13.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:14:13.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:14:13.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:14:13.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:14:13.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:14:13.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:14:13.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:14:13.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:14:13.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:14:13.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:14:13.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:14:13.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:14:13.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:14:13.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:14:13.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:14:13.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:14:13.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:14:13.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:14:13.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:14:13.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:14:13.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:14:13.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:14:13.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:14:13.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:14:13.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:14:13.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:14:13.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:14:13.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:14:13.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:14:13.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:14:13.982 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:14:13.982 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:14:13.982 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:14:13.982 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:14:13.982 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:14:13.989 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:13.990 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:13.990 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:13.990 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:14:14.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:14:14.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:14:14.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:14.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:14.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:14.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:14:14.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:14.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:14.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:14.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:14:14.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:14:14.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:14:14.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:14:14.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:14.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:14:14.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:14.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 33 2026-02-18T08:14:14.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 38 2026-02-18T08:14:14.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:14:14.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=153, tag_tid=9, entry_tid=1], [object_number=152, tag_tid=9, entry_tid=0], [object_number=154, tag_tid=8, entry_tid=58722], [object_number=155, tag_tid=8, entry_tid=58719]]], state=connected] 2026-02-18T08:14:14.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=134, tag_tid=8, entry_tid=37402], [object_number=133, tag_tid=8, entry_tid=37401], [object_number=132, tag_tid=8, entry_tid=37400], [object_number=135, tag_tid=8, entry_tid=37399]]], state=connected] 2026-02-18T08:14:14.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:14.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:14.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:14:14.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:14.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:14.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:14.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:14:14.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:14.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:14.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:14.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:14:14.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:14:14.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:14:14.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:14:14.086 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:14.086 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:14.086 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:14:14.113 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 33 2026-02-18T08:14:14.113 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 38 2026-02-18T08:14:14.113 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:14:14.113 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=153, tag_tid=9, entry_tid=1], [object_number=152, tag_tid=9, entry_tid=0], [object_number=154, tag_tid=8, entry_tid=58722], [object_number=155, tag_tid=8, entry_tid=58719]]], state=connected] 2026-02-18T08:14:14.113 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=134, tag_tid=8, entry_tid=37402], [object_number=133, tag_tid=8, entry_tid=37401], [object_number=132, tag_tid=8, entry_tid=37400], [object_number=135, tag_tid=8, entry_tid=37399]]], state=connected] 2026-02-18T08:14:14.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:14.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=134, tag_tid=8, entry_tid=37402' 2026-02-18T08:14:14.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=153, tag_tid=9, entry_tid=1' -a 'object_number=153, tag_tid=9, entry_tid=1' = 'object_number=134, tag_tid=8, entry_tid=37402' 2026-02-18T08:14:14.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=134, tag_tid=8, entry_tid=37402' '!=' 'object_number=134, tag_tid=8, entry_tid=37186' 2026-02-18T08:14:14.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:14:14.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=134, tag_tid=8, entry_tid=37402' = 'object_number=134, tag_tid=8, entry_tid=37186' 2026-02-18T08:14:14.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=134, tag_tid=8, entry_tid=37402' 2026-02-18T08:14:14.118 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:14.118 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:14:14.118 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:14.118 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=9 2026-02-18T08:14:14.119 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=134, tag_tid=8, entry_tid=37402' 2026-02-18T08:14:14.119 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:14:14.120 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:14.121 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=8 2026-02-18T08:14:14.122 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:14.122 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:14:14.122 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:14.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:14:14.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=134, tag_tid=8, entry_tid=37402' 2026-02-18T08:14:14.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:14:14.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:14.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=37402 2026-02-18T08:14:14.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 9 = 8 -a 1 -le 37402 2026-02-18T08:14:14.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:14:14.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:14:14.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:14:14.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:14:14.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:14:14.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:14:14.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:14:14.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:14:14.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:14:14.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:14:14.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:14:14.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:14:14.329 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:14:14.329 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:14:14.329 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:14:14.329 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:14:14.329 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:14:14.329 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:14:14.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:14:14.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:14:14.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:14:14.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:14:14.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:14:14.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:14:14.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:14:14.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:14:14.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:14:14.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:14:14.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:14:14.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:14:14.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:14:14.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:14:14.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:14:14.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:14:14.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:14:14.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:14:14.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:14:14.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:14:14.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:14:14.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:14:14.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:14:14.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:14:14.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:14:14.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:14:14.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:14:14.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:14:14.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:14:14.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:14:14.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:14:14.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:14:14.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:14:14.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:14:14.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:14.344 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:14.344 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:14.344 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:14:14.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:14:14.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:14:14.405 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:14.405 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:14.405 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:14.405 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:14:14.405 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:14.405 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:14.405 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:14.405 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:14:14.405 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:14:14.406 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:14:14.406 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:14:14.407 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:14.407 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:14:14.407 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:14.435 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 33 2026-02-18T08:14:14.435 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 38 2026-02-18T08:14:14.435 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:14:14.436 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=153, tag_tid=9, entry_tid=1], [object_number=152, tag_tid=9, entry_tid=0], [object_number=154, tag_tid=8, entry_tid=58722], [object_number=155, tag_tid=8, entry_tid=58719]]], state=connected] 2026-02-18T08:14:14.436 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=135, tag_tid=8, entry_tid=37623], [object_number=134, tag_tid=8, entry_tid=37622], [object_number=133, tag_tid=8, entry_tid=37621], [object_number=132, tag_tid=8, entry_tid=37620]]], state=connected] 2026-02-18T08:14:14.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:14.440 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:14.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:14:14.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:14.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:14.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:14.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:14:14.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:14.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:14.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:14.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:14:14.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:14:14.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:14:14.443 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:14:14.443 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:14.443 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:14:14.443 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:14.470 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 33 2026-02-18T08:14:14.470 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 38 2026-02-18T08:14:14.470 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:14:14.470 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=153, tag_tid=9, entry_tid=1], [object_number=152, tag_tid=9, entry_tid=0], [object_number=154, tag_tid=8, entry_tid=58722], [object_number=155, tag_tid=8, entry_tid=58719]]], state=connected] 2026-02-18T08:14:14.470 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=135, tag_tid=8, entry_tid=37623], [object_number=134, tag_tid=8, entry_tid=37622], [object_number=133, tag_tid=8, entry_tid=37621], [object_number=132, tag_tid=8, entry_tid=37620]]], state=connected] 2026-02-18T08:14:14.472 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:14.474 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=135, tag_tid=8, entry_tid=37623' 2026-02-18T08:14:14.474 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=153, tag_tid=9, entry_tid=1' -a 'object_number=153, tag_tid=9, entry_tid=1' = 'object_number=135, tag_tid=8, entry_tid=37623' 2026-02-18T08:14:14.474 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=135, tag_tid=8, entry_tid=37623' '!=' 'object_number=134, tag_tid=8, entry_tid=37402' 2026-02-18T08:14:14.474 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:14:14.474 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=135, tag_tid=8, entry_tid=37623' = 'object_number=134, tag_tid=8, entry_tid=37402' 2026-02-18T08:14:14.475 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=135, tag_tid=8, entry_tid=37623' 2026-02-18T08:14:14.475 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:14.475 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:14:14.475 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:14.476 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=9 2026-02-18T08:14:14.477 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=135, tag_tid=8, entry_tid=37623' 2026-02-18T08:14:14.477 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:14:14.477 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:14.478 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=8 2026-02-18T08:14:14.479 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:14.479 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:14:14.479 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:14.481 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:14:14.481 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=135, tag_tid=8, entry_tid=37623' 2026-02-18T08:14:14.481 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:14:14.482 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:14.483 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=37623 2026-02-18T08:14:14.483 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 9 = 8 -a 1 -le 37623 2026-02-18T08:14:14.483 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:14:14.483 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:14:14.483 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:14:14.684 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:14:14.684 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:14:14.684 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:14:14.684 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:14:14.684 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:14:14.684 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:14:14.684 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:14:14.684 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:14:14.684 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:14:14.685 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:14:14.685 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:14:14.685 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:14:14.685 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:14:14.685 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:14:14.685 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:14:14.685 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:14:14.685 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:14:14.686 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:14:14.686 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:14:14.686 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:14:14.686 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:14:14.686 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:14:14.686 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:14:14.686 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:14:14.686 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:14:14.686 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:14:14.686 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:14:14.686 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:14:14.686 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:14:14.686 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:14:14.687 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:14:14.687 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:14:14.687 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:14:14.687 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:14:14.687 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:14:14.687 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:14:14.687 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:14:14.687 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:14:14.687 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:14:14.687 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:14:14.687 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:14:14.687 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:14:14.687 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:14:14.687 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:14:14.687 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:14:14.687 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:14:14.687 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:14:14.687 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:14:14.688 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:14:14.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:14.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:14.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:14.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:14:14.770 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:14:14.771 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:14:14.771 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:14.771 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:14.771 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:14.771 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:14:14.771 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:14.772 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:14.772 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:14.772 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:14:14.772 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:14:14.772 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:14:14.772 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:14:14.773 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:14.773 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:14:14.773 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:14.801 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 33 2026-02-18T08:14:14.801 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 38 2026-02-18T08:14:14.801 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:14:14.801 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=153, tag_tid=9, entry_tid=1], [object_number=152, tag_tid=9, entry_tid=0], [object_number=154, tag_tid=8, entry_tid=58722], [object_number=155, tag_tid=8, entry_tid=58719]]], state=connected] 2026-02-18T08:14:14.801 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=135, tag_tid=8, entry_tid=37855], [object_number=134, tag_tid=8, entry_tid=37854], [object_number=133, tag_tid=8, entry_tid=37853], [object_number=132, tag_tid=8, entry_tid=37852]]], state=connected] 2026-02-18T08:14:14.803 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:14.805 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:14.805 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:14:14.805 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:14.806 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:14.806 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:14.806 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:14:14.806 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:14.806 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:14.806 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:14.806 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:14:14.806 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:14:14.806 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:14:14.806 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:14:14.808 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:14.808 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:14:14.808 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:14.845 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 33 2026-02-18T08:14:14.845 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 38 2026-02-18T08:14:14.845 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:14:14.845 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=153, tag_tid=9, entry_tid=1], [object_number=152, tag_tid=9, entry_tid=0], [object_number=154, tag_tid=8, entry_tid=58722], [object_number=155, tag_tid=8, entry_tid=58719]]], state=connected] 2026-02-18T08:14:14.845 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=135, tag_tid=8, entry_tid=37855], [object_number=134, tag_tid=8, entry_tid=37854], [object_number=133, tag_tid=8, entry_tid=37853], [object_number=132, tag_tid=8, entry_tid=37852]]], state=connected] 2026-02-18T08:14:14.848 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:14.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=135, tag_tid=8, entry_tid=37855' 2026-02-18T08:14:14.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=153, tag_tid=9, entry_tid=1' -a 'object_number=153, tag_tid=9, entry_tid=1' = 'object_number=135, tag_tid=8, entry_tid=37855' 2026-02-18T08:14:14.850 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=135, tag_tid=8, entry_tid=37855' '!=' 'object_number=135, tag_tid=8, entry_tid=37623' 2026-02-18T08:14:14.850 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:14:14.850 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=135, tag_tid=8, entry_tid=37855' = 'object_number=135, tag_tid=8, entry_tid=37623' 2026-02-18T08:14:14.850 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=135, tag_tid=8, entry_tid=37855' 2026-02-18T08:14:14.850 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:14.850 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:14:14.850 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:14.852 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=9 2026-02-18T08:14:14.852 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=135, tag_tid=8, entry_tid=37855' 2026-02-18T08:14:14.852 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:14:14.853 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:14.854 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=8 2026-02-18T08:14:14.855 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:14.855 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:14:14.855 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:14.856 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:14:14.858 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=135, tag_tid=8, entry_tid=37855' 2026-02-18T08:14:14.858 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:14:14.858 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:14.859 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=37855 2026-02-18T08:14:14.859 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 9 = 8 -a 1 -le 37855 2026-02-18T08:14:14.859 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:14:14.859 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:14:14.860 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:14:15.062 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:14:15.062 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:14:15.062 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:14:15.062 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:14:15.062 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:14:15.062 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:14:15.062 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:14:15.062 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:14:15.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:14:15.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:14:15.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:14:15.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:14:15.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:14:15.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:14:15.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:14:15.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:14:15.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:14:15.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:14:15.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:14:15.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:14:15.067 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:14:15.067 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:14:15.067 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:14:15.067 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:14:15.067 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:14:15.067 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:14:15.067 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:14:15.067 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:14:15.067 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:14:15.067 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:14:15.067 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:14:15.068 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:14:15.068 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:14:15.068 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:14:15.068 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:14:15.068 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:14:15.068 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:14:15.068 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:14:15.068 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:14:15.068 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:14:15.068 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:14:15.068 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:14:15.068 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:14:15.069 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:14:15.069 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:14:15.069 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:14:15.069 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:14:15.069 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:14:15.069 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:14:15.076 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:15.076 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:15.076 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:15.076 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:14:15.136 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:14:15.137 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:14:15.138 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:15.138 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:15.138 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:15.138 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:14:15.138 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:15.138 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:15.138 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:15.138 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:14:15.138 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:14:15.138 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:14:15.139 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:14:15.139 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:15.139 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:14:15.140 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:15.173 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 33 2026-02-18T08:14:15.173 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 38 2026-02-18T08:14:15.173 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:14:15.173 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=153, tag_tid=9, entry_tid=1], [object_number=152, tag_tid=9, entry_tid=0], [object_number=154, tag_tid=8, entry_tid=58722], [object_number=155, tag_tid=8, entry_tid=58719]]], state=connected] 2026-02-18T08:14:15.173 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=132, tag_tid=8, entry_tid=38096], [object_number=135, tag_tid=8, entry_tid=38095], [object_number=134, tag_tid=8, entry_tid=38094], [object_number=133, tag_tid=8, entry_tid=38093]]], state=connected] 2026-02-18T08:14:15.177 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:15.179 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:15.180 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:14:15.180 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:15.180 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:15.180 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:15.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:14:15.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:15.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:15.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:15.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:14:15.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:14:15.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:14:15.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:14:15.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:15.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:14:15.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:15.210 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 33 2026-02-18T08:14:15.210 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 38 2026-02-18T08:14:15.211 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:14:15.211 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=153, tag_tid=9, entry_tid=1], [object_number=152, tag_tid=9, entry_tid=0], [object_number=154, tag_tid=8, entry_tid=58722], [object_number=155, tag_tid=8, entry_tid=58719]]], state=connected] 2026-02-18T08:14:15.211 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=132, tag_tid=8, entry_tid=38096], [object_number=135, tag_tid=8, entry_tid=38095], [object_number=134, tag_tid=8, entry_tid=38094], [object_number=133, tag_tid=8, entry_tid=38093]]], state=connected] 2026-02-18T08:14:15.213 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:15.215 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=132, tag_tid=8, entry_tid=38096' 2026-02-18T08:14:15.215 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=153, tag_tid=9, entry_tid=1' -a 'object_number=153, tag_tid=9, entry_tid=1' = 'object_number=132, tag_tid=8, entry_tid=38096' 2026-02-18T08:14:15.215 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=132, tag_tid=8, entry_tid=38096' '!=' 'object_number=135, tag_tid=8, entry_tid=37855' 2026-02-18T08:14:15.215 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:14:15.215 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=132, tag_tid=8, entry_tid=38096' = 'object_number=135, tag_tid=8, entry_tid=37855' 2026-02-18T08:14:15.215 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=132, tag_tid=8, entry_tid=38096' 2026-02-18T08:14:15.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:15.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:14:15.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:15.217 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=9 2026-02-18T08:14:15.218 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=132, tag_tid=8, entry_tid=38096' 2026-02-18T08:14:15.218 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:14:15.218 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:15.219 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=8 2026-02-18T08:14:15.220 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:15.220 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:14:15.220 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:15.221 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:14:15.222 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=132, tag_tid=8, entry_tid=38096' 2026-02-18T08:14:15.222 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:14:15.222 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:15.224 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=38096 2026-02-18T08:14:15.224 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 9 = 8 -a 1 -le 38096 2026-02-18T08:14:15.224 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:14:15.224 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:14:15.224 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:14:15.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:14:15.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:14:15.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:14:15.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:14:15.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:14:15.427 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:14:15.427 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:14:15.427 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:14:15.427 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:14:15.427 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:14:15.427 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:14:15.427 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:14:15.427 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:14:15.427 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:14:15.427 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:14:15.428 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:14:15.430 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:14:15.430 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:14:15.430 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:14:15.431 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:14:15.431 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:14:15.431 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:14:15.431 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:14:15.431 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:14:15.431 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:14:15.431 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:14:15.431 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:14:15.431 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:14:15.431 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:14:15.431 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:14:15.431 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:14:15.432 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:14:15.432 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:14:15.432 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:14:15.432 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:14:15.432 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:14:15.432 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:14:15.432 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:14:15.432 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:14:15.432 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:14:15.432 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:14:15.432 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:14:15.432 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:14:15.433 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:14:15.433 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:14:15.433 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:14:15.433 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:14:15.433 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:14:15.433 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:14:15.440 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:15.440 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:15.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:15.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:14:15.500 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:14:15.502 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:14:15.502 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:15.502 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:15.502 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:15.502 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:14:15.502 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:15.502 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:15.502 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:15.503 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:14:15.503 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:14:15.503 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:14:15.503 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:14:15.503 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:15.504 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:14:15.504 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:15.539 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 33 2026-02-18T08:14:15.539 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 38 2026-02-18T08:14:15.539 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:14:15.539 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=153, tag_tid=9, entry_tid=1], [object_number=152, tag_tid=9, entry_tid=0], [object_number=154, tag_tid=8, entry_tid=58722], [object_number=155, tag_tid=8, entry_tid=58719]]], state=connected] 2026-02-18T08:14:15.539 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=133, tag_tid=8, entry_tid=38357], [object_number=132, tag_tid=8, entry_tid=38356], [object_number=135, tag_tid=8, entry_tid=38355], [object_number=134, tag_tid=8, entry_tid=38354]]], state=connected] 2026-02-18T08:14:15.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:15.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:15.545 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:14:15.545 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:15.545 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:15.545 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:15.545 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:14:15.545 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:15.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:15.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:15.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:14:15.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:14:15.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:14:15.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:14:15.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:15.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:14:15.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:15.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 33 2026-02-18T08:14:15.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 38 2026-02-18T08:14:15.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:14:15.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=153, tag_tid=9, entry_tid=1], [object_number=152, tag_tid=9, entry_tid=0], [object_number=154, tag_tid=8, entry_tid=58722], [object_number=155, tag_tid=8, entry_tid=58719]]], state=connected] 2026-02-18T08:14:15.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=133, tag_tid=8, entry_tid=38357], [object_number=132, tag_tid=8, entry_tid=38356], [object_number=135, tag_tid=8, entry_tid=38355], [object_number=134, tag_tid=8, entry_tid=38354]]], state=connected] 2026-02-18T08:14:15.585 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:15.587 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=133, tag_tid=8, entry_tid=38357' 2026-02-18T08:14:15.587 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=153, tag_tid=9, entry_tid=1' -a 'object_number=153, tag_tid=9, entry_tid=1' = 'object_number=133, tag_tid=8, entry_tid=38357' 2026-02-18T08:14:15.587 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=133, tag_tid=8, entry_tid=38357' '!=' 'object_number=132, tag_tid=8, entry_tid=38096' 2026-02-18T08:14:15.587 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:14:15.587 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=133, tag_tid=8, entry_tid=38357' = 'object_number=132, tag_tid=8, entry_tid=38096' 2026-02-18T08:14:15.587 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=133, tag_tid=8, entry_tid=38357' 2026-02-18T08:14:15.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:15.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:14:15.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:15.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=9 2026-02-18T08:14:15.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=133, tag_tid=8, entry_tid=38357' 2026-02-18T08:14:15.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:14:15.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:15.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=8 2026-02-18T08:14:15.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:15.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:14:15.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:15.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:14:15.594 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=133, tag_tid=8, entry_tid=38357' 2026-02-18T08:14:15.594 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:14:15.594 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:15.596 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=38357 2026-02-18T08:14:15.596 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 9 = 8 -a 1 -le 38357 2026-02-18T08:14:15.596 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:14:15.596 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:14:15.596 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:14:15.798 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:14:15.798 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:14:15.798 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:14:15.798 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:14:15.798 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:14:15.798 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:14:15.798 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:14:15.799 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:14:15.799 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:14:15.799 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:14:15.799 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:14:15.799 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:14:15.799 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:14:15.799 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:14:15.799 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:14:15.799 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:14:15.802 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:14:15.802 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:14:15.802 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:14:15.802 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:14:15.802 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:14:15.802 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:14:15.802 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:14:15.802 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:14:15.803 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:14:15.803 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:14:15.803 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:14:15.803 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:14:15.803 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:14:15.803 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:14:15.803 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:14:15.803 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:14:15.803 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:14:15.803 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:14:15.803 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:14:15.803 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:14:15.803 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:14:15.804 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:14:15.804 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:14:15.804 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:14:15.804 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:14:15.804 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:14:15.804 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:14:15.804 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:14:15.804 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:14:15.804 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:14:15.804 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:14:15.804 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:14:15.804 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:14:15.812 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:15.812 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:15.813 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:15.813 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:14:15.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:14:15.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:14:15.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:15.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:15.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:15.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:14:15.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:15.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:15.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:15.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:14:15.874 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:14:15.874 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:14:15.874 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:14:15.874 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:15.875 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:14:15.875 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:15.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 33 2026-02-18T08:14:15.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 38 2026-02-18T08:14:15.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:14:15.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=153, tag_tid=9, entry_tid=1], [object_number=152, tag_tid=9, entry_tid=0], [object_number=154, tag_tid=8, entry_tid=58722], [object_number=155, tag_tid=8, entry_tid=58719]]], state=connected] 2026-02-18T08:14:15.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=132, tag_tid=8, entry_tid=38596], [object_number=135, tag_tid=8, entry_tid=38595], [object_number=134, tag_tid=8, entry_tid=38594], [object_number=133, tag_tid=8, entry_tid=38593]]], state=connected] 2026-02-18T08:14:15.905 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:15.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:15.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:14:15.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:15.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:15.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:15.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:14:15.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:15.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:15.909 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:15.909 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:14:15.909 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:14:15.909 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:14:15.909 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:14:15.909 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:15.910 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:14:15.910 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:16.144 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 33 2026-02-18T08:14:16.144 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 38 2026-02-18T08:14:16.144 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:14:16.144 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=153, tag_tid=9, entry_tid=1], [object_number=152, tag_tid=9, entry_tid=0], [object_number=154, tag_tid=8, entry_tid=58722], [object_number=155, tag_tid=8, entry_tid=58719]]], state=connected] 2026-02-18T08:14:16.144 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=132, tag_tid=8, entry_tid=38596], [object_number=135, tag_tid=8, entry_tid=38595], [object_number=134, tag_tid=8, entry_tid=38594], [object_number=133, tag_tid=8, entry_tid=38593]]], state=connected] 2026-02-18T08:14:16.147 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:16.149 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=132, tag_tid=8, entry_tid=38596' 2026-02-18T08:14:16.149 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=153, tag_tid=9, entry_tid=1' -a 'object_number=153, tag_tid=9, entry_tid=1' = 'object_number=132, tag_tid=8, entry_tid=38596' 2026-02-18T08:14:16.149 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=132, tag_tid=8, entry_tid=38596' '!=' 'object_number=133, tag_tid=8, entry_tid=38357' 2026-02-18T08:14:16.150 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:14:16.150 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=132, tag_tid=8, entry_tid=38596' = 'object_number=133, tag_tid=8, entry_tid=38357' 2026-02-18T08:14:16.150 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=132, tag_tid=8, entry_tid=38596' 2026-02-18T08:14:16.150 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:16.150 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:14:16.150 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:16.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=9 2026-02-18T08:14:16.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=132, tag_tid=8, entry_tid=38596' 2026-02-18T08:14:16.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:14:16.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:16.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=8 2026-02-18T08:14:16.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:16.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:14:16.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:16.156 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:14:16.157 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=132, tag_tid=8, entry_tid=38596' 2026-02-18T08:14:16.157 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:14:16.157 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:16.158 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=38596 2026-02-18T08:14:16.158 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 9 = 8 -a 1 -le 38596 2026-02-18T08:14:16.158 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:14:16.158 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:14:16.158 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:14:16.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:14:16.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:14:16.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:14:16.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:14:16.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:14:16.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:14:16.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:14:16.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:14:16.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:14:16.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:14:16.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:14:16.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:14:16.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:14:16.360 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:14:16.360 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:14:16.360 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:14:16.360 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:14:16.361 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:14:16.361 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:14:16.361 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:14:16.361 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:14:16.361 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:14:16.361 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:14:16.361 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:14:16.361 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:14:16.361 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:14:16.361 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:14:16.362 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:14:16.362 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:14:16.362 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:14:16.362 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:14:16.362 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:14:16.362 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:14:16.362 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:14:16.362 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:14:16.362 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:14:16.362 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:14:16.362 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:14:16.362 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:14:16.362 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:14:16.362 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:14:16.362 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:14:16.362 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:14:16.362 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:14:16.362 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:14:16.362 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:14:16.362 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:14:16.362 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:14:16.362 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:14:16.373 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:16.374 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:16.374 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:16.374 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:14:16.433 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:14:16.434 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:14:16.435 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:16.435 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:16.435 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:16.435 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:14:16.435 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:16.435 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:16.435 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:16.435 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:14:16.435 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:14:16.435 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:14:16.435 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:14:16.436 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:16.437 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:14:16.437 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:16.473 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 33 2026-02-18T08:14:16.473 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 38 2026-02-18T08:14:16.473 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:14:16.473 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=153, tag_tid=9, entry_tid=1], [object_number=152, tag_tid=9, entry_tid=0], [object_number=154, tag_tid=8, entry_tid=58722], [object_number=155, tag_tid=8, entry_tid=58719]]], state=connected] 2026-02-18T08:14:16.473 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=133, tag_tid=8, entry_tid=38969], [object_number=132, tag_tid=8, entry_tid=38968], [object_number=135, tag_tid=8, entry_tid=38967], [object_number=134, tag_tid=8, entry_tid=38966]]], state=connected] 2026-02-18T08:14:16.477 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:16.479 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:16.480 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:14:16.480 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:16.480 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:16.480 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:16.480 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:14:16.480 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:16.480 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:16.480 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:16.480 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:14:16.480 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:14:16.481 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:14:16.481 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:14:16.481 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:16.482 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:14:16.482 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:16.510 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 33 2026-02-18T08:14:16.510 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 38 2026-02-18T08:14:16.510 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:14:16.511 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=153, tag_tid=9, entry_tid=1], [object_number=152, tag_tid=9, entry_tid=0], [object_number=154, tag_tid=8, entry_tid=58722], [object_number=155, tag_tid=8, entry_tid=58719]]], state=connected] 2026-02-18T08:14:16.511 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=133, tag_tid=8, entry_tid=38969], [object_number=132, tag_tid=8, entry_tid=38968], [object_number=135, tag_tid=8, entry_tid=38967], [object_number=134, tag_tid=8, entry_tid=38966]]], state=connected] 2026-02-18T08:14:16.513 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:16.514 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=133, tag_tid=8, entry_tid=38969' 2026-02-18T08:14:16.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=153, tag_tid=9, entry_tid=1' -a 'object_number=153, tag_tid=9, entry_tid=1' = 'object_number=133, tag_tid=8, entry_tid=38969' 2026-02-18T08:14:16.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=133, tag_tid=8, entry_tid=38969' '!=' 'object_number=132, tag_tid=8, entry_tid=38596' 2026-02-18T08:14:16.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:14:16.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=133, tag_tid=8, entry_tid=38969' = 'object_number=132, tag_tid=8, entry_tid=38596' 2026-02-18T08:14:16.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=133, tag_tid=8, entry_tid=38969' 2026-02-18T08:14:16.516 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:16.516 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:14:16.516 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:16.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=9 2026-02-18T08:14:16.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=133, tag_tid=8, entry_tid=38969' 2026-02-18T08:14:16.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:14:16.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:16.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=8 2026-02-18T08:14:16.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:16.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:14:16.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:16.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:14:16.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=133, tag_tid=8, entry_tid=38969' 2026-02-18T08:14:16.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:16.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:14:16.523 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=38969 2026-02-18T08:14:16.523 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 9 = 8 -a 1 -le 38969 2026-02-18T08:14:16.524 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:14:16.524 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:14:16.524 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:14:16.724 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:14:16.724 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:14:16.725 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:14:16.725 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:14:16.725 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:14:16.725 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:14:16.725 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:14:16.725 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:14:16.725 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:14:16.725 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:14:16.725 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:14:16.725 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:14:16.725 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:14:16.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:14:16.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:14:16.729 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:14:16.729 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:14:16.729 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:14:16.729 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:14:16.729 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:14:16.729 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:14:16.729 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:14:16.729 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:14:16.729 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:14:16.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:14:16.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:14:16.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:14:16.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:14:16.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:14:16.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:14:16.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:14:16.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:14:16.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:14:16.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:14:16.731 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:14:16.731 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:14:16.731 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:14:16.731 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:14:16.731 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:14:16.731 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:14:16.731 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:14:16.731 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:14:16.731 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:14:16.731 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:14:16.731 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:14:16.731 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:14:16.732 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:14:16.732 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:14:16.732 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:14:16.740 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:16.740 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:16.740 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:16.741 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:14:16.811 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:14:16.812 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:14:16.812 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:16.812 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:16.813 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:16.813 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:14:16.813 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:16.813 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:16.813 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:16.813 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:14:16.813 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:14:16.813 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:14:16.813 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:14:16.814 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:16.814 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:14:16.814 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:16.843 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 33 2026-02-18T08:14:16.843 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 38 2026-02-18T08:14:16.843 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:14:16.843 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=153, tag_tid=9, entry_tid=1], [object_number=152, tag_tid=9, entry_tid=0], [object_number=154, tag_tid=8, entry_tid=58722], [object_number=155, tag_tid=8, entry_tid=58719]]], state=connected] 2026-02-18T08:14:16.843 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=133, tag_tid=8, entry_tid=39257], [object_number=132, tag_tid=8, entry_tid=39256], [object_number=135, tag_tid=8, entry_tid=39255], [object_number=134, tag_tid=8, entry_tid=39254]]], state=connected] 2026-02-18T08:14:16.846 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:16.847 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:16.848 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:14:16.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:16.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:16.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:16.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:14:16.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:16.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:16.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:16.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:14:16.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:14:16.850 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:14:16.850 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:14:16.850 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:16.851 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:14:16.851 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:16.877 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 33 2026-02-18T08:14:16.877 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 38 2026-02-18T08:14:16.877 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:14:16.877 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=153, tag_tid=9, entry_tid=1], [object_number=152, tag_tid=9, entry_tid=0], [object_number=154, tag_tid=8, entry_tid=58722], [object_number=155, tag_tid=8, entry_tid=58719]]], state=connected] 2026-02-18T08:14:16.877 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=133, tag_tid=8, entry_tid=39257], [object_number=132, tag_tid=8, entry_tid=39256], [object_number=135, tag_tid=8, entry_tid=39255], [object_number=134, tag_tid=8, entry_tid=39254]]], state=connected] 2026-02-18T08:14:16.879 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:16.881 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=133, tag_tid=8, entry_tid=39257' 2026-02-18T08:14:16.881 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=153, tag_tid=9, entry_tid=1' -a 'object_number=153, tag_tid=9, entry_tid=1' = 'object_number=133, tag_tid=8, entry_tid=39257' 2026-02-18T08:14:16.881 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=133, tag_tid=8, entry_tid=39257' '!=' 'object_number=133, tag_tid=8, entry_tid=38969' 2026-02-18T08:14:16.881 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:14:16.882 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=133, tag_tid=8, entry_tid=39257' = 'object_number=133, tag_tid=8, entry_tid=38969' 2026-02-18T08:14:16.882 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=133, tag_tid=8, entry_tid=39257' 2026-02-18T08:14:16.882 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:16.882 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:14:16.882 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:16.883 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=9 2026-02-18T08:14:16.884 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=133, tag_tid=8, entry_tid=39257' 2026-02-18T08:14:16.884 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:14:16.884 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:16.885 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=8 2026-02-18T08:14:16.886 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:16.886 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:14:16.886 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:16.887 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:14:16.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=133, tag_tid=8, entry_tid=39257' 2026-02-18T08:14:16.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:14:16.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:16.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=39257 2026-02-18T08:14:16.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 9 = 8 -a 1 -le 39257 2026-02-18T08:14:16.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:14:16.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:14:16.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:14:17.092 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:14:17.092 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:14:17.093 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:14:17.093 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:14:17.093 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:14:17.093 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:14:17.093 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:14:17.093 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:14:17.093 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:14:17.093 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:14:17.093 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:14:17.093 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:14:17.094 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:14:17.094 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:14:17.094 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:14:17.094 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:14:17.096 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:14:17.096 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:14:17.096 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:14:17.096 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:14:17.096 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:14:17.097 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:14:17.097 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:14:17.097 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:14:17.097 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:14:17.097 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:14:17.097 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:14:17.097 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:14:17.097 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:14:17.097 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:14:17.097 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:14:17.097 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:14:17.097 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:14:17.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:14:17.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:14:17.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:14:17.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:14:17.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:14:17.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:14:17.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:14:17.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:14:17.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:14:17.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:14:17.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:14:17.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:14:17.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:14:17.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:14:17.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:14:17.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:14:17.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:17.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:17.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:17.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:14:17.178 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:14:17.179 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:14:17.179 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:17.179 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:17.179 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:17.179 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:14:17.179 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:17.179 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:17.179 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:17.179 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:14:17.179 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:14:17.179 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:14:17.179 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:14:17.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:17.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:14:17.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:17.210 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 33 2026-02-18T08:14:17.210 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 38 2026-02-18T08:14:17.210 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:14:17.210 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=153, tag_tid=9, entry_tid=1], [object_number=152, tag_tid=9, entry_tid=0], [object_number=154, tag_tid=8, entry_tid=58722], [object_number=155, tag_tid=8, entry_tid=58719]]], state=connected] 2026-02-18T08:14:17.210 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=137, tag_tid=8, entry_tid=39489], [object_number=132, tag_tid=8, entry_tid=39488], [object_number=135, tag_tid=8, entry_tid=39487], [object_number=134, tag_tid=8, entry_tid=39486]]], state=connected] 2026-02-18T08:14:17.212 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:17.213 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:17.214 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:14:17.214 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:17.214 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:17.214 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:17.214 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:14:17.214 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:17.214 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:17.214 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:17.214 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:14:17.214 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:14:17.214 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:14:17.215 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:14:17.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:17.217 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:14:17.217 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:17.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 33 2026-02-18T08:14:17.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 38 2026-02-18T08:14:17.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:14:17.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=153, tag_tid=9, entry_tid=1], [object_number=152, tag_tid=9, entry_tid=0], [object_number=154, tag_tid=8, entry_tid=58722], [object_number=155, tag_tid=8, entry_tid=58719]]], state=connected] 2026-02-18T08:14:17.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=137, tag_tid=8, entry_tid=39489], [object_number=132, tag_tid=8, entry_tid=39488], [object_number=135, tag_tid=8, entry_tid=39487], [object_number=134, tag_tid=8, entry_tid=39486]]], state=connected] 2026-02-18T08:14:17.245 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:17.246 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=137, tag_tid=8, entry_tid=39489' 2026-02-18T08:14:17.246 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=153, tag_tid=9, entry_tid=1' -a 'object_number=153, tag_tid=9, entry_tid=1' = 'object_number=137, tag_tid=8, entry_tid=39489' 2026-02-18T08:14:17.246 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=137, tag_tid=8, entry_tid=39489' '!=' 'object_number=133, tag_tid=8, entry_tid=39257' 2026-02-18T08:14:17.246 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:14:17.246 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=137, tag_tid=8, entry_tid=39489' = 'object_number=133, tag_tid=8, entry_tid=39257' 2026-02-18T08:14:17.246 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=137, tag_tid=8, entry_tid=39489' 2026-02-18T08:14:17.247 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:17.247 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:14:17.247 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:17.249 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=9 2026-02-18T08:14:17.249 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=137, tag_tid=8, entry_tid=39489' 2026-02-18T08:14:17.250 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:14:17.250 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:17.251 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=8 2026-02-18T08:14:17.251 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:17.252 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:14:17.252 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:17.253 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:14:17.253 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=137, tag_tid=8, entry_tid=39489' 2026-02-18T08:14:17.254 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:14:17.254 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:17.255 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=39489 2026-02-18T08:14:17.255 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 9 = 8 -a 1 -le 39489 2026-02-18T08:14:17.255 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:14:17.255 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:14:17.255 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:14:17.458 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:14:17.458 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:14:17.458 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:14:17.458 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:14:17.458 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:14:17.458 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:14:17.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:14:17.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:14:17.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:14:17.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:14:17.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:14:17.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:14:17.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:14:17.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:14:17.460 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:14:17.460 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:14:17.463 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:14:17.463 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:14:17.463 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:14:17.463 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:14:17.463 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:14:17.463 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:14:17.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:14:17.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:14:17.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:14:17.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:14:17.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:14:17.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:14:17.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:14:17.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:14:17.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:14:17.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:14:17.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:14:17.465 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:14:17.465 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:14:17.465 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:14:17.465 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:14:17.465 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:14:17.465 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:14:17.465 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:14:17.465 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:14:17.465 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:14:17.465 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:14:17.465 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:14:17.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:14:17.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:14:17.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:14:17.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:14:17.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:14:17.472 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:17.472 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:17.472 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:17.473 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:14:17.532 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:14:17.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:14:17.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:17.534 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:17.534 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:17.534 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:14:17.534 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:17.534 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:17.534 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:17.534 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:14:17.534 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:14:17.534 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:14:17.534 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:14:17.535 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:17.535 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:17.535 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:14:17.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 34 2026-02-18T08:14:17.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 38 2026-02-18T08:14:17.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:14:17.565 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=153, tag_tid=9, entry_tid=1], [object_number=152, tag_tid=9, entry_tid=0], [object_number=154, tag_tid=8, entry_tid=58722], [object_number=155, tag_tid=8, entry_tid=58719]]], state=connected] 2026-02-18T08:14:17.565 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=137, tag_tid=8, entry_tid=39645], [object_number=136, tag_tid=8, entry_tid=39644], [object_number=139, tag_tid=8, entry_tid=39643], [object_number=138, tag_tid=8, entry_tid=39642]]], state=connected] 2026-02-18T08:14:17.567 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:17.568 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:17.569 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:14:17.569 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:17.569 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:17.570 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:17.570 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:14:17.570 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:17.570 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:17.570 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:17.570 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:14:17.570 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:14:17.572 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:14:17.572 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:14:17.572 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:17.572 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:14:17.572 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:17.599 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 34 2026-02-18T08:14:17.599 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 38 2026-02-18T08:14:17.599 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:14:17.599 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=153, tag_tid=9, entry_tid=1], [object_number=152, tag_tid=9, entry_tid=0], [object_number=154, tag_tid=8, entry_tid=58722], [object_number=155, tag_tid=8, entry_tid=58719]]], state=connected] 2026-02-18T08:14:17.599 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=137, tag_tid=8, entry_tid=39645], [object_number=136, tag_tid=8, entry_tid=39644], [object_number=139, tag_tid=8, entry_tid=39643], [object_number=138, tag_tid=8, entry_tid=39642]]], state=connected] 2026-02-18T08:14:17.601 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:17.603 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=137, tag_tid=8, entry_tid=39645' 2026-02-18T08:14:17.603 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=153, tag_tid=9, entry_tid=1' -a 'object_number=153, tag_tid=9, entry_tid=1' = 'object_number=137, tag_tid=8, entry_tid=39645' 2026-02-18T08:14:17.603 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=137, tag_tid=8, entry_tid=39645' '!=' 'object_number=137, tag_tid=8, entry_tid=39489' 2026-02-18T08:14:17.603 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:14:17.603 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=137, tag_tid=8, entry_tid=39645' = 'object_number=137, tag_tid=8, entry_tid=39489' 2026-02-18T08:14:17.603 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=137, tag_tid=8, entry_tid=39645' 2026-02-18T08:14:17.604 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:17.604 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:14:17.604 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:17.604 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=9 2026-02-18T08:14:17.605 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=137, tag_tid=8, entry_tid=39645' 2026-02-18T08:14:17.605 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:14:17.605 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:17.606 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=8 2026-02-18T08:14:17.607 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:17.607 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:14:17.607 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:17.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:14:17.610 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=137, tag_tid=8, entry_tid=39645' 2026-02-18T08:14:17.610 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:14:17.610 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:17.611 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=39645 2026-02-18T08:14:17.611 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 9 = 8 -a 1 -le 39645 2026-02-18T08:14:17.612 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:14:17.612 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:14:17.612 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:14:17.814 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:14:17.814 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:14:17.814 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:14:17.814 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:14:17.814 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:14:17.815 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:14:17.815 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:14:17.815 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:14:17.815 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:14:17.815 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:14:17.815 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:14:17.815 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:14:17.815 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:14:17.815 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:14:17.815 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:14:17.815 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:14:17.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:14:17.818 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:14:17.819 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:14:17.819 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:14:17.819 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:14:17.819 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:14:17.819 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:14:17.819 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:14:17.819 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:14:17.819 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:14:17.819 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:14:17.819 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:14:17.819 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:14:17.820 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:14:17.820 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:14:17.820 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:14:17.820 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:14:17.820 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:14:17.820 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:14:17.820 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:14:17.820 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:14:17.820 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:14:17.820 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:14:17.820 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:14:17.820 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:14:17.821 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:14:17.821 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:14:17.821 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:14:17.821 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:14:17.821 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:14:17.821 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:14:17.821 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:14:17.821 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:14:17.828 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:17.828 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:17.828 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:17.829 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:14:17.887 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:14:17.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:14:17.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:17.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:17.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:17.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:14:17.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:17.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:17.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:17.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:14:17.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:14:17.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:14:17.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:14:17.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:17.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:14:17.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:17.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 34 2026-02-18T08:14:17.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 38 2026-02-18T08:14:17.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:14:17.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=153, tag_tid=9, entry_tid=1], [object_number=152, tag_tid=9, entry_tid=0], [object_number=154, tag_tid=8, entry_tid=58722], [object_number=155, tag_tid=8, entry_tid=58719]]], state=connected] 2026-02-18T08:14:17.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=137, tag_tid=8, entry_tid=39925], [object_number=136, tag_tid=8, entry_tid=39924], [object_number=139, tag_tid=8, entry_tid=39923], [object_number=138, tag_tid=8, entry_tid=39922]]], state=connected] 2026-02-18T08:14:17.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:17.923 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:17.924 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:14:17.924 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:17.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:17.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:17.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:14:17.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:17.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:17.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:17.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:14:17.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:14:17.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:14:17.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:14:17.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:17.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:14:17.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:17.955 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 34 2026-02-18T08:14:17.955 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 38 2026-02-18T08:14:17.955 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:14:17.956 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=153, tag_tid=9, entry_tid=1], [object_number=152, tag_tid=9, entry_tid=0], [object_number=154, tag_tid=8, entry_tid=58722], [object_number=155, tag_tid=8, entry_tid=58719]]], state=connected] 2026-02-18T08:14:17.956 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=137, tag_tid=8, entry_tid=39925], [object_number=136, tag_tid=8, entry_tid=39924], [object_number=139, tag_tid=8, entry_tid=39923], [object_number=138, tag_tid=8, entry_tid=39922]]], state=connected] 2026-02-18T08:14:17.958 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:17.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=137, tag_tid=8, entry_tid=39925' 2026-02-18T08:14:17.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=153, tag_tid=9, entry_tid=1' -a 'object_number=153, tag_tid=9, entry_tid=1' = 'object_number=137, tag_tid=8, entry_tid=39925' 2026-02-18T08:14:17.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=137, tag_tid=8, entry_tid=39925' '!=' 'object_number=137, tag_tid=8, entry_tid=39645' 2026-02-18T08:14:17.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:14:17.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=137, tag_tid=8, entry_tid=39925' = 'object_number=137, tag_tid=8, entry_tid=39645' 2026-02-18T08:14:17.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=137, tag_tid=8, entry_tid=39925' 2026-02-18T08:14:17.961 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:17.961 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:14:17.961 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:17.962 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=9 2026-02-18T08:14:17.963 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=137, tag_tid=8, entry_tid=39925' 2026-02-18T08:14:17.963 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:14:17.963 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:17.964 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=8 2026-02-18T08:14:17.965 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:17.965 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:14:17.965 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:17.966 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:14:17.967 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=137, tag_tid=8, entry_tid=39925' 2026-02-18T08:14:17.967 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:14:17.967 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:17.969 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=39925 2026-02-18T08:14:17.969 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 9 = 8 -a 1 -le 39925 2026-02-18T08:14:17.969 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:14:17.969 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:14:17.969 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:14:18.171 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:14:18.171 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:14:18.171 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:14:18.171 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:14:18.171 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:14:18.171 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:14:18.171 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:14:18.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:14:18.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:14:18.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:14:18.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:14:18.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:14:18.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:14:18.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:14:18.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:14:18.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:14:18.176 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:14:18.176 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:14:18.176 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:14:18.176 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:14:18.176 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:14:18.176 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:14:18.176 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:14:18.176 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:14:18.176 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:14:18.176 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:14:18.177 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:14:18.177 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:14:18.177 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:14:18.177 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:14:18.177 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:14:18.177 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:14:18.177 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:14:18.177 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:14:18.177 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:14:18.177 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:14:18.177 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:14:18.178 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:14:18.178 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:14:18.178 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:14:18.178 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:14:18.178 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:14:18.178 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:14:18.178 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:14:18.178 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:14:18.178 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:14:18.178 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:14:18.178 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:14:18.179 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:14:18.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:18.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:18.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:18.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:14:18.256 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:14:18.257 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:14:18.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:18.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:18.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:18.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:14:18.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:18.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:18.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:18.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:14:18.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:14:18.259 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:14:18.259 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:14:18.259 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:18.260 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:14:18.260 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:18.286 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 34 2026-02-18T08:14:18.287 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 38 2026-02-18T08:14:18.287 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:14:18.287 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=153, tag_tid=9, entry_tid=1], [object_number=152, tag_tid=9, entry_tid=0], [object_number=154, tag_tid=8, entry_tid=58722], [object_number=155, tag_tid=8, entry_tid=58719]]], state=connected] 2026-02-18T08:14:18.287 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=136, tag_tid=8, entry_tid=40184], [object_number=139, tag_tid=8, entry_tid=40183], [object_number=138, tag_tid=8, entry_tid=40182], [object_number=137, tag_tid=8, entry_tid=40181]]], state=connected] 2026-02-18T08:14:18.289 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:18.290 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:18.291 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:14:18.291 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:18.292 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:18.292 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:18.292 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:14:18.292 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:18.292 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:18.292 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:18.292 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:14:18.292 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:14:18.292 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:14:18.292 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:14:18.293 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:18.293 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:14:18.293 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:18.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 34 2026-02-18T08:14:18.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 38 2026-02-18T08:14:18.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:14:18.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=153, tag_tid=9, entry_tid=1], [object_number=152, tag_tid=9, entry_tid=0], [object_number=154, tag_tid=8, entry_tid=58722], [object_number=155, tag_tid=8, entry_tid=58719]]], state=connected] 2026-02-18T08:14:18.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=136, tag_tid=8, entry_tid=40184], [object_number=139, tag_tid=8, entry_tid=40183], [object_number=138, tag_tid=8, entry_tid=40182], [object_number=137, tag_tid=8, entry_tid=40181]]], state=connected] 2026-02-18T08:14:18.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:18.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=136, tag_tid=8, entry_tid=40184' 2026-02-18T08:14:18.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=153, tag_tid=9, entry_tid=1' -a 'object_number=153, tag_tid=9, entry_tid=1' = 'object_number=136, tag_tid=8, entry_tid=40184' 2026-02-18T08:14:18.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=136, tag_tid=8, entry_tid=40184' '!=' 'object_number=137, tag_tid=8, entry_tid=39925' 2026-02-18T08:14:18.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:14:18.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=136, tag_tid=8, entry_tid=40184' = 'object_number=137, tag_tid=8, entry_tid=39925' 2026-02-18T08:14:18.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=136, tag_tid=8, entry_tid=40184' 2026-02-18T08:14:18.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:18.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:14:18.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:18.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=9 2026-02-18T08:14:18.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=136, tag_tid=8, entry_tid=40184' 2026-02-18T08:14:18.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:14:18.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:18.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=8 2026-02-18T08:14:18.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:18.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:14:18.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:18.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:14:18.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=136, tag_tid=8, entry_tid=40184' 2026-02-18T08:14:18.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:14:18.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:18.344 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=40184 2026-02-18T08:14:18.344 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 9 = 8 -a 1 -le 40184 2026-02-18T08:14:18.344 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:14:18.344 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:14:18.344 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:14:18.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:14:18.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:14:18.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:14:18.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:14:18.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:14:18.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:14:18.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:14:18.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:14:18.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:14:18.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:14:18.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:14:18.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:14:18.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:14:18.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:14:18.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:14:18.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:14:18.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:14:18.552 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:14:18.552 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:14:18.552 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:14:18.552 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:14:18.553 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:14:18.553 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:14:18.553 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:14:18.553 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:14:18.553 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:14:18.553 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:14:18.553 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:14:18.553 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:14:18.553 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:14:18.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:14:18.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:14:18.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:14:18.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:14:18.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:14:18.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:14:18.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:14:18.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:14:18.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:14:18.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:14:18.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:14:18.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:14:18.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:14:18.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:14:18.557 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:14:18.557 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:14:18.557 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:14:18.557 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:14:18.557 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:14:18.560 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:18.561 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:18.561 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:18.561 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:14:18.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:14:18.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:14:18.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:18.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:18.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:18.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:14:18.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:18.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:18.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:18.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:14:18.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:14:18.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:14:18.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:14:18.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:18.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:14:18.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:18.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 34 2026-02-18T08:14:18.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 38 2026-02-18T08:14:18.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:14:18.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=153, tag_tid=9, entry_tid=1], [object_number=152, tag_tid=9, entry_tid=0], [object_number=154, tag_tid=8, entry_tid=58722], [object_number=155, tag_tid=8, entry_tid=58719]]], state=connected] 2026-02-18T08:14:18.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=139, tag_tid=8, entry_tid=40423], [object_number=138, tag_tid=8, entry_tid=40422], [object_number=137, tag_tid=8, entry_tid=40421], [object_number=136, tag_tid=8, entry_tid=40420]]], state=connected] 2026-02-18T08:14:18.672 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:18.674 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:18.675 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:14:18.675 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:18.675 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:18.675 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:18.675 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:14:18.675 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:18.675 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:18.675 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:18.676 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:14:18.676 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:14:18.676 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:14:18.676 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:14:18.677 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:18.677 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:14:18.677 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:18.704 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 34 2026-02-18T08:14:18.704 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 38 2026-02-18T08:14:18.704 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:14:18.704 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=153, tag_tid=9, entry_tid=1], [object_number=152, tag_tid=9, entry_tid=0], [object_number=154, tag_tid=8, entry_tid=58722], [object_number=155, tag_tid=8, entry_tid=58719]]], state=connected] 2026-02-18T08:14:18.704 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=139, tag_tid=8, entry_tid=40423], [object_number=138, tag_tid=8, entry_tid=40422], [object_number=137, tag_tid=8, entry_tid=40421], [object_number=136, tag_tid=8, entry_tid=40420]]], state=connected] 2026-02-18T08:14:18.707 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:18.708 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=139, tag_tid=8, entry_tid=40423' 2026-02-18T08:14:18.708 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=153, tag_tid=9, entry_tid=1' -a 'object_number=153, tag_tid=9, entry_tid=1' = 'object_number=139, tag_tid=8, entry_tid=40423' 2026-02-18T08:14:18.709 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=139, tag_tid=8, entry_tid=40423' '!=' 'object_number=136, tag_tid=8, entry_tid=40184' 2026-02-18T08:14:18.709 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:14:18.709 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=139, tag_tid=8, entry_tid=40423' = 'object_number=136, tag_tid=8, entry_tid=40184' 2026-02-18T08:14:18.709 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=139, tag_tid=8, entry_tid=40423' 2026-02-18T08:14:18.709 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:18.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:14:18.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:18.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=9 2026-02-18T08:14:18.711 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=139, tag_tid=8, entry_tid=40423' 2026-02-18T08:14:18.711 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:14:18.712 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:18.713 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=8 2026-02-18T08:14:18.714 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:18.714 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:14:18.714 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:18.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:14:18.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=139, tag_tid=8, entry_tid=40423' 2026-02-18T08:14:18.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:14:18.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:18.717 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=40423 2026-02-18T08:14:18.717 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 9 = 8 -a 1 -le 40423 2026-02-18T08:14:18.717 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:14:18.717 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:14:18.717 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:14:18.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:14:18.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:14:18.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:14:18.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:14:18.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:14:18.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:14:18.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:14:18.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:14:18.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:14:18.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:14:18.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:14:18.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:14:18.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:14:18.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:14:18.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:14:18.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:14:18.924 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:14:18.924 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:14:18.924 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:14:18.924 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:14:18.924 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:14:18.924 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:14:18.924 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:14:18.924 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:14:18.924 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:14:18.924 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:14:18.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:14:18.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:14:18.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:14:18.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:14:18.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:14:18.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:14:18.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:14:18.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:14:18.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:14:18.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:14:18.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:14:18.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:14:18.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:14:18.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:14:18.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:14:18.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:14:18.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:14:18.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:14:18.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:14:18.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:14:18.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:14:18.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:14:18.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:14:18.933 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:18.933 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:18.933 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:18.933 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:14:18.990 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:14:18.992 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:14:18.992 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:18.992 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:18.992 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:18.992 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:14:18.992 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:18.992 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:18.992 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:18.992 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:14:18.992 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:14:18.993 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:14:18.993 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:14:18.993 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:18.994 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:14:18.994 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:19.020 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 34 2026-02-18T08:14:19.020 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 38 2026-02-18T08:14:19.020 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:14:19.020 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=153, tag_tid=9, entry_tid=1], [object_number=152, tag_tid=9, entry_tid=0], [object_number=154, tag_tid=8, entry_tid=58722], [object_number=155, tag_tid=8, entry_tid=58719]]], state=connected] 2026-02-18T08:14:19.020 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=136, tag_tid=8, entry_tid=40640], [object_number=139, tag_tid=8, entry_tid=40639], [object_number=138, tag_tid=8, entry_tid=40638], [object_number=137, tag_tid=8, entry_tid=40637]]], state=connected] 2026-02-18T08:14:19.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:19.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:19.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:14:19.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:19.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:19.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:19.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:14:19.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:19.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:19.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:19.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:14:19.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:14:19.027 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:14:19.027 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:14:19.027 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:19.028 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:14:19.028 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:19.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 34 2026-02-18T08:14:19.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 38 2026-02-18T08:14:19.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:14:19.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=153, tag_tid=9, entry_tid=1], [object_number=152, tag_tid=9, entry_tid=0], [object_number=154, tag_tid=8, entry_tid=58722], [object_number=155, tag_tid=8, entry_tid=58719]]], state=connected] 2026-02-18T08:14:19.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=136, tag_tid=8, entry_tid=40640], [object_number=139, tag_tid=8, entry_tid=40639], [object_number=138, tag_tid=8, entry_tid=40638], [object_number=137, tag_tid=8, entry_tid=40637]]], state=connected] 2026-02-18T08:14:19.068 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:19.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=136, tag_tid=8, entry_tid=40640' 2026-02-18T08:14:19.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=153, tag_tid=9, entry_tid=1' -a 'object_number=153, tag_tid=9, entry_tid=1' = 'object_number=136, tag_tid=8, entry_tid=40640' 2026-02-18T08:14:19.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=136, tag_tid=8, entry_tid=40640' '!=' 'object_number=139, tag_tid=8, entry_tid=40423' 2026-02-18T08:14:19.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:14:19.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=136, tag_tid=8, entry_tid=40640' = 'object_number=139, tag_tid=8, entry_tid=40423' 2026-02-18T08:14:19.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=136, tag_tid=8, entry_tid=40640' 2026-02-18T08:14:19.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:19.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:14:19.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:19.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=9 2026-02-18T08:14:19.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=136, tag_tid=8, entry_tid=40640' 2026-02-18T08:14:19.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:14:19.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:19.074 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=8 2026-02-18T08:14:19.075 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:19.075 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:14:19.075 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:19.076 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:14:19.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=136, tag_tid=8, entry_tid=40640' 2026-02-18T08:14:19.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:14:19.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:19.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=40640 2026-02-18T08:14:19.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 9 = 8 -a 1 -le 40640 2026-02-18T08:14:19.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:14:19.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:14:19.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:14:19.281 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:14:19.281 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:14:19.281 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:14:19.281 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:14:19.281 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:14:19.281 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:14:19.281 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:14:19.281 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:14:19.282 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:14:19.282 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:14:19.282 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:14:19.282 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:14:19.282 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:14:19.282 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:14:19.282 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:14:19.285 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:14:19.285 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:14:19.285 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:14:19.285 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:14:19.285 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:14:19.286 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:14:19.286 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:14:19.286 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:14:19.286 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:14:19.286 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:14:19.286 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:14:19.286 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:14:19.286 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:14:19.286 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:14:19.286 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:14:19.286 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:14:19.287 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:14:19.287 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:14:19.287 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:14:19.287 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:14:19.287 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:14:19.287 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:14:19.287 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:14:19.287 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:14:19.287 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:14:19.287 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:14:19.287 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:14:19.287 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:14:19.288 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:14:19.288 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:14:19.288 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:14:19.288 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:14:19.288 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:14:19.288 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:14:19.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:19.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:19.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:19.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:14:19.356 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:14:19.357 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:14:19.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:19.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:19.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:19.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:14:19.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:19.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:19.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:19.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:14:19.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:14:19.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:14:19.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:14:19.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:19.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:14:19.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:19.393 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 34 2026-02-18T08:14:19.394 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 38 2026-02-18T08:14:19.394 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:14:19.394 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=153, tag_tid=9, entry_tid=1], [object_number=152, tag_tid=9, entry_tid=0], [object_number=154, tag_tid=8, entry_tid=58722], [object_number=155, tag_tid=8, entry_tid=58719]]], state=connected] 2026-02-18T08:14:19.394 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=136, tag_tid=8, entry_tid=40860], [object_number=139, tag_tid=8, entry_tid=40859], [object_number=138, tag_tid=8, entry_tid=40858], [object_number=137, tag_tid=8, entry_tid=40857]]], state=connected] 2026-02-18T08:14:19.397 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:19.399 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:19.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:14:19.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:19.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:19.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:19.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:14:19.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:19.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:19.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:19.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:14:19.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:14:19.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:14:19.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:14:19.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:19.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:14:19.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:19.429 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 34 2026-02-18T08:14:19.429 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 38 2026-02-18T08:14:19.429 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:14:19.429 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=153, tag_tid=9, entry_tid=1], [object_number=152, tag_tid=9, entry_tid=0], [object_number=154, tag_tid=8, entry_tid=58722], [object_number=155, tag_tid=8, entry_tid=58719]]], state=connected] 2026-02-18T08:14:19.429 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=136, tag_tid=8, entry_tid=40860], [object_number=139, tag_tid=8, entry_tid=40859], [object_number=138, tag_tid=8, entry_tid=40858], [object_number=137, tag_tid=8, entry_tid=40857]]], state=connected] 2026-02-18T08:14:19.432 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:19.433 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=136, tag_tid=8, entry_tid=40860' 2026-02-18T08:14:19.433 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=153, tag_tid=9, entry_tid=1' -a 'object_number=153, tag_tid=9, entry_tid=1' = 'object_number=136, tag_tid=8, entry_tid=40860' 2026-02-18T08:14:19.433 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=136, tag_tid=8, entry_tid=40860' '!=' 'object_number=136, tag_tid=8, entry_tid=40640' 2026-02-18T08:14:19.433 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:14:19.433 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=136, tag_tid=8, entry_tid=40860' = 'object_number=136, tag_tid=8, entry_tid=40640' 2026-02-18T08:14:19.434 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=136, tag_tid=8, entry_tid=40860' 2026-02-18T08:14:19.434 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:19.434 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:14:19.435 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:19.435 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=9 2026-02-18T08:14:19.436 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=136, tag_tid=8, entry_tid=40860' 2026-02-18T08:14:19.436 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:14:19.436 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:19.437 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=8 2026-02-18T08:14:19.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:19.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:14:19.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:19.439 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:14:19.440 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=136, tag_tid=8, entry_tid=40860' 2026-02-18T08:14:19.440 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:14:19.440 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:19.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=40860 2026-02-18T08:14:19.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 9 = 8 -a 1 -le 40860 2026-02-18T08:14:19.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:14:19.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:14:19.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:14:19.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:14:19.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:14:19.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:14:19.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:14:19.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:14:19.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:14:19.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:14:19.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:14:19.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:14:19.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:14:19.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:14:19.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:14:19.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:14:19.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:14:19.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:14:19.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:14:19.646 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:14:19.648 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:14:19.649 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:14:19.649 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:14:19.649 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:14:19.649 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:14:19.649 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:14:19.649 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:14:19.649 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:14:19.649 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:14:19.649 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:14:19.649 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:14:19.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:14:19.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:14:19.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:14:19.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:14:19.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:14:19.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:14:19.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:14:19.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:14:19.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:14:19.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:14:19.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:14:19.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:14:19.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:14:19.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:14:19.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:14:19.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:14:19.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:14:19.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:14:19.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:14:19.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:14:19.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:14:19.658 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:19.658 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:19.658 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:19.658 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:14:19.719 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:14:19.720 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:14:19.720 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:19.720 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:19.720 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:19.721 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:14:19.721 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:19.721 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:19.721 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:19.721 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:14:19.721 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:14:19.721 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:14:19.721 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:14:19.722 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:19.722 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:14:19.722 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:19.758 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 34 2026-02-18T08:14:19.758 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 38 2026-02-18T08:14:19.758 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:14:19.759 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=153, tag_tid=9, entry_tid=1], [object_number=152, tag_tid=9, entry_tid=0], [object_number=154, tag_tid=8, entry_tid=58722], [object_number=155, tag_tid=8, entry_tid=58719]]], state=connected] 2026-02-18T08:14:19.759 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=139, tag_tid=8, entry_tid=41087], [object_number=138, tag_tid=8, entry_tid=41086], [object_number=137, tag_tid=8, entry_tid=41085], [object_number=136, tag_tid=8, entry_tid=41084]]], state=connected] 2026-02-18T08:14:19.762 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:19.763 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:19.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:14:19.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:19.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:19.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:19.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:14:19.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:19.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:19.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:19.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:14:19.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:14:19.766 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:14:19.766 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:14:19.766 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:19.767 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:14:19.767 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:19.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 34 2026-02-18T08:14:19.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 38 2026-02-18T08:14:19.999 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:14:19.999 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=153, tag_tid=9, entry_tid=1], [object_number=152, tag_tid=9, entry_tid=0], [object_number=154, tag_tid=8, entry_tid=58722], [object_number=155, tag_tid=8, entry_tid=58719]]], state=connected] 2026-02-18T08:14:19.999 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=139, tag_tid=8, entry_tid=41087], [object_number=138, tag_tid=8, entry_tid=41086], [object_number=137, tag_tid=8, entry_tid=41085], [object_number=136, tag_tid=8, entry_tid=41084]]], state=connected] 2026-02-18T08:14:20.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:20.003 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=139, tag_tid=8, entry_tid=41087' 2026-02-18T08:14:20.003 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=153, tag_tid=9, entry_tid=1' -a 'object_number=153, tag_tid=9, entry_tid=1' = 'object_number=139, tag_tid=8, entry_tid=41087' 2026-02-18T08:14:20.003 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=139, tag_tid=8, entry_tid=41087' '!=' 'object_number=136, tag_tid=8, entry_tid=40860' 2026-02-18T08:14:20.004 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:14:20.004 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=139, tag_tid=8, entry_tid=41087' = 'object_number=136, tag_tid=8, entry_tid=40860' 2026-02-18T08:14:20.004 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=139, tag_tid=8, entry_tid=41087' 2026-02-18T08:14:20.004 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:20.004 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:14:20.004 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:20.005 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=9 2026-02-18T08:14:20.006 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=139, tag_tid=8, entry_tid=41087' 2026-02-18T08:14:20.006 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:14:20.006 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:20.007 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=8 2026-02-18T08:14:20.008 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:20.008 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:14:20.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:20.010 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:14:20.010 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=139, tag_tid=8, entry_tid=41087' 2026-02-18T08:14:20.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:14:20.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:20.012 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=41087 2026-02-18T08:14:20.012 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 9 = 8 -a 1 -le 41087 2026-02-18T08:14:20.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:14:20.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:14:20.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:14:20.214 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:14:20.215 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:14:20.215 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:14:20.215 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:14:20.215 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:14:20.215 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:14:20.215 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:14:20.215 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:14:20.215 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:14:20.215 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:14:20.215 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:14:20.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:14:20.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:14:20.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:14:20.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:14:20.218 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:14:20.219 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:14:20.219 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:14:20.219 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:14:20.219 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:14:20.219 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:14:20.219 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:14:20.219 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:14:20.219 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:14:20.219 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:14:20.219 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:14:20.220 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:14:20.220 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:14:20.220 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:14:20.220 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:14:20.220 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:14:20.220 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:14:20.220 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:14:20.220 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:14:20.220 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:14:20.220 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:14:20.220 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:14:20.221 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:14:20.221 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:14:20.221 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:14:20.221 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:14:20.221 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:14:20.221 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:14:20.221 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:14:20.221 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:14:20.221 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:14:20.221 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:14:20.221 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:14:20.221 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:14:20.229 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:20.229 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:20.229 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:20.229 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:14:20.287 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:14:20.289 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:14:20.289 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:20.289 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:20.289 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:20.289 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:14:20.289 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:20.289 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:20.289 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:20.289 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:14:20.290 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:14:20.290 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:14:20.290 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:14:20.291 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:20.291 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:14:20.291 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:20.318 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 34 2026-02-18T08:14:20.318 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 38 2026-02-18T08:14:20.318 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:14:20.318 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=153, tag_tid=9, entry_tid=1], [object_number=152, tag_tid=9, entry_tid=0], [object_number=154, tag_tid=8, entry_tid=58722], [object_number=155, tag_tid=8, entry_tid=58719]]], state=connected] 2026-02-18T08:14:20.318 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=139, tag_tid=8, entry_tid=41467], [object_number=138, tag_tid=8, entry_tid=41466], [object_number=137, tag_tid=8, entry_tid=41465], [object_number=136, tag_tid=8, entry_tid=41464]]], state=connected] 2026-02-18T08:14:20.320 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:20.322 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:20.323 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:14:20.323 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:20.323 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:20.324 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:20.324 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:14:20.324 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:20.324 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:20.324 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:20.324 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:14:20.324 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:14:20.325 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:14:20.325 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:14:20.325 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:20.326 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:14:20.326 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:20.361 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 34 2026-02-18T08:14:20.361 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 38 2026-02-18T08:14:20.361 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:14:20.361 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=153, tag_tid=9, entry_tid=1], [object_number=152, tag_tid=9, entry_tid=0], [object_number=154, tag_tid=8, entry_tid=58722], [object_number=155, tag_tid=8, entry_tid=58719]]], state=connected] 2026-02-18T08:14:20.361 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=139, tag_tid=8, entry_tid=41467], [object_number=138, tag_tid=8, entry_tid=41466], [object_number=137, tag_tid=8, entry_tid=41465], [object_number=136, tag_tid=8, entry_tid=41464]]], state=connected] 2026-02-18T08:14:20.364 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:20.366 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=139, tag_tid=8, entry_tid=41467' 2026-02-18T08:14:20.366 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=153, tag_tid=9, entry_tid=1' -a 'object_number=153, tag_tid=9, entry_tid=1' = 'object_number=139, tag_tid=8, entry_tid=41467' 2026-02-18T08:14:20.366 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=139, tag_tid=8, entry_tid=41467' '!=' 'object_number=139, tag_tid=8, entry_tid=41087' 2026-02-18T08:14:20.366 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:14:20.367 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=139, tag_tid=8, entry_tid=41467' = 'object_number=139, tag_tid=8, entry_tid=41087' 2026-02-18T08:14:20.367 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=139, tag_tid=8, entry_tid=41467' 2026-02-18T08:14:20.367 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:20.367 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:14:20.368 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:20.368 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=9 2026-02-18T08:14:20.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=139, tag_tid=8, entry_tid=41467' 2026-02-18T08:14:20.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:14:20.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:20.371 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=8 2026-02-18T08:14:20.372 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:20.372 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:14:20.372 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:20.373 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:14:20.374 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=139, tag_tid=8, entry_tid=41467' 2026-02-18T08:14:20.374 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:14:20.374 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:20.376 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=41467 2026-02-18T08:14:20.376 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 9 = 8 -a 1 -le 41467 2026-02-18T08:14:20.376 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:14:20.376 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:14:20.376 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:14:20.578 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:14:20.578 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:14:20.578 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:14:20.578 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:14:20.578 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:14:20.578 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:14:20.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:14:20.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:14:20.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:14:20.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:14:20.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:14:20.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:14:20.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:14:20.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:14:20.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:14:20.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:14:20.582 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:14:20.582 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:14:20.582 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:14:20.582 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:14:20.582 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:14:20.582 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:14:20.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:14:20.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:14:20.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:14:20.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:14:20.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:14:20.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:14:20.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:14:20.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:14:20.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:14:20.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:14:20.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:14:20.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:14:20.584 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:14:20.584 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:14:20.584 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:14:20.584 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:14:20.584 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:14:20.584 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:14:20.584 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:14:20.584 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:14:20.584 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:14:20.584 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:14:20.584 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:14:20.584 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:14:20.585 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:14:20.585 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:14:20.585 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:14:20.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:20.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:20.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:20.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:14:20.659 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:14:20.661 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:14:20.661 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:20.661 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:20.661 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:20.661 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:14:20.661 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:20.661 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:20.661 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:20.661 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:14:20.662 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:14:20.662 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:14:20.662 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:14:20.662 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:20.663 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:14:20.663 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:20.691 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 34 2026-02-18T08:14:20.691 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 38 2026-02-18T08:14:20.691 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:14:20.692 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=153, tag_tid=9, entry_tid=1], [object_number=152, tag_tid=9, entry_tid=0], [object_number=154, tag_tid=8, entry_tid=58722], [object_number=155, tag_tid=8, entry_tid=58719]]], state=connected] 2026-02-18T08:14:20.692 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=137, tag_tid=8, entry_tid=41717], [object_number=136, tag_tid=8, entry_tid=41716], [object_number=139, tag_tid=8, entry_tid=41715], [object_number=138, tag_tid=8, entry_tid=41714]]], state=connected] 2026-02-18T08:14:20.694 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:20.695 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:20.697 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:14:20.697 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:20.697 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:20.697 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:20.697 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:14:20.697 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:20.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:20.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:20.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:14:20.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:14:20.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:14:20.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:14:20.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:20.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:14:20.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:20.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 34 2026-02-18T08:14:20.728 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 38 2026-02-18T08:14:20.728 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:14:20.728 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=153, tag_tid=9, entry_tid=1], [object_number=152, tag_tid=9, entry_tid=0], [object_number=154, tag_tid=8, entry_tid=58722], [object_number=155, tag_tid=8, entry_tid=58719]]], state=connected] 2026-02-18T08:14:20.728 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=137, tag_tid=8, entry_tid=41717], [object_number=136, tag_tid=8, entry_tid=41716], [object_number=139, tag_tid=8, entry_tid=41715], [object_number=138, tag_tid=8, entry_tid=41714]]], state=connected] 2026-02-18T08:14:20.731 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:20.733 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=137, tag_tid=8, entry_tid=41717' 2026-02-18T08:14:20.733 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=153, tag_tid=9, entry_tid=1' -a 'object_number=153, tag_tid=9, entry_tid=1' = 'object_number=137, tag_tid=8, entry_tid=41717' 2026-02-18T08:14:20.733 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=137, tag_tid=8, entry_tid=41717' '!=' 'object_number=139, tag_tid=8, entry_tid=41467' 2026-02-18T08:14:20.734 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:14:20.734 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=137, tag_tid=8, entry_tid=41717' = 'object_number=139, tag_tid=8, entry_tid=41467' 2026-02-18T08:14:20.734 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=137, tag_tid=8, entry_tid=41717' 2026-02-18T08:14:20.734 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:20.734 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:14:20.735 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:20.735 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=9 2026-02-18T08:14:20.736 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=137, tag_tid=8, entry_tid=41717' 2026-02-18T08:14:20.736 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:14:20.736 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:20.737 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=8 2026-02-18T08:14:20.738 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:20.738 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:14:20.738 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:20.739 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:14:20.740 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=137, tag_tid=8, entry_tid=41717' 2026-02-18T08:14:20.740 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:14:20.741 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:20.742 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=41717 2026-02-18T08:14:20.742 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 9 = 8 -a 1 -le 41717 2026-02-18T08:14:20.742 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:14:20.742 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:14:20.742 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:14:20.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:14:20.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:14:20.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:14:20.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:14:20.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:14:20.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:14:20.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:14:20.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:14:20.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:14:20.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:14:20.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:14:20.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:14:20.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:14:20.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:14:20.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:14:20.944 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:14:20.944 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:14:20.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:14:20.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:14:20.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:14:20.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:14:20.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:14:20.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:14:20.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:14:20.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:14:20.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:14:20.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:14:20.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:14:20.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:14:20.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:14:20.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:14:20.946 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:14:20.946 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:14:20.946 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:14:20.946 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:14:20.946 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:14:20.946 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:14:20.946 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:14:20.946 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:14:20.946 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:14:20.946 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:14:20.946 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:14:20.946 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:14:20.946 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:14:20.946 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:14:20.946 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:14:20.946 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:14:20.946 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:14:20.947 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:14:20.959 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:20.959 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:20.959 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:20.959 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:14:21.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:14:21.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:14:21.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:21.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:21.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:21.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:14:21.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:21.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:21.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:21.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:14:21.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:14:21.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:14:21.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:14:21.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:21.035 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:21.035 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:14:21.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 34 2026-02-18T08:14:21.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 38 2026-02-18T08:14:21.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:14:21.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=153, tag_tid=9, entry_tid=1], [object_number=152, tag_tid=9, entry_tid=0], [object_number=154, tag_tid=8, entry_tid=58722], [object_number=155, tag_tid=8, entry_tid=58719]]], state=connected] 2026-02-18T08:14:21.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=137, tag_tid=8, entry_tid=41957], [object_number=136, tag_tid=8, entry_tid=41956], [object_number=139, tag_tid=8, entry_tid=41955], [object_number=138, tag_tid=8, entry_tid=41954]]], state=connected] 2026-02-18T08:14:21.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:21.067 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:21.068 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:14:21.068 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:21.069 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:21.069 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:21.069 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:14:21.069 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:21.069 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:21.069 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:21.069 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:14:21.069 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:14:21.069 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:14:21.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:14:21.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:21.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:14:21.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:21.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 34 2026-02-18T08:14:21.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 38 2026-02-18T08:14:21.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:14:21.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=153, tag_tid=9, entry_tid=1], [object_number=152, tag_tid=9, entry_tid=0], [object_number=154, tag_tid=8, entry_tid=58722], [object_number=155, tag_tid=8, entry_tid=58719]]], state=connected] 2026-02-18T08:14:21.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=137, tag_tid=8, entry_tid=41957], [object_number=136, tag_tid=8, entry_tid=41956], [object_number=139, tag_tid=8, entry_tid=41955], [object_number=138, tag_tid=8, entry_tid=41954]]], state=connected] 2026-02-18T08:14:21.101 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:21.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=137, tag_tid=8, entry_tid=41957' 2026-02-18T08:14:21.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=153, tag_tid=9, entry_tid=1' -a 'object_number=153, tag_tid=9, entry_tid=1' = 'object_number=137, tag_tid=8, entry_tid=41957' 2026-02-18T08:14:21.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=137, tag_tid=8, entry_tid=41957' '!=' 'object_number=137, tag_tid=8, entry_tid=41717' 2026-02-18T08:14:21.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:14:21.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=137, tag_tid=8, entry_tid=41957' = 'object_number=137, tag_tid=8, entry_tid=41717' 2026-02-18T08:14:21.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=137, tag_tid=8, entry_tid=41957' 2026-02-18T08:14:21.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:21.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:14:21.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:21.105 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=9 2026-02-18T08:14:21.105 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=137, tag_tid=8, entry_tid=41957' 2026-02-18T08:14:21.106 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:14:21.106 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:21.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=8 2026-02-18T08:14:21.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:21.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:14:21.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:21.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:14:21.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=137, tag_tid=8, entry_tid=41957' 2026-02-18T08:14:21.110 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:14:21.110 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:21.111 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=41957 2026-02-18T08:14:21.111 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 9 = 8 -a 1 -le 41957 2026-02-18T08:14:21.111 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:14:21.111 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:14:21.112 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:14:21.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:14:21.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:14:21.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:14:21.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:14:21.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:14:21.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:14:21.314 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:14:21.314 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:14:21.314 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:14:21.314 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:14:21.314 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:14:21.314 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:14:21.314 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:14:21.314 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:14:21.314 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:14:21.317 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:14:21.317 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:14:21.317 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:14:21.317 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:14:21.317 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:14:21.317 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:14:21.318 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:14:21.318 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:14:21.318 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:14:21.318 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:14:21.318 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:14:21.318 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:14:21.318 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:14:21.318 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:14:21.318 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:14:21.318 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:14:21.318 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:14:21.318 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:14:21.319 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:14:21.319 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:14:21.319 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:14:21.319 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:14:21.319 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:14:21.319 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:14:21.319 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:14:21.319 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:14:21.319 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:14:21.319 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:14:21.319 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:14:21.319 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:14:21.320 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:14:21.320 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:14:21.320 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:14:21.320 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:14:21.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:21.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:21.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:21.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:14:21.396 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:14:21.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:14:21.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:21.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:21.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:21.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:14:21.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:21.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:21.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:21.399 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:14:21.399 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:14:21.399 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:14:21.399 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:14:21.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:21.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:14:21.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:21.435 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 34 2026-02-18T08:14:21.435 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 38 2026-02-18T08:14:21.435 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:14:21.435 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=153, tag_tid=9, entry_tid=1], [object_number=152, tag_tid=9, entry_tid=0], [object_number=154, tag_tid=8, entry_tid=58722], [object_number=155, tag_tid=8, entry_tid=58719]]], state=connected] 2026-02-18T08:14:21.435 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=137, tag_tid=8, entry_tid=42197], [object_number=136, tag_tid=8, entry_tid=42196], [object_number=139, tag_tid=8, entry_tid=42195], [object_number=138, tag_tid=8, entry_tid=42194]]], state=connected] 2026-02-18T08:14:21.439 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:21.440 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:21.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:14:21.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:21.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:21.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:21.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:14:21.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:21.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:21.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:21.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:14:21.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:14:21.443 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:14:21.443 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:14:21.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:21.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:14:21.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:21.478 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 34 2026-02-18T08:14:21.478 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 38 2026-02-18T08:14:21.478 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:14:21.478 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=153, tag_tid=9, entry_tid=1], [object_number=152, tag_tid=9, entry_tid=0], [object_number=154, tag_tid=8, entry_tid=58722], [object_number=155, tag_tid=8, entry_tid=58719]]], state=connected] 2026-02-18T08:14:21.478 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=137, tag_tid=8, entry_tid=42197], [object_number=136, tag_tid=8, entry_tid=42196], [object_number=139, tag_tid=8, entry_tid=42195], [object_number=138, tag_tid=8, entry_tid=42194]]], state=connected] 2026-02-18T08:14:21.481 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:21.484 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=137, tag_tid=8, entry_tid=42197' 2026-02-18T08:14:21.484 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=153, tag_tid=9, entry_tid=1' -a 'object_number=153, tag_tid=9, entry_tid=1' = 'object_number=137, tag_tid=8, entry_tid=42197' 2026-02-18T08:14:21.484 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=137, tag_tid=8, entry_tid=42197' '!=' 'object_number=137, tag_tid=8, entry_tid=41957' 2026-02-18T08:14:21.484 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:14:21.484 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=137, tag_tid=8, entry_tid=42197' = 'object_number=137, tag_tid=8, entry_tid=41957' 2026-02-18T08:14:21.484 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=137, tag_tid=8, entry_tid=42197' 2026-02-18T08:14:21.485 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:21.485 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:14:21.485 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:21.486 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=9 2026-02-18T08:14:21.486 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=137, tag_tid=8, entry_tid=42197' 2026-02-18T08:14:21.487 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:14:21.487 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:21.488 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=8 2026-02-18T08:14:21.489 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:21.489 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:14:21.489 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:21.490 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:14:21.491 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=137, tag_tid=8, entry_tid=42197' 2026-02-18T08:14:21.491 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:14:21.491 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:21.493 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=42197 2026-02-18T08:14:21.493 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 9 = 8 -a 1 -le 42197 2026-02-18T08:14:21.493 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:14:21.493 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:14:21.493 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:14:21.695 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:14:21.695 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:14:21.695 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:14:21.695 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:14:21.695 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:14:21.695 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:14:21.696 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:14:21.696 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:14:21.696 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:14:21.696 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:14:21.696 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:14:21.696 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:14:21.696 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:14:21.696 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:14:21.696 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:14:21.696 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:14:21.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:14:21.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:14:21.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:14:21.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:14:21.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:14:21.700 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:14:21.700 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:14:21.700 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:14:21.700 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:14:21.700 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:14:21.700 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:14:21.700 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:14:21.700 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:14:21.700 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:14:21.700 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:14:21.700 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:14:21.701 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:14:21.701 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:14:21.701 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:14:21.701 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:14:21.701 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:14:21.701 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:14:21.701 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:14:21.701 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:14:21.701 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:14:21.701 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:14:21.701 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:14:21.702 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:14:21.702 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:14:21.702 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:14:21.702 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:14:21.702 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:14:21.702 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:14:21.709 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:21.709 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:21.709 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:21.709 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:14:21.768 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:14:21.769 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:14:21.769 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:21.770 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:21.770 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:21.770 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:14:21.770 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:21.770 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:21.770 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:21.770 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:14:21.770 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:14:21.770 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:14:21.770 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:14:21.771 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:21.771 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:14:21.771 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:21.809 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 34 2026-02-18T08:14:21.809 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 38 2026-02-18T08:14:21.809 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:14:21.809 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=153, tag_tid=9, entry_tid=1], [object_number=152, tag_tid=9, entry_tid=0], [object_number=154, tag_tid=8, entry_tid=58722], [object_number=155, tag_tid=8, entry_tid=58719]]], state=connected] 2026-02-18T08:14:21.809 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=138, tag_tid=8, entry_tid=42442], [object_number=137, tag_tid=8, entry_tid=42441], [object_number=136, tag_tid=8, entry_tid=42440], [object_number=139, tag_tid=8, entry_tid=42439]]], state=connected] 2026-02-18T08:14:21.812 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:21.814 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:21.815 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:14:21.815 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:21.815 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:21.815 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:21.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:14:21.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:21.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:21.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:21.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:14:21.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:14:21.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:14:21.817 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:14:21.817 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:21.817 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:14:21.818 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:21.853 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 34 2026-02-18T08:14:21.853 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 38 2026-02-18T08:14:21.854 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:14:21.854 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=153, tag_tid=9, entry_tid=1], [object_number=152, tag_tid=9, entry_tid=0], [object_number=154, tag_tid=8, entry_tid=58722], [object_number=155, tag_tid=8, entry_tid=58719]]], state=connected] 2026-02-18T08:14:21.854 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=138, tag_tid=8, entry_tid=42442], [object_number=137, tag_tid=8, entry_tid=42441], [object_number=136, tag_tid=8, entry_tid=42440], [object_number=139, tag_tid=8, entry_tid=42439]]], state=connected] 2026-02-18T08:14:21.857 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:21.859 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=138, tag_tid=8, entry_tid=42442' 2026-02-18T08:14:21.859 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=153, tag_tid=9, entry_tid=1' -a 'object_number=153, tag_tid=9, entry_tid=1' = 'object_number=138, tag_tid=8, entry_tid=42442' 2026-02-18T08:14:21.859 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=138, tag_tid=8, entry_tid=42442' '!=' 'object_number=137, tag_tid=8, entry_tid=42197' 2026-02-18T08:14:21.859 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:14:21.859 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=138, tag_tid=8, entry_tid=42442' = 'object_number=137, tag_tid=8, entry_tid=42197' 2026-02-18T08:14:21.859 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=138, tag_tid=8, entry_tid=42442' 2026-02-18T08:14:21.859 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:21.860 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:14:21.860 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:21.861 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=9 2026-02-18T08:14:21.861 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=138, tag_tid=8, entry_tid=42442' 2026-02-18T08:14:21.862 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:14:21.862 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:21.863 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=8 2026-02-18T08:14:21.863 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:21.864 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:14:21.864 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:21.865 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:14:21.866 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=138, tag_tid=8, entry_tid=42442' 2026-02-18T08:14:21.866 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:14:21.866 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:21.868 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=42442 2026-02-18T08:14:21.868 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 9 = 8 -a 1 -le 42442 2026-02-18T08:14:21.868 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:14:21.868 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:14:21.868 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:14:22.069 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:14:22.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:14:22.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:14:22.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:14:22.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:14:22.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:14:22.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:14:22.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:14:22.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:14:22.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:14:22.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:14:22.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:14:22.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:14:22.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:14:22.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:14:22.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:14:22.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:14:22.074 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:14:22.074 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:14:22.074 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:14:22.074 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:14:22.074 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:14:22.074 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:14:22.074 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:14:22.074 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:14:22.074 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:14:22.074 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:14:22.075 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:14:22.075 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:14:22.075 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:14:22.075 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:14:22.075 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:14:22.075 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:14:22.075 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:14:22.075 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:14:22.075 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:14:22.075 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:14:22.075 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:14:22.075 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:14:22.076 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:14:22.076 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:14:22.076 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:14:22.076 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:14:22.076 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:14:22.076 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:14:22.076 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:14:22.076 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:14:22.076 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:14:22.076 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:14:22.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:22.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:22.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:22.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:14:22.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:14:22.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:14:22.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:22.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:22.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:22.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:14:22.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:22.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:22.155 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:22.155 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:14:22.155 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:14:22.155 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:14:22.155 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:14:22.156 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:22.156 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:14:22.156 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:22.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 34 2026-02-18T08:14:22.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 38 2026-02-18T08:14:22.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:14:22.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=153, tag_tid=9, entry_tid=1], [object_number=152, tag_tid=9, entry_tid=0], [object_number=154, tag_tid=8, entry_tid=58722], [object_number=155, tag_tid=8, entry_tid=58719]]], state=connected] 2026-02-18T08:14:22.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=136, tag_tid=8, entry_tid=42696], [object_number=139, tag_tid=8, entry_tid=42695], [object_number=138, tag_tid=8, entry_tid=42694], [object_number=137, tag_tid=8, entry_tid=42693]]], state=connected] 2026-02-18T08:14:22.194 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:22.196 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:22.197 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:14:22.197 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:22.197 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:22.197 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:22.197 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:14:22.197 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:22.197 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:22.197 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:22.197 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:14:22.198 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:14:22.198 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:14:22.198 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:14:22.199 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:22.199 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:14:22.199 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:22.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 34 2026-02-18T08:14:22.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 38 2026-02-18T08:14:22.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:14:22.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=153, tag_tid=9, entry_tid=1], [object_number=152, tag_tid=9, entry_tid=0], [object_number=154, tag_tid=8, entry_tid=58722], [object_number=155, tag_tid=8, entry_tid=58719]]], state=connected] 2026-02-18T08:14:22.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=136, tag_tid=8, entry_tid=42696], [object_number=139, tag_tid=8, entry_tid=42695], [object_number=138, tag_tid=8, entry_tid=42694], [object_number=137, tag_tid=8, entry_tid=42693]]], state=connected] 2026-02-18T08:14:22.229 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:22.230 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=136, tag_tid=8, entry_tid=42696' 2026-02-18T08:14:22.230 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=153, tag_tid=9, entry_tid=1' -a 'object_number=153, tag_tid=9, entry_tid=1' = 'object_number=136, tag_tid=8, entry_tid=42696' 2026-02-18T08:14:22.230 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=136, tag_tid=8, entry_tid=42696' '!=' 'object_number=138, tag_tid=8, entry_tid=42442' 2026-02-18T08:14:22.230 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:14:22.231 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=136, tag_tid=8, entry_tid=42696' = 'object_number=138, tag_tid=8, entry_tid=42442' 2026-02-18T08:14:22.231 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=136, tag_tid=8, entry_tid=42696' 2026-02-18T08:14:22.231 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:22.231 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:14:22.231 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:22.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=9 2026-02-18T08:14:22.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=136, tag_tid=8, entry_tid=42696' 2026-02-18T08:14:22.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:14:22.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:22.234 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=8 2026-02-18T08:14:22.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:22.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:14:22.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:22.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:14:22.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=136, tag_tid=8, entry_tid=42696' 2026-02-18T08:14:22.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:14:22.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:22.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=42696 2026-02-18T08:14:22.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 9 = 8 -a 1 -le 42696 2026-02-18T08:14:22.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:14:22.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:14:22.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:14:22.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:14:22.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:14:22.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:14:22.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:14:22.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:14:22.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:14:22.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:14:22.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:14:22.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:14:22.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:14:22.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:14:22.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:14:22.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:14:22.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:14:22.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:14:22.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:14:22.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:14:22.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:14:22.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:14:22.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:14:22.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:14:22.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:14:22.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:14:22.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:14:22.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:14:22.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:14:22.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:14:22.446 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:14:22.446 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:14:22.446 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:14:22.446 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:14:22.446 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:14:22.446 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:14:22.446 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:14:22.446 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:14:22.446 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:14:22.446 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:14:22.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:14:22.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:14:22.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:14:22.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:14:22.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:14:22.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:14:22.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:14:22.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:14:22.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:14:22.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:14:22.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:14:22.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:14:22.455 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:22.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:22.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:22.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:14:22.524 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:14:22.525 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:14:22.525 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:22.525 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:22.525 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:22.525 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:14:22.525 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:22.525 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:22.525 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:22.525 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:14:22.526 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:14:22.526 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:14:22.526 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:14:22.527 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:22.527 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:14:22.527 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:22.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 34 2026-02-18T08:14:22.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 38 2026-02-18T08:14:22.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:14:22.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=153, tag_tid=9, entry_tid=1], [object_number=152, tag_tid=9, entry_tid=0], [object_number=154, tag_tid=8, entry_tid=58722], [object_number=155, tag_tid=8, entry_tid=58719]]], state=connected] 2026-02-18T08:14:22.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=138, tag_tid=8, entry_tid=42954], [object_number=137, tag_tid=8, entry_tid=42953], [object_number=136, tag_tid=8, entry_tid=42952], [object_number=139, tag_tid=8, entry_tid=42951]]], state=connected] 2026-02-18T08:14:22.558 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:22.560 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:22.561 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:14:22.561 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:22.561 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:22.561 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:22.561 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:14:22.561 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:22.561 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:22.561 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:22.561 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:14:22.561 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:14:22.562 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:14:22.562 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:14:22.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:22.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:14:22.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:22.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 34 2026-02-18T08:14:22.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 38 2026-02-18T08:14:22.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:14:22.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=153, tag_tid=9, entry_tid=1], [object_number=152, tag_tid=9, entry_tid=0], [object_number=154, tag_tid=8, entry_tid=58722], [object_number=155, tag_tid=8, entry_tid=58719]]], state=connected] 2026-02-18T08:14:22.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=138, tag_tid=8, entry_tid=42954], [object_number=137, tag_tid=8, entry_tid=42953], [object_number=136, tag_tid=8, entry_tid=42952], [object_number=139, tag_tid=8, entry_tid=42951]]], state=connected] 2026-02-18T08:14:22.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:22.595 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=138, tag_tid=8, entry_tid=42954' 2026-02-18T08:14:22.595 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=153, tag_tid=9, entry_tid=1' -a 'object_number=153, tag_tid=9, entry_tid=1' = 'object_number=138, tag_tid=8, entry_tid=42954' 2026-02-18T08:14:22.595 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=138, tag_tid=8, entry_tid=42954' '!=' 'object_number=136, tag_tid=8, entry_tid=42696' 2026-02-18T08:14:22.595 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:14:22.595 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=138, tag_tid=8, entry_tid=42954' = 'object_number=136, tag_tid=8, entry_tid=42696' 2026-02-18T08:14:22.595 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=138, tag_tid=8, entry_tid=42954' 2026-02-18T08:14:22.596 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:22.596 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:14:22.596 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:22.597 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=9 2026-02-18T08:14:22.597 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=138, tag_tid=8, entry_tid=42954' 2026-02-18T08:14:22.598 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:14:22.598 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:22.599 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=8 2026-02-18T08:14:22.599 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:22.600 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:14:22.600 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:22.601 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:14:22.602 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=138, tag_tid=8, entry_tid=42954' 2026-02-18T08:14:22.602 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:14:22.602 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:22.603 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=42954 2026-02-18T08:14:22.603 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 9 = 8 -a 1 -le 42954 2026-02-18T08:14:22.604 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:14:22.604 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:14:22.604 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:14:22.805 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:14:22.805 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:14:22.805 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:14:22.806 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:14:22.806 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:14:22.806 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:14:22.806 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:14:22.806 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:14:22.806 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:14:22.806 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:14:22.806 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:14:22.806 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:14:22.806 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:14:22.806 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:14:22.806 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:14:22.809 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:14:22.809 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:14:22.809 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:14:22.809 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:14:22.809 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:14:22.810 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:14:22.810 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:14:22.810 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:14:22.810 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:14:22.810 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:14:22.810 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:14:22.810 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:14:22.810 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:14:22.810 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:14:22.810 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:14:22.810 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:14:22.810 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:14:22.811 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:14:22.811 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:14:22.811 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:14:22.811 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:14:22.811 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:14:22.811 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:14:22.811 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:14:22.811 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:14:22.811 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:14:22.811 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:14:22.811 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:14:22.811 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:14:22.812 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:14:22.812 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:14:22.812 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:14:22.812 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:14:22.812 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:14:22.819 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:22.820 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:22.820 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:22.820 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:14:22.887 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:14:22.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:14:22.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:22.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:22.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:22.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:14:22.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:22.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:22.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:22.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:14:22.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:14:22.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:14:22.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:14:22.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:22.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:14:22.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:22.918 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 34 2026-02-18T08:14:22.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 38 2026-02-18T08:14:22.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:14:22.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=153, tag_tid=9, entry_tid=1], [object_number=152, tag_tid=9, entry_tid=0], [object_number=154, tag_tid=8, entry_tid=58722], [object_number=155, tag_tid=8, entry_tid=58719]]], state=connected] 2026-02-18T08:14:22.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=136, tag_tid=8, entry_tid=43192], [object_number=139, tag_tid=8, entry_tid=43191], [object_number=138, tag_tid=8, entry_tid=43190], [object_number=137, tag_tid=8, entry_tid=43189]]], state=connected] 2026-02-18T08:14:22.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:22.923 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:22.924 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:14:22.924 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:22.924 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:22.924 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:22.924 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:14:22.924 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:22.924 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:22.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:22.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:14:22.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:14:22.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:14:22.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:14:22.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:22.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:14:22.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:22.961 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 34 2026-02-18T08:14:22.961 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 38 2026-02-18T08:14:22.961 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:14:22.962 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=153, tag_tid=9, entry_tid=1], [object_number=152, tag_tid=9, entry_tid=0], [object_number=154, tag_tid=8, entry_tid=58722], [object_number=155, tag_tid=8, entry_tid=58719]]], state=connected] 2026-02-18T08:14:22.962 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=136, tag_tid=8, entry_tid=43192], [object_number=139, tag_tid=8, entry_tid=43191], [object_number=138, tag_tid=8, entry_tid=43190], [object_number=137, tag_tid=8, entry_tid=43189]]], state=connected] 2026-02-18T08:14:22.965 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:22.967 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=136, tag_tid=8, entry_tid=43192' 2026-02-18T08:14:22.967 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=153, tag_tid=9, entry_tid=1' -a 'object_number=153, tag_tid=9, entry_tid=1' = 'object_number=136, tag_tid=8, entry_tid=43192' 2026-02-18T08:14:22.967 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=136, tag_tid=8, entry_tid=43192' '!=' 'object_number=138, tag_tid=8, entry_tid=42954' 2026-02-18T08:14:22.967 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:14:22.967 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=136, tag_tid=8, entry_tid=43192' = 'object_number=138, tag_tid=8, entry_tid=42954' 2026-02-18T08:14:22.967 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=136, tag_tid=8, entry_tid=43192' 2026-02-18T08:14:22.968 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:22.968 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:14:22.968 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:22.969 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=9 2026-02-18T08:14:22.970 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=136, tag_tid=8, entry_tid=43192' 2026-02-18T08:14:22.970 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:14:22.970 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:22.971 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=8 2026-02-18T08:14:22.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:22.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:14:22.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:22.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:14:22.974 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=136, tag_tid=8, entry_tid=43192' 2026-02-18T08:14:22.974 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:14:22.974 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:22.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=43192 2026-02-18T08:14:22.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 9 = 8 -a 1 -le 43192 2026-02-18T08:14:22.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:14:22.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:14:22.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:14:23.178 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:14:23.178 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:14:23.178 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:14:23.178 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:14:23.178 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:14:23.178 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:14:23.178 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:14:23.178 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:14:23.179 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:14:23.179 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:14:23.179 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:14:23.179 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:14:23.179 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:14:23.179 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:14:23.179 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:14:23.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:14:23.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:14:23.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:14:23.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:14:23.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:14:23.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:14:23.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:14:23.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:14:23.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:14:23.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:14:23.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:14:23.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:14:23.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:14:23.183 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:14:23.183 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:14:23.183 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:14:23.183 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:14:23.183 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:14:23.183 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:14:23.183 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:14:23.183 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:14:23.183 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:14:23.183 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:14:23.183 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:14:23.183 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:14:23.183 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:14:23.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:14:23.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:14:23.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:14:23.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:14:23.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:14:23.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:14:23.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:14:23.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:14:23.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:23.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:23.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:23.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:14:23.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:14:23.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:14:23.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:23.266 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:23.266 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:23.266 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:14:23.266 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:23.266 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:23.266 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:23.266 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:14:23.266 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:14:23.266 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:14:23.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:14:23.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:23.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:14:23.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:23.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 34 2026-02-18T08:14:23.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 38 2026-02-18T08:14:23.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:14:23.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=153, tag_tid=9, entry_tid=1], [object_number=152, tag_tid=9, entry_tid=0], [object_number=154, tag_tid=8, entry_tid=58722], [object_number=155, tag_tid=8, entry_tid=58719]]], state=connected] 2026-02-18T08:14:23.297 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=138, tag_tid=8, entry_tid=43438], [object_number=137, tag_tid=8, entry_tid=43437], [object_number=136, tag_tid=8, entry_tid=43436], [object_number=139, tag_tid=8, entry_tid=43435]]], state=connected] 2026-02-18T08:14:23.299 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:23.300 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:23.301 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:14:23.302 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:23.302 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:23.302 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:23.302 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:14:23.302 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:23.302 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:23.302 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:23.302 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:14:23.302 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:14:23.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:14:23.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:14:23.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:23.304 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:14:23.304 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:23.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 34 2026-02-18T08:14:23.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 38 2026-02-18T08:14:23.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:14:23.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=153, tag_tid=9, entry_tid=1], [object_number=152, tag_tid=9, entry_tid=0], [object_number=154, tag_tid=8, entry_tid=58722], [object_number=155, tag_tid=8, entry_tid=58719]]], state=connected] 2026-02-18T08:14:23.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=138, tag_tid=8, entry_tid=43438], [object_number=137, tag_tid=8, entry_tid=43437], [object_number=136, tag_tid=8, entry_tid=43436], [object_number=139, tag_tid=8, entry_tid=43435]]], state=connected] 2026-02-18T08:14:23.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:23.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=138, tag_tid=8, entry_tid=43438' 2026-02-18T08:14:23.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=153, tag_tid=9, entry_tid=1' -a 'object_number=153, tag_tid=9, entry_tid=1' = 'object_number=138, tag_tid=8, entry_tid=43438' 2026-02-18T08:14:23.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=138, tag_tid=8, entry_tid=43438' '!=' 'object_number=136, tag_tid=8, entry_tid=43192' 2026-02-18T08:14:23.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:14:23.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=138, tag_tid=8, entry_tid=43438' = 'object_number=136, tag_tid=8, entry_tid=43192' 2026-02-18T08:14:23.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=138, tag_tid=8, entry_tid=43438' 2026-02-18T08:14:23.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:23.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:14:23.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:23.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=9 2026-02-18T08:14:23.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=138, tag_tid=8, entry_tid=43438' 2026-02-18T08:14:23.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:14:23.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:23.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=8 2026-02-18T08:14:23.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:23.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:14:23.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:23.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:14:23.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=138, tag_tid=8, entry_tid=43438' 2026-02-18T08:14:23.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:14:23.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:23.344 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=43438 2026-02-18T08:14:23.344 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 9 = 8 -a 1 -le 43438 2026-02-18T08:14:23.344 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:14:23.344 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:14:23.344 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:14:23.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:14:23.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:14:23.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:14:23.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:14:23.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:14:23.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:14:23.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:14:23.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:14:23.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:14:23.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:14:23.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:14:23.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:14:23.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:14:23.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:14:23.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:14:23.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:14:23.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:14:23.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:14:23.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:14:23.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:14:23.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:14:23.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:14:23.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:14:23.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:14:23.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:14:23.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:14:23.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:14:23.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:14:23.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:14:23.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:14:23.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:14:23.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:14:23.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:14:23.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:14:23.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:14:23.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:14:23.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:14:23.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:14:23.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:14:23.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:14:23.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:14:23.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:14:23.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:14:23.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:14:23.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:14:23.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:14:23.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:14:23.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:14:23.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:14:23.560 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:23.561 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:23.561 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:23.561 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:14:23.620 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:14:23.620 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:14:23.620 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:23.620 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:23.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:23.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:14:23.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:23.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:23.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:23.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:14:23.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:14:23.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:14:23.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:14:23.622 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:23.622 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:14:23.623 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:23.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 34 2026-02-18T08:14:23.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 38 2026-02-18T08:14:23.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:14:23.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=153, tag_tid=9, entry_tid=1], [object_number=152, tag_tid=9, entry_tid=0], [object_number=154, tag_tid=8, entry_tid=58722], [object_number=155, tag_tid=8, entry_tid=58719]]], state=connected] 2026-02-18T08:14:23.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=142, tag_tid=8, entry_tid=43622], [object_number=137, tag_tid=8, entry_tid=43621], [object_number=136, tag_tid=8, entry_tid=43620], [object_number=139, tag_tid=8, entry_tid=43619]]], state=connected] 2026-02-18T08:14:23.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:23.654 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:23.654 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:14:23.654 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:23.654 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:23.655 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:23.655 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:14:23.655 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:23.655 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:23.655 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:23.655 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:14:23.655 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:14:23.655 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:14:23.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:14:23.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:23.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:14:23.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:23.682 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 34 2026-02-18T08:14:23.682 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 38 2026-02-18T08:14:23.683 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:14:23.683 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=153, tag_tid=9, entry_tid=1], [object_number=152, tag_tid=9, entry_tid=0], [object_number=154, tag_tid=8, entry_tid=58722], [object_number=155, tag_tid=8, entry_tid=58719]]], state=connected] 2026-02-18T08:14:23.683 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=142, tag_tid=8, entry_tid=43622], [object_number=137, tag_tid=8, entry_tid=43621], [object_number=136, tag_tid=8, entry_tid=43620], [object_number=139, tag_tid=8, entry_tid=43619]]], state=connected] 2026-02-18T08:14:23.685 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:23.686 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=142, tag_tid=8, entry_tid=43622' 2026-02-18T08:14:23.686 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=153, tag_tid=9, entry_tid=1' -a 'object_number=153, tag_tid=9, entry_tid=1' = 'object_number=142, tag_tid=8, entry_tid=43622' 2026-02-18T08:14:23.687 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=142, tag_tid=8, entry_tid=43622' '!=' 'object_number=138, tag_tid=8, entry_tid=43438' 2026-02-18T08:14:23.687 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:14:23.687 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=142, tag_tid=8, entry_tid=43622' = 'object_number=138, tag_tid=8, entry_tid=43438' 2026-02-18T08:14:23.687 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=142, tag_tid=8, entry_tid=43622' 2026-02-18T08:14:23.687 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:23.688 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:14:23.688 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:23.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=9 2026-02-18T08:14:23.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=142, tag_tid=8, entry_tid=43622' 2026-02-18T08:14:23.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:14:23.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:23.691 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=8 2026-02-18T08:14:23.692 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:23.692 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:14:23.692 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:23.693 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:14:23.694 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=142, tag_tid=8, entry_tid=43622' 2026-02-18T08:14:23.694 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:14:23.694 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:23.696 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=43622 2026-02-18T08:14:23.696 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 9 = 8 -a 1 -le 43622 2026-02-18T08:14:23.696 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:14:23.697 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:14:23.697 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:14:23.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:14:23.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:14:23.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:14:23.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:14:23.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:14:23.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:14:23.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:14:23.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:14:23.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:14:23.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:14:23.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:14:23.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:14:23.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:14:23.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:14:23.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:14:23.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:14:23.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:14:23.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:14:23.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:14:23.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:14:23.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:14:23.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:14:23.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:14:23.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:14:23.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:14:23.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:14:23.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:14:23.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:14:23.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:14:23.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:14:23.904 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:14:23.904 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:14:23.904 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:14:23.904 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:14:23.904 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:14:23.904 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:14:23.904 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:14:23.904 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:14:23.904 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:14:23.904 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:14:23.904 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:14:23.904 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:14:23.905 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:14:23.905 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:14:23.905 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:14:23.905 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:14:23.905 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:14:23.905 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:14:23.905 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:14:23.911 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:23.912 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:23.912 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:23.912 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:14:23.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:14:23.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:14:23.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:23.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:23.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:23.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:14:23.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:23.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:23.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:23.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:14:23.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:14:23.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:14:23.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:14:23.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:23.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:14:23.999 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:24.027 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 35 2026-02-18T08:14:24.028 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 38 2026-02-18T08:14:24.028 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:14:24.028 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=153, tag_tid=9, entry_tid=1], [object_number=152, tag_tid=9, entry_tid=0], [object_number=154, tag_tid=8, entry_tid=58722], [object_number=155, tag_tid=8, entry_tid=58719]]], state=connected] 2026-02-18T08:14:24.028 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=143, tag_tid=8, entry_tid=43803], [object_number=142, tag_tid=8, entry_tid=43802], [object_number=141, tag_tid=8, entry_tid=43801], [object_number=140, tag_tid=8, entry_tid=43800]]], state=connected] 2026-02-18T08:14:24.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:24.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:24.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:14:24.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:24.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:24.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:24.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:14:24.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:24.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:24.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:24.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:14:24.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:14:24.035 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:14:24.035 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:14:24.036 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:24.036 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:14:24.036 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:24.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 35 2026-02-18T08:14:24.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 38 2026-02-18T08:14:24.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:14:24.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=153, tag_tid=9, entry_tid=1], [object_number=152, tag_tid=9, entry_tid=0], [object_number=154, tag_tid=8, entry_tid=58722], [object_number=155, tag_tid=8, entry_tid=58719]]], state=connected] 2026-02-18T08:14:24.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=143, tag_tid=8, entry_tid=43803], [object_number=142, tag_tid=8, entry_tid=43802], [object_number=141, tag_tid=8, entry_tid=43801], [object_number=140, tag_tid=8, entry_tid=43800]]], state=connected] 2026-02-18T08:14:24.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:24.067 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=143, tag_tid=8, entry_tid=43803' 2026-02-18T08:14:24.067 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=153, tag_tid=9, entry_tid=1' -a 'object_number=153, tag_tid=9, entry_tid=1' = 'object_number=143, tag_tid=8, entry_tid=43803' 2026-02-18T08:14:24.068 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=143, tag_tid=8, entry_tid=43803' '!=' 'object_number=142, tag_tid=8, entry_tid=43622' 2026-02-18T08:14:24.068 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:14:24.068 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=143, tag_tid=8, entry_tid=43803' = 'object_number=142, tag_tid=8, entry_tid=43622' 2026-02-18T08:14:24.068 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=143, tag_tid=8, entry_tid=43803' 2026-02-18T08:14:24.069 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:24.069 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:14:24.069 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:24.069 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=9 2026-02-18T08:14:24.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=143, tag_tid=8, entry_tid=43803' 2026-02-18T08:14:24.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:14:24.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:24.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=8 2026-02-18T08:14:24.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:24.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:14:24.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:24.074 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:14:24.075 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=143, tag_tid=8, entry_tid=43803' 2026-02-18T08:14:24.075 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:14:24.075 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:24.076 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=43803 2026-02-18T08:14:24.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 9 = 8 -a 1 -le 43803 2026-02-18T08:14:24.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:14:24.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:14:24.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:14:24.278 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:14:24.279 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:14:24.279 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:14:24.279 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:14:24.279 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:14:24.279 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:14:24.279 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:14:24.279 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:14:24.279 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:14:24.279 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:14:24.279 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:14:24.279 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:14:24.280 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:14:24.280 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:14:24.280 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:14:24.280 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:14:24.282 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:14:24.282 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:14:24.283 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:14:24.283 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:14:24.283 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:14:24.283 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:14:24.283 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:14:24.283 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:14:24.283 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:14:24.283 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:14:24.283 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:14:24.283 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:14:24.284 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:14:24.284 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:14:24.284 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:14:24.284 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:14:24.284 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:14:24.284 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:14:24.284 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:14:24.284 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:14:24.284 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:14:24.284 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:14:24.285 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:14:24.285 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:14:24.285 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:14:24.285 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:14:24.285 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:14:24.285 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:14:24.285 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:14:24.285 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:14:24.285 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:14:24.285 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:14:24.285 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:14:24.292 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:24.292 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:24.293 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:24.293 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:14:24.352 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:14:24.353 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:14:24.353 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:24.353 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:24.353 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:24.353 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:14:24.353 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:24.354 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:24.354 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:24.354 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:14:24.354 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:14:24.354 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:14:24.354 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:14:24.354 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:24.355 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:14:24.355 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:24.383 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 35 2026-02-18T08:14:24.383 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 38 2026-02-18T08:14:24.383 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:14:24.383 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=153, tag_tid=9, entry_tid=1], [object_number=152, tag_tid=9, entry_tid=0], [object_number=154, tag_tid=8, entry_tid=58722], [object_number=155, tag_tid=8, entry_tid=58719]]], state=connected] 2026-02-18T08:14:24.383 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=143, tag_tid=8, entry_tid=44067], [object_number=142, tag_tid=8, entry_tid=44066], [object_number=141, tag_tid=8, entry_tid=44065], [object_number=140, tag_tid=8, entry_tid=44064]]], state=connected] 2026-02-18T08:14:24.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:24.387 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:24.388 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:14:24.388 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:24.388 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:24.388 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:24.389 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:14:24.389 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:24.389 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:24.389 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:24.389 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:14:24.389 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:14:24.389 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:14:24.390 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:14:24.390 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:24.390 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:14:24.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:24.417 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 35 2026-02-18T08:14:24.417 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 38 2026-02-18T08:14:24.417 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:14:24.417 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=153, tag_tid=9, entry_tid=1], [object_number=152, tag_tid=9, entry_tid=0], [object_number=154, tag_tid=8, entry_tid=58722], [object_number=155, tag_tid=8, entry_tid=58719]]], state=connected] 2026-02-18T08:14:24.417 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=143, tag_tid=8, entry_tid=44067], [object_number=142, tag_tid=8, entry_tid=44066], [object_number=141, tag_tid=8, entry_tid=44065], [object_number=140, tag_tid=8, entry_tid=44064]]], state=connected] 2026-02-18T08:14:24.419 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:24.422 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=143, tag_tid=8, entry_tid=44067' 2026-02-18T08:14:24.422 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=153, tag_tid=9, entry_tid=1' -a 'object_number=153, tag_tid=9, entry_tid=1' = 'object_number=143, tag_tid=8, entry_tid=44067' 2026-02-18T08:14:24.422 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=143, tag_tid=8, entry_tid=44067' '!=' 'object_number=143, tag_tid=8, entry_tid=43803' 2026-02-18T08:14:24.422 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:14:24.422 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=143, tag_tid=8, entry_tid=44067' = 'object_number=143, tag_tid=8, entry_tid=43803' 2026-02-18T08:14:24.422 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=143, tag_tid=8, entry_tid=44067' 2026-02-18T08:14:24.423 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:24.423 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:14:24.423 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:24.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=9 2026-02-18T08:14:24.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=143, tag_tid=8, entry_tid=44067' 2026-02-18T08:14:24.425 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:14:24.425 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:24.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=8 2026-02-18T08:14:24.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:24.427 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:14:24.427 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:24.428 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:14:24.429 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=143, tag_tid=8, entry_tid=44067' 2026-02-18T08:14:24.429 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:14:24.429 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:24.431 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=44067 2026-02-18T08:14:24.431 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 9 = 8 -a 1 -le 44067 2026-02-18T08:14:24.431 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:14:24.431 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:14:24.431 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:14:24.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:14:24.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:14:24.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:14:24.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:14:24.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:14:24.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:14:24.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:14:24.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:14:24.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:14:24.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:14:24.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:14:24.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:14:24.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:14:24.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:14:24.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:14:24.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:14:24.637 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:14:24.637 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:14:24.637 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:14:24.637 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:14:24.637 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:14:24.638 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:14:24.638 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:14:24.638 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:14:24.638 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:14:24.638 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:14:24.638 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:14:24.638 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:14:24.638 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:14:24.638 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:14:24.638 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:14:24.638 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:14:24.638 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:14:24.639 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:14:24.639 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:14:24.639 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:14:24.639 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:14:24.639 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:14:24.639 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:14:24.639 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:14:24.639 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:14:24.639 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:14:24.639 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:14:24.639 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:14:24.639 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:14:24.640 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:14:24.640 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:14:24.640 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:14:24.640 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:14:24.648 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:24.648 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:24.648 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:24.649 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:14:24.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:14:24.717 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:14:24.717 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:24.717 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:24.717 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:24.717 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:14:24.717 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:24.718 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:24.718 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:24.718 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:14:24.718 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:14:24.718 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:14:24.718 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:14:24.719 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:24.719 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:14:24.719 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:24.746 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 35 2026-02-18T08:14:24.746 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 38 2026-02-18T08:14:24.746 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:14:24.746 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=153, tag_tid=9, entry_tid=1], [object_number=152, tag_tid=9, entry_tid=0], [object_number=154, tag_tid=8, entry_tid=58722], [object_number=155, tag_tid=8, entry_tid=58719]]], state=connected] 2026-02-18T08:14:24.747 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=141, tag_tid=8, entry_tid=44325], [object_number=140, tag_tid=8, entry_tid=44324], [object_number=143, tag_tid=8, entry_tid=44323], [object_number=142, tag_tid=8, entry_tid=44322]]], state=connected] 2026-02-18T08:14:24.749 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:24.750 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:24.751 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:14:24.752 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:24.752 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:24.752 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:24.752 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:14:24.752 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:24.752 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:24.752 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:24.752 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:14:24.752 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:14:24.753 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:14:24.753 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:14:24.753 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:24.754 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:14:24.754 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:24.781 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 35 2026-02-18T08:14:24.781 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 38 2026-02-18T08:14:24.781 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:14:24.781 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=153, tag_tid=9, entry_tid=1], [object_number=152, tag_tid=9, entry_tid=0], [object_number=154, tag_tid=8, entry_tid=58722], [object_number=155, tag_tid=8, entry_tid=58719]]], state=connected] 2026-02-18T08:14:24.781 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=141, tag_tid=8, entry_tid=44325], [object_number=140, tag_tid=8, entry_tid=44324], [object_number=143, tag_tid=8, entry_tid=44323], [object_number=142, tag_tid=8, entry_tid=44322]]], state=connected] 2026-02-18T08:14:24.783 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:24.785 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=141, tag_tid=8, entry_tid=44325' 2026-02-18T08:14:24.785 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=153, tag_tid=9, entry_tid=1' -a 'object_number=153, tag_tid=9, entry_tid=1' = 'object_number=141, tag_tid=8, entry_tid=44325' 2026-02-18T08:14:24.785 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=141, tag_tid=8, entry_tid=44325' '!=' 'object_number=143, tag_tid=8, entry_tid=44067' 2026-02-18T08:14:24.785 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:14:24.785 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=141, tag_tid=8, entry_tid=44325' = 'object_number=143, tag_tid=8, entry_tid=44067' 2026-02-18T08:14:24.785 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=141, tag_tid=8, entry_tid=44325' 2026-02-18T08:14:24.786 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:24.786 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:14:24.786 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:24.787 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=9 2026-02-18T08:14:24.788 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=141, tag_tid=8, entry_tid=44325' 2026-02-18T08:14:24.788 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:14:24.788 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:24.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=8 2026-02-18T08:14:24.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:24.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:14:24.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:24.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:14:24.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=141, tag_tid=8, entry_tid=44325' 2026-02-18T08:14:24.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:14:24.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:24.794 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=44325 2026-02-18T08:14:24.794 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 9 = 8 -a 1 -le 44325 2026-02-18T08:14:24.794 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:14:24.794 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:14:24.794 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:14:24.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:14:24.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:14:24.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:14:24.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:14:24.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:14:24.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:14:24.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:14:24.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:14:24.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:14:24.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:14:24.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:14:24.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:14:24.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:14:24.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:14:24.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:14:24.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:14:24.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:14:25.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:14:25.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:14:25.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:14:25.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:14:25.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:14:25.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:14:25.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:14:25.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:14:25.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:14:25.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:14:25.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:14:25.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:14:25.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:14:25.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:14:25.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:14:25.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:14:25.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:14:25.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:14:25.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:14:25.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:14:25.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:14:25.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:14:25.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:14:25.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:14:25.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:14:25.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:14:25.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:14:25.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:14:25.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:14:25.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:14:25.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:14:25.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:14:25.012 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:25.012 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:25.012 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:25.012 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:14:25.062 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.0 with skip (0.31245137580310955) > weight (0.25) 2026-02-18T08:14:25.062 INFO:tasks.rbd_mirror_thrash:kill cluster1.client.mirror.1 2026-02-18T08:14:25.062 INFO:tasks.rbd_mirror.cluster1.client.mirror.1:Sent signal 15 2026-02-18T08:14:25.062 INFO:tasks.rbd_mirror_thrash:waiting for 5.836876622406329 secs before reviving daemons 2026-02-18T08:14:25.063 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.trial163.stderr:2026-02-18T08:14:25.059+0000 7f3aad038640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.1 (PID: 14903) UID: 1000 2026-02-18T08:14:25.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:14:25.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:14:25.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:25.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:25.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:25.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:14:25.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:25.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:25.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:25.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:14:25.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:14:25.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:14:25.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:14:25.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:25.074 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:14:25.074 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:25.101 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 35 2026-02-18T08:14:25.101 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 38 2026-02-18T08:14:25.101 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:14:25.101 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=153, tag_tid=9, entry_tid=1], [object_number=152, tag_tid=9, entry_tid=0], [object_number=154, tag_tid=8, entry_tid=58722], [object_number=155, tag_tid=8, entry_tid=58719]]], state=connected] 2026-02-18T08:14:25.102 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=140, tag_tid=8, entry_tid=44568], [object_number=143, tag_tid=8, entry_tid=44567], [object_number=142, tag_tid=8, entry_tid=44566], [object_number=141, tag_tid=8, entry_tid=44565]]], state=connected] 2026-02-18T08:14:25.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:25.105 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:25.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:14:25.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:25.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:25.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:25.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:14:25.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:25.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:25.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:25.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:14:25.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:14:25.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:14:25.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:14:25.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:25.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:14:25.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:25.137 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 35 2026-02-18T08:14:25.137 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 38 2026-02-18T08:14:25.137 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:14:25.137 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=153, tag_tid=9, entry_tid=1], [object_number=152, tag_tid=9, entry_tid=0], [object_number=154, tag_tid=8, entry_tid=58722], [object_number=155, tag_tid=8, entry_tid=58719]]], state=connected] 2026-02-18T08:14:25.137 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=140, tag_tid=8, entry_tid=44568], [object_number=143, tag_tid=8, entry_tid=44567], [object_number=142, tag_tid=8, entry_tid=44566], [object_number=141, tag_tid=8, entry_tid=44565]]], state=connected] 2026-02-18T08:14:25.139 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:25.141 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=140, tag_tid=8, entry_tid=44568' 2026-02-18T08:14:25.141 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=153, tag_tid=9, entry_tid=1' -a 'object_number=153, tag_tid=9, entry_tid=1' = 'object_number=140, tag_tid=8, entry_tid=44568' 2026-02-18T08:14:25.141 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=140, tag_tid=8, entry_tid=44568' '!=' 'object_number=141, tag_tid=8, entry_tid=44325' 2026-02-18T08:14:25.141 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:14:25.141 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=140, tag_tid=8, entry_tid=44568' = 'object_number=141, tag_tid=8, entry_tid=44325' 2026-02-18T08:14:25.141 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=140, tag_tid=8, entry_tid=44568' 2026-02-18T08:14:25.142 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:25.142 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:14:25.142 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:25.143 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=9 2026-02-18T08:14:25.144 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=140, tag_tid=8, entry_tid=44568' 2026-02-18T08:14:25.144 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:14:25.144 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:25.145 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=8 2026-02-18T08:14:25.146 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:25.146 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:14:25.146 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:25.147 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:14:25.148 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=140, tag_tid=8, entry_tid=44568' 2026-02-18T08:14:25.148 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:14:25.148 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:25.150 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=44568 2026-02-18T08:14:25.150 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 9 = 8 -a 1 -le 44568 2026-02-18T08:14:25.150 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:14:25.150 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:14:25.150 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:14:25.352 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:14:25.352 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:14:25.352 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:14:25.353 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:14:25.353 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:14:25.353 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:14:25.353 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:14:25.353 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:14:25.353 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:14:25.353 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:14:25.353 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:14:25.353 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:14:25.353 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:14:25.354 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:14:25.354 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:14:25.354 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:14:25.354 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:14:25.356 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:14:25.357 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:14:25.357 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:14:25.357 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:14:25.357 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:14:25.357 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:14:25.357 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:14:25.357 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:14:25.357 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:14:25.357 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:14:25.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:14:25.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:14:25.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:14:25.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:14:25.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:14:25.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:14:25.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:14:25.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:14:25.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:14:25.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:14:25.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:14:25.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:14:25.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:14:25.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:14:25.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:14:25.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:14:25.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:14:25.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:14:25.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:14:25.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:14:25.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:14:25.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:14:25.368 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:25.368 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:25.368 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:25.368 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:14:25.428 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:14:25.428 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:14:25.428 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:25.428 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:25.428 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:25.428 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:14:25.429 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:25.429 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:25.429 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:25.429 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:14:25.429 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:14:25.429 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:14:25.429 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:14:25.430 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:25.430 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:14:25.430 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:25.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 35 2026-02-18T08:14:25.458 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 38 2026-02-18T08:14:25.458 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:14:25.458 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=153, tag_tid=9, entry_tid=1], [object_number=152, tag_tid=9, entry_tid=0], [object_number=154, tag_tid=8, entry_tid=58722], [object_number=155, tag_tid=8, entry_tid=58719]]], state=connected] 2026-02-18T08:14:25.458 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=141, tag_tid=8, entry_tid=44789], [object_number=140, tag_tid=8, entry_tid=44788], [object_number=143, tag_tid=8, entry_tid=44787], [object_number=142, tag_tid=8, entry_tid=44786]]], state=connected] 2026-02-18T08:14:25.460 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:25.462 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:25.463 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:14:25.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:25.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:25.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:25.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:14:25.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:25.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:25.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:25.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:14:25.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:14:25.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:14:25.465 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:14:25.465 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:25.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:14:25.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:25.492 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 35 2026-02-18T08:14:25.492 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 38 2026-02-18T08:14:25.492 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:14:25.492 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=153, tag_tid=9, entry_tid=1], [object_number=152, tag_tid=9, entry_tid=0], [object_number=154, tag_tid=8, entry_tid=58722], [object_number=155, tag_tid=8, entry_tid=58719]]], state=connected] 2026-02-18T08:14:25.493 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=141, tag_tid=8, entry_tid=44789], [object_number=140, tag_tid=8, entry_tid=44788], [object_number=143, tag_tid=8, entry_tid=44787], [object_number=142, tag_tid=8, entry_tid=44786]]], state=connected] 2026-02-18T08:14:25.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:25.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=141, tag_tid=8, entry_tid=44789' 2026-02-18T08:14:25.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=153, tag_tid=9, entry_tid=1' -a 'object_number=153, tag_tid=9, entry_tid=1' = 'object_number=141, tag_tid=8, entry_tid=44789' 2026-02-18T08:14:25.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=141, tag_tid=8, entry_tid=44789' '!=' 'object_number=140, tag_tid=8, entry_tid=44568' 2026-02-18T08:14:25.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:14:25.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=141, tag_tid=8, entry_tid=44789' = 'object_number=140, tag_tid=8, entry_tid=44568' 2026-02-18T08:14:25.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=141, tag_tid=8, entry_tid=44789' 2026-02-18T08:14:25.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:25.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:14:25.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:25.500 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=9 2026-02-18T08:14:25.500 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=141, tag_tid=8, entry_tid=44789' 2026-02-18T08:14:25.500 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:14:25.500 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:25.501 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=8 2026-02-18T08:14:25.502 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:25.502 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:14:25.502 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:25.503 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:14:25.504 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=141, tag_tid=8, entry_tid=44789' 2026-02-18T08:14:25.504 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:14:25.504 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:25.505 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=44789 2026-02-18T08:14:25.505 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 9 = 8 -a 1 -le 44789 2026-02-18T08:14:25.505 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:14:25.506 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:14:25.506 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:14:25.707 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:14:25.708 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:14:25.708 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:14:25.708 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:14:25.708 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:14:25.708 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:14:25.708 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:14:25.708 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:14:25.708 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:14:25.708 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:14:25.708 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:14:25.709 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:14:25.709 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:14:25.709 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:14:25.709 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:14:25.709 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:14:25.709 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:14:25.712 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:14:25.712 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:14:25.712 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:14:25.712 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:14:25.712 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:14:25.712 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:14:25.712 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:14:25.713 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:14:25.713 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:14:25.713 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:14:25.713 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:14:25.713 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:14:25.713 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:14:25.713 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:14:25.713 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:14:25.713 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:14:25.713 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:14:25.713 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:14:25.714 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:14:25.714 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:14:25.714 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:14:25.714 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:14:25.714 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:14:25.714 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:14:25.714 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:14:25.714 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:14:25.714 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:14:25.714 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:14:25.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:14:25.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:14:25.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:14:25.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:14:25.722 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:25.722 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:25.722 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:25.722 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:14:25.782 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:14:25.782 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:14:25.782 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:25.783 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:25.783 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:25.783 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:14:25.783 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:25.783 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:25.783 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:25.783 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:14:25.783 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:14:25.784 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:14:25.784 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:14:25.784 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:25.785 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:14:25.785 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:25.821 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 35 2026-02-18T08:14:25.821 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 38 2026-02-18T08:14:25.821 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:14:25.821 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=153, tag_tid=9, entry_tid=1], [object_number=152, tag_tid=9, entry_tid=0], [object_number=154, tag_tid=8, entry_tid=58722], [object_number=155, tag_tid=8, entry_tid=58719]]], state=connected] 2026-02-18T08:14:25.821 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=142, tag_tid=8, entry_tid=45010], [object_number=141, tag_tid=8, entry_tid=45009], [object_number=140, tag_tid=8, entry_tid=45008], [object_number=143, tag_tid=8, entry_tid=45007]]], state=connected] 2026-02-18T08:14:25.825 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:25.826 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:25.827 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:14:25.828 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:25.828 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:25.828 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:25.828 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:14:25.828 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:25.828 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:25.828 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:25.828 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:14:25.828 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:14:25.828 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:14:25.829 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:14:25.829 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:25.830 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:14:25.830 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:25.865 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 35 2026-02-18T08:14:25.865 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 38 2026-02-18T08:14:25.865 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:14:25.865 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=153, tag_tid=9, entry_tid=1], [object_number=152, tag_tid=9, entry_tid=0], [object_number=154, tag_tid=8, entry_tid=58722], [object_number=155, tag_tid=8, entry_tid=58719]]], state=connected] 2026-02-18T08:14:25.865 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=142, tag_tid=8, entry_tid=45010], [object_number=141, tag_tid=8, entry_tid=45009], [object_number=140, tag_tid=8, entry_tid=45008], [object_number=143, tag_tid=8, entry_tid=45007]]], state=connected] 2026-02-18T08:14:25.868 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:25.870 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=142, tag_tid=8, entry_tid=45010' 2026-02-18T08:14:25.870 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=153, tag_tid=9, entry_tid=1' -a 'object_number=153, tag_tid=9, entry_tid=1' = 'object_number=142, tag_tid=8, entry_tid=45010' 2026-02-18T08:14:25.870 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=142, tag_tid=8, entry_tid=45010' '!=' 'object_number=141, tag_tid=8, entry_tid=44789' 2026-02-18T08:14:25.870 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:14:25.870 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=142, tag_tid=8, entry_tid=45010' = 'object_number=141, tag_tid=8, entry_tid=44789' 2026-02-18T08:14:25.870 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=142, tag_tid=8, entry_tid=45010' 2026-02-18T08:14:25.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:25.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:14:25.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:25.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=9 2026-02-18T08:14:25.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=142, tag_tid=8, entry_tid=45010' 2026-02-18T08:14:25.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:14:25.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:25.874 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=8 2026-02-18T08:14:25.875 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:25.875 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:14:25.876 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:25.877 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:14:25.877 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=142, tag_tid=8, entry_tid=45010' 2026-02-18T08:14:25.878 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:14:25.878 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:25.879 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=45010 2026-02-18T08:14:25.879 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 9 = 8 -a 1 -le 45010 2026-02-18T08:14:25.879 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:14:25.879 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:14:25.880 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:14:26.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:14:26.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:14:26.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:14:26.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:14:26.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:14:26.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:14:26.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:14:26.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:14:26.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:14:26.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:14:26.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:14:26.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:14:26.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:14:26.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:14:26.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:14:26.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:14:26.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:14:26.086 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:14:26.086 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:14:26.086 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:14:26.086 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:14:26.086 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:14:26.086 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:14:26.086 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:14:26.086 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:14:26.086 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:14:26.086 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:14:26.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:14:26.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:14:26.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:14:26.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:14:26.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:14:26.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:14:26.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:14:26.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:14:26.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:14:26.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:14:26.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:14:26.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:14:26.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:14:26.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:14:26.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:14:26.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:14:26.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:14:26.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:14:26.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:14:26.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:14:26.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:14:26.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:14:26.096 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:26.096 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:26.096 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:26.096 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:14:26.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:14:26.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:14:26.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:26.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:26.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:26.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:14:26.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:26.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:26.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:26.155 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:14:26.155 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:14:26.155 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:14:26.155 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:14:26.156 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:26.156 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:14:26.156 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:26.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 35 2026-02-18T08:14:26.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 38 2026-02-18T08:14:26.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:14:26.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=153, tag_tid=9, entry_tid=1], [object_number=152, tag_tid=9, entry_tid=0], [object_number=154, tag_tid=8, entry_tid=58722], [object_number=155, tag_tid=8, entry_tid=58719]]], state=connected] 2026-02-18T08:14:26.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=141, tag_tid=8, entry_tid=45205], [object_number=140, tag_tid=8, entry_tid=45204], [object_number=143, tag_tid=8, entry_tid=45203], [object_number=142, tag_tid=8, entry_tid=45202]]], state=connected] 2026-02-18T08:14:26.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:26.188 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:26.189 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:14:26.189 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:26.189 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:26.189 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:26.190 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:14:26.190 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:26.190 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:26.190 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:26.190 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:14:26.190 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:14:26.190 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:14:26.190 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:14:26.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:26.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:14:26.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:26.217 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 35 2026-02-18T08:14:26.217 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 38 2026-02-18T08:14:26.217 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:14:26.217 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=153, tag_tid=9, entry_tid=1], [object_number=152, tag_tid=9, entry_tid=0], [object_number=154, tag_tid=8, entry_tid=58722], [object_number=155, tag_tid=8, entry_tid=58719]]], state=connected] 2026-02-18T08:14:26.217 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=141, tag_tid=8, entry_tid=45205], [object_number=140, tag_tid=8, entry_tid=45204], [object_number=143, tag_tid=8, entry_tid=45203], [object_number=142, tag_tid=8, entry_tid=45202]]], state=connected] 2026-02-18T08:14:26.219 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:26.222 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=141, tag_tid=8, entry_tid=45205' 2026-02-18T08:14:26.222 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=153, tag_tid=9, entry_tid=1' -a 'object_number=153, tag_tid=9, entry_tid=1' = 'object_number=141, tag_tid=8, entry_tid=45205' 2026-02-18T08:14:26.222 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=141, tag_tid=8, entry_tid=45205' '!=' 'object_number=142, tag_tid=8, entry_tid=45010' 2026-02-18T08:14:26.222 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:14:26.222 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=141, tag_tid=8, entry_tid=45205' = 'object_number=142, tag_tid=8, entry_tid=45010' 2026-02-18T08:14:26.222 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=141, tag_tid=8, entry_tid=45205' 2026-02-18T08:14:26.222 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:26.222 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:14:26.222 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:26.223 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=9 2026-02-18T08:14:26.224 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=141, tag_tid=8, entry_tid=45205' 2026-02-18T08:14:26.224 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:14:26.224 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:26.225 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=8 2026-02-18T08:14:26.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:26.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:14:26.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:26.227 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:14:26.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=141, tag_tid=8, entry_tid=45205' 2026-02-18T08:14:26.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:14:26.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:26.230 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=45205 2026-02-18T08:14:26.230 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 9 = 8 -a 1 -le 45205 2026-02-18T08:14:26.230 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:14:26.230 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:14:26.230 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:14:26.431 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:14:26.432 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:14:26.432 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:14:26.432 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:14:26.432 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:14:26.432 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:14:26.432 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:14:26.432 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:14:26.433 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:14:26.433 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:14:26.433 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:14:26.433 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:14:26.433 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:14:26.433 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:14:26.433 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:14:26.433 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:14:26.436 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:14:26.436 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:14:26.436 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:14:26.436 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:14:26.436 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:14:26.436 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:14:26.436 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:14:26.437 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:14:26.437 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:14:26.437 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:14:26.437 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:14:26.437 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:14:26.437 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:14:26.437 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:14:26.437 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:14:26.437 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:14:26.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:14:26.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:14:26.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:14:26.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:14:26.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:14:26.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:14:26.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:14:26.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:14:26.439 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:14:26.439 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:14:26.439 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:14:26.439 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:14:26.439 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:14:26.439 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:14:26.439 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:14:26.439 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:14:26.440 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:14:26.446 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:26.446 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:26.446 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:26.446 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:14:26.539 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:14:26.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:14:26.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:26.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:26.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:26.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:14:26.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:26.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:26.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:26.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:14:26.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:14:26.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:14:26.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:14:26.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:26.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:14:26.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:26.569 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 35 2026-02-18T08:14:26.569 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 38 2026-02-18T08:14:26.569 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:14:26.569 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=153, tag_tid=9, entry_tid=1], [object_number=152, tag_tid=9, entry_tid=0], [object_number=154, tag_tid=8, entry_tid=58722], [object_number=155, tag_tid=8, entry_tid=58719]]], state=connected] 2026-02-18T08:14:26.570 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=141, tag_tid=8, entry_tid=45205], [object_number=140, tag_tid=8, entry_tid=45204], [object_number=143, tag_tid=8, entry_tid=45203], [object_number=142, tag_tid=8, entry_tid=45202]]], state=connected] 2026-02-18T08:14:26.572 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:26.573 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:26.574 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:14:26.574 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:26.574 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:26.575 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:26.575 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:14:26.575 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:26.575 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:26.575 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:26.575 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:14:26.575 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:14:26.576 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:14:26.576 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:14:26.577 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:26.577 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:14:26.577 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:26.604 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 35 2026-02-18T08:14:26.604 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 38 2026-02-18T08:14:26.604 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:14:26.604 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=153, tag_tid=9, entry_tid=1], [object_number=152, tag_tid=9, entry_tid=0], [object_number=154, tag_tid=8, entry_tid=58722], [object_number=155, tag_tid=8, entry_tid=58719]]], state=connected] 2026-02-18T08:14:26.604 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=141, tag_tid=8, entry_tid=45205], [object_number=140, tag_tid=8, entry_tid=45204], [object_number=143, tag_tid=8, entry_tid=45203], [object_number=142, tag_tid=8, entry_tid=45202]]], state=connected] 2026-02-18T08:14:26.606 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:26.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=141, tag_tid=8, entry_tid=45205' 2026-02-18T08:14:26.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=153, tag_tid=9, entry_tid=1' -a 'object_number=153, tag_tid=9, entry_tid=1' = 'object_number=141, tag_tid=8, entry_tid=45205' 2026-02-18T08:14:26.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=141, tag_tid=8, entry_tid=45205' '!=' 'object_number=141, tag_tid=8, entry_tid=45205' 2026-02-18T08:14:26.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:14:26.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.4 2026-02-18T08:14:27.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:14:27.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:14:27.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:14:27.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:14:27.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:14:27.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:14:27.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:14:27.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:14:27.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:14:27.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:14:27.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:14:27.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:14:27.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:14:27.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:14:27.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:14:27.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:14:27.010 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:14:27.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:14:27.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:14:27.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:14:27.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:14:27.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:14:27.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:14:27.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:14:27.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:14:27.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:14:27.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:14:27.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:14:27.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:14:27.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:14:27.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:14:27.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:14:27.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:14:27.012 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:14:27.012 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:14:27.012 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:14:27.012 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:14:27.012 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:14:27.012 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:14:27.012 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:14:27.012 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:14:27.012 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:14:27.012 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:14:27.012 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:14:27.012 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:14:27.012 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:14:27.012 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:14:27.012 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:14:27.012 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:14:27.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:27.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:27.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:27.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:14:27.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:14:27.086 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:14:27.086 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:27.086 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:27.086 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:27.086 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:14:27.086 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:27.086 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:27.086 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:27.086 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:14:27.086 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:14:27.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:14:27.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:14:27.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:27.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:14:27.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:27.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 35 2026-02-18T08:14:27.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 38 2026-02-18T08:14:27.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:14:27.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=153, tag_tid=9, entry_tid=1], [object_number=152, tag_tid=9, entry_tid=0], [object_number=154, tag_tid=8, entry_tid=58722], [object_number=155, tag_tid=8, entry_tid=58719]]], state=connected] 2026-02-18T08:14:27.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=141, tag_tid=8, entry_tid=45565], [object_number=140, tag_tid=8, entry_tid=45564], [object_number=143, tag_tid=8, entry_tid=45563], [object_number=142, tag_tid=8, entry_tid=45562]]], state=connected] 2026-02-18T08:14:27.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:27.129 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:27.130 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:14:27.130 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:27.131 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:27.131 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:27.131 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:14:27.131 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:27.131 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:27.131 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:27.131 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:14:27.131 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:14:27.132 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:14:27.132 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:14:27.132 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:27.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:14:27.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:27.169 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 35 2026-02-18T08:14:27.169 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 38 2026-02-18T08:14:27.169 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:14:27.169 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=153, tag_tid=9, entry_tid=1], [object_number=152, tag_tid=9, entry_tid=0], [object_number=154, tag_tid=8, entry_tid=58722], [object_number=155, tag_tid=8, entry_tid=58719]]], state=connected] 2026-02-18T08:14:27.169 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=141, tag_tid=8, entry_tid=45565], [object_number=140, tag_tid=8, entry_tid=45564], [object_number=143, tag_tid=8, entry_tid=45563], [object_number=142, tag_tid=8, entry_tid=45562]]], state=connected] 2026-02-18T08:14:27.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:27.174 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=141, tag_tid=8, entry_tid=45565' 2026-02-18T08:14:27.174 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=153, tag_tid=9, entry_tid=1' -a 'object_number=153, tag_tid=9, entry_tid=1' = 'object_number=141, tag_tid=8, entry_tid=45565' 2026-02-18T08:14:27.174 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=141, tag_tid=8, entry_tid=45565' '!=' 'object_number=141, tag_tid=8, entry_tid=45205' 2026-02-18T08:14:27.174 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:14:27.174 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=141, tag_tid=8, entry_tid=45565' = 'object_number=141, tag_tid=8, entry_tid=45205' 2026-02-18T08:14:27.174 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=141, tag_tid=8, entry_tid=45565' 2026-02-18T08:14:27.175 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:27.175 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:14:27.175 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:27.176 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=9 2026-02-18T08:14:27.177 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=141, tag_tid=8, entry_tid=45565' 2026-02-18T08:14:27.177 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:14:27.177 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:27.178 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=8 2026-02-18T08:14:27.179 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:27.179 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:14:27.179 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:27.180 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:14:27.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=141, tag_tid=8, entry_tid=45565' 2026-02-18T08:14:27.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:14:27.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:27.183 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=45565 2026-02-18T08:14:27.183 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 9 = 8 -a 1 -le 45565 2026-02-18T08:14:27.183 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:14:27.183 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:14:27.183 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:14:27.385 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:14:27.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:14:27.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:14:27.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:14:27.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:14:27.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:14:27.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:14:27.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:14:27.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:14:27.387 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:14:27.387 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:14:27.387 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:14:27.387 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:14:27.387 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:14:27.387 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:14:27.387 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:14:27.387 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:14:27.390 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:14:27.390 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:14:27.390 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:14:27.390 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:14:27.390 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:14:27.390 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:14:27.390 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:14:27.390 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:14:27.390 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:14:27.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:14:27.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:14:27.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:14:27.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:14:27.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:14:27.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:14:27.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:14:27.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:14:27.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:14:27.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:14:27.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:14:27.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:14:27.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:14:27.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:14:27.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:14:27.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:14:27.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:14:27.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:14:27.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:14:27.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:14:27.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:14:27.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:14:27.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:14:27.399 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:27.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:27.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:27.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:14:27.460 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:14:27.462 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:14:27.462 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:27.462 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:27.462 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:27.462 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:14:27.462 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:27.462 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:27.462 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:27.462 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:14:27.462 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:14:27.463 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:14:27.463 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:14:27.463 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:27.463 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:14:27.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:27.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 35 2026-02-18T08:14:27.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 38 2026-02-18T08:14:27.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:14:27.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=153, tag_tid=9, entry_tid=1], [object_number=152, tag_tid=9, entry_tid=0], [object_number=154, tag_tid=8, entry_tid=58722], [object_number=155, tag_tid=8, entry_tid=58719]]], state=connected] 2026-02-18T08:14:27.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=143, tag_tid=8, entry_tid=45827], [object_number=142, tag_tid=8, entry_tid=45826], [object_number=141, tag_tid=8, entry_tid=45825], [object_number=140, tag_tid=8, entry_tid=45824]]], state=connected] 2026-02-18T08:14:27.503 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:27.506 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:27.506 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:14:27.506 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:27.506 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:27.506 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:27.507 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:14:27.507 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:27.507 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:27.507 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:27.507 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:14:27.507 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:14:27.507 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:14:27.507 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:14:27.508 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:27.508 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:14:27.508 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:27.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 35 2026-02-18T08:14:27.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 38 2026-02-18T08:14:27.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:14:27.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=153, tag_tid=9, entry_tid=1], [object_number=152, tag_tid=9, entry_tid=0], [object_number=154, tag_tid=8, entry_tid=58722], [object_number=155, tag_tid=8, entry_tid=58719]]], state=connected] 2026-02-18T08:14:27.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=143, tag_tid=8, entry_tid=45827], [object_number=142, tag_tid=8, entry_tid=45826], [object_number=141, tag_tid=8, entry_tid=45825], [object_number=140, tag_tid=8, entry_tid=45824]]], state=connected] 2026-02-18T08:14:27.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:27.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=143, tag_tid=8, entry_tid=45827' 2026-02-18T08:14:27.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=153, tag_tid=9, entry_tid=1' -a 'object_number=153, tag_tid=9, entry_tid=1' = 'object_number=143, tag_tid=8, entry_tid=45827' 2026-02-18T08:14:27.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=143, tag_tid=8, entry_tid=45827' '!=' 'object_number=141, tag_tid=8, entry_tid=45565' 2026-02-18T08:14:27.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:14:27.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=143, tag_tid=8, entry_tid=45827' = 'object_number=141, tag_tid=8, entry_tid=45565' 2026-02-18T08:14:27.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=143, tag_tid=8, entry_tid=45827' 2026-02-18T08:14:27.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:27.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:14:27.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:27.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=9 2026-02-18T08:14:27.551 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=143, tag_tid=8, entry_tid=45827' 2026-02-18T08:14:27.551 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:14:27.551 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:27.552 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=8 2026-02-18T08:14:27.553 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:27.553 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:14:27.553 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:27.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:14:27.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=143, tag_tid=8, entry_tid=45827' 2026-02-18T08:14:27.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:14:27.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:27.557 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=45827 2026-02-18T08:14:27.557 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 9 = 8 -a 1 -le 45827 2026-02-18T08:14:27.557 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:14:27.557 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:14:27.557 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:14:27.759 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:14:27.759 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:14:27.759 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:14:27.759 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:14:27.759 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:14:27.760 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:14:27.760 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:14:27.760 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:14:27.760 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:14:27.760 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:14:27.760 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:14:27.760 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:14:27.760 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:14:27.760 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:14:27.760 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:14:27.761 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:14:27.763 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:14:27.763 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:14:27.763 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:14:27.763 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:14:27.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:14:27.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:14:27.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:14:27.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:14:27.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:14:27.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:14:27.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:14:27.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:14:27.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:14:27.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:14:27.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:14:27.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:14:27.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:14:27.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:14:27.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:14:27.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:14:27.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:14:27.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:14:27.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:14:27.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:14:27.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:14:27.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:14:27.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:14:27.766 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:14:27.766 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:14:27.766 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:14:27.766 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:14:27.766 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:14:27.766 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:14:27.773 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:27.774 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:27.774 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:27.774 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:14:27.843 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:14:27.845 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:14:27.845 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:27.845 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:27.845 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:27.845 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:14:27.845 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:27.845 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:27.845 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:27.846 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:14:27.846 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:14:27.846 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:14:27.846 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:14:27.846 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:27.847 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:14:27.847 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:27.875 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 35 2026-02-18T08:14:27.875 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 38 2026-02-18T08:14:27.875 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:14:27.875 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=153, tag_tid=9, entry_tid=1], [object_number=152, tag_tid=9, entry_tid=0], [object_number=154, tag_tid=8, entry_tid=58722], [object_number=155, tag_tid=8, entry_tid=58719]]], state=connected] 2026-02-18T08:14:27.875 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=140, tag_tid=8, entry_tid=46096], [object_number=143, tag_tid=8, entry_tid=46095], [object_number=142, tag_tid=8, entry_tid=46094], [object_number=141, tag_tid=8, entry_tid=46093]]], state=connected] 2026-02-18T08:14:27.877 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:27.879 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:27.880 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:14:27.880 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:27.880 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:27.880 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:27.881 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:14:27.881 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:27.881 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:27.881 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:27.881 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:14:27.881 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:14:27.881 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:14:27.882 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:14:27.882 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:27.882 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:14:27.883 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:28.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 35 2026-02-18T08:14:28.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 38 2026-02-18T08:14:28.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:14:28.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=153, tag_tid=9, entry_tid=1], [object_number=152, tag_tid=9, entry_tid=0], [object_number=154, tag_tid=8, entry_tid=58722], [object_number=155, tag_tid=8, entry_tid=58719]]], state=connected] 2026-02-18T08:14:28.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=140, tag_tid=8, entry_tid=46096], [object_number=143, tag_tid=8, entry_tid=46095], [object_number=142, tag_tid=8, entry_tid=46094], [object_number=141, tag_tid=8, entry_tid=46093]]], state=connected] 2026-02-18T08:14:28.119 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:28.121 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=140, tag_tid=8, entry_tid=46096' 2026-02-18T08:14:28.121 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=153, tag_tid=9, entry_tid=1' -a 'object_number=153, tag_tid=9, entry_tid=1' = 'object_number=140, tag_tid=8, entry_tid=46096' 2026-02-18T08:14:28.121 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=140, tag_tid=8, entry_tid=46096' '!=' 'object_number=143, tag_tid=8, entry_tid=45827' 2026-02-18T08:14:28.121 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:14:28.121 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=140, tag_tid=8, entry_tid=46096' = 'object_number=143, tag_tid=8, entry_tid=45827' 2026-02-18T08:14:28.121 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=140, tag_tid=8, entry_tid=46096' 2026-02-18T08:14:28.122 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:28.122 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:14:28.122 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:28.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=9 2026-02-18T08:14:28.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=140, tag_tid=8, entry_tid=46096' 2026-02-18T08:14:28.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:14:28.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:28.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=8 2026-02-18T08:14:28.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:28.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:14:28.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:28.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:14:28.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=140, tag_tid=8, entry_tid=46096' 2026-02-18T08:14:28.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:14:28.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:28.130 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=46096 2026-02-18T08:14:28.130 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 9 = 8 -a 1 -le 46096 2026-02-18T08:14:28.130 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:14:28.130 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:14:28.130 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:14:28.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:14:28.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:14:28.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:14:28.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:14:28.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:14:28.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:14:28.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:14:28.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:14:28.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:14:28.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:14:28.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:14:28.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:14:28.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:14:28.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:14:28.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:14:28.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:14:28.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:14:28.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:14:28.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:14:28.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:14:28.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:14:28.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:14:28.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:14:28.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:14:28.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:14:28.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:14:28.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:14:28.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:14:28.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:14:28.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:14:28.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:14:28.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:14:28.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:14:28.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:14:28.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:14:28.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:14:28.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:14:28.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:14:28.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:14:28.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:14:28.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:14:28.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:14:28.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:14:28.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:14:28.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:14:28.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:14:28.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:14:28.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:14:28.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:14:28.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:28.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:28.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:28.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:14:28.409 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:14:28.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:14:28.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:28.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:28.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:28.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:14:28.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:28.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:28.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:28.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:14:28.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:14:28.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:14:28.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:14:28.412 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:28.412 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:14:28.412 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:28.448 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 35 2026-02-18T08:14:28.448 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 38 2026-02-18T08:14:28.448 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:14:28.448 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=153, tag_tid=9, entry_tid=1], [object_number=152, tag_tid=9, entry_tid=0], [object_number=154, tag_tid=8, entry_tid=58722], [object_number=155, tag_tid=8, entry_tid=58719]]], state=connected] 2026-02-18T08:14:28.448 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=142, tag_tid=8, entry_tid=46574], [object_number=141, tag_tid=8, entry_tid=46573], [object_number=140, tag_tid=8, entry_tid=46572], [object_number=143, tag_tid=8, entry_tid=46571]]], state=connected] 2026-02-18T08:14:28.451 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:28.453 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:28.454 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:14:28.454 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:28.454 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:28.454 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:28.454 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:14:28.454 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:28.454 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:28.454 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:28.454 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:14:28.454 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:14:28.455 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:14:28.455 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:14:28.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:28.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:14:28.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:28.489 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 35 2026-02-18T08:14:28.489 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 38 2026-02-18T08:14:28.489 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:14:28.490 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=153, tag_tid=9, entry_tid=1], [object_number=152, tag_tid=9, entry_tid=0], [object_number=154, tag_tid=8, entry_tid=58722], [object_number=155, tag_tid=8, entry_tid=58719]]], state=connected] 2026-02-18T08:14:28.490 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=142, tag_tid=8, entry_tid=46574], [object_number=141, tag_tid=8, entry_tid=46573], [object_number=140, tag_tid=8, entry_tid=46572], [object_number=143, tag_tid=8, entry_tid=46571]]], state=connected] 2026-02-18T08:14:28.493 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:28.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=142, tag_tid=8, entry_tid=46574' 2026-02-18T08:14:28.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=153, tag_tid=9, entry_tid=1' -a 'object_number=153, tag_tid=9, entry_tid=1' = 'object_number=142, tag_tid=8, entry_tid=46574' 2026-02-18T08:14:28.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=142, tag_tid=8, entry_tid=46574' '!=' 'object_number=140, tag_tid=8, entry_tid=46096' 2026-02-18T08:14:28.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:14:28.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=142, tag_tid=8, entry_tid=46574' = 'object_number=140, tag_tid=8, entry_tid=46096' 2026-02-18T08:14:28.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=142, tag_tid=8, entry_tid=46574' 2026-02-18T08:14:28.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:28.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:14:28.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:28.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=9 2026-02-18T08:14:28.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=142, tag_tid=8, entry_tid=46574' 2026-02-18T08:14:28.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:14:28.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:28.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=8 2026-02-18T08:14:28.500 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:28.500 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:14:28.500 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:28.501 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:14:28.502 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=142, tag_tid=8, entry_tid=46574' 2026-02-18T08:14:28.502 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:14:28.502 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:28.504 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=46574 2026-02-18T08:14:28.504 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 9 = 8 -a 1 -le 46574 2026-02-18T08:14:28.504 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:14:28.504 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:14:28.504 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:14:28.706 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:14:28.706 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:14:28.706 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:14:28.706 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:14:28.706 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:14:28.706 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:14:28.706 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:14:28.707 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:14:28.707 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:14:28.707 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:14:28.707 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:14:28.707 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:14:28.707 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:14:28.707 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:14:28.707 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:14:28.707 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:14:28.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:14:28.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:14:28.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:14:28.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:14:28.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:14:28.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:14:28.711 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:14:28.711 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:14:28.711 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:14:28.711 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:14:28.711 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:14:28.711 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:14:28.711 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:14:28.711 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:14:28.711 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:14:28.711 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:14:28.711 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:14:28.711 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:14:28.712 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:14:28.712 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:14:28.712 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:14:28.712 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:14:28.712 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:14:28.712 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:14:28.712 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:14:28.712 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:14:28.712 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:14:28.712 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:14:28.712 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:14:28.712 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:14:28.713 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:14:28.713 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:14:28.713 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:14:28.721 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:28.721 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:28.721 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:28.721 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:14:28.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:14:28.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:14:28.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:28.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:28.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:28.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:14:28.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:28.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:28.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:28.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:14:28.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:14:28.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:14:28.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:14:28.793 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:28.793 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:14:28.793 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:28.831 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 35 2026-02-18T08:14:28.831 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 38 2026-02-18T08:14:28.831 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:14:28.832 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=153, tag_tid=9, entry_tid=1], [object_number=152, tag_tid=9, entry_tid=0], [object_number=154, tag_tid=8, entry_tid=58722], [object_number=155, tag_tid=8, entry_tid=58719]]], state=connected] 2026-02-18T08:14:28.832 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=143, tag_tid=8, entry_tid=46839], [object_number=142, tag_tid=8, entry_tid=46838], [object_number=141, tag_tid=8, entry_tid=46837], [object_number=140, tag_tid=8, entry_tid=46836]]], state=connected] 2026-02-18T08:14:28.835 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:28.836 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:28.838 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:14:28.838 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:28.838 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:28.838 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:28.838 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:14:28.838 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:28.838 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:28.838 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:28.839 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:14:28.839 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:14:28.839 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:14:28.839 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:14:28.839 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:28.840 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:14:28.840 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:28.876 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 35 2026-02-18T08:14:28.876 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 38 2026-02-18T08:14:28.876 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:14:28.876 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=153, tag_tid=9, entry_tid=1], [object_number=152, tag_tid=9, entry_tid=0], [object_number=154, tag_tid=8, entry_tid=58722], [object_number=155, tag_tid=8, entry_tid=58719]]], state=connected] 2026-02-18T08:14:28.876 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=143, tag_tid=8, entry_tid=46839], [object_number=142, tag_tid=8, entry_tid=46838], [object_number=141, tag_tid=8, entry_tid=46837], [object_number=140, tag_tid=8, entry_tid=46836]]], state=connected] 2026-02-18T08:14:28.879 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:28.882 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=143, tag_tid=8, entry_tid=46839' 2026-02-18T08:14:28.882 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=153, tag_tid=9, entry_tid=1' -a 'object_number=153, tag_tid=9, entry_tid=1' = 'object_number=143, tag_tid=8, entry_tid=46839' 2026-02-18T08:14:28.882 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=143, tag_tid=8, entry_tid=46839' '!=' 'object_number=142, tag_tid=8, entry_tid=46574' 2026-02-18T08:14:28.882 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:14:28.882 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=143, tag_tid=8, entry_tid=46839' = 'object_number=142, tag_tid=8, entry_tid=46574' 2026-02-18T08:14:28.883 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=143, tag_tid=8, entry_tid=46839' 2026-02-18T08:14:28.883 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:28.883 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:14:28.883 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:28.884 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=9 2026-02-18T08:14:28.885 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=143, tag_tid=8, entry_tid=46839' 2026-02-18T08:14:28.885 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:14:28.885 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:28.885 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=8 2026-02-18T08:14:28.886 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:28.887 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:14:28.887 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:28.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:14:28.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=143, tag_tid=8, entry_tid=46839' 2026-02-18T08:14:28.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:14:28.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:28.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=46839 2026-02-18T08:14:28.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 9 = 8 -a 1 -le 46839 2026-02-18T08:14:28.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:14:28.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:14:28.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:14:29.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:14:29.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:14:29.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:14:29.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:14:29.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:14:29.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:14:29.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:14:29.092 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:14:29.092 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:14:29.092 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:14:29.092 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:14:29.092 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:14:29.092 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:14:29.092 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:14:29.092 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:14:29.092 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:14:29.092 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:14:29.094 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:14:29.094 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:14:29.094 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:14:29.095 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:14:29.095 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:14:29.095 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:14:29.095 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:14:29.095 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:14:29.095 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:14:29.095 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:14:29.095 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:14:29.095 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:14:29.095 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:14:29.095 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:14:29.095 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:14:29.095 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:14:29.095 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:14:29.096 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:14:29.096 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:14:29.096 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:14:29.096 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:14:29.096 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:14:29.096 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:14:29.096 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:14:29.096 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:14:29.096 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:14:29.096 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:14:29.096 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:14:29.096 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:14:29.096 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:14:29.096 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:14:29.096 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:14:29.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:29.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:29.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:29.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:14:29.169 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:14:29.170 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:14:29.171 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:29.171 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:29.171 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:29.171 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:14:29.171 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:29.171 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:29.171 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:29.171 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:14:29.171 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:14:29.171 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:14:29.171 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:14:29.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:29.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:14:29.173 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:29.200 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 35 2026-02-18T08:14:29.200 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 38 2026-02-18T08:14:29.200 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:14:29.200 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=153, tag_tid=9, entry_tid=1], [object_number=152, tag_tid=9, entry_tid=0], [object_number=154, tag_tid=8, entry_tid=58722], [object_number=155, tag_tid=8, entry_tid=58719]]], state=connected] 2026-02-18T08:14:29.200 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=142, tag_tid=8, entry_tid=47114], [object_number=141, tag_tid=8, entry_tid=47113], [object_number=140, tag_tid=8, entry_tid=47112], [object_number=143, tag_tid=8, entry_tid=47111]]], state=connected] 2026-02-18T08:14:29.203 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:29.204 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:29.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:14:29.206 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:29.206 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:29.206 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:29.206 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:14:29.206 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:29.206 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:29.206 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:29.206 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:14:29.206 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:14:29.206 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:14:29.207 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:14:29.207 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:29.208 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:14:29.208 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:29.245 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 35 2026-02-18T08:14:29.245 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 38 2026-02-18T08:14:29.245 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:14:29.245 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=153, tag_tid=9, entry_tid=1], [object_number=152, tag_tid=9, entry_tid=0], [object_number=154, tag_tid=8, entry_tid=58722], [object_number=155, tag_tid=8, entry_tid=58719]]], state=connected] 2026-02-18T08:14:29.245 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=142, tag_tid=8, entry_tid=47114], [object_number=141, tag_tid=8, entry_tid=47113], [object_number=140, tag_tid=8, entry_tid=47112], [object_number=143, tag_tid=8, entry_tid=47111]]], state=connected] 2026-02-18T08:14:29.249 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:29.250 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=142, tag_tid=8, entry_tid=47114' 2026-02-18T08:14:29.250 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=153, tag_tid=9, entry_tid=1' -a 'object_number=153, tag_tid=9, entry_tid=1' = 'object_number=142, tag_tid=8, entry_tid=47114' 2026-02-18T08:14:29.250 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=142, tag_tid=8, entry_tid=47114' '!=' 'object_number=143, tag_tid=8, entry_tid=46839' 2026-02-18T08:14:29.250 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:14:29.250 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=142, tag_tid=8, entry_tid=47114' = 'object_number=143, tag_tid=8, entry_tid=46839' 2026-02-18T08:14:29.251 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=142, tag_tid=8, entry_tid=47114' 2026-02-18T08:14:29.251 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:29.251 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:14:29.251 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:29.252 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=9 2026-02-18T08:14:29.253 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=142, tag_tid=8, entry_tid=47114' 2026-02-18T08:14:29.253 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:14:29.253 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:29.255 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=8 2026-02-18T08:14:29.255 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:29.255 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:14:29.255 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:29.257 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:14:29.257 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=142, tag_tid=8, entry_tid=47114' 2026-02-18T08:14:29.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:14:29.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:29.260 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=47114 2026-02-18T08:14:29.260 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 9 = 8 -a 1 -le 47114 2026-02-18T08:14:29.260 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:14:29.260 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:14:29.260 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:14:29.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:14:29.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:14:29.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:14:29.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:14:29.462 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:14:29.462 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:14:29.462 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:14:29.462 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:14:29.462 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:14:29.462 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:14:29.462 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:14:29.462 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:14:29.462 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:14:29.462 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:14:29.462 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:14:29.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:14:29.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:14:29.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:14:29.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:14:29.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:14:29.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:14:29.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:14:29.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:14:29.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:14:29.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:14:29.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:14:29.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:14:29.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:14:29.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:14:29.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:14:29.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:14:29.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:14:29.465 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:14:29.465 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:14:29.465 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:14:29.465 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:14:29.465 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:14:29.465 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:14:29.465 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:14:29.465 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:14:29.465 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:14:29.465 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:14:29.465 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:14:29.465 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:14:29.465 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:14:29.465 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:14:29.465 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:14:29.465 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:14:29.465 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:14:29.475 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:29.476 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:29.476 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:29.476 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:14:29.536 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:14:29.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:14:29.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:29.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:29.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:29.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:14:29.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:29.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:29.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:29.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:14:29.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:14:29.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:14:29.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:14:29.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:29.539 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:14:29.539 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:29.769 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 35 2026-02-18T08:14:29.769 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 38 2026-02-18T08:14:29.769 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:14:29.770 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=153, tag_tid=9, entry_tid=1], [object_number=152, tag_tid=9, entry_tid=0], [object_number=154, tag_tid=8, entry_tid=58722], [object_number=155, tag_tid=8, entry_tid=58719]]], state=connected] 2026-02-18T08:14:29.770 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=143, tag_tid=8, entry_tid=47363], [object_number=142, tag_tid=8, entry_tid=47362], [object_number=141, tag_tid=8, entry_tid=47361], [object_number=140, tag_tid=8, entry_tid=47360]]], state=connected] 2026-02-18T08:14:29.773 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:29.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:29.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:14:29.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:29.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:29.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:29.776 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:14:29.776 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:29.776 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:29.776 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:29.776 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:14:29.776 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:14:29.776 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:14:29.776 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:14:29.778 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:29.778 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:14:29.778 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:29.806 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 35 2026-02-18T08:14:29.806 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 38 2026-02-18T08:14:29.806 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:14:29.806 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=153, tag_tid=9, entry_tid=1], [object_number=152, tag_tid=9, entry_tid=0], [object_number=154, tag_tid=8, entry_tid=58722], [object_number=155, tag_tid=8, entry_tid=58719]]], state=connected] 2026-02-18T08:14:29.806 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=143, tag_tid=8, entry_tid=47363], [object_number=142, tag_tid=8, entry_tid=47362], [object_number=141, tag_tid=8, entry_tid=47361], [object_number=140, tag_tid=8, entry_tid=47360]]], state=connected] 2026-02-18T08:14:29.809 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:29.810 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=143, tag_tid=8, entry_tid=47363' 2026-02-18T08:14:29.810 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=153, tag_tid=9, entry_tid=1' -a 'object_number=153, tag_tid=9, entry_tid=1' = 'object_number=143, tag_tid=8, entry_tid=47363' 2026-02-18T08:14:29.811 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=143, tag_tid=8, entry_tid=47363' '!=' 'object_number=142, tag_tid=8, entry_tid=47114' 2026-02-18T08:14:29.811 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:14:29.811 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=143, tag_tid=8, entry_tid=47363' = 'object_number=142, tag_tid=8, entry_tid=47114' 2026-02-18T08:14:29.811 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=143, tag_tid=8, entry_tid=47363' 2026-02-18T08:14:29.811 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:29.811 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:14:29.811 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:29.813 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=9 2026-02-18T08:14:29.814 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=143, tag_tid=8, entry_tid=47363' 2026-02-18T08:14:29.814 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:14:29.814 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:29.815 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=8 2026-02-18T08:14:29.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:29.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:14:29.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:29.817 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:14:29.818 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=143, tag_tid=8, entry_tid=47363' 2026-02-18T08:14:29.818 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:14:29.818 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:29.820 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=47363 2026-02-18T08:14:29.820 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 9 = 8 -a 1 -le 47363 2026-02-18T08:14:29.820 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:14:29.820 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:14:29.820 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:14:30.021 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:14:30.021 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:14:30.021 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:14:30.021 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:14:30.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:14:30.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:14:30.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:14:30.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:14:30.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:14:30.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:14:30.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:14:30.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:14:30.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:14:30.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:14:30.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:14:30.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:14:30.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:14:30.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:14:30.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:14:30.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:14:30.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:14:30.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:14:30.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:14:30.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:14:30.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:14:30.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:14:30.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:14:30.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:14:30.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:14:30.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:14:30.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:14:30.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:14:30.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:14:30.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:14:30.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:14:30.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:14:30.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:14:30.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:14:30.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:14:30.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:14:30.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:14:30.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:14:30.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:14:30.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:14:30.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:14:30.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:14:30.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:14:30.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:14:30.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:14:30.035 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:30.036 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:30.036 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:30.036 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:14:30.105 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:14:30.106 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:14:30.106 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:30.106 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:30.106 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:30.106 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:14:30.106 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:30.106 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:30.106 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:30.106 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:14:30.106 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:14:30.106 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:14:30.106 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:14:30.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:30.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:14:30.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:30.144 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 35 2026-02-18T08:14:30.144 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 38 2026-02-18T08:14:30.144 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:14:30.144 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=153, tag_tid=9, entry_tid=1], [object_number=152, tag_tid=9, entry_tid=0], [object_number=154, tag_tid=8, entry_tid=58722], [object_number=155, tag_tid=8, entry_tid=58719]]], state=connected] 2026-02-18T08:14:30.144 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=140, tag_tid=8, entry_tid=47732], [object_number=143, tag_tid=8, entry_tid=47731], [object_number=142, tag_tid=8, entry_tid=47730], [object_number=141, tag_tid=8, entry_tid=47729]]], state=connected] 2026-02-18T08:14:30.148 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:30.149 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:30.150 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:14:30.150 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:30.150 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:30.150 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:30.150 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:14:30.150 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:30.150 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:30.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:30.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:14:30.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:14:30.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:14:30.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:14:30.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:30.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:14:30.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:30.179 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 35 2026-02-18T08:14:30.179 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 38 2026-02-18T08:14:30.179 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:14:30.179 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=153, tag_tid=9, entry_tid=1], [object_number=152, tag_tid=9, entry_tid=0], [object_number=154, tag_tid=8, entry_tid=58722], [object_number=155, tag_tid=8, entry_tid=58719]]], state=connected] 2026-02-18T08:14:30.179 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=140, tag_tid=8, entry_tid=47732], [object_number=143, tag_tid=8, entry_tid=47731], [object_number=142, tag_tid=8, entry_tid=47730], [object_number=141, tag_tid=8, entry_tid=47729]]], state=connected] 2026-02-18T08:14:30.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:30.183 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=140, tag_tid=8, entry_tid=47732' 2026-02-18T08:14:30.183 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=153, tag_tid=9, entry_tid=1' -a 'object_number=153, tag_tid=9, entry_tid=1' = 'object_number=140, tag_tid=8, entry_tid=47732' 2026-02-18T08:14:30.183 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=140, tag_tid=8, entry_tid=47732' '!=' 'object_number=143, tag_tid=8, entry_tid=47363' 2026-02-18T08:14:30.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:14:30.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=140, tag_tid=8, entry_tid=47732' = 'object_number=143, tag_tid=8, entry_tid=47363' 2026-02-18T08:14:30.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=140, tag_tid=8, entry_tid=47732' 2026-02-18T08:14:30.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:30.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:14:30.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:30.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=9 2026-02-18T08:14:30.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=140, tag_tid=8, entry_tid=47732' 2026-02-18T08:14:30.187 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:14:30.187 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:30.188 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=8 2026-02-18T08:14:30.189 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:30.189 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:30.189 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:14:30.190 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:14:30.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=140, tag_tid=8, entry_tid=47732' 2026-02-18T08:14:30.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:14:30.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:30.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=47732 2026-02-18T08:14:30.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 9 = 8 -a 1 -le 47732 2026-02-18T08:14:30.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:14:30.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:14:30.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:14:30.395 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:14:30.395 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:14:30.395 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:14:30.395 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:14:30.396 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:14:30.396 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:14:30.396 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:14:30.396 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:14:30.396 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:14:30.396 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:14:30.396 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:14:30.396 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:14:30.396 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:14:30.396 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:14:30.397 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:14:30.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:14:30.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:14:30.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:14:30.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:14:30.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:14:30.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:14:30.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:14:30.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:14:30.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:14:30.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:14:30.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:14:30.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:14:30.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:14:30.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:14:30.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:14:30.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:14:30.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:14:30.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:14:30.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:14:30.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:14:30.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:14:30.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:14:30.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:14:30.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:14:30.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:14:30.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:14:30.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:14:30.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:14:30.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:14:30.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:14:30.403 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:14:30.403 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:14:30.403 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:14:30.403 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:14:30.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:30.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:30.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:30.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:14:30.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:14:30.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:14:30.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:30.472 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:30.472 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:30.472 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:14:30.472 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:30.472 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:30.472 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:30.472 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:14:30.472 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:14:30.472 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:14:30.472 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:14:30.473 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:30.473 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:14:30.474 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:30.502 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 36 2026-02-18T08:14:30.502 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 38 2026-02-18T08:14:30.502 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:14:30.502 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=153, tag_tid=9, entry_tid=1], [object_number=152, tag_tid=9, entry_tid=0], [object_number=154, tag_tid=8, entry_tid=58722], [object_number=155, tag_tid=8, entry_tid=58719]]], state=connected] 2026-02-18T08:14:30.502 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=144, tag_tid=8, entry_tid=47888], [object_number=147, tag_tid=8, entry_tid=47887], [object_number=146, tag_tid=8, entry_tid=47886], [object_number=145, tag_tid=8, entry_tid=47885]]], state=connected] 2026-02-18T08:14:30.504 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:30.505 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:30.507 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:14:30.507 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:30.507 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:30.507 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:30.507 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:14:30.507 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:30.508 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:30.508 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:30.508 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:14:30.508 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:14:30.508 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:14:30.508 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:14:30.509 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:30.509 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:14:30.509 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:30.536 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 36 2026-02-18T08:14:30.536 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 38 2026-02-18T08:14:30.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:14:30.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=153, tag_tid=9, entry_tid=1], [object_number=152, tag_tid=9, entry_tid=0], [object_number=154, tag_tid=8, entry_tid=58722], [object_number=155, tag_tid=8, entry_tid=58719]]], state=connected] 2026-02-18T08:14:30.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=144, tag_tid=8, entry_tid=47888], [object_number=147, tag_tid=8, entry_tid=47887], [object_number=146, tag_tid=8, entry_tid=47886], [object_number=145, tag_tid=8, entry_tid=47885]]], state=connected] 2026-02-18T08:14:30.539 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:30.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=144, tag_tid=8, entry_tid=47888' 2026-02-18T08:14:30.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=153, tag_tid=9, entry_tid=1' -a 'object_number=153, tag_tid=9, entry_tid=1' = 'object_number=144, tag_tid=8, entry_tid=47888' 2026-02-18T08:14:30.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=144, tag_tid=8, entry_tid=47888' '!=' 'object_number=140, tag_tid=8, entry_tid=47732' 2026-02-18T08:14:30.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:14:30.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=144, tag_tid=8, entry_tid=47888' = 'object_number=140, tag_tid=8, entry_tid=47732' 2026-02-18T08:14:30.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=144, tag_tid=8, entry_tid=47888' 2026-02-18T08:14:30.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:30.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:14:30.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:30.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=9 2026-02-18T08:14:30.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=144, tag_tid=8, entry_tid=47888' 2026-02-18T08:14:30.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:14:30.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:30.545 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=8 2026-02-18T08:14:30.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:30.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:14:30.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:30.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:14:30.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=144, tag_tid=8, entry_tid=47888' 2026-02-18T08:14:30.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:14:30.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:30.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=47888 2026-02-18T08:14:30.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 9 = 8 -a 1 -le 47888 2026-02-18T08:14:30.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:14:30.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:14:30.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:14:30.751 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:14:30.751 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:14:30.751 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:14:30.751 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:14:30.752 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:14:30.752 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:14:30.752 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:14:30.752 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:14:30.752 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:14:30.752 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:14:30.752 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:14:30.752 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:14:30.752 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:14:30.752 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:14:30.753 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:14:30.756 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:14:30.756 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:14:30.756 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:14:30.756 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:14:30.756 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:14:30.756 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:14:30.756 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:14:30.756 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:14:30.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:14:30.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:14:30.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:14:30.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:14:30.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:14:30.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:14:30.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:14:30.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:14:30.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:14:30.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:14:30.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:14:30.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:14:30.758 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:14:30.758 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:14:30.758 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:14:30.758 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:14:30.758 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:14:30.758 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:14:30.758 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:14:30.758 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:14:30.758 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:14:30.758 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:14:30.758 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:14:30.758 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:14:30.758 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:14:30.759 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:14:30.767 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:30.768 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:30.768 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:30.768 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:14:30.836 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:14:30.836 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:14:30.836 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:30.836 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:30.836 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:30.836 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:14:30.836 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:30.836 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:30.836 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:30.836 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:14:30.837 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:14:30.837 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:14:30.837 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:14:30.838 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:30.839 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:14:30.839 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:30.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 36 2026-02-18T08:14:30.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 38 2026-02-18T08:14:30.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:14:30.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=153, tag_tid=9, entry_tid=1], [object_number=152, tag_tid=9, entry_tid=0], [object_number=154, tag_tid=8, entry_tid=58722], [object_number=155, tag_tid=8, entry_tid=58719]]], state=connected] 2026-02-18T08:14:30.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=144, tag_tid=8, entry_tid=48120], [object_number=147, tag_tid=8, entry_tid=48119], [object_number=146, tag_tid=8, entry_tid=48118], [object_number=145, tag_tid=8, entry_tid=48117]]], state=connected] 2026-02-18T08:14:30.877 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:30.878 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:30.878 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:14:30.878 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:30.878 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:30.878 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:30.878 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:14:30.879 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:30.879 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:30.879 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:30.879 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:14:30.879 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:14:30.879 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:14:30.879 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:14:30.881 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:30.881 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:14:30.881 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:30.900 INFO:tasks.rbd_mirror_thrash:waiting for cluster1.client.mirror.1 2026-02-18T08:14:30.900 INFO:teuthology.orchestra.run:waiting for 600 2026-02-18T08:14:30.901 INFO:tasks.rbd_mirror_thrash:reviving cluster1.client.mirror.1 2026-02-18T08:14:30.901 INFO:tasks.rbd_mirror.cluster1.client.mirror.1:Restarting daemon 2026-02-18T08:14:30.901 DEBUG:teuthology.orchestra.run.trial163:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.1 2026-02-18T08:14:30.902 INFO:tasks.rbd_mirror.cluster1.client.mirror.1:Started 2026-02-18T08:14:30.902 INFO:tasks.rbd_mirror_thrash:waiting for 78.44950716992696 secs before thrashing 2026-02-18T08:14:30.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 36 2026-02-18T08:14:30.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 38 2026-02-18T08:14:30.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:14:30.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=153, tag_tid=9, entry_tid=1], [object_number=152, tag_tid=9, entry_tid=0], [object_number=154, tag_tid=8, entry_tid=58722], [object_number=155, tag_tid=8, entry_tid=58719]]], state=connected] 2026-02-18T08:14:30.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=144, tag_tid=8, entry_tid=48120], [object_number=147, tag_tid=8, entry_tid=48119], [object_number=146, tag_tid=8, entry_tid=48118], [object_number=145, tag_tid=8, entry_tid=48117]]], state=connected] 2026-02-18T08:14:30.910 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:30.912 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=144, tag_tid=8, entry_tid=48120' 2026-02-18T08:14:30.912 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=153, tag_tid=9, entry_tid=1' -a 'object_number=153, tag_tid=9, entry_tid=1' = 'object_number=144, tag_tid=8, entry_tid=48120' 2026-02-18T08:14:30.912 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=144, tag_tid=8, entry_tid=48120' '!=' 'object_number=144, tag_tid=8, entry_tid=47888' 2026-02-18T08:14:30.912 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:14:30.912 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=144, tag_tid=8, entry_tid=48120' = 'object_number=144, tag_tid=8, entry_tid=47888' 2026-02-18T08:14:30.912 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=144, tag_tid=8, entry_tid=48120' 2026-02-18T08:14:30.912 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:30.912 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:14:30.913 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:30.914 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=9 2026-02-18T08:14:30.914 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=144, tag_tid=8, entry_tid=48120' 2026-02-18T08:14:30.915 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:14:30.915 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:30.916 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=8 2026-02-18T08:14:30.917 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:30.917 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:30.917 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:14:30.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:14:30.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=144, tag_tid=8, entry_tid=48120' 2026-02-18T08:14:30.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:14:30.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:30.922 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=48120 2026-02-18T08:14:30.922 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 9 = 8 -a 1 -le 48120 2026-02-18T08:14:30.922 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:14:30.922 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:14:30.922 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:14:31.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:14:31.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:14:31.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:14:31.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:14:31.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:14:31.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:14:31.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:14:31.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:14:31.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:14:31.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:14:31.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:14:31.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:14:31.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:14:31.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:14:31.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:14:31.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:14:31.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:14:31.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:14:31.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:14:31.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:14:31.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:14:31.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:14:31.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:14:31.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:14:31.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:14:31.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:14:31.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:14:31.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:14:31.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:14:31.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:14:31.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:14:31.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:14:31.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:14:31.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:14:31.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:14:31.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:14:31.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:14:31.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:14:31.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:14:31.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:14:31.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:14:31.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:14:31.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:14:31.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:14:31.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:14:31.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:14:31.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:14:31.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:14:31.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:14:31.138 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:31.138 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:31.138 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:31.138 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:14:31.199 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:14:31.199 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:14:31.199 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:31.199 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:31.199 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:31.200 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:14:31.200 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:31.200 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:31.200 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:31.200 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:14:31.200 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:14:31.200 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:14:31.200 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:14:31.202 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:31.202 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:14:31.202 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:31.230 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 36 2026-02-18T08:14:31.230 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 38 2026-02-18T08:14:31.230 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:14:31.230 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=153, tag_tid=9, entry_tid=1], [object_number=152, tag_tid=9, entry_tid=0], [object_number=154, tag_tid=8, entry_tid=58722], [object_number=155, tag_tid=8, entry_tid=58719]]], state=connected] 2026-02-18T08:14:31.230 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=145, tag_tid=8, entry_tid=48373], [object_number=144, tag_tid=8, entry_tid=48372], [object_number=147, tag_tid=8, entry_tid=48371], [object_number=146, tag_tid=8, entry_tid=48370]]], state=connected] 2026-02-18T08:14:31.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:31.234 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:31.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:14:31.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:31.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:31.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:31.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:14:31.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:31.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:31.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:31.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:14:31.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:14:31.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:14:31.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:14:31.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:31.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:14:31.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:31.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 36 2026-02-18T08:14:31.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 38 2026-02-18T08:14:31.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:14:31.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=153, tag_tid=9, entry_tid=1], [object_number=152, tag_tid=9, entry_tid=0], [object_number=154, tag_tid=8, entry_tid=58722], [object_number=155, tag_tid=8, entry_tid=58719]]], state=connected] 2026-02-18T08:14:31.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=145, tag_tid=8, entry_tid=48373], [object_number=144, tag_tid=8, entry_tid=48372], [object_number=147, tag_tid=8, entry_tid=48371], [object_number=146, tag_tid=8, entry_tid=48370]]], state=connected] 2026-02-18T08:14:31.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:31.269 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=145, tag_tid=8, entry_tid=48373' 2026-02-18T08:14:31.269 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=153, tag_tid=9, entry_tid=1' -a 'object_number=153, tag_tid=9, entry_tid=1' = 'object_number=145, tag_tid=8, entry_tid=48373' 2026-02-18T08:14:31.269 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=145, tag_tid=8, entry_tid=48373' '!=' 'object_number=144, tag_tid=8, entry_tid=48120' 2026-02-18T08:14:31.270 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:14:31.270 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=145, tag_tid=8, entry_tid=48373' = 'object_number=144, tag_tid=8, entry_tid=48120' 2026-02-18T08:14:31.270 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=145, tag_tid=8, entry_tid=48373' 2026-02-18T08:14:31.270 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:31.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:14:31.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:31.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=9 2026-02-18T08:14:31.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=145, tag_tid=8, entry_tid=48373' 2026-02-18T08:14:31.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:14:31.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:31.274 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=8 2026-02-18T08:14:31.274 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:31.274 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:14:31.275 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:31.275 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:14:31.276 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=145, tag_tid=8, entry_tid=48373' 2026-02-18T08:14:31.277 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:14:31.277 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:31.278 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=48373 2026-02-18T08:14:31.278 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 9 = 8 -a 1 -le 48373 2026-02-18T08:14:31.278 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:14:31.279 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:14:31.279 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:14:31.480 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:14:31.480 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:14:31.480 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:14:31.480 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:14:31.481 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:14:31.481 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:14:31.481 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:14:31.481 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:14:31.481 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:14:31.481 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:14:31.481 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:14:31.481 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:14:31.481 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:14:31.481 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:14:31.482 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:14:31.482 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:14:31.484 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:14:31.484 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:14:31.485 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:14:31.485 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:14:31.485 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:14:31.485 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:14:31.485 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:14:31.485 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:14:31.485 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:14:31.485 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:14:31.485 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:14:31.485 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:14:31.485 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:14:31.486 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:14:31.486 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:14:31.486 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:14:31.486 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:14:31.486 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:14:31.486 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:14:31.486 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:14:31.486 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:14:31.486 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:14:31.486 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:14:31.486 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:14:31.487 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:14:31.487 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:14:31.487 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:14:31.487 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:14:31.487 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:14:31.487 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:14:31.487 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:14:31.487 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:14:31.487 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:14:31.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:31.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:31.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:31.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:14:31.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:14:31.557 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:14:31.557 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:31.557 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:31.557 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:31.557 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:14:31.557 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:31.557 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:31.557 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:31.558 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:14:31.558 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:14:31.558 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:14:31.558 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:14:31.559 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:31.559 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:14:31.559 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:31.587 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 36 2026-02-18T08:14:31.587 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 38 2026-02-18T08:14:31.587 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:14:31.587 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=153, tag_tid=9, entry_tid=1], [object_number=152, tag_tid=9, entry_tid=0], [object_number=154, tag_tid=8, entry_tid=58722], [object_number=155, tag_tid=8, entry_tid=58719]]], state=connected] 2026-02-18T08:14:31.587 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=144, tag_tid=8, entry_tid=48612], [object_number=147, tag_tid=8, entry_tid=48611], [object_number=146, tag_tid=8, entry_tid=48610], [object_number=145, tag_tid=8, entry_tid=48609]]], state=connected] 2026-02-18T08:14:31.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:31.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:31.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:14:31.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:31.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:31.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:31.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:14:31.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:31.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:31.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:31.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:14:31.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:14:31.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:14:31.594 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:14:31.594 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:31.594 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:14:31.595 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:31.622 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 36 2026-02-18T08:14:31.622 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 38 2026-02-18T08:14:31.622 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:14:31.623 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=153, tag_tid=9, entry_tid=1], [object_number=152, tag_tid=9, entry_tid=0], [object_number=154, tag_tid=8, entry_tid=58722], [object_number=155, tag_tid=8, entry_tid=58719]]], state=connected] 2026-02-18T08:14:31.623 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=144, tag_tid=8, entry_tid=48612], [object_number=147, tag_tid=8, entry_tid=48611], [object_number=146, tag_tid=8, entry_tid=48610], [object_number=145, tag_tid=8, entry_tid=48609]]], state=connected] 2026-02-18T08:14:31.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:31.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=144, tag_tid=8, entry_tid=48612' 2026-02-18T08:14:31.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=153, tag_tid=9, entry_tid=1' -a 'object_number=153, tag_tid=9, entry_tid=1' = 'object_number=144, tag_tid=8, entry_tid=48612' 2026-02-18T08:14:31.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=144, tag_tid=8, entry_tid=48612' '!=' 'object_number=145, tag_tid=8, entry_tid=48373' 2026-02-18T08:14:31.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:14:31.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=144, tag_tid=8, entry_tid=48612' = 'object_number=145, tag_tid=8, entry_tid=48373' 2026-02-18T08:14:31.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=144, tag_tid=8, entry_tid=48612' 2026-02-18T08:14:31.628 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:31.628 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:14:31.628 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:31.629 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=9 2026-02-18T08:14:31.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=144, tag_tid=8, entry_tid=48612' 2026-02-18T08:14:31.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:14:31.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:31.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=8 2026-02-18T08:14:31.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:31.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:14:31.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:31.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:14:31.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=144, tag_tid=8, entry_tid=48612' 2026-02-18T08:14:31.635 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:14:31.635 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:31.637 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=48612 2026-02-18T08:14:31.637 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 9 = 8 -a 1 -le 48612 2026-02-18T08:14:31.637 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:14:31.637 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:14:31.637 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:14:31.839 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:14:31.839 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:14:31.839 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:14:31.839 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:14:31.840 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:14:31.840 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:14:31.840 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:14:31.840 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:14:31.840 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:14:31.840 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:14:31.840 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:14:31.840 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:14:31.840 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:14:31.840 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:14:31.840 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:14:31.841 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:14:31.841 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:14:31.844 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:14:31.844 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:14:31.844 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:14:31.844 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:14:31.844 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:14:31.844 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:14:31.844 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:14:31.844 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:14:31.844 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:14:31.845 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:14:31.845 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:14:31.845 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:14:31.845 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:14:31.845 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:14:31.845 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:14:31.845 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:14:31.845 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:14:31.845 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:14:31.845 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:14:31.846 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:14:31.846 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:14:31.846 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:14:31.846 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:14:31.846 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:14:31.846 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:14:31.846 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:14:31.846 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:14:31.846 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:14:31.846 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:14:31.846 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:14:31.847 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:14:31.847 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:14:31.853 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:31.853 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:31.853 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:31.853 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:14:31.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:14:31.922 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:14:31.922 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:31.922 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:31.923 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:31.923 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:14:31.923 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:31.923 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:31.923 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:31.923 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:14:31.923 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:14:31.923 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:14:31.923 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:14:31.924 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:31.924 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:14:31.924 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:31.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 36 2026-02-18T08:14:31.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 38 2026-02-18T08:14:31.961 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:14:31.961 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=153, tag_tid=9, entry_tid=1], [object_number=152, tag_tid=9, entry_tid=0], [object_number=154, tag_tid=8, entry_tid=58722], [object_number=155, tag_tid=8, entry_tid=58719]]], state=connected] 2026-02-18T08:14:31.961 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=145, tag_tid=8, entry_tid=48861], [object_number=144, tag_tid=8, entry_tid=48860], [object_number=147, tag_tid=8, entry_tid=48859], [object_number=146, tag_tid=8, entry_tid=48858]]], state=connected] 2026-02-18T08:14:31.964 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:31.966 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:31.967 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:14:31.967 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:31.967 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:31.967 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:31.968 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:14:31.968 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:31.968 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:31.968 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:31.968 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:14:31.968 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:14:31.969 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:14:31.969 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:14:31.970 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:31.970 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:14:31.970 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:32.006 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 36 2026-02-18T08:14:32.006 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 38 2026-02-18T08:14:32.007 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:14:32.007 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=153, tag_tid=9, entry_tid=1], [object_number=152, tag_tid=9, entry_tid=0], [object_number=154, tag_tid=8, entry_tid=58722], [object_number=155, tag_tid=8, entry_tid=58719]]], state=connected] 2026-02-18T08:14:32.007 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=145, tag_tid=8, entry_tid=48861], [object_number=144, tag_tid=8, entry_tid=48860], [object_number=147, tag_tid=8, entry_tid=48859], [object_number=146, tag_tid=8, entry_tid=48858]]], state=connected] 2026-02-18T08:14:32.010 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:32.012 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=145, tag_tid=8, entry_tid=48861' 2026-02-18T08:14:32.012 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=153, tag_tid=9, entry_tid=1' -a 'object_number=153, tag_tid=9, entry_tid=1' = 'object_number=145, tag_tid=8, entry_tid=48861' 2026-02-18T08:14:32.012 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=145, tag_tid=8, entry_tid=48861' '!=' 'object_number=144, tag_tid=8, entry_tid=48612' 2026-02-18T08:14:32.012 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:14:32.012 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=145, tag_tid=8, entry_tid=48861' = 'object_number=144, tag_tid=8, entry_tid=48612' 2026-02-18T08:14:32.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=145, tag_tid=8, entry_tid=48861' 2026-02-18T08:14:32.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:32.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:14:32.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:32.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=9 2026-02-18T08:14:32.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=145, tag_tid=8, entry_tid=48861' 2026-02-18T08:14:32.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:14:32.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:32.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=8 2026-02-18T08:14:32.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:32.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:14:32.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:32.018 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:14:32.019 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=145, tag_tid=8, entry_tid=48861' 2026-02-18T08:14:32.019 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:14:32.019 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:32.020 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=48861 2026-02-18T08:14:32.020 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 9 = 8 -a 1 -le 48861 2026-02-18T08:14:32.020 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:14:32.021 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:14:32.021 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:14:32.222 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:14:32.222 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:14:32.222 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:14:32.223 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:14:32.223 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:14:32.223 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:14:32.223 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:14:32.223 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:14:32.223 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:14:32.223 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:14:32.223 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:14:32.223 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:14:32.223 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:14:32.223 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:14:32.223 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:14:32.225 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:14:32.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:14:32.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:14:32.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:14:32.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:14:32.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:14:32.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:14:32.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:14:32.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:14:32.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:14:32.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:14:32.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:14:32.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:14:32.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:14:32.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:14:32.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:14:32.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:14:32.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:14:32.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:14:32.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:14:32.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:14:32.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:14:32.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:14:32.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:14:32.227 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:14:32.227 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:14:32.227 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:14:32.227 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:14:32.227 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:14:32.227 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:14:32.227 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:14:32.227 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:14:32.227 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:14:32.227 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:14:32.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:32.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:32.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:32.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:14:32.298 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:14:32.298 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:14:32.298 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:32.298 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:32.298 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:32.298 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:14:32.299 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:32.299 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:32.299 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:32.299 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:14:32.299 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:14:32.300 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:14:32.300 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:14:32.300 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:32.301 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:14:32.301 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:32.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 36 2026-02-18T08:14:32.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 38 2026-02-18T08:14:32.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:14:32.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=153, tag_tid=9, entry_tid=1], [object_number=152, tag_tid=9, entry_tid=0], [object_number=154, tag_tid=8, entry_tid=58722], [object_number=155, tag_tid=8, entry_tid=58719]]], state=connected] 2026-02-18T08:14:32.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=145, tag_tid=8, entry_tid=49105], [object_number=144, tag_tid=8, entry_tid=49104], [object_number=147, tag_tid=8, entry_tid=49103], [object_number=146, tag_tid=8, entry_tid=49102]]], state=connected] 2026-02-18T08:14:32.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:32.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:32.344 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:14:32.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:32.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:32.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:32.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:14:32.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:32.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:32.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:32.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:14:32.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:14:32.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:14:32.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:14:32.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:32.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:14:32.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:32.374 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 36 2026-02-18T08:14:32.374 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 38 2026-02-18T08:14:32.374 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:14:32.374 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=153, tag_tid=9, entry_tid=1], [object_number=152, tag_tid=9, entry_tid=0], [object_number=154, tag_tid=8, entry_tid=58722], [object_number=155, tag_tid=8, entry_tid=58719]]], state=connected] 2026-02-18T08:14:32.374 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=145, tag_tid=8, entry_tid=49105], [object_number=144, tag_tid=8, entry_tid=49104], [object_number=147, tag_tid=8, entry_tid=49103], [object_number=146, tag_tid=8, entry_tid=49102]]], state=connected] 2026-02-18T08:14:32.376 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:32.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=145, tag_tid=8, entry_tid=49105' 2026-02-18T08:14:32.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=153, tag_tid=9, entry_tid=1' -a 'object_number=153, tag_tid=9, entry_tid=1' = 'object_number=145, tag_tid=8, entry_tid=49105' 2026-02-18T08:14:32.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=145, tag_tid=8, entry_tid=49105' '!=' 'object_number=145, tag_tid=8, entry_tid=48861' 2026-02-18T08:14:32.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:14:32.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=145, tag_tid=8, entry_tid=49105' = 'object_number=145, tag_tid=8, entry_tid=48861' 2026-02-18T08:14:32.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=145, tag_tid=8, entry_tid=49105' 2026-02-18T08:14:32.379 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:32.379 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:14:32.379 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:32.380 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=9 2026-02-18T08:14:32.381 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=145, tag_tid=8, entry_tid=49105' 2026-02-18T08:14:32.381 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:14:32.381 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:32.383 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=8 2026-02-18T08:14:32.383 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:32.383 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:14:32.384 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:32.385 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:14:32.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=145, tag_tid=8, entry_tid=49105' 2026-02-18T08:14:32.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:14:32.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:32.387 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=49105 2026-02-18T08:14:32.387 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 9 = 8 -a 1 -le 49105 2026-02-18T08:14:32.387 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:14:32.387 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:14:32.387 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:14:32.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:14:32.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:14:32.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:14:32.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:14:32.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:14:32.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:14:32.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:14:32.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:14:32.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:14:32.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:14:32.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:14:32.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:14:32.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:14:32.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:14:32.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:14:32.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:14:32.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:14:32.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:14:32.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:14:32.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:14:32.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:14:32.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:14:32.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:14:32.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:14:32.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:14:32.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:14:32.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:14:32.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:14:32.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:14:32.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:14:32.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:14:32.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:14:32.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:14:32.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:14:32.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:14:32.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:14:32.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:14:32.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:14:32.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:14:32.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:14:32.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:14:32.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:14:32.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:14:32.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:14:32.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:14:32.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:14:32.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:14:32.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:14:32.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:14:32.603 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:32.605 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:32.605 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:32.605 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:14:32.682 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:14:32.682 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:14:32.682 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:32.682 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:32.682 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:32.682 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:14:32.683 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:32.683 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:32.683 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:32.683 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:14:32.683 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:14:32.683 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:14:32.683 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:14:32.683 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:32.684 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:14:32.684 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:32.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 36 2026-02-18T08:14:32.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 38 2026-02-18T08:14:32.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:14:32.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=153, tag_tid=9, entry_tid=1], [object_number=152, tag_tid=9, entry_tid=0], [object_number=154, tag_tid=8, entry_tid=58722], [object_number=155, tag_tid=8, entry_tid=58719]]], state=connected] 2026-02-18T08:14:32.717 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=146, tag_tid=8, entry_tid=49342], [object_number=145, tag_tid=8, entry_tid=49341], [object_number=144, tag_tid=8, entry_tid=49340], [object_number=147, tag_tid=8, entry_tid=49339]]], state=connected] 2026-02-18T08:14:32.720 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:32.721 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:32.722 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:14:32.722 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:32.722 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:32.722 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:32.722 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:14:32.722 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:32.722 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:32.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:32.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:14:32.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:14:32.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:14:32.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:14:32.724 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:32.724 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:14:32.724 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:32.752 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 36 2026-02-18T08:14:32.752 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 38 2026-02-18T08:14:32.752 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:14:32.752 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=153, tag_tid=9, entry_tid=1], [object_number=152, tag_tid=9, entry_tid=0], [object_number=154, tag_tid=8, entry_tid=58722], [object_number=155, tag_tid=8, entry_tid=58719]]], state=connected] 2026-02-18T08:14:32.752 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=146, tag_tid=8, entry_tid=49342], [object_number=145, tag_tid=8, entry_tid=49341], [object_number=144, tag_tid=8, entry_tid=49340], [object_number=147, tag_tid=8, entry_tid=49339]]], state=connected] 2026-02-18T08:14:32.755 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:32.756 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=146, tag_tid=8, entry_tid=49342' 2026-02-18T08:14:32.756 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=153, tag_tid=9, entry_tid=1' -a 'object_number=153, tag_tid=9, entry_tid=1' = 'object_number=146, tag_tid=8, entry_tid=49342' 2026-02-18T08:14:32.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=146, tag_tid=8, entry_tid=49342' '!=' 'object_number=145, tag_tid=8, entry_tid=49105' 2026-02-18T08:14:32.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:14:32.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=146, tag_tid=8, entry_tid=49342' = 'object_number=145, tag_tid=8, entry_tid=49105' 2026-02-18T08:14:32.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=146, tag_tid=8, entry_tid=49342' 2026-02-18T08:14:32.758 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:32.758 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:14:32.758 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:32.758 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=9 2026-02-18T08:14:32.759 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=146, tag_tid=8, entry_tid=49342' 2026-02-18T08:14:32.759 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:14:32.759 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:32.761 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=8 2026-02-18T08:14:32.762 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:32.762 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:14:32.762 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:32.763 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:14:32.763 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=146, tag_tid=8, entry_tid=49342' 2026-02-18T08:14:32.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:14:32.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:32.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=49342 2026-02-18T08:14:32.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 9 = 8 -a 1 -le 49342 2026-02-18T08:14:32.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:14:32.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:14:32.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:14:32.966 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:14:32.966 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:14:32.966 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:14:32.966 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:14:32.966 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:14:32.967 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:14:32.967 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:14:32.967 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:14:32.967 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:14:32.967 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:14:32.967 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:14:32.967 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:14:32.967 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:14:32.967 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:14:32.967 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:14:32.967 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:14:32.967 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:14:32.970 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:14:32.970 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:14:32.970 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:14:32.970 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:14:32.970 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:14:32.970 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:14:32.970 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:14:32.970 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:14:32.970 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:14:32.970 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:14:32.970 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:14:32.970 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:14:32.970 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:14:32.970 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:14:32.970 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:14:32.971 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:14:32.971 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:14:32.971 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:14:32.971 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:14:32.971 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:14:32.971 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:14:32.971 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:14:32.971 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:14:32.971 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:14:32.971 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:14:32.971 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:14:32.971 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:14:32.971 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:14:32.971 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:14:32.971 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:14:32.971 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:14:32.971 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:14:32.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:32.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:32.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:32.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:14:33.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:14:33.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:14:33.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:33.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:33.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:33.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:14:33.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:33.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:33.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:33.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:14:33.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:14:33.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:14:33.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:14:33.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:33.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:14:33.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:33.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 36 2026-02-18T08:14:33.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 38 2026-02-18T08:14:33.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:14:33.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=153, tag_tid=9, entry_tid=1], [object_number=152, tag_tid=9, entry_tid=0], [object_number=154, tag_tid=8, entry_tid=58722], [object_number=155, tag_tid=8, entry_tid=58719]]], state=connected] 2026-02-18T08:14:33.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=146, tag_tid=8, entry_tid=49574], [object_number=145, tag_tid=8, entry_tid=49573], [object_number=144, tag_tid=8, entry_tid=49572], [object_number=147, tag_tid=8, entry_tid=49571]]], state=connected] 2026-02-18T08:14:33.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:33.093 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:33.094 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:14:33.094 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:33.094 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:33.094 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:33.094 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:14:33.094 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:33.094 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:33.094 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:33.094 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:14:33.094 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:14:33.095 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:14:33.095 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:14:33.096 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:33.097 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:14:33.097 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:33.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 36 2026-02-18T08:14:33.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 38 2026-02-18T08:14:33.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:14:33.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=153, tag_tid=9, entry_tid=1], [object_number=152, tag_tid=9, entry_tid=0], [object_number=154, tag_tid=8, entry_tid=58722], [object_number=155, tag_tid=8, entry_tid=58719]]], state=connected] 2026-02-18T08:14:33.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=146, tag_tid=8, entry_tid=49574], [object_number=145, tag_tid=8, entry_tid=49573], [object_number=144, tag_tid=8, entry_tid=49572], [object_number=147, tag_tid=8, entry_tid=49571]]], state=connected] 2026-02-18T08:14:33.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:33.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=146, tag_tid=8, entry_tid=49574' 2026-02-18T08:14:33.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=153, tag_tid=9, entry_tid=1' -a 'object_number=153, tag_tid=9, entry_tid=1' = 'object_number=146, tag_tid=8, entry_tid=49574' 2026-02-18T08:14:33.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=146, tag_tid=8, entry_tid=49574' '!=' 'object_number=146, tag_tid=8, entry_tid=49342' 2026-02-18T08:14:33.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:14:33.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=146, tag_tid=8, entry_tid=49574' = 'object_number=146, tag_tid=8, entry_tid=49342' 2026-02-18T08:14:33.129 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=146, tag_tid=8, entry_tid=49574' 2026-02-18T08:14:33.129 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:33.129 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:14:33.129 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:33.130 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=9 2026-02-18T08:14:33.131 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=146, tag_tid=8, entry_tid=49574' 2026-02-18T08:14:33.131 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:14:33.131 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:33.132 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=8 2026-02-18T08:14:33.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:33.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:14:33.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:33.134 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:14:33.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=146, tag_tid=8, entry_tid=49574' 2026-02-18T08:14:33.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:14:33.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:33.137 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=49574 2026-02-18T08:14:33.137 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 9 = 8 -a 1 -le 49574 2026-02-18T08:14:33.137 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:14:33.137 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:14:33.137 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:14:33.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:14:33.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:14:33.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:14:33.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:14:33.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:14:33.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:14:33.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:14:33.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:14:33.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:14:33.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:14:33.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:14:33.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:14:33.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:14:33.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:14:33.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:14:33.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:14:33.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:14:33.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:14:33.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:14:33.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:14:33.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:14:33.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:14:33.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:14:33.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:14:33.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:14:33.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:14:33.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:14:33.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:14:33.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:14:33.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:14:33.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:14:33.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:14:33.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:14:33.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:14:33.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:14:33.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:14:33.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:14:33.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:14:33.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:14:33.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:14:33.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:14:33.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:14:33.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:14:33.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:14:33.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:14:33.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:14:33.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:14:33.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:14:33.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:14:33.354 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:33.354 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:33.354 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:33.354 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:14:33.414 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:14:33.415 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:14:33.415 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:33.415 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:33.415 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:33.415 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:14:33.415 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:33.415 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:33.415 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:33.415 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:14:33.415 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:14:33.416 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:14:33.416 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:14:33.417 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:33.417 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:14:33.418 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:33.455 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 36 2026-02-18T08:14:33.455 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 38 2026-02-18T08:14:33.455 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:14:33.455 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=153, tag_tid=9, entry_tid=1], [object_number=152, tag_tid=9, entry_tid=0], [object_number=154, tag_tid=8, entry_tid=58722], [object_number=155, tag_tid=8, entry_tid=58719]]], state=connected] 2026-02-18T08:14:33.455 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=147, tag_tid=8, entry_tid=49811], [object_number=146, tag_tid=8, entry_tid=49810], [object_number=145, tag_tid=8, entry_tid=49809], [object_number=144, tag_tid=8, entry_tid=49808]]], state=connected] 2026-02-18T08:14:33.458 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:33.460 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:33.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:14:33.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:33.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:33.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:33.462 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:14:33.462 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:33.462 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:33.462 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:33.462 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:14:33.462 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:14:33.462 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:14:33.462 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:14:33.463 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:33.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:14:33.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:33.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 36 2026-02-18T08:14:33.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 38 2026-02-18T08:14:33.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:14:33.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=153, tag_tid=9, entry_tid=1], [object_number=152, tag_tid=9, entry_tid=0], [object_number=154, tag_tid=8, entry_tid=58722], [object_number=155, tag_tid=8, entry_tid=58719]]], state=connected] 2026-02-18T08:14:33.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=147, tag_tid=8, entry_tid=49811], [object_number=146, tag_tid=8, entry_tid=49810], [object_number=145, tag_tid=8, entry_tid=49809], [object_number=144, tag_tid=8, entry_tid=49808]]], state=connected] 2026-02-18T08:14:33.502 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:33.504 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=147, tag_tid=8, entry_tid=49811' 2026-02-18T08:14:33.504 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=153, tag_tid=9, entry_tid=1' -a 'object_number=153, tag_tid=9, entry_tid=1' = 'object_number=147, tag_tid=8, entry_tid=49811' 2026-02-18T08:14:33.504 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=147, tag_tid=8, entry_tid=49811' '!=' 'object_number=146, tag_tid=8, entry_tid=49574' 2026-02-18T08:14:33.505 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:14:33.505 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=147, tag_tid=8, entry_tid=49811' = 'object_number=146, tag_tid=8, entry_tid=49574' 2026-02-18T08:14:33.505 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=147, tag_tid=8, entry_tid=49811' 2026-02-18T08:14:33.505 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:33.505 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:14:33.505 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:33.507 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=9 2026-02-18T08:14:33.507 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=147, tag_tid=8, entry_tid=49811' 2026-02-18T08:14:33.507 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:14:33.508 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:33.509 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=8 2026-02-18T08:14:33.510 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:33.510 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:14:33.510 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:33.511 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:14:33.512 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=147, tag_tid=8, entry_tid=49811' 2026-02-18T08:14:33.512 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:14:33.512 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:33.514 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=49811 2026-02-18T08:14:33.514 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 9 = 8 -a 1 -le 49811 2026-02-18T08:14:33.514 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:14:33.514 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:14:33.514 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:14:33.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:14:33.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:14:33.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:14:33.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:14:33.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:14:33.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:14:33.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:14:33.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:14:33.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:14:33.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:14:33.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:14:33.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:14:33.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:14:33.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:14:33.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:14:33.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:14:33.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:14:33.718 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:14:33.718 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:14:33.718 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:14:33.718 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:14:33.718 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:14:33.719 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:14:33.719 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:14:33.719 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:14:33.719 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:14:33.719 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:14:33.719 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:14:33.719 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:14:33.719 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:14:33.719 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:14:33.719 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:14:33.719 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:14:33.719 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:14:33.719 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:14:33.719 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:14:33.719 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:14:33.719 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:14:33.720 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:14:33.720 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:14:33.720 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:14:33.720 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:14:33.720 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:14:33.720 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:14:33.720 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:14:33.720 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:14:33.720 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:14:33.720 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:14:33.720 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:14:33.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:33.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:33.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:33.731 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:14:33.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:14:33.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:14:33.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:33.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:33.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:33.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:14:33.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:33.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:33.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:33.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:14:33.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:14:33.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:14:33.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:14:33.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:33.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:14:33.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:33.828 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 36 2026-02-18T08:14:33.829 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 38 2026-02-18T08:14:33.829 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:14:33.829 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=153, tag_tid=9, entry_tid=1], [object_number=152, tag_tid=9, entry_tid=0], [object_number=154, tag_tid=8, entry_tid=58722], [object_number=155, tag_tid=8, entry_tid=58719]]], state=connected] 2026-02-18T08:14:33.829 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=145, tag_tid=8, entry_tid=50097], [object_number=144, tag_tid=8, entry_tid=50096], [object_number=147, tag_tid=8, entry_tid=50095], [object_number=146, tag_tid=8, entry_tid=50094]]], state=connected] 2026-02-18T08:14:33.832 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:33.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:33.835 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:14:33.835 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:33.835 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:33.835 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:33.835 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:14:33.835 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:33.835 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:33.835 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:33.835 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:14:33.835 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:14:33.836 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:14:33.836 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:14:33.837 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:33.837 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:14:33.837 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:33.866 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 36 2026-02-18T08:14:33.866 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 38 2026-02-18T08:14:33.866 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:14:33.866 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=153, tag_tid=9, entry_tid=1], [object_number=152, tag_tid=9, entry_tid=0], [object_number=154, tag_tid=8, entry_tid=58722], [object_number=155, tag_tid=8, entry_tid=58719]]], state=connected] 2026-02-18T08:14:33.866 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=145, tag_tid=8, entry_tid=50097], [object_number=144, tag_tid=8, entry_tid=50096], [object_number=147, tag_tid=8, entry_tid=50095], [object_number=146, tag_tid=8, entry_tid=50094]]], state=connected] 2026-02-18T08:14:33.869 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:33.870 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=145, tag_tid=8, entry_tid=50097' 2026-02-18T08:14:33.870 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=153, tag_tid=9, entry_tid=1' -a 'object_number=153, tag_tid=9, entry_tid=1' = 'object_number=145, tag_tid=8, entry_tid=50097' 2026-02-18T08:14:33.870 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=145, tag_tid=8, entry_tid=50097' '!=' 'object_number=147, tag_tid=8, entry_tid=49811' 2026-02-18T08:14:33.870 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:14:33.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=145, tag_tid=8, entry_tid=50097' = 'object_number=147, tag_tid=8, entry_tid=49811' 2026-02-18T08:14:33.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=145, tag_tid=8, entry_tid=50097' 2026-02-18T08:14:33.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:33.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:14:33.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:33.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=9 2026-02-18T08:14:33.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=145, tag_tid=8, entry_tid=50097' 2026-02-18T08:14:33.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:14:33.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:33.875 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=8 2026-02-18T08:14:33.875 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:33.876 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:14:33.876 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:33.877 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:14:33.878 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=145, tag_tid=8, entry_tid=50097' 2026-02-18T08:14:33.878 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:14:33.878 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:33.879 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=50097 2026-02-18T08:14:33.879 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 9 = 8 -a 1 -le 50097 2026-02-18T08:14:33.879 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:14:33.879 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:14:33.879 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:14:34.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:14:34.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:14:34.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:14:34.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:14:34.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:14:34.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:14:34.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:14:34.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:14:34.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:14:34.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:14:34.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:14:34.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:14:34.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:14:34.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:14:34.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:14:34.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:14:34.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:14:34.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:14:34.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:14:34.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:14:34.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:14:34.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:14:34.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:14:34.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:14:34.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:14:34.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:14:34.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:14:34.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:14:34.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:14:34.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:14:34.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:14:34.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:14:34.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:14:34.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:14:34.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:14:34.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:14:34.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:14:34.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:14:34.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:14:34.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:14:34.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:14:34.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:14:34.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:14:34.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:14:34.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:14:34.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:14:34.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:14:34.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:14:34.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:14:34.096 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:34.096 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:34.096 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:34.096 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:14:34.164 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:14:34.165 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:14:34.165 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:34.165 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:34.165 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:34.165 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:14:34.165 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:34.165 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:34.165 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:34.165 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:14:34.165 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:14:34.165 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:14:34.166 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:14:34.167 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:34.167 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:14:34.167 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:34.194 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 36 2026-02-18T08:14:34.194 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 38 2026-02-18T08:14:34.195 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:14:34.195 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=153, tag_tid=9, entry_tid=1], [object_number=152, tag_tid=9, entry_tid=0], [object_number=154, tag_tid=8, entry_tid=58722], [object_number=155, tag_tid=8, entry_tid=58719]]], state=connected] 2026-02-18T08:14:34.195 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=146, tag_tid=8, entry_tid=50342], [object_number=145, tag_tid=8, entry_tid=50341], [object_number=144, tag_tid=8, entry_tid=50340], [object_number=147, tag_tid=8, entry_tid=50339]]], state=connected] 2026-02-18T08:14:34.197 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:34.199 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:34.199 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:14:34.199 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:34.199 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:34.199 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:34.200 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:14:34.200 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:34.200 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:34.200 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:34.200 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:14:34.200 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:14:34.200 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:14:34.200 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:14:34.202 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:34.202 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:14:34.202 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:34.229 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 36 2026-02-18T08:14:34.229 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 38 2026-02-18T08:14:34.229 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:14:34.229 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=153, tag_tid=9, entry_tid=1], [object_number=152, tag_tid=9, entry_tid=0], [object_number=154, tag_tid=8, entry_tid=58722], [object_number=155, tag_tid=8, entry_tid=58719]]], state=connected] 2026-02-18T08:14:34.229 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=146, tag_tid=8, entry_tid=50342], [object_number=145, tag_tid=8, entry_tid=50341], [object_number=144, tag_tid=8, entry_tid=50340], [object_number=147, tag_tid=8, entry_tid=50339]]], state=connected] 2026-02-18T08:14:34.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:34.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=146, tag_tid=8, entry_tid=50342' 2026-02-18T08:14:34.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=153, tag_tid=9, entry_tid=1' -a 'object_number=153, tag_tid=9, entry_tid=1' = 'object_number=146, tag_tid=8, entry_tid=50342' 2026-02-18T08:14:34.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=146, tag_tid=8, entry_tid=50342' '!=' 'object_number=145, tag_tid=8, entry_tid=50097' 2026-02-18T08:14:34.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:14:34.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=146, tag_tid=8, entry_tid=50342' = 'object_number=145, tag_tid=8, entry_tid=50097' 2026-02-18T08:14:34.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=146, tag_tid=8, entry_tid=50342' 2026-02-18T08:14:34.234 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:34.234 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:14:34.234 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:34.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=9 2026-02-18T08:14:34.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=146, tag_tid=8, entry_tid=50342' 2026-02-18T08:14:34.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:14:34.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:34.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=8 2026-02-18T08:14:34.238 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:34.238 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:14:34.238 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:34.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:14:34.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=146, tag_tid=8, entry_tid=50342' 2026-02-18T08:14:34.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:14:34.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:34.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=50342 2026-02-18T08:14:34.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 9 = 8 -a 1 -le 50342 2026-02-18T08:14:34.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:14:34.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:14:34.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:14:34.443 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:14:34.443 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:14:34.443 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:14:34.443 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:14:34.443 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:14:34.443 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:14:34.443 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:14:34.443 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:14:34.443 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:14:34.443 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:14:34.443 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:14:34.443 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:14:34.443 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:14:34.443 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:14:34.443 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:14:34.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:14:34.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:14:34.446 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:14:34.446 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:14:34.446 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:14:34.446 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:14:34.446 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:14:34.446 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:14:34.446 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:14:34.446 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:14:34.446 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:14:34.446 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:14:34.446 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:14:34.446 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:14:34.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:14:34.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:14:34.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:14:34.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:14:34.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:14:34.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:14:34.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:14:34.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:14:34.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:14:34.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:14:34.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:14:34.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:14:34.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:14:34.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:14:34.448 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:14:34.448 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:14:34.448 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:14:34.448 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:14:34.448 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:14:34.448 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:14:34.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:34.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:34.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:34.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:14:34.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:14:34.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:14:34.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:34.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:34.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:34.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:14:34.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:34.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:34.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:34.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:14:34.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:14:34.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:14:34.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:14:34.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:34.523 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:14:34.523 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:34.557 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 36 2026-02-18T08:14:34.557 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 38 2026-02-18T08:14:34.557 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:14:34.558 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=153, tag_tid=9, entry_tid=1], [object_number=152, tag_tid=9, entry_tid=0], [object_number=154, tag_tid=8, entry_tid=58722], [object_number=155, tag_tid=8, entry_tid=58719]]], state=connected] 2026-02-18T08:14:34.558 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=147, tag_tid=8, entry_tid=50575], [object_number=146, tag_tid=8, entry_tid=50574], [object_number=145, tag_tid=8, entry_tid=50573], [object_number=144, tag_tid=8, entry_tid=50572]]], state=connected] 2026-02-18T08:14:34.561 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:34.562 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:34.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:14:34.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:34.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:34.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:34.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:14:34.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:34.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:34.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:34.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:14:34.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:14:34.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:14:34.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:14:34.565 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:34.566 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:14:34.566 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:34.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 36 2026-02-18T08:14:34.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 38 2026-02-18T08:14:34.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:14:34.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=153, tag_tid=9, entry_tid=1], [object_number=152, tag_tid=9, entry_tid=0], [object_number=154, tag_tid=8, entry_tid=58722], [object_number=155, tag_tid=8, entry_tid=58719]]], state=connected] 2026-02-18T08:14:34.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=147, tag_tid=8, entry_tid=50575], [object_number=146, tag_tid=8, entry_tid=50574], [object_number=145, tag_tid=8, entry_tid=50573], [object_number=144, tag_tid=8, entry_tid=50572]]], state=connected] 2026-02-18T08:14:34.595 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:34.597 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=147, tag_tid=8, entry_tid=50575' 2026-02-18T08:14:34.597 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=153, tag_tid=9, entry_tid=1' -a 'object_number=153, tag_tid=9, entry_tid=1' = 'object_number=147, tag_tid=8, entry_tid=50575' 2026-02-18T08:14:34.597 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=147, tag_tid=8, entry_tid=50575' '!=' 'object_number=146, tag_tid=8, entry_tid=50342' 2026-02-18T08:14:34.597 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:14:34.597 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=147, tag_tid=8, entry_tid=50575' = 'object_number=146, tag_tid=8, entry_tid=50342' 2026-02-18T08:14:34.597 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=147, tag_tid=8, entry_tid=50575' 2026-02-18T08:14:34.597 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:34.597 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:14:34.597 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:34.599 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=9 2026-02-18T08:14:34.600 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=147, tag_tid=8, entry_tid=50575' 2026-02-18T08:14:34.600 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:14:34.600 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:34.601 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=8 2026-02-18T08:14:34.602 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:34.602 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:14:34.602 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:34.603 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:14:34.604 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=147, tag_tid=8, entry_tid=50575' 2026-02-18T08:14:34.604 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:14:34.604 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:34.606 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=50575 2026-02-18T08:14:34.606 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 9 = 8 -a 1 -le 50575 2026-02-18T08:14:34.606 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:14:34.606 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:14:34.606 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:14:34.808 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:14:34.808 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:14:34.808 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:14:34.808 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:14:34.808 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:14:34.808 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:14:34.808 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:14:34.808 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:14:34.808 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:14:34.808 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:14:34.808 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:14:34.808 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:14:34.808 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:14:34.808 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:14:34.808 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:14:34.809 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:14:34.809 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:14:34.811 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:14:34.811 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:14:34.811 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:14:34.811 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:14:34.812 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:14:34.812 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:14:34.812 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:14:34.812 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:14:34.812 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:14:34.812 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:14:34.812 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:14:34.812 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:14:34.812 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:14:34.812 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:14:34.812 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:14:34.812 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:14:34.812 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:14:34.812 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:14:34.812 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:14:34.813 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:14:34.813 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:14:34.813 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:14:34.813 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:14:34.813 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:14:34.813 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:14:34.813 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:14:34.813 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:14:34.813 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:14:34.813 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:14:34.813 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:14:34.813 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:14:34.813 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:14:34.824 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:34.824 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:34.824 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:34.824 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:14:34.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:14:34.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:14:34.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:34.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:34.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:34.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:14:34.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:34.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:34.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:34.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:14:34.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:14:34.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:14:34.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:14:34.894 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:34.895 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:14:34.895 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:34.930 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 36 2026-02-18T08:14:34.930 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 38 2026-02-18T08:14:34.930 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:14:34.930 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=153, tag_tid=9, entry_tid=1], [object_number=152, tag_tid=9, entry_tid=0], [object_number=154, tag_tid=8, entry_tid=58722], [object_number=155, tag_tid=8, entry_tid=58719]]], state=connected] 2026-02-18T08:14:34.930 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=146, tag_tid=8, entry_tid=50802], [object_number=145, tag_tid=8, entry_tid=50801], [object_number=144, tag_tid=8, entry_tid=50800], [object_number=147, tag_tid=8, entry_tid=50799]]], state=connected] 2026-02-18T08:14:34.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:34.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:34.936 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:14:34.936 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:34.936 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:34.936 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:34.936 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:14:34.936 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:34.936 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:34.936 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:34.936 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:14:34.936 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:14:34.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:14:34.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:14:34.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:34.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:14:34.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:34.967 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 36 2026-02-18T08:14:34.967 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 38 2026-02-18T08:14:34.967 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:14:34.967 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=153, tag_tid=9, entry_tid=1], [object_number=152, tag_tid=9, entry_tid=0], [object_number=154, tag_tid=8, entry_tid=58722], [object_number=155, tag_tid=8, entry_tid=58719]]], state=connected] 2026-02-18T08:14:34.967 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=146, tag_tid=8, entry_tid=50802], [object_number=145, tag_tid=8, entry_tid=50801], [object_number=144, tag_tid=8, entry_tid=50800], [object_number=147, tag_tid=8, entry_tid=50799]]], state=connected] 2026-02-18T08:14:34.969 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:34.971 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=146, tag_tid=8, entry_tid=50802' 2026-02-18T08:14:34.971 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=153, tag_tid=9, entry_tid=1' -a 'object_number=153, tag_tid=9, entry_tid=1' = 'object_number=146, tag_tid=8, entry_tid=50802' 2026-02-18T08:14:34.971 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=146, tag_tid=8, entry_tid=50802' '!=' 'object_number=147, tag_tid=8, entry_tid=50575' 2026-02-18T08:14:34.971 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:14:34.971 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=146, tag_tid=8, entry_tid=50802' = 'object_number=147, tag_tid=8, entry_tid=50575' 2026-02-18T08:14:34.971 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=146, tag_tid=8, entry_tid=50802' 2026-02-18T08:14:34.971 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:34.971 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:14:34.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:34.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=9 2026-02-18T08:14:34.974 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=146, tag_tid=8, entry_tid=50802' 2026-02-18T08:14:34.974 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:14:34.974 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:34.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=8 2026-02-18T08:14:34.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:34.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:14:34.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:34.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:14:34.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=146, tag_tid=8, entry_tid=50802' 2026-02-18T08:14:34.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:14:34.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:34.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=50802 2026-02-18T08:14:34.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 9 = 8 -a 1 -le 50802 2026-02-18T08:14:34.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:14:34.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:14:34.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:14:35.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:14:35.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:14:35.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:14:35.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:14:35.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:14:35.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:14:35.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:14:35.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:14:35.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:14:35.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:14:35.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:14:35.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:14:35.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:14:35.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:14:35.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:14:35.183 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:14:35.183 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:14:35.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:14:35.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:14:35.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:14:35.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:14:35.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:14:35.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:14:35.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:14:35.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:14:35.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:14:35.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:14:35.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:14:35.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:14:35.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:14:35.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:14:35.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:14:35.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:14:35.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:14:35.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:14:35.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:14:35.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:14:35.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:14:35.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:14:35.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:14:35.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:14:35.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:14:35.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:14:35.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:14:35.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:14:35.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:14:35.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:14:35.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:14:35.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:14:35.196 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:35.196 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:35.196 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:35.197 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:14:35.257 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:14:35.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:14:35.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:35.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:35.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:35.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:14:35.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:35.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:35.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:35.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:14:35.259 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:14:35.259 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:14:35.259 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:14:35.260 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:35.260 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:14:35.261 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:35.288 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 36 2026-02-18T08:14:35.288 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 38 2026-02-18T08:14:35.288 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:14:35.288 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=153, tag_tid=9, entry_tid=1], [object_number=152, tag_tid=9, entry_tid=0], [object_number=154, tag_tid=8, entry_tid=58722], [object_number=155, tag_tid=8, entry_tid=58719]]], state=connected] 2026-02-18T08:14:35.288 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=145, tag_tid=8, entry_tid=51037], [object_number=144, tag_tid=8, entry_tid=51036], [object_number=147, tag_tid=8, entry_tid=51035], [object_number=146, tag_tid=8, entry_tid=51034]]], state=connected] 2026-02-18T08:14:35.291 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:35.292 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:35.293 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:14:35.293 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:35.293 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:35.293 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:35.293 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:14:35.293 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:35.293 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:35.293 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:35.293 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:14:35.294 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:14:35.294 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:14:35.294 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:14:35.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:35.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:14:35.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:35.322 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 36 2026-02-18T08:14:35.323 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 38 2026-02-18T08:14:35.323 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:14:35.323 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=153, tag_tid=9, entry_tid=1], [object_number=152, tag_tid=9, entry_tid=0], [object_number=154, tag_tid=8, entry_tid=58722], [object_number=155, tag_tid=8, entry_tid=58719]]], state=connected] 2026-02-18T08:14:35.323 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=145, tag_tid=8, entry_tid=51037], [object_number=144, tag_tid=8, entry_tid=51036], [object_number=147, tag_tid=8, entry_tid=51035], [object_number=146, tag_tid=8, entry_tid=51034]]], state=connected] 2026-02-18T08:14:35.325 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:35.327 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=145, tag_tid=8, entry_tid=51037' 2026-02-18T08:14:35.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=153, tag_tid=9, entry_tid=1' -a 'object_number=153, tag_tid=9, entry_tid=1' = 'object_number=145, tag_tid=8, entry_tid=51037' 2026-02-18T08:14:35.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=145, tag_tid=8, entry_tid=51037' '!=' 'object_number=146, tag_tid=8, entry_tid=50802' 2026-02-18T08:14:35.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:14:35.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=145, tag_tid=8, entry_tid=51037' = 'object_number=146, tag_tid=8, entry_tid=50802' 2026-02-18T08:14:35.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=145, tag_tid=8, entry_tid=51037' 2026-02-18T08:14:35.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:35.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:14:35.329 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:35.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=9 2026-02-18T08:14:35.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=145, tag_tid=8, entry_tid=51037' 2026-02-18T08:14:35.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:14:35.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:35.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=8 2026-02-18T08:14:35.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:35.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:14:35.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:35.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:14:35.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=145, tag_tid=8, entry_tid=51037' 2026-02-18T08:14:35.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:14:35.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:35.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=51037 2026-02-18T08:14:35.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 9 = 8 -a 1 -le 51037 2026-02-18T08:14:35.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:14:35.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:14:35.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:14:35.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:14:35.539 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:14:35.539 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:14:35.539 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:14:35.539 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:14:35.539 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:14:35.539 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:14:35.539 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:14:35.539 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:14:35.539 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:14:35.539 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:14:35.539 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:14:35.539 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:14:35.539 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:14:35.539 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:14:35.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:14:35.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:14:35.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:14:35.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:14:35.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:14:35.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:14:35.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:14:35.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:14:35.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:14:35.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:14:35.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:14:35.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:14:35.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:14:35.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:14:35.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:14:35.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:14:35.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:14:35.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:14:35.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:14:35.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:14:35.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:14:35.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:14:35.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:14:35.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:14:35.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:14:35.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:14:35.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:14:35.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:14:35.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:14:35.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:14:35.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:14:35.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:14:35.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:14:35.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:14:35.552 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:35.553 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:35.553 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:35.553 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:14:35.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:14:35.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:14:35.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:35.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:35.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:35.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:14:35.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:35.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:35.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:35.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:14:35.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:14:35.628 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:14:35.628 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:14:35.628 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:35.629 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:14:35.629 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:35.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 36 2026-02-18T08:14:35.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 38 2026-02-18T08:14:35.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:14:35.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=153, tag_tid=9, entry_tid=1], [object_number=152, tag_tid=9, entry_tid=0], [object_number=154, tag_tid=8, entry_tid=58722], [object_number=155, tag_tid=8, entry_tid=58719]]], state=connected] 2026-02-18T08:14:35.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=146, tag_tid=8, entry_tid=51274], [object_number=145, tag_tid=8, entry_tid=51273], [object_number=144, tag_tid=8, entry_tid=51272], [object_number=147, tag_tid=8, entry_tid=51271]]], state=connected] 2026-02-18T08:14:35.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:35.671 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:35.672 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:14:35.672 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:35.672 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:35.672 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:35.672 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:14:35.672 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:35.672 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:35.672 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:35.672 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:14:35.672 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:14:35.673 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:14:35.673 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:14:35.674 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:35.674 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:14:35.674 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:35.702 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 36 2026-02-18T08:14:35.703 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 38 2026-02-18T08:14:35.703 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:14:35.703 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=153, tag_tid=9, entry_tid=1], [object_number=152, tag_tid=9, entry_tid=0], [object_number=154, tag_tid=8, entry_tid=58722], [object_number=155, tag_tid=8, entry_tid=58719]]], state=connected] 2026-02-18T08:14:35.703 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=146, tag_tid=8, entry_tid=51274], [object_number=145, tag_tid=8, entry_tid=51273], [object_number=144, tag_tid=8, entry_tid=51272], [object_number=147, tag_tid=8, entry_tid=51271]]], state=connected] 2026-02-18T08:14:35.705 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:35.706 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=146, tag_tid=8, entry_tid=51274' 2026-02-18T08:14:35.707 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=153, tag_tid=9, entry_tid=1' -a 'object_number=153, tag_tid=9, entry_tid=1' = 'object_number=146, tag_tid=8, entry_tid=51274' 2026-02-18T08:14:35.707 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=146, tag_tid=8, entry_tid=51274' '!=' 'object_number=145, tag_tid=8, entry_tid=51037' 2026-02-18T08:14:35.707 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:14:35.707 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=146, tag_tid=8, entry_tid=51274' = 'object_number=145, tag_tid=8, entry_tid=51037' 2026-02-18T08:14:35.707 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=146, tag_tid=8, entry_tid=51274' 2026-02-18T08:14:35.707 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:35.707 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:14:35.707 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:35.709 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=9 2026-02-18T08:14:35.709 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=146, tag_tid=8, entry_tid=51274' 2026-02-18T08:14:35.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:14:35.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:35.711 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=8 2026-02-18T08:14:35.712 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:35.712 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:14:35.712 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:35.714 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:14:35.714 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=146, tag_tid=8, entry_tid=51274' 2026-02-18T08:14:35.714 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:14:35.714 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:35.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=51274 2026-02-18T08:14:35.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 9 = 8 -a 1 -le 51274 2026-02-18T08:14:35.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:14:35.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:14:35.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:14:35.917 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:14:35.917 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:14:35.917 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:14:35.917 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:14:35.917 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:14:35.917 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:14:35.917 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:14:35.917 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:14:35.917 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:14:35.918 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:14:35.918 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:14:35.918 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:14:35.918 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:14:35.918 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:14:35.918 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:14:35.918 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:14:35.918 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:14:35.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:14:35.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:14:35.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:14:35.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:14:35.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:14:35.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:14:35.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:14:35.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:14:35.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:14:35.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:14:35.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:14:35.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:14:35.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:14:35.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:14:35.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:14:35.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:14:35.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:14:35.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:14:35.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:14:35.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:14:35.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:14:35.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:14:35.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:14:35.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:14:35.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:14:35.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:14:35.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:14:35.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:14:35.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:14:35.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:14:35.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:14:35.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:14:35.932 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:35.933 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:35.933 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:35.933 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:14:35.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:14:35.999 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:14:35.999 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:35.999 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:35.999 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:36.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:14:36.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:36.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:36.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:36.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:14:36.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:14:36.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:14:36.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:14:36.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:36.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:14:36.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:36.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 36 2026-02-18T08:14:36.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 38 2026-02-18T08:14:36.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:14:36.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=153, tag_tid=9, entry_tid=1], [object_number=152, tag_tid=9, entry_tid=0], [object_number=154, tag_tid=8, entry_tid=58722], [object_number=155, tag_tid=8, entry_tid=58719]]], state=connected] 2026-02-18T08:14:36.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=144, tag_tid=8, entry_tid=51532], [object_number=147, tag_tid=8, entry_tid=51531], [object_number=146, tag_tid=8, entry_tid=51530], [object_number=145, tag_tid=8, entry_tid=51529]]], state=connected] 2026-02-18T08:14:36.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:36.035 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:36.036 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:14:36.036 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:36.036 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:36.036 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:36.036 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:14:36.036 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:36.036 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:36.036 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:36.037 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:14:36.037 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:14:36.037 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:14:36.037 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:14:36.038 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:36.038 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:14:36.038 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:36.074 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 36 2026-02-18T08:14:36.074 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 38 2026-02-18T08:14:36.074 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:14:36.074 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=153, tag_tid=9, entry_tid=1], [object_number=152, tag_tid=9, entry_tid=0], [object_number=154, tag_tid=8, entry_tid=58722], [object_number=155, tag_tid=8, entry_tid=58719]]], state=connected] 2026-02-18T08:14:36.074 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=144, tag_tid=8, entry_tid=51532], [object_number=147, tag_tid=8, entry_tid=51531], [object_number=146, tag_tid=8, entry_tid=51530], [object_number=145, tag_tid=8, entry_tid=51529]]], state=connected] 2026-02-18T08:14:36.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:36.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=144, tag_tid=8, entry_tid=51532' 2026-02-18T08:14:36.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=153, tag_tid=9, entry_tid=1' -a 'object_number=153, tag_tid=9, entry_tid=1' = 'object_number=144, tag_tid=8, entry_tid=51532' 2026-02-18T08:14:36.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=144, tag_tid=8, entry_tid=51532' '!=' 'object_number=146, tag_tid=8, entry_tid=51274' 2026-02-18T08:14:36.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:14:36.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=144, tag_tid=8, entry_tid=51532' = 'object_number=146, tag_tid=8, entry_tid=51274' 2026-02-18T08:14:36.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=144, tag_tid=8, entry_tid=51532' 2026-02-18T08:14:36.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:36.080 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:14:36.080 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:36.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=9 2026-02-18T08:14:36.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=144, tag_tid=8, entry_tid=51532' 2026-02-18T08:14:36.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:14:36.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:36.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=8 2026-02-18T08:14:36.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:36.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:14:36.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:36.086 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:14:36.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=144, tag_tid=8, entry_tid=51532' 2026-02-18T08:14:36.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:14:36.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:36.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=51532 2026-02-18T08:14:36.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 9 = 8 -a 1 -le 51532 2026-02-18T08:14:36.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:14:36.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:14:36.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:14:36.290 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:14:36.290 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:14:36.290 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:14:36.290 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:14:36.291 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:14:36.291 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:14:36.291 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:14:36.291 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:14:36.291 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:14:36.291 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:14:36.291 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:14:36.291 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:14:36.291 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:14:36.291 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:14:36.291 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:14:36.291 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:14:36.291 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:14:36.293 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:14:36.293 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:14:36.294 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:14:36.294 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:14:36.294 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:14:36.294 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:14:36.294 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:14:36.294 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:14:36.294 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:14:36.294 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:14:36.294 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:14:36.294 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:14:36.294 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:14:36.294 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:14:36.294 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:14:36.294 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:14:36.294 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:14:36.294 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:14:36.294 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:14:36.294 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:14:36.294 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:14:36.294 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:14:36.294 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:14:36.294 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:14:36.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:14:36.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:14:36.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:14:36.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:14:36.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:14:36.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:14:36.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:14:36.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:14:36.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:36.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:36.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:36.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:14:36.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:14:36.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:14:36.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:36.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:36.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:36.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:14:36.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:36.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:36.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:36.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:14:36.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:14:36.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:14:36.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:14:36.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:36.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:14:36.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:36.585 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 36 2026-02-18T08:14:36.585 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 38 2026-02-18T08:14:36.585 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:14:36.585 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=153, tag_tid=9, entry_tid=1], [object_number=152, tag_tid=9, entry_tid=0], [object_number=154, tag_tid=8, entry_tid=58722], [object_number=155, tag_tid=8, entry_tid=58719]]], state=connected] 2026-02-18T08:14:36.585 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=147, tag_tid=8, entry_tid=51799], [object_number=146, tag_tid=8, entry_tid=51798], [object_number=145, tag_tid=8, entry_tid=51797], [object_number=144, tag_tid=8, entry_tid=51796]]], state=connected] 2026-02-18T08:14:36.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:36.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:36.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:14:36.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:36.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:36.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:36.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:14:36.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:36.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:36.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:36.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:14:36.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:14:36.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:14:36.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:14:36.594 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:36.594 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:14:36.594 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:36.622 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 36 2026-02-18T08:14:36.622 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 38 2026-02-18T08:14:36.622 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:14:36.622 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=153, tag_tid=9, entry_tid=1], [object_number=152, tag_tid=9, entry_tid=0], [object_number=154, tag_tid=8, entry_tid=58722], [object_number=155, tag_tid=8, entry_tid=58719]]], state=connected] 2026-02-18T08:14:36.622 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=147, tag_tid=8, entry_tid=51799], [object_number=146, tag_tid=8, entry_tid=51798], [object_number=145, tag_tid=8, entry_tid=51797], [object_number=144, tag_tid=8, entry_tid=51796]]], state=connected] 2026-02-18T08:14:36.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:36.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=147, tag_tid=8, entry_tid=51799' 2026-02-18T08:14:36.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=153, tag_tid=9, entry_tid=1' -a 'object_number=153, tag_tid=9, entry_tid=1' = 'object_number=147, tag_tid=8, entry_tid=51799' 2026-02-18T08:14:36.628 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=147, tag_tid=8, entry_tid=51799' '!=' 'object_number=144, tag_tid=8, entry_tid=51532' 2026-02-18T08:14:36.628 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:14:36.628 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=147, tag_tid=8, entry_tid=51799' = 'object_number=144, tag_tid=8, entry_tid=51532' 2026-02-18T08:14:36.628 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=147, tag_tid=8, entry_tid=51799' 2026-02-18T08:14:36.628 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:36.628 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:14:36.628 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:36.629 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=9 2026-02-18T08:14:36.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=147, tag_tid=8, entry_tid=51799' 2026-02-18T08:14:36.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:14:36.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:36.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=8 2026-02-18T08:14:36.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:36.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:14:36.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:36.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:14:36.635 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=147, tag_tid=8, entry_tid=51799' 2026-02-18T08:14:36.635 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:14:36.635 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:36.637 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=51799 2026-02-18T08:14:36.637 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 9 = 8 -a 1 -le 51799 2026-02-18T08:14:36.637 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:14:36.637 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:14:36.637 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:14:36.838 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:14:36.838 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:14:36.838 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:14:36.838 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:14:36.838 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:14:36.838 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:14:36.838 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:14:36.839 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:14:36.839 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:14:36.839 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:14:36.839 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:14:36.839 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:14:36.839 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:14:36.839 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:14:36.839 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:14:36.839 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:14:36.839 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:14:36.841 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:14:36.841 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:14:36.842 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:14:36.842 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:14:36.842 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:14:36.842 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:14:36.842 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:14:36.842 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:14:36.842 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:14:36.842 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:14:36.842 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:14:36.842 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:14:36.842 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:14:36.842 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:14:36.842 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:14:36.842 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:14:36.842 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:14:36.843 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:14:36.843 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:14:36.843 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:14:36.843 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:14:36.843 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:14:36.843 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:14:36.843 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:14:36.843 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:14:36.843 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:14:36.843 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:14:36.843 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:14:36.843 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:14:36.843 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:14:36.843 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:14:36.843 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:14:36.853 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:36.853 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:36.853 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:36.853 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:14:36.949 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:14:36.950 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:14:36.950 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:36.950 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:36.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:36.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:14:36.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:36.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:36.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:36.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:14:36.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:14:36.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:14:36.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:14:36.952 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:36.953 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:14:36.953 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:36.982 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 36 2026-02-18T08:14:36.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 38 2026-02-18T08:14:36.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:14:36.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=153, tag_tid=9, entry_tid=1], [object_number=152, tag_tid=9, entry_tid=0], [object_number=154, tag_tid=8, entry_tid=58722], [object_number=155, tag_tid=8, entry_tid=58719]]], state=connected] 2026-02-18T08:14:36.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=149, tag_tid=8, entry_tid=52017], [object_number=144, tag_tid=8, entry_tid=52016], [object_number=147, tag_tid=8, entry_tid=52015], [object_number=146, tag_tid=8, entry_tid=52014]]], state=connected] 2026-02-18T08:14:36.985 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:36.986 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:36.987 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:14:36.987 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:36.987 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:36.987 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:36.987 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:14:36.987 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:36.987 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:36.987 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:36.987 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:14:36.988 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:14:36.988 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:14:36.988 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:14:36.989 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:36.990 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:14:36.990 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:37.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 36 2026-02-18T08:14:37.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 38 2026-02-18T08:14:37.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:14:37.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=153, tag_tid=9, entry_tid=1], [object_number=152, tag_tid=9, entry_tid=0], [object_number=154, tag_tid=8, entry_tid=58722], [object_number=155, tag_tid=8, entry_tid=58719]]], state=connected] 2026-02-18T08:14:37.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=149, tag_tid=8, entry_tid=52017], [object_number=144, tag_tid=8, entry_tid=52016], [object_number=147, tag_tid=8, entry_tid=52015], [object_number=146, tag_tid=8, entry_tid=52014]]], state=connected] 2026-02-18T08:14:37.019 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:37.021 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=149, tag_tid=8, entry_tid=52017' 2026-02-18T08:14:37.021 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=153, tag_tid=9, entry_tid=1' -a 'object_number=153, tag_tid=9, entry_tid=1' = 'object_number=149, tag_tid=8, entry_tid=52017' 2026-02-18T08:14:37.021 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=149, tag_tid=8, entry_tid=52017' '!=' 'object_number=147, tag_tid=8, entry_tid=51799' 2026-02-18T08:14:37.021 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:14:37.021 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=149, tag_tid=8, entry_tid=52017' = 'object_number=147, tag_tid=8, entry_tid=51799' 2026-02-18T08:14:37.021 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=149, tag_tid=8, entry_tid=52017' 2026-02-18T08:14:37.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:37.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:14:37.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:37.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=9 2026-02-18T08:14:37.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=149, tag_tid=8, entry_tid=52017' 2026-02-18T08:14:37.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:14:37.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:37.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=8 2026-02-18T08:14:37.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:37.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:14:37.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:37.028 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:14:37.028 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=149, tag_tid=8, entry_tid=52017' 2026-02-18T08:14:37.029 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:14:37.029 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:37.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=52017 2026-02-18T08:14:37.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 9 = 8 -a 1 -le 52017 2026-02-18T08:14:37.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:14:37.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:14:37.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:14:37.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:14:37.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:14:37.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:14:37.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:14:37.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:14:37.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:14:37.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:14:37.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:14:37.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:14:37.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:14:37.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:14:37.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:14:37.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:14:37.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:14:37.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:14:37.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:14:37.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:14:37.234 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:14:37.234 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:14:37.234 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:14:37.234 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:14:37.234 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:14:37.234 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:14:37.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:14:37.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:14:37.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:14:37.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:14:37.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:14:37.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:14:37.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:14:37.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:14:37.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:14:37.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:14:37.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:14:37.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:14:37.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:14:37.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:14:37.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:14:37.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:14:37.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:14:37.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:14:37.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:14:37.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:14:37.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:14:37.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:14:37.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:14:37.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:14:37.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:14:37.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:14:37.247 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:37.247 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:37.247 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:37.247 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:14:37.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:14:37.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:14:37.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:37.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:37.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:37.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:14:37.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:37.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:37.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:37.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:14:37.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:14:37.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:14:37.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:14:37.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:37.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:14:37.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:37.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 37 2026-02-18T08:14:37.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 38 2026-02-18T08:14:37.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:14:37.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=153, tag_tid=9, entry_tid=1], [object_number=152, tag_tid=9, entry_tid=0], [object_number=154, tag_tid=8, entry_tid=58722], [object_number=155, tag_tid=8, entry_tid=58719]]], state=connected] 2026-02-18T08:14:37.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=149, tag_tid=8, entry_tid=52209], [object_number=148, tag_tid=8, entry_tid=52208], [object_number=151, tag_tid=8, entry_tid=52207], [object_number=150, tag_tid=8, entry_tid=52206]]], state=connected] 2026-02-18T08:14:37.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:37.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:37.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:14:37.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:37.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:37.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:37.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:14:37.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:37.344 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:37.344 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:37.344 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:14:37.344 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:14:37.344 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:14:37.344 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:14:37.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:37.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:14:37.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:37.373 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 37 2026-02-18T08:14:37.373 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 38 2026-02-18T08:14:37.373 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:14:37.373 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=153, tag_tid=9, entry_tid=1], [object_number=152, tag_tid=9, entry_tid=0], [object_number=154, tag_tid=8, entry_tid=58722], [object_number=155, tag_tid=8, entry_tid=58719]]], state=connected] 2026-02-18T08:14:37.373 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=149, tag_tid=8, entry_tid=52209], [object_number=148, tag_tid=8, entry_tid=52208], [object_number=151, tag_tid=8, entry_tid=52207], [object_number=150, tag_tid=8, entry_tid=52206]]], state=connected] 2026-02-18T08:14:37.376 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:37.377 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=149, tag_tid=8, entry_tid=52209' 2026-02-18T08:14:37.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=153, tag_tid=9, entry_tid=1' -a 'object_number=153, tag_tid=9, entry_tid=1' = 'object_number=149, tag_tid=8, entry_tid=52209' 2026-02-18T08:14:37.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=149, tag_tid=8, entry_tid=52209' '!=' 'object_number=149, tag_tid=8, entry_tid=52017' 2026-02-18T08:14:37.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:14:37.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=149, tag_tid=8, entry_tid=52209' = 'object_number=149, tag_tid=8, entry_tid=52017' 2026-02-18T08:14:37.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=149, tag_tid=8, entry_tid=52209' 2026-02-18T08:14:37.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:37.379 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:14:37.379 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:37.380 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=9 2026-02-18T08:14:37.380 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=149, tag_tid=8, entry_tid=52209' 2026-02-18T08:14:37.381 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:14:37.381 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:37.382 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=8 2026-02-18T08:14:37.382 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:37.383 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:14:37.383 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:37.384 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:14:37.385 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=149, tag_tid=8, entry_tid=52209' 2026-02-18T08:14:37.385 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:14:37.385 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:37.387 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=52209 2026-02-18T08:14:37.387 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 9 = 8 -a 1 -le 52209 2026-02-18T08:14:37.387 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:14:37.387 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:14:37.387 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:14:37.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:14:37.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:14:37.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:14:37.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:14:37.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:14:37.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:14:37.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:14:37.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:14:37.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:14:37.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:14:37.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:14:37.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:14:37.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:14:37.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:14:37.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:14:37.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:14:37.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:14:37.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:14:37.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:14:37.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:14:37.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:14:37.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:14:37.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:14:37.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:14:37.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:14:37.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:14:37.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:14:37.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:14:37.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:14:37.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:14:37.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:14:37.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:14:37.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:14:37.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:14:37.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:14:37.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:14:37.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:14:37.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:14:37.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:14:37.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:14:37.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:14:37.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:14:37.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:14:37.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:14:37.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:14:37.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:14:37.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:14:37.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:14:37.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:14:37.603 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:37.604 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:37.604 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:37.604 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:14:37.664 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:14:37.664 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:14:37.664 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:37.664 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:37.664 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:37.664 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:14:37.664 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:37.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:37.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:37.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:14:37.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:14:37.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:14:37.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:14:37.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:37.667 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:14:37.667 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:37.694 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 37 2026-02-18T08:14:37.694 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 38 2026-02-18T08:14:37.694 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:14:37.694 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=153, tag_tid=9, entry_tid=1], [object_number=152, tag_tid=9, entry_tid=0], [object_number=154, tag_tid=8, entry_tid=58722], [object_number=155, tag_tid=8, entry_tid=58719]]], state=connected] 2026-02-18T08:14:37.694 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=148, tag_tid=8, entry_tid=52432], [object_number=151, tag_tid=8, entry_tid=52431], [object_number=150, tag_tid=8, entry_tid=52430], [object_number=149, tag_tid=8, entry_tid=52429]]], state=connected] 2026-02-18T08:14:37.697 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:37.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:37.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:14:37.700 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:37.700 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:37.700 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:37.700 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:14:37.700 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:37.700 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:37.700 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:37.700 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:14:37.700 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:14:37.700 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:14:37.700 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:14:37.702 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:37.702 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:14:37.702 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:37.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 37 2026-02-18T08:14:37.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 38 2026-02-18T08:14:37.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:14:37.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=153, tag_tid=9, entry_tid=1], [object_number=152, tag_tid=9, entry_tid=0], [object_number=154, tag_tid=8, entry_tid=58722], [object_number=155, tag_tid=8, entry_tid=58719]]], state=connected] 2026-02-18T08:14:37.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=148, tag_tid=8, entry_tid=52432], [object_number=151, tag_tid=8, entry_tid=52431], [object_number=150, tag_tid=8, entry_tid=52430], [object_number=149, tag_tid=8, entry_tid=52429]]], state=connected] 2026-02-18T08:14:37.732 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:37.734 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=148, tag_tid=8, entry_tid=52432' 2026-02-18T08:14:37.734 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=153, tag_tid=9, entry_tid=1' -a 'object_number=153, tag_tid=9, entry_tid=1' = 'object_number=148, tag_tid=8, entry_tid=52432' 2026-02-18T08:14:37.734 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=148, tag_tid=8, entry_tid=52432' '!=' 'object_number=149, tag_tid=8, entry_tid=52209' 2026-02-18T08:14:37.734 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:14:37.734 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=148, tag_tid=8, entry_tid=52432' = 'object_number=149, tag_tid=8, entry_tid=52209' 2026-02-18T08:14:37.734 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=148, tag_tid=8, entry_tid=52432' 2026-02-18T08:14:37.735 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:37.735 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:14:37.735 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:37.736 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=9 2026-02-18T08:14:37.737 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=148, tag_tid=8, entry_tid=52432' 2026-02-18T08:14:37.737 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:14:37.737 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:37.739 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=8 2026-02-18T08:14:37.739 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:37.739 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:14:37.739 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:37.741 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:14:37.742 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=148, tag_tid=8, entry_tid=52432' 2026-02-18T08:14:37.742 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:14:37.742 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:37.743 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=52432 2026-02-18T08:14:37.743 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 9 = 8 -a 1 -le 52432 2026-02-18T08:14:37.743 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:14:37.744 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:14:37.744 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:14:37.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:14:37.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:14:37.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:14:37.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:14:37.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:14:37.946 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:14:37.946 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:14:37.946 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:14:37.946 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:14:37.946 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:14:37.946 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:14:37.946 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:14:37.946 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:14:37.946 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:14:37.946 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:14:37.946 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:14:37.947 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:14:37.948 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:14:37.948 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:14:37.948 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:14:37.948 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:14:37.948 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:14:37.948 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:14:37.949 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:14:37.949 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:14:37.949 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:14:37.949 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:14:37.949 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:14:37.949 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:14:37.949 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:14:37.949 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:14:37.949 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:14:37.949 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:14:37.949 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:14:37.949 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:14:37.949 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:14:37.949 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:14:37.949 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:14:37.949 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:14:37.949 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:14:37.949 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:14:37.949 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:14:37.949 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:14:37.949 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:14:37.949 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:14:37.950 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:14:37.950 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:14:37.950 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:14:37.950 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:14:37.959 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:37.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:37.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:37.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:14:38.028 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:14:38.029 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:14:38.029 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:38.029 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:38.029 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:38.029 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:14:38.029 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:38.029 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:38.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:38.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:14:38.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:14:38.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:14:38.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:14:38.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:38.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:14:38.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:38.058 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 37 2026-02-18T08:14:38.058 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 38 2026-02-18T08:14:38.058 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:14:38.058 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=153, tag_tid=9, entry_tid=1], [object_number=152, tag_tid=9, entry_tid=0], [object_number=154, tag_tid=8, entry_tid=58722], [object_number=155, tag_tid=8, entry_tid=58719]]], state=connected] 2026-02-18T08:14:38.058 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=148, tag_tid=8, entry_tid=52680], [object_number=151, tag_tid=8, entry_tid=52679], [object_number=150, tag_tid=8, entry_tid=52678], [object_number=149, tag_tid=8, entry_tid=52677]]], state=connected] 2026-02-18T08:14:38.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:38.062 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:38.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:14:38.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:38.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:38.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:38.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:14:38.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:38.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:38.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:38.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:14:38.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:14:38.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:14:38.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:14:38.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:38.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:14:38.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:38.093 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 37 2026-02-18T08:14:38.093 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 38 2026-02-18T08:14:38.093 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:14:38.093 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=153, tag_tid=9, entry_tid=1], [object_number=152, tag_tid=9, entry_tid=0], [object_number=154, tag_tid=8, entry_tid=58722], [object_number=155, tag_tid=8, entry_tid=58719]]], state=connected] 2026-02-18T08:14:38.094 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=148, tag_tid=8, entry_tid=52680], [object_number=151, tag_tid=8, entry_tid=52679], [object_number=150, tag_tid=8, entry_tid=52678], [object_number=149, tag_tid=8, entry_tid=52677]]], state=connected] 2026-02-18T08:14:38.096 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:38.097 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=148, tag_tid=8, entry_tid=52680' 2026-02-18T08:14:38.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=153, tag_tid=9, entry_tid=1' -a 'object_number=153, tag_tid=9, entry_tid=1' = 'object_number=148, tag_tid=8, entry_tid=52680' 2026-02-18T08:14:38.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=148, tag_tid=8, entry_tid=52680' '!=' 'object_number=148, tag_tid=8, entry_tid=52432' 2026-02-18T08:14:38.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:14:38.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=148, tag_tid=8, entry_tid=52680' = 'object_number=148, tag_tid=8, entry_tid=52432' 2026-02-18T08:14:38.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=148, tag_tid=8, entry_tid=52680' 2026-02-18T08:14:38.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:38.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:14:38.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:38.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=9 2026-02-18T08:14:38.100 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=148, tag_tid=8, entry_tid=52680' 2026-02-18T08:14:38.100 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:14:38.100 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:38.102 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=8 2026-02-18T08:14:38.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:38.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:14:38.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:38.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:14:38.105 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=148, tag_tid=8, entry_tid=52680' 2026-02-18T08:14:38.105 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:14:38.105 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:38.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=52680 2026-02-18T08:14:38.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 9 = 8 -a 1 -le 52680 2026-02-18T08:14:38.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:14:38.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:14:38.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:14:38.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:14:38.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:14:38.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:14:38.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:14:38.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:14:38.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:14:38.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:14:38.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:14:38.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:14:38.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:14:38.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:14:38.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:14:38.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:14:38.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:14:38.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:14:38.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:14:38.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:14:38.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:14:38.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:14:38.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:14:38.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:14:38.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:14:38.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:14:38.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:14:38.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:14:38.312 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:14:38.312 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:14:38.312 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:14:38.312 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:14:38.312 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:14:38.312 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:14:38.312 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:14:38.312 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:14:38.312 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:14:38.312 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:14:38.312 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:14:38.312 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:14:38.312 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:14:38.312 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:14:38.312 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:14:38.312 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:14:38.312 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:14:38.312 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:14:38.312 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:14:38.312 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:14:38.312 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:14:38.312 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:14:38.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:14:38.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:14:38.323 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:38.323 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:38.323 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:38.323 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:14:38.384 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:14:38.384 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:14:38.384 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:38.385 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:38.385 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:38.385 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:14:38.385 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:38.385 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:38.385 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:38.385 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:14:38.385 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:14:38.385 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:14:38.385 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:14:38.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:38.387 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:14:38.387 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:38.414 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 37 2026-02-18T08:14:38.414 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 38 2026-02-18T08:14:38.414 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:14:38.414 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=153, tag_tid=9, entry_tid=1], [object_number=152, tag_tid=9, entry_tid=0], [object_number=154, tag_tid=8, entry_tid=58722], [object_number=155, tag_tid=8, entry_tid=58719]]], state=connected] 2026-02-18T08:14:38.414 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=150, tag_tid=8, entry_tid=52942], [object_number=149, tag_tid=8, entry_tid=52941], [object_number=148, tag_tid=8, entry_tid=52940], [object_number=151, tag_tid=8, entry_tid=52939]]], state=connected] 2026-02-18T08:14:38.416 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:38.418 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:38.419 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:14:38.419 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:38.419 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:38.419 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:38.419 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:14:38.419 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:38.419 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:38.419 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:38.419 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:14:38.420 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:14:38.420 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:14:38.420 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:14:38.421 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:38.421 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:14:38.421 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:38.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 37 2026-02-18T08:14:38.448 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 38 2026-02-18T08:14:38.448 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:14:38.448 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=153, tag_tid=9, entry_tid=1], [object_number=152, tag_tid=9, entry_tid=0], [object_number=154, tag_tid=8, entry_tid=58722], [object_number=155, tag_tid=8, entry_tid=58719]]], state=connected] 2026-02-18T08:14:38.448 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=150, tag_tid=8, entry_tid=52942], [object_number=149, tag_tid=8, entry_tid=52941], [object_number=148, tag_tid=8, entry_tid=52940], [object_number=151, tag_tid=8, entry_tid=52939]]], state=connected] 2026-02-18T08:14:38.450 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:38.452 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=150, tag_tid=8, entry_tid=52942' 2026-02-18T08:14:38.452 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=153, tag_tid=9, entry_tid=1' -a 'object_number=153, tag_tid=9, entry_tid=1' = 'object_number=150, tag_tid=8, entry_tid=52942' 2026-02-18T08:14:38.452 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=150, tag_tid=8, entry_tid=52942' '!=' 'object_number=148, tag_tid=8, entry_tid=52680' 2026-02-18T08:14:38.452 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:14:38.452 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=150, tag_tid=8, entry_tid=52942' = 'object_number=148, tag_tid=8, entry_tid=52680' 2026-02-18T08:14:38.452 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=150, tag_tid=8, entry_tid=52942' 2026-02-18T08:14:38.453 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:38.453 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:14:38.453 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:38.454 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=9 2026-02-18T08:14:38.455 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=150, tag_tid=8, entry_tid=52942' 2026-02-18T08:14:38.455 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:14:38.455 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:38.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=8 2026-02-18T08:14:38.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:38.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:14:38.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:38.458 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:14:38.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=150, tag_tid=8, entry_tid=52942' 2026-02-18T08:14:38.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:14:38.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:38.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=52942 2026-02-18T08:14:38.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 9 = 8 -a 1 -le 52942 2026-02-18T08:14:38.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:14:38.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:14:38.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:14:38.663 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:14:38.663 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:14:38.663 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:14:38.663 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:14:38.663 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:14:38.663 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:14:38.663 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:14:38.663 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:14:38.663 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:14:38.663 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:14:38.663 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:14:38.663 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:14:38.663 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:14:38.664 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:14:38.664 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:14:38.664 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:14:38.664 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:14:38.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:14:38.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:14:38.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:14:38.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:14:38.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:14:38.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:14:38.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:14:38.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:14:38.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:14:38.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:14:38.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:14:38.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:14:38.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:14:38.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:14:38.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:14:38.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:14:38.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:14:38.667 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:14:38.667 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:14:38.667 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:14:38.667 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:14:38.667 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:14:38.667 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:14:38.667 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:14:38.667 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:14:38.667 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:14:38.667 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:14:38.667 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:14:38.667 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:14:38.667 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:14:38.667 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:14:38.667 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:14:38.679 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:38.679 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:38.679 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:38.679 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:14:38.740 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:14:38.740 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:14:38.740 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:38.740 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:38.740 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:38.740 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:14:38.740 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:38.741 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:38.741 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:38.741 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:14:38.741 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:14:38.741 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:14:38.741 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:14:38.742 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:38.743 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:14:38.743 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:38.769 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 37 2026-02-18T08:14:38.769 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 38 2026-02-18T08:14:38.770 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:14:38.770 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=153, tag_tid=9, entry_tid=1], [object_number=152, tag_tid=9, entry_tid=0], [object_number=154, tag_tid=8, entry_tid=58722], [object_number=155, tag_tid=8, entry_tid=58719]]], state=connected] 2026-02-18T08:14:38.770 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=150, tag_tid=8, entry_tid=53206], [object_number=149, tag_tid=8, entry_tid=53205], [object_number=148, tag_tid=8, entry_tid=53204], [object_number=151, tag_tid=8, entry_tid=53203]]], state=connected] 2026-02-18T08:14:38.772 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:38.774 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:38.774 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:14:38.774 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:38.774 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:38.774 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:38.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:14:38.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:38.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:38.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:38.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:14:38.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:14:38.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:14:38.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:14:38.777 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:38.777 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:14:38.777 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:38.803 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 37 2026-02-18T08:14:38.803 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 38 2026-02-18T08:14:38.803 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:14:38.803 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=153, tag_tid=9, entry_tid=1], [object_number=152, tag_tid=9, entry_tid=0], [object_number=154, tag_tid=8, entry_tid=58722], [object_number=155, tag_tid=8, entry_tid=58719]]], state=connected] 2026-02-18T08:14:38.804 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=150, tag_tid=8, entry_tid=53206], [object_number=149, tag_tid=8, entry_tid=53205], [object_number=148, tag_tid=8, entry_tid=53204], [object_number=151, tag_tid=8, entry_tid=53203]]], state=connected] 2026-02-18T08:14:38.806 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:38.807 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=150, tag_tid=8, entry_tid=53206' 2026-02-18T08:14:38.807 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=153, tag_tid=9, entry_tid=1' -a 'object_number=153, tag_tid=9, entry_tid=1' = 'object_number=150, tag_tid=8, entry_tid=53206' 2026-02-18T08:14:38.807 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=150, tag_tid=8, entry_tid=53206' '!=' 'object_number=150, tag_tid=8, entry_tid=52942' 2026-02-18T08:14:38.807 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:14:38.807 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=150, tag_tid=8, entry_tid=53206' = 'object_number=150, tag_tid=8, entry_tid=52942' 2026-02-18T08:14:38.807 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=150, tag_tid=8, entry_tid=53206' 2026-02-18T08:14:38.808 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:38.808 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:14:38.808 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:38.809 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=9 2026-02-18T08:14:38.810 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=150, tag_tid=8, entry_tid=53206' 2026-02-18T08:14:38.810 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:14:38.811 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:38.812 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=8 2026-02-18T08:14:38.813 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:38.813 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:14:38.813 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:38.814 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:14:38.815 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=150, tag_tid=8, entry_tid=53206' 2026-02-18T08:14:38.815 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:14:38.815 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:38.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=53206 2026-02-18T08:14:38.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 9 = 8 -a 1 -le 53206 2026-02-18T08:14:38.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:14:38.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:14:38.817 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:14:39.018 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:14:39.018 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:14:39.018 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:14:39.018 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:14:39.018 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:14:39.018 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:14:39.018 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:14:39.018 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:14:39.018 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:14:39.018 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:14:39.019 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:14:39.019 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:14:39.019 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:14:39.019 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:14:39.019 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:14:39.021 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:14:39.021 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:14:39.021 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:14:39.021 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:14:39.021 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:14:39.021 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:14:39.021 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:14:39.021 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:14:39.021 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:14:39.021 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:14:39.021 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:14:39.021 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:14:39.021 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:14:39.021 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:14:39.021 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:14:39.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:14:39.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:14:39.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:14:39.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:14:39.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:14:39.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:14:39.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:14:39.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:14:39.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:14:39.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:14:39.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:14:39.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:14:39.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:14:39.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:14:39.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:14:39.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:14:39.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:14:39.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:14:39.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:14:39.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:39.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:39.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:39.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:14:39.094 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:14:39.095 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:14:39.095 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:39.095 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:39.095 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:39.095 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:14:39.095 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:39.095 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:39.095 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:39.095 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:14:39.095 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:14:39.095 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:14:39.095 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:14:39.097 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:39.097 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:14:39.097 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:39.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 37 2026-02-18T08:14:39.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 38 2026-02-18T08:14:39.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:14:39.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=153, tag_tid=9, entry_tid=1], [object_number=152, tag_tid=9, entry_tid=0], [object_number=154, tag_tid=8, entry_tid=58722], [object_number=155, tag_tid=8, entry_tid=58719]]], state=connected] 2026-02-18T08:14:39.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=148, tag_tid=8, entry_tid=53440], [object_number=151, tag_tid=8, entry_tid=53439], [object_number=150, tag_tid=8, entry_tid=53438], [object_number=149, tag_tid=8, entry_tid=53437]]], state=connected] 2026-02-18T08:14:39.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:39.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:39.130 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:14:39.130 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:39.130 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:39.130 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:39.130 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:14:39.130 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:39.131 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:39.131 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:39.131 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:14:39.131 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:14:39.131 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:14:39.131 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:14:39.132 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:39.132 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:14:39.132 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:39.158 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 37 2026-02-18T08:14:39.163 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 38 2026-02-18T08:14:39.163 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:14:39.163 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=153, tag_tid=9, entry_tid=1], [object_number=152, tag_tid=9, entry_tid=0], [object_number=154, tag_tid=8, entry_tid=58722], [object_number=155, tag_tid=8, entry_tid=58719]]], state=connected] 2026-02-18T08:14:39.163 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=148, tag_tid=8, entry_tid=53440], [object_number=151, tag_tid=8, entry_tid=53439], [object_number=150, tag_tid=8, entry_tid=53438], [object_number=149, tag_tid=8, entry_tid=53437]]], state=connected] 2026-02-18T08:14:39.165 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:39.165 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=148, tag_tid=8, entry_tid=53440' 2026-02-18T08:14:39.165 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=153, tag_tid=9, entry_tid=1' -a 'object_number=153, tag_tid=9, entry_tid=1' = 'object_number=148, tag_tid=8, entry_tid=53440' 2026-02-18T08:14:39.165 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=148, tag_tid=8, entry_tid=53440' '!=' 'object_number=150, tag_tid=8, entry_tid=53206' 2026-02-18T08:14:39.165 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:14:39.166 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=148, tag_tid=8, entry_tid=53440' = 'object_number=150, tag_tid=8, entry_tid=53206' 2026-02-18T08:14:39.166 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=148, tag_tid=8, entry_tid=53440' 2026-02-18T08:14:39.166 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:39.166 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:14:39.166 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:39.166 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=9 2026-02-18T08:14:39.167 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=148, tag_tid=8, entry_tid=53440' 2026-02-18T08:14:39.167 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:14:39.167 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:39.167 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=8 2026-02-18T08:14:39.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:39.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:14:39.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:39.169 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:14:39.170 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=148, tag_tid=8, entry_tid=53440' 2026-02-18T08:14:39.170 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:14:39.170 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:39.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=53440 2026-02-18T08:14:39.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 9 = 8 -a 1 -le 53440 2026-02-18T08:14:39.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:14:39.173 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:14:39.173 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:14:39.374 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:14:39.374 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:14:39.374 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:14:39.374 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:14:39.374 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:14:39.374 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:14:39.375 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:14:39.375 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:14:39.375 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:14:39.375 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:14:39.375 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:14:39.375 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:14:39.375 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:14:39.375 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:14:39.375 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:14:39.375 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:14:39.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:14:39.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:14:39.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:14:39.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:14:39.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:14:39.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:14:39.379 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:14:39.379 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:14:39.379 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:14:39.379 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:14:39.379 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:14:39.379 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:14:39.379 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:14:39.379 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:14:39.379 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:14:39.379 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:14:39.379 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:14:39.379 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:14:39.380 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:14:39.380 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:14:39.380 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:14:39.380 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:14:39.380 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:14:39.380 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:14:39.380 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:14:39.380 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:14:39.380 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:14:39.380 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:14:39.380 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:14:39.380 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:14:39.381 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:14:39.381 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:14:39.381 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:14:39.389 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:39.389 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:39.389 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:39.389 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:14:39.450 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:14:39.451 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:14:39.451 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:39.451 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:39.452 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:39.452 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:14:39.452 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:39.452 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:39.452 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:39.452 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:14:39.452 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:14:39.452 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:14:39.452 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:14:39.453 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:39.453 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:14:39.453 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:39.480 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 37 2026-02-18T08:14:39.480 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 38 2026-02-18T08:14:39.480 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:14:39.480 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=153, tag_tid=9, entry_tid=1], [object_number=152, tag_tid=9, entry_tid=0], [object_number=154, tag_tid=8, entry_tid=58722], [object_number=155, tag_tid=8, entry_tid=58719]]], state=connected] 2026-02-18T08:14:39.481 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=150, tag_tid=8, entry_tid=53718], [object_number=149, tag_tid=8, entry_tid=53717], [object_number=148, tag_tid=8, entry_tid=53716], [object_number=151, tag_tid=8, entry_tid=53715]]], state=connected] 2026-02-18T08:14:39.483 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:39.484 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:39.485 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:14:39.485 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:39.486 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:39.486 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:39.486 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:14:39.486 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:39.486 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:39.486 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:39.486 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:14:39.486 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:14:39.487 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:14:39.487 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:14:39.487 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:39.488 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:14:39.488 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:39.514 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 37 2026-02-18T08:14:39.514 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 38 2026-02-18T08:14:39.514 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:14:39.514 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=153, tag_tid=9, entry_tid=1], [object_number=152, tag_tid=9, entry_tid=0], [object_number=154, tag_tid=8, entry_tid=58722], [object_number=155, tag_tid=8, entry_tid=58719]]], state=connected] 2026-02-18T08:14:39.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=150, tag_tid=8, entry_tid=53718], [object_number=149, tag_tid=8, entry_tid=53717], [object_number=148, tag_tid=8, entry_tid=53716], [object_number=151, tag_tid=8, entry_tid=53715]]], state=connected] 2026-02-18T08:14:39.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:39.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=150, tag_tid=8, entry_tid=53718' 2026-02-18T08:14:39.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=153, tag_tid=9, entry_tid=1' -a 'object_number=153, tag_tid=9, entry_tid=1' = 'object_number=150, tag_tid=8, entry_tid=53718' 2026-02-18T08:14:39.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=150, tag_tid=8, entry_tid=53718' '!=' 'object_number=148, tag_tid=8, entry_tid=53440' 2026-02-18T08:14:39.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:14:39.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=150, tag_tid=8, entry_tid=53718' = 'object_number=148, tag_tid=8, entry_tid=53440' 2026-02-18T08:14:39.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=150, tag_tid=8, entry_tid=53718' 2026-02-18T08:14:39.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:39.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:14:39.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:39.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=9 2026-02-18T08:14:39.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=150, tag_tid=8, entry_tid=53718' 2026-02-18T08:14:39.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:14:39.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:39.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=8 2026-02-18T08:14:39.523 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:39.523 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:14:39.523 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:39.524 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:14:39.525 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=150, tag_tid=8, entry_tid=53718' 2026-02-18T08:14:39.525 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:14:39.526 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:39.527 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=53718 2026-02-18T08:14:39.527 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 9 = 8 -a 1 -le 53718 2026-02-18T08:14:39.527 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:14:39.527 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:14:39.527 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:14:39.729 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:14:39.729 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:14:39.729 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:14:39.729 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:14:39.729 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:14:39.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:14:39.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:14:39.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:14:39.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:14:39.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:14:39.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:14:39.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:14:39.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:14:39.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:14:39.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:14:39.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:14:39.731 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:14:39.733 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:14:39.733 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:14:39.733 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:14:39.733 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:14:39.733 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:14:39.733 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:14:39.733 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:14:39.734 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:14:39.734 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:14:39.734 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:14:39.734 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:14:39.734 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:14:39.734 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:14:39.734 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:14:39.734 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:14:39.734 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:14:39.734 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:14:39.734 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:14:39.734 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:14:39.735 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:14:39.735 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:14:39.735 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:14:39.735 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:14:39.735 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:14:39.735 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:14:39.735 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:14:39.735 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:14:39.735 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:14:39.735 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:14:39.735 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:14:39.735 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:14:39.735 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:14:39.743 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:39.744 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:39.744 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:39.744 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:14:39.805 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:14:39.806 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:14:39.806 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:39.806 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:39.806 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:39.806 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:14:39.806 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:39.806 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:39.806 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:39.806 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:14:39.806 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:14:39.807 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:14:39.807 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:14:39.807 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:39.808 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:14:39.808 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:39.836 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 37 2026-02-18T08:14:39.836 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 38 2026-02-18T08:14:39.836 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:14:39.836 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=153, tag_tid=9, entry_tid=1], [object_number=152, tag_tid=9, entry_tid=0], [object_number=154, tag_tid=8, entry_tid=58722], [object_number=155, tag_tid=8, entry_tid=58719]]], state=connected] 2026-02-18T08:14:39.837 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=149, tag_tid=8, entry_tid=53949], [object_number=148, tag_tid=8, entry_tid=53948], [object_number=151, tag_tid=8, entry_tid=53947], [object_number=150, tag_tid=8, entry_tid=53946]]], state=connected] 2026-02-18T08:14:39.839 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:39.840 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:39.841 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:14:39.841 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:39.841 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:39.841 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:39.841 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:14:39.842 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:39.842 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:39.842 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:39.842 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:14:39.842 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:14:39.842 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:14:39.843 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:14:39.843 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:39.843 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:14:39.844 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:39.870 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 37 2026-02-18T08:14:39.870 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 38 2026-02-18T08:14:39.870 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:14:39.870 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=153, tag_tid=9, entry_tid=1], [object_number=152, tag_tid=9, entry_tid=0], [object_number=154, tag_tid=8, entry_tid=58722], [object_number=155, tag_tid=8, entry_tid=58719]]], state=connected] 2026-02-18T08:14:39.870 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=149, tag_tid=8, entry_tid=53949], [object_number=148, tag_tid=8, entry_tid=53948], [object_number=151, tag_tid=8, entry_tid=53947], [object_number=150, tag_tid=8, entry_tid=53946]]], state=connected] 2026-02-18T08:14:39.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:39.874 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=149, tag_tid=8, entry_tid=53949' 2026-02-18T08:14:39.875 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=153, tag_tid=9, entry_tid=1' -a 'object_number=153, tag_tid=9, entry_tid=1' = 'object_number=149, tag_tid=8, entry_tid=53949' 2026-02-18T08:14:39.875 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=149, tag_tid=8, entry_tid=53949' '!=' 'object_number=150, tag_tid=8, entry_tid=53718' 2026-02-18T08:14:39.875 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:14:39.875 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=149, tag_tid=8, entry_tid=53949' = 'object_number=150, tag_tid=8, entry_tid=53718' 2026-02-18T08:14:39.875 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=149, tag_tid=8, entry_tid=53949' 2026-02-18T08:14:39.876 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:39.876 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:14:39.876 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:39.877 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=9 2026-02-18T08:14:39.877 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=149, tag_tid=8, entry_tid=53949' 2026-02-18T08:14:39.878 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:14:39.878 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:39.879 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=8 2026-02-18T08:14:39.880 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:39.880 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:14:39.880 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:39.881 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:14:39.882 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=149, tag_tid=8, entry_tid=53949' 2026-02-18T08:14:39.882 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:14:39.882 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:39.884 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=53949 2026-02-18T08:14:39.884 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 9 = 8 -a 1 -le 53949 2026-02-18T08:14:39.884 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:14:39.884 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:14:39.884 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:14:40.086 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:14:40.086 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:14:40.086 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:14:40.086 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:14:40.086 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:14:40.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:14:40.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:14:40.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:14:40.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:14:40.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:14:40.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:14:40.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:14:40.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:14:40.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:14:40.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:14:40.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:14:40.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:14:40.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:14:40.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:14:40.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:14:40.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:14:40.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:14:40.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:14:40.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:14:40.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:14:40.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:14:40.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:14:40.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:14:40.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:14:40.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:14:40.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:14:40.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:14:40.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:14:40.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:14:40.092 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:14:40.092 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:14:40.092 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:14:40.092 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:14:40.092 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:14:40.092 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:14:40.092 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:14:40.092 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:14:40.092 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:14:40.092 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:14:40.092 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:14:40.092 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:14:40.093 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:14:40.093 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:14:40.093 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:14:40.100 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:40.100 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:40.100 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:40.101 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:14:40.161 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:14:40.162 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:14:40.162 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:40.162 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:40.162 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:40.162 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:14:40.163 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:40.163 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:40.163 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:40.163 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:14:40.163 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:14:40.163 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:14:40.163 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:14:40.163 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:40.164 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:14:40.164 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:40.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 37 2026-02-18T08:14:40.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 38 2026-02-18T08:14:40.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:14:40.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=153, tag_tid=9, entry_tid=1], [object_number=152, tag_tid=9, entry_tid=0], [object_number=154, tag_tid=8, entry_tid=58722], [object_number=155, tag_tid=8, entry_tid=58719]]], state=connected] 2026-02-18T08:14:40.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=149, tag_tid=8, entry_tid=54201], [object_number=148, tag_tid=8, entry_tid=54200], [object_number=151, tag_tid=8, entry_tid=54199], [object_number=150, tag_tid=8, entry_tid=54198]]], state=connected] 2026-02-18T08:14:40.194 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:40.195 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:40.196 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:14:40.197 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:40.197 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:40.197 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:40.197 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:14:40.197 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:40.197 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:40.197 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:40.197 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:14:40.197 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:14:40.198 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:14:40.198 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:14:40.198 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:40.199 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:14:40.199 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:40.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 37 2026-02-18T08:14:40.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 38 2026-02-18T08:14:40.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:14:40.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=153, tag_tid=9, entry_tid=1], [object_number=152, tag_tid=9, entry_tid=0], [object_number=154, tag_tid=8, entry_tid=58722], [object_number=155, tag_tid=8, entry_tid=58719]]], state=connected] 2026-02-18T08:14:40.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=149, tag_tid=8, entry_tid=54201], [object_number=148, tag_tid=8, entry_tid=54200], [object_number=151, tag_tid=8, entry_tid=54199], [object_number=150, tag_tid=8, entry_tid=54198]]], state=connected] 2026-02-18T08:14:40.229 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:40.230 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=149, tag_tid=8, entry_tid=54201' 2026-02-18T08:14:40.230 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=153, tag_tid=9, entry_tid=1' -a 'object_number=153, tag_tid=9, entry_tid=1' = 'object_number=149, tag_tid=8, entry_tid=54201' 2026-02-18T08:14:40.230 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=149, tag_tid=8, entry_tid=54201' '!=' 'object_number=149, tag_tid=8, entry_tid=53949' 2026-02-18T08:14:40.231 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:14:40.231 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=149, tag_tid=8, entry_tid=54201' = 'object_number=149, tag_tid=8, entry_tid=53949' 2026-02-18T08:14:40.231 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=149, tag_tid=8, entry_tid=54201' 2026-02-18T08:14:40.231 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:40.231 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:14:40.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:40.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=9 2026-02-18T08:14:40.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=149, tag_tid=8, entry_tid=54201' 2026-02-18T08:14:40.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:14:40.234 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:40.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=8 2026-02-18T08:14:40.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:40.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:14:40.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:40.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:14:40.238 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=149, tag_tid=8, entry_tid=54201' 2026-02-18T08:14:40.238 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:14:40.238 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:40.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=54201 2026-02-18T08:14:40.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 9 = 8 -a 1 -le 54201 2026-02-18T08:14:40.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:14:40.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:14:40.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:14:40.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:14:40.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:14:40.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:14:40.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:14:40.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:14:40.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:14:40.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:14:40.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:14:40.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:14:40.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:14:40.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:14:40.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:14:40.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:14:40.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:14:40.443 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:14:40.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:14:40.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:14:40.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:14:40.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:14:40.446 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:14:40.446 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:14:40.446 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:14:40.446 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:14:40.446 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:14:40.446 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:14:40.446 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:14:40.446 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:14:40.446 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:14:40.446 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:14:40.446 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:14:40.446 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:14:40.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:14:40.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:14:40.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:14:40.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:14:40.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:14:40.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:14:40.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:14:40.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:14:40.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:14:40.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:14:40.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:14:40.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:14:40.448 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:14:40.448 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:14:40.448 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:14:40.448 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:14:40.448 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:14:40.448 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:14:40.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:40.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:40.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:40.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:14:40.530 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:14:40.531 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:14:40.531 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:40.531 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:40.532 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:40.532 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:14:40.532 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:40.532 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:40.532 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:40.532 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:14:40.532 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:14:40.532 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:14:40.532 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:14:40.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:40.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:14:40.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:40.560 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 37 2026-02-18T08:14:40.560 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 38 2026-02-18T08:14:40.560 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:14:40.560 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=153, tag_tid=9, entry_tid=1], [object_number=152, tag_tid=9, entry_tid=0], [object_number=154, tag_tid=8, entry_tid=58722], [object_number=155, tag_tid=8, entry_tid=58719]]], state=connected] 2026-02-18T08:14:40.560 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=150, tag_tid=8, entry_tid=54454], [object_number=149, tag_tid=8, entry_tid=54453], [object_number=148, tag_tid=8, entry_tid=54452], [object_number=151, tag_tid=8, entry_tid=54451]]], state=connected] 2026-02-18T08:14:40.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:40.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:40.565 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:14:40.565 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:40.566 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:40.566 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:40.566 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:14:40.566 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:40.566 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:40.566 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:40.566 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:14:40.566 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:14:40.567 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:14:40.567 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:14:40.567 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:40.567 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:14:40.568 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:40.602 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 37 2026-02-18T08:14:40.602 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 38 2026-02-18T08:14:40.602 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:14:40.602 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=153, tag_tid=9, entry_tid=1], [object_number=152, tag_tid=9, entry_tid=0], [object_number=154, tag_tid=8, entry_tid=58722], [object_number=155, tag_tid=8, entry_tid=58719]]], state=connected] 2026-02-18T08:14:40.602 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=150, tag_tid=8, entry_tid=54454], [object_number=149, tag_tid=8, entry_tid=54453], [object_number=148, tag_tid=8, entry_tid=54452], [object_number=151, tag_tid=8, entry_tid=54451]]], state=connected] 2026-02-18T08:14:40.606 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:40.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=150, tag_tid=8, entry_tid=54454' 2026-02-18T08:14:40.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=153, tag_tid=9, entry_tid=1' -a 'object_number=153, tag_tid=9, entry_tid=1' = 'object_number=150, tag_tid=8, entry_tid=54454' 2026-02-18T08:14:40.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=150, tag_tid=8, entry_tid=54454' '!=' 'object_number=149, tag_tid=8, entry_tid=54201' 2026-02-18T08:14:40.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:14:40.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=150, tag_tid=8, entry_tid=54454' = 'object_number=149, tag_tid=8, entry_tid=54201' 2026-02-18T08:14:40.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=150, tag_tid=8, entry_tid=54454' 2026-02-18T08:14:40.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:40.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:14:40.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:40.610 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=9 2026-02-18T08:14:40.611 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=150, tag_tid=8, entry_tid=54454' 2026-02-18T08:14:40.611 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:14:40.611 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:40.612 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=8 2026-02-18T08:14:40.613 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:40.613 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:14:40.613 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:40.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:14:40.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=150, tag_tid=8, entry_tid=54454' 2026-02-18T08:14:40.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:14:40.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:40.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=54454 2026-02-18T08:14:40.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 9 = 8 -a 1 -le 54454 2026-02-18T08:14:40.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:14:40.618 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:14:40.618 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:14:40.819 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:14:40.819 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:14:40.819 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:14:40.819 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:14:40.819 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:14:40.819 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:14:40.819 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:14:40.820 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:14:40.820 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:14:40.820 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:14:40.820 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:14:40.820 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:14:40.820 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:14:40.820 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:14:40.820 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:14:40.820 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:14:40.823 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:14:40.823 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:14:40.823 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:14:40.823 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:14:40.823 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:14:40.824 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:14:40.824 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:14:40.824 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:14:40.824 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:14:40.824 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:14:40.824 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:14:40.824 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:14:40.824 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:14:40.824 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:14:40.824 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:14:40.824 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:14:40.824 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:14:40.825 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:14:40.825 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:14:40.825 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:14:40.825 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:14:40.825 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:14:40.825 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:14:40.825 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:14:40.825 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:14:40.825 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:14:40.825 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:14:40.825 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:14:40.825 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:14:40.826 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:14:40.826 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:14:40.826 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:14:40.826 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:14:40.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:40.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:40.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:40.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:14:40.895 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:14:40.896 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:14:40.897 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:40.897 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:40.897 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:40.897 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:14:40.897 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:40.897 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:40.897 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:40.897 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:14:40.897 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:14:40.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:14:40.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:14:40.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:40.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:14:40.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:40.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 37 2026-02-18T08:14:40.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 38 2026-02-18T08:14:40.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:14:40.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=153, tag_tid=9, entry_tid=1], [object_number=152, tag_tid=9, entry_tid=0], [object_number=154, tag_tid=8, entry_tid=58722], [object_number=155, tag_tid=8, entry_tid=58719]]], state=connected] 2026-02-18T08:14:40.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=149, tag_tid=8, entry_tid=54717], [object_number=148, tag_tid=8, entry_tid=54716], [object_number=151, tag_tid=8, entry_tid=54715], [object_number=150, tag_tid=8, entry_tid=54714]]], state=connected] 2026-02-18T08:14:40.929 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:40.930 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:40.931 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:14:40.931 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:40.932 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:40.932 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:40.932 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:14:40.932 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:40.932 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:40.932 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:40.932 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:14:40.932 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:14:40.933 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:14:40.933 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:14:40.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:40.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:14:40.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:40.961 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 37 2026-02-18T08:14:40.961 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 38 2026-02-18T08:14:40.961 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:14:40.961 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=153, tag_tid=9, entry_tid=1], [object_number=152, tag_tid=9, entry_tid=0], [object_number=154, tag_tid=8, entry_tid=58722], [object_number=155, tag_tid=8, entry_tid=58719]]], state=connected] 2026-02-18T08:14:40.961 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=149, tag_tid=8, entry_tid=54717], [object_number=148, tag_tid=8, entry_tid=54716], [object_number=151, tag_tid=8, entry_tid=54715], [object_number=150, tag_tid=8, entry_tid=54714]]], state=connected] 2026-02-18T08:14:40.963 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:40.965 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=149, tag_tid=8, entry_tid=54717' 2026-02-18T08:14:40.965 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=153, tag_tid=9, entry_tid=1' -a 'object_number=153, tag_tid=9, entry_tid=1' = 'object_number=149, tag_tid=8, entry_tid=54717' 2026-02-18T08:14:40.965 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=149, tag_tid=8, entry_tid=54717' '!=' 'object_number=150, tag_tid=8, entry_tid=54454' 2026-02-18T08:14:40.965 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:14:40.965 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=149, tag_tid=8, entry_tid=54717' = 'object_number=150, tag_tid=8, entry_tid=54454' 2026-02-18T08:14:40.966 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=149, tag_tid=8, entry_tid=54717' 2026-02-18T08:14:40.966 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:40.966 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:14:40.966 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:40.967 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=9 2026-02-18T08:14:40.968 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=149, tag_tid=8, entry_tid=54717' 2026-02-18T08:14:40.968 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:14:40.968 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:40.969 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=8 2026-02-18T08:14:40.970 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:40.970 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:14:40.970 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:40.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:14:40.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=149, tag_tid=8, entry_tid=54717' 2026-02-18T08:14:40.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:14:40.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:40.974 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=54717 2026-02-18T08:14:40.974 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 9 = 8 -a 1 -le 54717 2026-02-18T08:14:40.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:14:40.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:14:40.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:14:41.176 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:14:41.176 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:14:41.177 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:14:41.177 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:14:41.177 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:14:41.177 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:14:41.177 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:14:41.177 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:14:41.177 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:14:41.177 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:14:41.177 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:14:41.177 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:14:41.178 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:14:41.178 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:14:41.178 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:14:41.178 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:14:41.180 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:14:41.180 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:14:41.180 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:14:41.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:14:41.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:14:41.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:14:41.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:14:41.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:14:41.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:14:41.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:14:41.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:14:41.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:14:41.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:14:41.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:14:41.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:14:41.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:14:41.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:14:41.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:14:41.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:14:41.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:14:41.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:14:41.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:14:41.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:14:41.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:14:41.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:14:41.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:14:41.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:14:41.183 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:14:41.183 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:14:41.183 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:14:41.183 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:14:41.183 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:14:41.183 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:14:41.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:41.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:41.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:41.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:14:41.252 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:14:41.254 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:14:41.254 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:41.254 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:41.254 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:41.254 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:14:41.254 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:41.254 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:41.254 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:41.254 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:14:41.255 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:14:41.255 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:14:41.255 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:14:41.255 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:41.256 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:14:41.256 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:41.292 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 37 2026-02-18T08:14:41.292 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 38 2026-02-18T08:14:41.292 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:14:41.292 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=153, tag_tid=9, entry_tid=1], [object_number=152, tag_tid=9, entry_tid=0], [object_number=154, tag_tid=8, entry_tid=58722], [object_number=155, tag_tid=8, entry_tid=58719]]], state=connected] 2026-02-18T08:14:41.293 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=150, tag_tid=8, entry_tid=54966], [object_number=149, tag_tid=8, entry_tid=54965], [object_number=148, tag_tid=8, entry_tid=54964], [object_number=151, tag_tid=8, entry_tid=54963]]], state=connected] 2026-02-18T08:14:41.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:41.297 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:41.298 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:14:41.298 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:41.299 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:41.299 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:41.299 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:14:41.299 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:41.299 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:41.299 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:41.299 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:14:41.299 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:14:41.299 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:14:41.299 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:14:41.300 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:41.300 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:14:41.301 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:41.327 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 37 2026-02-18T08:14:41.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 38 2026-02-18T08:14:41.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:14:41.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=153, tag_tid=9, entry_tid=1], [object_number=152, tag_tid=9, entry_tid=0], [object_number=154, tag_tid=8, entry_tid=58722], [object_number=155, tag_tid=8, entry_tid=58719]]], state=connected] 2026-02-18T08:14:41.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=150, tag_tid=8, entry_tid=54966], [object_number=149, tag_tid=8, entry_tid=54965], [object_number=148, tag_tid=8, entry_tid=54964], [object_number=151, tag_tid=8, entry_tid=54963]]], state=connected] 2026-02-18T08:14:41.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:41.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=150, tag_tid=8, entry_tid=54966' 2026-02-18T08:14:41.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=153, tag_tid=9, entry_tid=1' -a 'object_number=153, tag_tid=9, entry_tid=1' = 'object_number=150, tag_tid=8, entry_tid=54966' 2026-02-18T08:14:41.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=150, tag_tid=8, entry_tid=54966' '!=' 'object_number=149, tag_tid=8, entry_tid=54717' 2026-02-18T08:14:41.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:14:41.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=150, tag_tid=8, entry_tid=54966' = 'object_number=149, tag_tid=8, entry_tid=54717' 2026-02-18T08:14:41.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=150, tag_tid=8, entry_tid=54966' 2026-02-18T08:14:41.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:41.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:14:41.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:41.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=9 2026-02-18T08:14:41.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=150, tag_tid=8, entry_tid=54966' 2026-02-18T08:14:41.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:14:41.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:41.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=8 2026-02-18T08:14:41.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:41.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:14:41.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:41.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:14:41.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=150, tag_tid=8, entry_tid=54966' 2026-02-18T08:14:41.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:14:41.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:41.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=54966 2026-02-18T08:14:41.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 9 = 8 -a 1 -le 54966 2026-02-18T08:14:41.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:14:41.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:14:41.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:14:41.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:14:41.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:14:41.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:14:41.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:14:41.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:14:41.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:14:41.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:14:41.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:14:41.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:14:41.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:14:41.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:14:41.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:14:41.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:14:41.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:14:41.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:14:41.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:14:41.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:14:41.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:14:41.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:14:41.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:14:41.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:14:41.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:14:41.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:14:41.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:14:41.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:14:41.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:14:41.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:14:41.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:14:41.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:14:41.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:14:41.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:14:41.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:14:41.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:14:41.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:14:41.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:14:41.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:14:41.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:14:41.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:14:41.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:14:41.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:14:41.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:14:41.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:14:41.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:14:41.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:14:41.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:14:41.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:14:41.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:14:41.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:14:41.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:14:41.557 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:41.558 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:41.558 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:41.558 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:14:41.620 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:14:41.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:14:41.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:41.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:41.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:41.622 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:14:41.622 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:41.622 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:41.622 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:41.622 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:14:41.622 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:14:41.622 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:14:41.622 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:14:41.623 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:41.623 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:14:41.623 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:41.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 37 2026-02-18T08:14:41.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 38 2026-02-18T08:14:41.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:14:41.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=153, tag_tid=9, entry_tid=1], [object_number=152, tag_tid=9, entry_tid=0], [object_number=154, tag_tid=8, entry_tid=58722], [object_number=155, tag_tid=8, entry_tid=58719]]], state=connected] 2026-02-18T08:14:41.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=150, tag_tid=8, entry_tid=55218], [object_number=149, tag_tid=8, entry_tid=55217], [object_number=148, tag_tid=8, entry_tid=55216], [object_number=151, tag_tid=8, entry_tid=55215]]], state=connected] 2026-02-18T08:14:41.653 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:41.655 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:41.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:14:41.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:41.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:41.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:41.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:14:41.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:41.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:41.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:41.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:14:41.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:14:41.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:14:41.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:14:41.658 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:41.658 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:14:41.658 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:41.685 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 37 2026-02-18T08:14:41.685 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 38 2026-02-18T08:14:41.685 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:14:41.686 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=153, tag_tid=9, entry_tid=1], [object_number=152, tag_tid=9, entry_tid=0], [object_number=154, tag_tid=8, entry_tid=58722], [object_number=155, tag_tid=8, entry_tid=58719]]], state=connected] 2026-02-18T08:14:41.686 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=150, tag_tid=8, entry_tid=55218], [object_number=149, tag_tid=8, entry_tid=55217], [object_number=148, tag_tid=8, entry_tid=55216], [object_number=151, tag_tid=8, entry_tid=55215]]], state=connected] 2026-02-18T08:14:41.688 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:41.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=150, tag_tid=8, entry_tid=55218' 2026-02-18T08:14:41.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=153, tag_tid=9, entry_tid=1' -a 'object_number=153, tag_tid=9, entry_tid=1' = 'object_number=150, tag_tid=8, entry_tid=55218' 2026-02-18T08:14:41.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=150, tag_tid=8, entry_tid=55218' '!=' 'object_number=150, tag_tid=8, entry_tid=54966' 2026-02-18T08:14:41.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:14:41.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=150, tag_tid=8, entry_tid=55218' = 'object_number=150, tag_tid=8, entry_tid=54966' 2026-02-18T08:14:41.691 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=150, tag_tid=8, entry_tid=55218' 2026-02-18T08:14:41.691 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:41.691 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:14:41.691 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:41.692 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=9 2026-02-18T08:14:41.693 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=150, tag_tid=8, entry_tid=55218' 2026-02-18T08:14:41.693 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:14:41.693 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:41.694 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=8 2026-02-18T08:14:41.695 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:41.695 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:14:41.695 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:41.697 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:14:41.697 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=150, tag_tid=8, entry_tid=55218' 2026-02-18T08:14:41.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:14:41.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:41.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=55218 2026-02-18T08:14:41.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 9 = 8 -a 1 -le 55218 2026-02-18T08:14:41.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:14:41.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:14:41.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:14:41.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:14:41.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:14:41.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:14:41.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:14:41.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:14:41.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:14:41.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:14:41.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:14:41.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:14:41.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:14:41.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:14:41.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:14:41.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:14:41.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:14:41.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:14:41.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:14:41.905 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:14:41.905 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:14:41.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:14:41.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:14:41.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:14:41.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:14:41.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:14:41.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:14:41.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:14:41.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:14:41.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:14:41.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:14:41.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:14:41.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:14:41.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:14:41.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:14:41.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:14:41.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:14:41.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:14:41.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:14:41.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:14:41.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:14:41.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:14:41.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:14:41.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:14:41.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:14:41.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:14:41.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:14:41.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:14:41.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:14:41.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:14:41.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:14:41.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:14:41.915 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:41.916 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:41.916 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:41.916 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:14:41.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:14:41.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:14:41.982 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:41.982 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:41.982 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:41.982 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:14:41.982 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:41.982 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:41.982 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:41.982 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:14:41.982 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:14:41.982 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:14:41.982 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:14:41.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:41.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:14:41.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:42.010 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 37 2026-02-18T08:14:42.010 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 38 2026-02-18T08:14:42.010 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:14:42.010 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=153, tag_tid=9, entry_tid=1], [object_number=152, tag_tid=9, entry_tid=0], [object_number=154, tag_tid=8, entry_tid=58722], [object_number=155, tag_tid=8, entry_tid=58719]]], state=connected] 2026-02-18T08:14:42.010 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=150, tag_tid=8, entry_tid=55450], [object_number=149, tag_tid=8, entry_tid=55449], [object_number=148, tag_tid=8, entry_tid=55448], [object_number=151, tag_tid=8, entry_tid=55447]]], state=connected] 2026-02-18T08:14:42.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:42.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:42.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:14:42.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:42.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:42.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:42.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:14:42.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:42.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:42.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:42.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:14:42.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:14:42.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:14:42.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:14:42.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:42.018 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:42.018 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:14:42.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 37 2026-02-18T08:14:42.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 38 2026-02-18T08:14:42.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:14:42.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=153, tag_tid=9, entry_tid=1], [object_number=152, tag_tid=9, entry_tid=0], [object_number=154, tag_tid=8, entry_tid=58722], [object_number=155, tag_tid=8, entry_tid=58719]]], state=connected] 2026-02-18T08:14:42.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=150, tag_tid=8, entry_tid=55450], [object_number=149, tag_tid=8, entry_tid=55449], [object_number=148, tag_tid=8, entry_tid=55448], [object_number=151, tag_tid=8, entry_tid=55447]]], state=connected] 2026-02-18T08:14:42.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:42.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=150, tag_tid=8, entry_tid=55450' 2026-02-18T08:14:42.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=153, tag_tid=9, entry_tid=1' -a 'object_number=153, tag_tid=9, entry_tid=1' = 'object_number=150, tag_tid=8, entry_tid=55450' 2026-02-18T08:14:42.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=150, tag_tid=8, entry_tid=55450' '!=' 'object_number=150, tag_tid=8, entry_tid=55218' 2026-02-18T08:14:42.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:14:42.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=150, tag_tid=8, entry_tid=55450' = 'object_number=150, tag_tid=8, entry_tid=55218' 2026-02-18T08:14:42.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=150, tag_tid=8, entry_tid=55450' 2026-02-18T08:14:42.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:42.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:14:42.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:42.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=9 2026-02-18T08:14:42.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=150, tag_tid=8, entry_tid=55450' 2026-02-18T08:14:42.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:14:42.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:42.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=8 2026-02-18T08:14:42.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:42.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:14:42.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:42.055 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:14:42.056 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=150, tag_tid=8, entry_tid=55450' 2026-02-18T08:14:42.056 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:14:42.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:42.058 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=55450 2026-02-18T08:14:42.058 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 9 = 8 -a 1 -le 55450 2026-02-18T08:14:42.058 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:14:42.058 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:14:42.058 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:14:42.259 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:14:42.259 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:14:42.260 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:14:42.260 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:14:42.260 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:14:42.260 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:14:42.260 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:14:42.260 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:14:42.260 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:14:42.260 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:14:42.260 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:14:42.260 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:14:42.260 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:14:42.260 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:14:42.260 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:14:42.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:14:42.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:14:42.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:14:42.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:14:42.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:14:42.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:14:42.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:14:42.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:14:42.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:14:42.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:14:42.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:14:42.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:14:42.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:14:42.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:14:42.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:14:42.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:14:42.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:14:42.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:14:42.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:14:42.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:14:42.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:14:42.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:14:42.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:14:42.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:14:42.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:14:42.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:14:42.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:14:42.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:14:42.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:14:42.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:14:42.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:14:42.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:14:42.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:14:42.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:14:42.274 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:42.275 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:42.275 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:42.275 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:14:42.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:14:42.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:14:42.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:42.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:42.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:42.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:14:42.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:42.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:42.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:42.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:14:42.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:14:42.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:14:42.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:14:42.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:42.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:14:42.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:42.366 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 37 2026-02-18T08:14:42.366 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 38 2026-02-18T08:14:42.366 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:14:42.366 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=153, tag_tid=9, entry_tid=1], [object_number=152, tag_tid=9, entry_tid=0], [object_number=154, tag_tid=8, entry_tid=58722], [object_number=155, tag_tid=8, entry_tid=58719]]], state=connected] 2026-02-18T08:14:42.366 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=151, tag_tid=8, entry_tid=55683], [object_number=150, tag_tid=8, entry_tid=55682], [object_number=149, tag_tid=8, entry_tid=55681], [object_number=148, tag_tid=8, entry_tid=55680]]], state=connected] 2026-02-18T08:14:42.368 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:42.370 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:42.371 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:14:42.371 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:42.371 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:42.371 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:42.371 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:14:42.371 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:42.371 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:42.372 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:42.372 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:14:42.372 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:14:42.372 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:14:42.372 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:14:42.373 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:42.373 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:14:42.373 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:42.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 37 2026-02-18T08:14:42.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 38 2026-02-18T08:14:42.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:14:42.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=153, tag_tid=9, entry_tid=1], [object_number=152, tag_tid=9, entry_tid=0], [object_number=154, tag_tid=8, entry_tid=58722], [object_number=155, tag_tid=8, entry_tid=58719]]], state=connected] 2026-02-18T08:14:42.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=151, tag_tid=8, entry_tid=55683], [object_number=150, tag_tid=8, entry_tid=55682], [object_number=149, tag_tid=8, entry_tid=55681], [object_number=148, tag_tid=8, entry_tid=55680]]], state=connected] 2026-02-18T08:14:42.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:42.405 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=151, tag_tid=8, entry_tid=55683' 2026-02-18T08:14:42.406 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=153, tag_tid=9, entry_tid=1' -a 'object_number=153, tag_tid=9, entry_tid=1' = 'object_number=151, tag_tid=8, entry_tid=55683' 2026-02-18T08:14:42.406 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=151, tag_tid=8, entry_tid=55683' '!=' 'object_number=150, tag_tid=8, entry_tid=55450' 2026-02-18T08:14:42.406 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:14:42.406 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=151, tag_tid=8, entry_tid=55683' = 'object_number=150, tag_tid=8, entry_tid=55450' 2026-02-18T08:14:42.406 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=151, tag_tid=8, entry_tid=55683' 2026-02-18T08:14:42.406 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:42.406 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:14:42.407 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:42.407 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=9 2026-02-18T08:14:42.408 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=151, tag_tid=8, entry_tid=55683' 2026-02-18T08:14:42.408 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:14:42.408 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:42.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=8 2026-02-18T08:14:42.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:42.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:14:42.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:42.412 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:14:42.413 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=151, tag_tid=8, entry_tid=55683' 2026-02-18T08:14:42.413 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:14:42.413 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:42.414 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=55683 2026-02-18T08:14:42.415 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 9 = 8 -a 1 -le 55683 2026-02-18T08:14:42.415 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:14:42.415 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:14:42.415 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:14:42.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:14:42.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:14:42.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:14:42.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:14:42.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:14:42.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:14:42.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:14:42.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:14:42.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:14:42.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:14:42.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:14:42.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:14:42.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:14:42.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:14:42.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:14:42.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:14:42.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:14:42.620 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:14:42.620 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:14:42.620 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:14:42.620 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:14:42.620 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:14:42.620 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:14:42.620 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:14:42.620 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:14:42.620 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:14:42.620 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:14:42.620 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:14:42.620 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:14:42.620 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:14:42.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:14:42.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:14:42.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:14:42.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:14:42.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:14:42.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:14:42.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:14:42.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:14:42.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:14:42.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:14:42.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:14:42.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:14:42.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:14:42.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:14:42.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:14:42.622 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:14:42.622 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:14:42.622 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:14:42.622 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:14:42.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:42.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:42.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:42.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:14:42.691 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:14:42.692 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:14:42.692 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:42.692 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:42.692 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:42.692 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:14:42.692 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:42.692 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:42.692 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:42.692 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:14:42.693 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:14:42.693 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:14:42.693 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:14:42.694 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:42.694 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:14:42.694 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:42.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 37 2026-02-18T08:14:42.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 38 2026-02-18T08:14:42.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:14:42.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=153, tag_tid=9, entry_tid=1], [object_number=152, tag_tid=9, entry_tid=0], [object_number=154, tag_tid=8, entry_tid=58722], [object_number=155, tag_tid=8, entry_tid=58719]]], state=connected] 2026-02-18T08:14:42.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=151, tag_tid=8, entry_tid=55783], [object_number=150, tag_tid=8, entry_tid=55782], [object_number=149, tag_tid=8, entry_tid=55781], [object_number=148, tag_tid=8, entry_tid=55780]]], state=connected] 2026-02-18T08:14:42.725 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:42.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:42.728 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:14:42.728 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:42.728 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:42.728 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:42.728 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:14:42.728 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:42.728 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:42.728 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:42.728 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:14:42.728 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:14:42.729 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:14:42.729 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:14:42.729 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:42.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:14:42.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:42.758 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 37 2026-02-18T08:14:42.759 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 38 2026-02-18T08:14:42.759 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:14:42.759 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=153, tag_tid=9, entry_tid=1], [object_number=152, tag_tid=9, entry_tid=0], [object_number=154, tag_tid=8, entry_tid=58722], [object_number=155, tag_tid=8, entry_tid=58719]]], state=connected] 2026-02-18T08:14:42.759 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=151, tag_tid=8, entry_tid=55783], [object_number=150, tag_tid=8, entry_tid=55782], [object_number=149, tag_tid=8, entry_tid=55781], [object_number=148, tag_tid=8, entry_tid=55780]]], state=connected] 2026-02-18T08:14:42.761 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:42.763 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=151, tag_tid=8, entry_tid=55783' 2026-02-18T08:14:42.763 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=153, tag_tid=9, entry_tid=1' -a 'object_number=153, tag_tid=9, entry_tid=1' = 'object_number=151, tag_tid=8, entry_tid=55783' 2026-02-18T08:14:42.763 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=151, tag_tid=8, entry_tid=55783' '!=' 'object_number=151, tag_tid=8, entry_tid=55683' 2026-02-18T08:14:42.763 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:14:42.763 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=151, tag_tid=8, entry_tid=55783' = 'object_number=151, tag_tid=8, entry_tid=55683' 2026-02-18T08:14:42.763 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=151, tag_tid=8, entry_tid=55783' 2026-02-18T08:14:42.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:42.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:14:42.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:42.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=9 2026-02-18T08:14:42.766 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:14:42.766 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=151, tag_tid=8, entry_tid=55783' 2026-02-18T08:14:42.766 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:42.767 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=8 2026-02-18T08:14:42.768 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:42.768 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:14:42.768 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:42.769 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:14:42.770 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=151, tag_tid=8, entry_tid=55783' 2026-02-18T08:14:42.770 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:14:42.770 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:42.771 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=55783 2026-02-18T08:14:42.771 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 9 = 8 -a 1 -le 55783 2026-02-18T08:14:42.771 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:14:42.771 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:14:42.771 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:14:42.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:14:42.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:14:42.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:14:42.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:14:42.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:14:42.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:14:42.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:14:42.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:14:42.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:14:42.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:14:42.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:14:42.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:14:42.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:14:42.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:14:42.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:14:42.974 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:14:42.974 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:14:42.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:14:42.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:14:42.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:14:42.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:14:42.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:14:42.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:14:42.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:14:42.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:14:42.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:14:42.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:14:42.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:14:42.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:14:42.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:14:42.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:14:42.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:14:42.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:14:42.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:14:42.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:14:42.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:14:42.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:14:42.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:14:42.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:14:42.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:14:42.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:14:42.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:14:42.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:14:42.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:14:42.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:14:42.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:14:42.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:14:42.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:14:42.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:14:42.988 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:42.988 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:42.989 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:42.989 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:14:43.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:14:43.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:14:43.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:43.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:43.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:43.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:14:43.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:43.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:43.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:43.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:14:43.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:14:43.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:14:43.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:14:43.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:43.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:14:43.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:43.080 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 37 2026-02-18T08:14:43.080 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 38 2026-02-18T08:14:43.080 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:14:43.080 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=153, tag_tid=9, entry_tid=1], [object_number=152, tag_tid=9, entry_tid=0], [object_number=154, tag_tid=8, entry_tid=58722], [object_number=155, tag_tid=8, entry_tid=58719]]], state=connected] 2026-02-18T08:14:43.080 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=151, tag_tid=8, entry_tid=56011], [object_number=150, tag_tid=8, entry_tid=56010], [object_number=149, tag_tid=8, entry_tid=56009], [object_number=148, tag_tid=8, entry_tid=56008]]], state=connected] 2026-02-18T08:14:43.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:43.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:43.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:14:43.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:43.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:43.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:43.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:14:43.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:43.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:43.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:43.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:14:43.086 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:14:43.086 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:14:43.086 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:14:43.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:43.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:14:43.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:43.114 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 37 2026-02-18T08:14:43.114 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 38 2026-02-18T08:14:43.114 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:14:43.114 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=153, tag_tid=9, entry_tid=1], [object_number=152, tag_tid=9, entry_tid=0], [object_number=154, tag_tid=8, entry_tid=58722], [object_number=155, tag_tid=8, entry_tid=58719]]], state=connected] 2026-02-18T08:14:43.114 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=151, tag_tid=8, entry_tid=56011], [object_number=150, tag_tid=8, entry_tid=56010], [object_number=149, tag_tid=8, entry_tid=56009], [object_number=148, tag_tid=8, entry_tid=56008]]], state=connected] 2026-02-18T08:14:43.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:43.118 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=151, tag_tid=8, entry_tid=56011' 2026-02-18T08:14:43.118 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=153, tag_tid=9, entry_tid=1' -a 'object_number=153, tag_tid=9, entry_tid=1' = 'object_number=151, tag_tid=8, entry_tid=56011' 2026-02-18T08:14:43.118 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=151, tag_tid=8, entry_tid=56011' '!=' 'object_number=151, tag_tid=8, entry_tid=55783' 2026-02-18T08:14:43.118 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:14:43.118 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=151, tag_tid=8, entry_tid=56011' = 'object_number=151, tag_tid=8, entry_tid=55783' 2026-02-18T08:14:43.118 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=151, tag_tid=8, entry_tid=56011' 2026-02-18T08:14:43.119 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:43.119 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:14:43.119 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:43.120 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=9 2026-02-18T08:14:43.121 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=151, tag_tid=8, entry_tid=56011' 2026-02-18T08:14:43.122 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:14:43.122 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:43.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=8 2026-02-18T08:14:43.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:43.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:14:43.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:43.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:14:43.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=151, tag_tid=8, entry_tid=56011' 2026-02-18T08:14:43.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:14:43.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:43.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=56011 2026-02-18T08:14:43.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 9 = 8 -a 1 -le 56011 2026-02-18T08:14:43.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:14:43.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:14:43.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:14:43.329 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:14:43.329 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:14:43.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:14:43.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:14:43.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:14:43.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:14:43.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:14:43.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:14:43.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:14:43.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:14:43.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:14:43.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:14:43.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:14:43.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:14:43.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:14:43.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:14:43.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:14:43.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:14:43.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:14:43.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:14:43.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:14:43.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:14:43.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:14:43.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:14:43.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:14:43.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:14:43.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:14:43.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:14:43.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:14:43.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:14:43.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:14:43.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:14:43.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:14:43.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:14:43.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:14:43.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:14:43.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:14:43.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:14:43.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:14:43.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:14:43.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:14:43.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:14:43.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:14:43.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:14:43.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:14:43.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:14:43.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:14:43.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:14:43.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:14:43.344 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:43.344 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:43.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:43.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:14:43.413 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:14:43.414 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:14:43.414 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:43.414 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:43.414 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:43.414 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:14:43.414 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:43.414 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:43.415 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:43.415 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:14:43.415 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:14:43.415 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:14:43.415 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:14:43.416 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:43.416 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:14:43.416 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:43.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 38 2026-02-18T08:14:43.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 38 2026-02-18T08:14:43.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:14:43.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=153, tag_tid=9, entry_tid=1], [object_number=152, tag_tid=9, entry_tid=0], [object_number=154, tag_tid=8, entry_tid=58722], [object_number=155, tag_tid=8, entry_tid=58719]]], state=connected] 2026-02-18T08:14:43.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=153, tag_tid=8, entry_tid=56213], [object_number=152, tag_tid=8, entry_tid=56212], [object_number=155, tag_tid=8, entry_tid=56211], [object_number=154, tag_tid=8, entry_tid=56210]]], state=connected] 2026-02-18T08:14:43.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:43.448 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:43.449 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:14:43.449 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:43.449 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:43.449 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:43.449 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:14:43.449 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:43.449 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:43.449 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:43.449 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:14:43.450 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:14:43.450 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:14:43.450 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:14:43.451 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:43.451 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:14:43.451 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:43.479 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 38 2026-02-18T08:14:43.479 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 38 2026-02-18T08:14:43.479 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:14:43.479 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=153, tag_tid=9, entry_tid=1], [object_number=152, tag_tid=9, entry_tid=0], [object_number=154, tag_tid=8, entry_tid=58722], [object_number=155, tag_tid=8, entry_tid=58719]]], state=connected] 2026-02-18T08:14:43.479 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=153, tag_tid=8, entry_tid=56213], [object_number=152, tag_tid=8, entry_tid=56212], [object_number=155, tag_tid=8, entry_tid=56211], [object_number=154, tag_tid=8, entry_tid=56210]]], state=connected] 2026-02-18T08:14:43.482 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:43.483 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=153, tag_tid=8, entry_tid=56213' 2026-02-18T08:14:43.483 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=153, tag_tid=9, entry_tid=1' -a 'object_number=153, tag_tid=9, entry_tid=1' = 'object_number=153, tag_tid=8, entry_tid=56213' 2026-02-18T08:14:43.483 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=153, tag_tid=8, entry_tid=56213' '!=' 'object_number=151, tag_tid=8, entry_tid=56011' 2026-02-18T08:14:43.483 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:14:43.483 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=153, tag_tid=8, entry_tid=56213' = 'object_number=151, tag_tid=8, entry_tid=56011' 2026-02-18T08:14:43.483 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=153, tag_tid=8, entry_tid=56213' 2026-02-18T08:14:43.484 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:43.484 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:14:43.484 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:43.485 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=9 2026-02-18T08:14:43.486 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=153, tag_tid=8, entry_tid=56213' 2026-02-18T08:14:43.486 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:14:43.486 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:43.488 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=8 2026-02-18T08:14:43.489 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:43.489 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:14:43.489 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:43.490 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:14:43.491 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=153, tag_tid=8, entry_tid=56213' 2026-02-18T08:14:43.491 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:14:43.491 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:43.492 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=56213 2026-02-18T08:14:43.492 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 9 = 8 -a 1 -le 56213 2026-02-18T08:14:43.492 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:14:43.493 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:14:43.493 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:14:43.693 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:14:43.694 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:14:43.694 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:14:43.694 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:14:43.694 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:14:43.694 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:14:43.694 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:14:43.694 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:14:43.694 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:14:43.694 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:14:43.694 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:14:43.694 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:14:43.694 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:14:43.695 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:14:43.695 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:14:43.697 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:14:43.697 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:14:43.697 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:14:43.697 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:14:43.697 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:14:43.697 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:14:43.697 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:14:43.697 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:14:43.697 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:14:43.697 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:14:43.697 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:14:43.697 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:14:43.697 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:14:43.697 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:14:43.697 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:14:43.697 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:14:43.697 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:14:43.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:14:43.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:14:43.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:14:43.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:14:43.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:14:43.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:14:43.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:14:43.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:14:43.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:14:43.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:14:43.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:14:43.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:14:43.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:14:43.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:14:43.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:14:43.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:14:43.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:14:43.708 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:43.708 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:43.709 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:43.709 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:14:43.767 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:14:43.768 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:14:43.768 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:43.769 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:43.769 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:43.769 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:14:43.769 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:43.769 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:43.769 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:43.769 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:14:43.769 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:14:43.769 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:14:43.769 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:14:43.770 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:43.770 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:14:43.771 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:43.799 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 38 2026-02-18T08:14:43.799 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 38 2026-02-18T08:14:43.799 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:14:43.799 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=153, tag_tid=9, entry_tid=1], [object_number=152, tag_tid=9, entry_tid=0], [object_number=154, tag_tid=8, entry_tid=58722], [object_number=155, tag_tid=8, entry_tid=58719]]], state=connected] 2026-02-18T08:14:43.799 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=152, tag_tid=8, entry_tid=56452], [object_number=155, tag_tid=8, entry_tid=56451], [object_number=154, tag_tid=8, entry_tid=56450], [object_number=153, tag_tid=8, entry_tid=56449]]], state=connected] 2026-02-18T08:14:43.801 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:43.803 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:43.804 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:14:43.804 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:43.804 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:43.804 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:43.804 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:14:43.804 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:43.804 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:43.804 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:43.804 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:14:43.804 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:14:43.805 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:14:43.805 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:14:43.806 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:43.806 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:14:43.806 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:43.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 38 2026-02-18T08:14:43.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 38 2026-02-18T08:14:43.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:14:43.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=153, tag_tid=9, entry_tid=1], [object_number=152, tag_tid=9, entry_tid=0], [object_number=154, tag_tid=8, entry_tid=58722], [object_number=155, tag_tid=8, entry_tid=58719]]], state=connected] 2026-02-18T08:14:43.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=152, tag_tid=8, entry_tid=56452], [object_number=155, tag_tid=8, entry_tid=56451], [object_number=154, tag_tid=8, entry_tid=56450], [object_number=153, tag_tid=8, entry_tid=56449]]], state=connected] 2026-02-18T08:14:43.836 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:43.838 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=152, tag_tid=8, entry_tid=56452' 2026-02-18T08:14:43.838 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=153, tag_tid=9, entry_tid=1' -a 'object_number=153, tag_tid=9, entry_tid=1' = 'object_number=152, tag_tid=8, entry_tid=56452' 2026-02-18T08:14:43.838 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=152, tag_tid=8, entry_tid=56452' '!=' 'object_number=153, tag_tid=8, entry_tid=56213' 2026-02-18T08:14:43.838 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:14:43.838 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=152, tag_tid=8, entry_tid=56452' = 'object_number=153, tag_tid=8, entry_tid=56213' 2026-02-18T08:14:43.838 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=152, tag_tid=8, entry_tid=56452' 2026-02-18T08:14:43.839 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:43.839 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:14:43.839 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:43.840 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=9 2026-02-18T08:14:43.841 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=152, tag_tid=8, entry_tid=56452' 2026-02-18T08:14:43.841 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:14:43.841 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:43.842 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=8 2026-02-18T08:14:43.843 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:43.843 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:14:43.843 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:43.845 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:14:43.845 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=152, tag_tid=8, entry_tid=56452' 2026-02-18T08:14:43.845 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:14:43.846 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:43.847 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=56452 2026-02-18T08:14:43.847 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 9 = 8 -a 1 -le 56452 2026-02-18T08:14:43.848 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:14:43.848 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:14:43.848 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:14:44.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:14:44.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:14:44.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:14:44.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:14:44.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:14:44.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:14:44.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:14:44.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:14:44.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:14:44.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:14:44.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:14:44.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:14:44.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:14:44.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:14:44.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:14:44.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:14:44.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:14:44.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:14:44.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:14:44.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:14:44.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:14:44.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:14:44.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:14:44.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:14:44.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:14:44.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:14:44.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:14:44.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:14:44.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:14:44.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:14:44.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:14:44.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:14:44.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:14:44.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:14:44.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:14:44.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:14:44.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:14:44.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:14:44.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:14:44.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:14:44.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:14:44.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:14:44.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:14:44.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:14:44.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:14:44.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:14:44.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:14:44.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:14:44.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:14:44.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:44.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:44.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:44.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:14:44.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:14:44.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:14:44.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:44.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:44.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:44.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:14:44.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:44.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:44.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:44.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:14:44.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:14:44.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:14:44.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:14:44.129 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:44.130 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:14:44.130 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:44.156 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 38 2026-02-18T08:14:44.156 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 38 2026-02-18T08:14:44.156 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:14:44.156 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=153, tag_tid=9, entry_tid=1], [object_number=152, tag_tid=9, entry_tid=0], [object_number=154, tag_tid=8, entry_tid=58722], [object_number=155, tag_tid=8, entry_tid=58719]]], state=connected] 2026-02-18T08:14:44.156 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=153, tag_tid=8, entry_tid=56725], [object_number=152, tag_tid=8, entry_tid=56724], [object_number=155, tag_tid=8, entry_tid=56723], [object_number=154, tag_tid=8, entry_tid=56722]]], state=connected] 2026-02-18T08:14:44.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:44.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:44.161 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:14:44.161 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:44.161 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:44.161 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:44.161 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:14:44.161 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:44.161 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:44.161 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:44.161 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:14:44.162 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:14:44.162 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:14:44.162 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:14:44.163 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:44.164 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:14:44.164 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:44.201 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 38 2026-02-18T08:14:44.201 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 38 2026-02-18T08:14:44.201 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:14:44.201 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=153, tag_tid=9, entry_tid=1], [object_number=152, tag_tid=9, entry_tid=0], [object_number=154, tag_tid=8, entry_tid=58722], [object_number=155, tag_tid=8, entry_tid=58719]]], state=connected] 2026-02-18T08:14:44.201 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=153, tag_tid=8, entry_tid=56725], [object_number=152, tag_tid=8, entry_tid=56724], [object_number=155, tag_tid=8, entry_tid=56723], [object_number=154, tag_tid=8, entry_tid=56722]]], state=connected] 2026-02-18T08:14:44.204 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:44.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=153, tag_tid=8, entry_tid=56725' 2026-02-18T08:14:44.206 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=153, tag_tid=9, entry_tid=1' -a 'object_number=153, tag_tid=9, entry_tid=1' = 'object_number=153, tag_tid=8, entry_tid=56725' 2026-02-18T08:14:44.206 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=153, tag_tid=8, entry_tid=56725' '!=' 'object_number=152, tag_tid=8, entry_tid=56452' 2026-02-18T08:14:44.206 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:14:44.206 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=153, tag_tid=8, entry_tid=56725' = 'object_number=152, tag_tid=8, entry_tid=56452' 2026-02-18T08:14:44.206 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=153, tag_tid=8, entry_tid=56725' 2026-02-18T08:14:44.206 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:44.206 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:14:44.206 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:44.208 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=9 2026-02-18T08:14:44.208 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=153, tag_tid=8, entry_tid=56725' 2026-02-18T08:14:44.209 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:14:44.209 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:44.210 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=8 2026-02-18T08:14:44.211 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:44.211 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:14:44.211 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:44.213 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:14:44.213 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=153, tag_tid=8, entry_tid=56725' 2026-02-18T08:14:44.213 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:14:44.213 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:44.215 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=56725 2026-02-18T08:14:44.215 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 9 = 8 -a 1 -le 56725 2026-02-18T08:14:44.215 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:14:44.215 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:14:44.215 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:14:44.416 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:14:44.417 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:14:44.417 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:14:44.417 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:14:44.417 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:14:44.417 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:14:44.417 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:14:44.417 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:14:44.417 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:14:44.417 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:14:44.417 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:14:44.417 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:14:44.417 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:14:44.417 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:14:44.417 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:14:44.417 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:14:44.417 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:14:44.419 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:14:44.419 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:14:44.419 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:14:44.419 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:14:44.419 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:14:44.419 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:14:44.420 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:14:44.420 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:14:44.420 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:14:44.420 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:14:44.420 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:14:44.420 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:14:44.420 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:14:44.420 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:14:44.420 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:14:44.420 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:14:44.420 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:14:44.420 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:14:44.420 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:14:44.420 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:14:44.420 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:14:44.420 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:14:44.420 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:14:44.420 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:14:44.420 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:14:44.420 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:14:44.420 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:14:44.420 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:14:44.421 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:14:44.421 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:14:44.421 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:14:44.421 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:14:44.432 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:44.432 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:44.432 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:44.432 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:14:44.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:14:44.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:14:44.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:44.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:44.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:44.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:14:44.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:44.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:44.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:44.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:14:44.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:14:44.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:14:44.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:14:44.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:44.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:14:44.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:44.524 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 38 2026-02-18T08:14:44.525 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 38 2026-02-18T08:14:44.525 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:14:44.525 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=153, tag_tid=9, entry_tid=1], [object_number=152, tag_tid=9, entry_tid=0], [object_number=154, tag_tid=8, entry_tid=58722], [object_number=155, tag_tid=8, entry_tid=58719]]], state=connected] 2026-02-18T08:14:44.525 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=153, tag_tid=8, entry_tid=56981], [object_number=152, tag_tid=8, entry_tid=56980], [object_number=155, tag_tid=8, entry_tid=56979], [object_number=154, tag_tid=8, entry_tid=56978]]], state=connected] 2026-02-18T08:14:44.527 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:44.529 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:44.530 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:14:44.530 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:44.530 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:44.530 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:44.530 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:14:44.530 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:44.530 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:44.530 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:44.530 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:14:44.530 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:14:44.530 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:14:44.530 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:14:44.532 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:44.532 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:14:44.532 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:44.568 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 38 2026-02-18T08:14:44.568 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 38 2026-02-18T08:14:44.568 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:14:44.568 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=153, tag_tid=9, entry_tid=1], [object_number=152, tag_tid=9, entry_tid=0], [object_number=154, tag_tid=8, entry_tid=58722], [object_number=155, tag_tid=8, entry_tid=58719]]], state=connected] 2026-02-18T08:14:44.568 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=153, tag_tid=8, entry_tid=56981], [object_number=152, tag_tid=8, entry_tid=56980], [object_number=155, tag_tid=8, entry_tid=56979], [object_number=154, tag_tid=8, entry_tid=56978]]], state=connected] 2026-02-18T08:14:44.572 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:44.573 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=153, tag_tid=8, entry_tid=56981' 2026-02-18T08:14:44.573 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=153, tag_tid=9, entry_tid=1' -a 'object_number=153, tag_tid=9, entry_tid=1' = 'object_number=153, tag_tid=8, entry_tid=56981' 2026-02-18T08:14:44.573 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=153, tag_tid=8, entry_tid=56981' '!=' 'object_number=153, tag_tid=8, entry_tid=56725' 2026-02-18T08:14:44.573 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:14:44.573 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=153, tag_tid=8, entry_tid=56981' = 'object_number=153, tag_tid=8, entry_tid=56725' 2026-02-18T08:14:44.574 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=153, tag_tid=8, entry_tid=56981' 2026-02-18T08:14:44.574 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:44.574 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:14:44.574 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:44.576 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=9 2026-02-18T08:14:44.576 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=153, tag_tid=8, entry_tid=56981' 2026-02-18T08:14:44.576 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:14:44.576 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:44.578 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=8 2026-02-18T08:14:44.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:44.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:14:44.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:44.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:14:44.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=153, tag_tid=8, entry_tid=56981' 2026-02-18T08:14:44.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:14:44.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:44.582 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=56981 2026-02-18T08:14:44.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 9 = 8 -a 1 -le 56981 2026-02-18T08:14:44.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:14:44.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:14:44.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:14:44.784 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:14:44.784 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:14:44.784 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:14:44.784 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:14:44.784 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:14:44.784 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:14:44.784 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:14:44.784 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:14:44.784 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:14:44.784 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:14:44.784 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:14:44.785 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:14:44.785 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:14:44.785 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:14:44.785 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:14:44.785 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:14:44.785 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:14:44.787 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:14:44.787 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:14:44.787 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:14:44.787 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:14:44.787 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:14:44.787 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:14:44.787 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:14:44.787 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:14:44.787 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:14:44.787 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:14:44.787 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:14:44.787 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:14:44.788 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:14:44.788 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:14:44.788 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:14:44.788 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:14:44.788 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:14:44.788 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:14:44.788 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:14:44.788 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:14:44.788 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:14:44.788 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:14:44.788 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:14:44.788 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:14:44.788 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:14:44.788 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:14:44.788 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:14:44.788 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:14:44.788 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:14:44.788 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:14:44.788 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:14:44.788 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:14:44.799 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:44.799 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:44.799 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:44.799 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:14:44.859 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:14:44.859 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:14:44.859 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:44.859 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:44.859 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:44.859 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:14:44.859 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:44.859 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:44.859 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:44.859 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:14:44.860 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:14:44.860 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:14:44.860 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:14:44.861 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:44.862 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:14:44.862 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:44.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 38 2026-02-18T08:14:44.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 38 2026-02-18T08:14:44.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:14:44.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=153, tag_tid=9, entry_tid=1], [object_number=152, tag_tid=9, entry_tid=0], [object_number=154, tag_tid=8, entry_tid=58722], [object_number=155, tag_tid=8, entry_tid=58719]]], state=connected] 2026-02-18T08:14:44.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=152, tag_tid=8, entry_tid=57236], [object_number=155, tag_tid=8, entry_tid=57235], [object_number=154, tag_tid=8, entry_tid=57234], [object_number=153, tag_tid=8, entry_tid=57233]]], state=connected] 2026-02-18T08:14:44.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:44.894 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:44.894 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:14:44.894 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:44.895 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:44.895 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:44.895 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:14:44.895 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:44.895 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:44.895 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:44.895 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:14:44.895 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:14:44.895 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:14:44.895 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:14:44.896 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:44.897 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:14:44.897 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:44.932 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 38 2026-02-18T08:14:44.933 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 38 2026-02-18T08:14:44.933 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:14:44.933 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=153, tag_tid=9, entry_tid=1], [object_number=152, tag_tid=9, entry_tid=0], [object_number=154, tag_tid=8, entry_tid=58722], [object_number=155, tag_tid=8, entry_tid=58719]]], state=connected] 2026-02-18T08:14:44.933 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=152, tag_tid=8, entry_tid=57236], [object_number=155, tag_tid=8, entry_tid=57235], [object_number=154, tag_tid=8, entry_tid=57234], [object_number=153, tag_tid=8, entry_tid=57233]]], state=connected] 2026-02-18T08:14:44.936 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:44.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=152, tag_tid=8, entry_tid=57236' 2026-02-18T08:14:44.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=153, tag_tid=9, entry_tid=1' -a 'object_number=153, tag_tid=9, entry_tid=1' = 'object_number=152, tag_tid=8, entry_tid=57236' 2026-02-18T08:14:44.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=152, tag_tid=8, entry_tid=57236' '!=' 'object_number=153, tag_tid=8, entry_tid=56981' 2026-02-18T08:14:44.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:14:44.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=152, tag_tid=8, entry_tid=57236' = 'object_number=153, tag_tid=8, entry_tid=56981' 2026-02-18T08:14:44.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=152, tag_tid=8, entry_tid=57236' 2026-02-18T08:14:44.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:44.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:14:44.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:44.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=9 2026-02-18T08:14:44.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=152, tag_tid=8, entry_tid=57236' 2026-02-18T08:14:44.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:14:44.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:44.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=8 2026-02-18T08:14:44.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:44.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:14:44.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:44.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:14:44.946 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=152, tag_tid=8, entry_tid=57236' 2026-02-18T08:14:44.946 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:14:44.946 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:44.947 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=57236 2026-02-18T08:14:44.947 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 9 = 8 -a 1 -le 57236 2026-02-18T08:14:44.947 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:14:44.947 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:14:44.947 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:14:45.149 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:14:45.149 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:14:45.149 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:14:45.149 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:14:45.149 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:14:45.149 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:14:45.149 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:14:45.149 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:14:45.149 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:14:45.149 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:14:45.150 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:14:45.150 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:14:45.150 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:14:45.150 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:14:45.150 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:14:45.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:14:45.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:14:45.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:14:45.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:14:45.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:14:45.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:14:45.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:14:45.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:14:45.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:14:45.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:14:45.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:14:45.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:14:45.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:14:45.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:14:45.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:14:45.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:14:45.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:14:45.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:14:45.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:14:45.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:14:45.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:14:45.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:14:45.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:14:45.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:14:45.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:14:45.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:14:45.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:14:45.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:14:45.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:14:45.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:14:45.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:14:45.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:14:45.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:14:45.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:14:45.164 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:45.164 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:45.164 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:45.164 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:14:45.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:14:45.234 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:14:45.234 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:45.234 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:45.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:45.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:14:45.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:45.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:45.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:45.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:14:45.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:14:45.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:14:45.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:14:45.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:45.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:14:45.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:45.273 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 38 2026-02-18T08:14:45.273 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 38 2026-02-18T08:14:45.273 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:14:45.273 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=153, tag_tid=9, entry_tid=1], [object_number=152, tag_tid=9, entry_tid=0], [object_number=154, tag_tid=8, entry_tid=58722], [object_number=155, tag_tid=8, entry_tid=58719]]], state=connected] 2026-02-18T08:14:45.273 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=155, tag_tid=8, entry_tid=57507], [object_number=154, tag_tid=8, entry_tid=57506], [object_number=153, tag_tid=8, entry_tid=57505], [object_number=152, tag_tid=8, entry_tid=57504]]], state=connected] 2026-02-18T08:14:45.277 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:45.279 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:45.279 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:14:45.280 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:45.280 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:45.280 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:45.280 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:14:45.280 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:45.280 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:45.280 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:45.280 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:14:45.280 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:14:45.280 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:14:45.281 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:14:45.281 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:45.282 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:14:45.282 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:45.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 38 2026-02-18T08:14:45.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 38 2026-02-18T08:14:45.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:14:45.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=153, tag_tid=9, entry_tid=1], [object_number=152, tag_tid=9, entry_tid=0], [object_number=154, tag_tid=8, entry_tid=58722], [object_number=155, tag_tid=8, entry_tid=58719]]], state=connected] 2026-02-18T08:14:45.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=155, tag_tid=8, entry_tid=57507], [object_number=154, tag_tid=8, entry_tid=57506], [object_number=153, tag_tid=8, entry_tid=57505], [object_number=152, tag_tid=8, entry_tid=57504]]], state=connected] 2026-02-18T08:14:45.312 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:45.314 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=155, tag_tid=8, entry_tid=57507' 2026-02-18T08:14:45.314 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=153, tag_tid=9, entry_tid=1' -a 'object_number=153, tag_tid=9, entry_tid=1' = 'object_number=155, tag_tid=8, entry_tid=57507' 2026-02-18T08:14:45.314 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=155, tag_tid=8, entry_tid=57507' '!=' 'object_number=152, tag_tid=8, entry_tid=57236' 2026-02-18T08:14:45.314 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:14:45.314 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=155, tag_tid=8, entry_tid=57507' = 'object_number=152, tag_tid=8, entry_tid=57236' 2026-02-18T08:14:45.315 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=155, tag_tid=8, entry_tid=57507' 2026-02-18T08:14:45.315 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:45.315 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:14:45.315 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:45.316 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=9 2026-02-18T08:14:45.317 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=155, tag_tid=8, entry_tid=57507' 2026-02-18T08:14:45.317 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:14:45.317 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:45.318 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=8 2026-02-18T08:14:45.319 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:45.319 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:14:45.319 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:45.321 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:14:45.322 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=155, tag_tid=8, entry_tid=57507' 2026-02-18T08:14:45.322 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:14:45.322 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:45.323 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=57507 2026-02-18T08:14:45.323 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 9 = 8 -a 1 -le 57507 2026-02-18T08:14:45.324 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:14:45.324 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:14:45.324 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:14:45.525 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:14:45.525 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:14:45.525 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:14:45.525 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:14:45.525 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:14:45.525 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:14:45.525 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:14:45.525 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:14:45.526 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:14:45.526 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:14:45.526 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:14:45.526 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:14:45.526 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:14:45.526 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:14:45.526 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:14:45.526 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:14:45.527 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:14:45.528 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:14:45.529 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:14:45.529 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:14:45.529 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:14:45.529 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:14:45.529 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:14:45.529 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:14:45.529 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:14:45.529 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:14:45.529 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:14:45.529 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:14:45.529 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:14:45.529 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:14:45.529 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:14:45.529 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:14:45.529 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:14:45.529 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:14:45.530 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:14:45.530 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:14:45.530 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:14:45.530 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:14:45.530 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:14:45.530 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:14:45.530 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:14:45.530 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:14:45.530 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:14:45.530 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:14:45.530 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:14:45.530 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:14:45.530 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:14:45.530 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:14:45.530 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:14:45.539 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:45.540 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:45.540 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:45.540 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:14:45.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:14:45.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:14:45.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:45.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:45.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:45.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:14:45.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:45.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:45.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:45.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:14:45.610 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:14:45.610 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:14:45.610 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:14:45.611 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:45.611 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:14:45.611 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:45.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 38 2026-02-18T08:14:45.646 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 38 2026-02-18T08:14:45.646 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:14:45.646 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=153, tag_tid=9, entry_tid=1], [object_number=152, tag_tid=9, entry_tid=0], [object_number=154, tag_tid=8, entry_tid=58722], [object_number=155, tag_tid=8, entry_tid=58719]]], state=connected] 2026-02-18T08:14:45.646 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=152, tag_tid=8, entry_tid=57732], [object_number=155, tag_tid=8, entry_tid=57731], [object_number=154, tag_tid=8, entry_tid=57730], [object_number=153, tag_tid=8, entry_tid=57729]]], state=connected] 2026-02-18T08:14:45.649 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:45.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:45.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:14:45.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:45.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:45.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:45.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:14:45.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:45.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:45.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:45.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:14:45.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:14:45.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:14:45.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:14:45.653 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:45.653 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:14:45.653 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:45.688 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 38 2026-02-18T08:14:45.688 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 38 2026-02-18T08:14:45.688 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:14:45.688 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=153, tag_tid=9, entry_tid=1], [object_number=152, tag_tid=9, entry_tid=0], [object_number=154, tag_tid=8, entry_tid=58722], [object_number=155, tag_tid=8, entry_tid=58719]]], state=connected] 2026-02-18T08:14:45.688 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=152, tag_tid=8, entry_tid=57732], [object_number=155, tag_tid=8, entry_tid=57731], [object_number=154, tag_tid=8, entry_tid=57730], [object_number=153, tag_tid=8, entry_tid=57729]]], state=connected] 2026-02-18T08:14:45.692 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:45.693 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=152, tag_tid=8, entry_tid=57732' 2026-02-18T08:14:45.693 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=153, tag_tid=9, entry_tid=1' -a 'object_number=153, tag_tid=9, entry_tid=1' = 'object_number=152, tag_tid=8, entry_tid=57732' 2026-02-18T08:14:45.693 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=152, tag_tid=8, entry_tid=57732' '!=' 'object_number=155, tag_tid=8, entry_tid=57507' 2026-02-18T08:14:45.694 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:14:45.694 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=152, tag_tid=8, entry_tid=57732' = 'object_number=155, tag_tid=8, entry_tid=57507' 2026-02-18T08:14:45.694 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=152, tag_tid=8, entry_tid=57732' 2026-02-18T08:14:45.694 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:45.694 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:14:45.694 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:45.695 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=9 2026-02-18T08:14:45.696 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=152, tag_tid=8, entry_tid=57732' 2026-02-18T08:14:45.696 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:14:45.696 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:45.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=8 2026-02-18T08:14:45.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:45.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:14:45.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:45.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:14:45.700 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=152, tag_tid=8, entry_tid=57732' 2026-02-18T08:14:45.700 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:14:45.700 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:45.702 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=57732 2026-02-18T08:14:45.702 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 9 = 8 -a 1 -le 57732 2026-02-18T08:14:45.702 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:14:45.702 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:14:45.702 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:14:45.904 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:14:45.904 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:14:45.904 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:14:45.904 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:14:45.904 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:14:45.904 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:14:45.904 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:14:45.904 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:14:45.904 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:14:45.904 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:14:45.904 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:14:45.904 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:14:45.904 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:14:45.905 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:14:45.905 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:14:45.905 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:14:45.905 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:14:45.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:14:45.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:14:45.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:14:45.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:14:45.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:14:45.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:14:45.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:14:45.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:14:45.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:14:45.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:14:45.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:14:45.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:14:45.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:14:45.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:14:45.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:14:45.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:14:45.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:14:45.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:14:45.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:14:45.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:14:45.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:14:45.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:14:45.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:14:45.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:14:45.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:14:45.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:14:45.909 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:14:45.909 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:14:45.909 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:14:45.909 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:14:45.909 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:14:45.909 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:14:45.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:45.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:45.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:45.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:14:45.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:14:45.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:14:45.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:45.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:45.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:45.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:14:45.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:45.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:45.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:45.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:14:45.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:14:45.982 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:14:45.982 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:14:45.982 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:45.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:14:45.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:46.010 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 38 2026-02-18T08:14:46.010 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 38 2026-02-18T08:14:46.010 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:14:46.010 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=153, tag_tid=9, entry_tid=1], [object_number=152, tag_tid=9, entry_tid=0], [object_number=154, tag_tid=8, entry_tid=58722], [object_number=155, tag_tid=8, entry_tid=58719]]], state=connected] 2026-02-18T08:14:46.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=155, tag_tid=8, entry_tid=57959], [object_number=154, tag_tid=8, entry_tid=57958], [object_number=153, tag_tid=8, entry_tid=57957], [object_number=152, tag_tid=8, entry_tid=57956]]], state=connected] 2026-02-18T08:14:46.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:46.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:46.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:14:46.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:46.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:46.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:46.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:14:46.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:46.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:46.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:46.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:14:46.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:14:46.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:14:46.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:14:46.018 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:46.018 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:14:46.018 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:46.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 38 2026-02-18T08:14:46.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 38 2026-02-18T08:14:46.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:14:46.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=153, tag_tid=9, entry_tid=1], [object_number=152, tag_tid=9, entry_tid=0], [object_number=154, tag_tid=8, entry_tid=58722], [object_number=155, tag_tid=8, entry_tid=58719]]], state=connected] 2026-02-18T08:14:46.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=155, tag_tid=8, entry_tid=57959], [object_number=154, tag_tid=8, entry_tid=57958], [object_number=153, tag_tid=8, entry_tid=57957], [object_number=152, tag_tid=8, entry_tid=57956]]], state=connected] 2026-02-18T08:14:46.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:46.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=155, tag_tid=8, entry_tid=57959' 2026-02-18T08:14:46.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=153, tag_tid=9, entry_tid=1' -a 'object_number=153, tag_tid=9, entry_tid=1' = 'object_number=155, tag_tid=8, entry_tid=57959' 2026-02-18T08:14:46.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=155, tag_tid=8, entry_tid=57959' '!=' 'object_number=152, tag_tid=8, entry_tid=57732' 2026-02-18T08:14:46.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:14:46.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=155, tag_tid=8, entry_tid=57959' = 'object_number=152, tag_tid=8, entry_tid=57732' 2026-02-18T08:14:46.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=155, tag_tid=8, entry_tid=57959' 2026-02-18T08:14:46.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:46.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:14:46.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:46.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=9 2026-02-18T08:14:46.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=155, tag_tid=8, entry_tid=57959' 2026-02-18T08:14:46.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:14:46.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:46.055 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=8 2026-02-18T08:14:46.056 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:46.056 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:14:46.056 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:46.058 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:14:46.058 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=155, tag_tid=8, entry_tid=57959' 2026-02-18T08:14:46.059 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:14:46.059 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:46.060 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=57959 2026-02-18T08:14:46.060 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 9 = 8 -a 1 -le 57959 2026-02-18T08:14:46.060 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:14:46.060 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:14:46.060 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:14:46.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:14:46.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:14:46.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:14:46.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:14:46.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:14:46.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:14:46.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:14:46.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:14:46.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:14:46.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:14:46.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:14:46.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:14:46.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:14:46.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:14:46.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:14:46.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:14:46.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:14:46.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:14:46.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:14:46.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:14:46.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:14:46.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:14:46.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:14:46.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:14:46.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:14:46.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:14:46.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:14:46.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:14:46.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:14:46.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:14:46.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:14:46.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:14:46.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:14:46.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:14:46.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:14:46.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:14:46.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:14:46.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:14:46.266 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:14:46.266 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:14:46.266 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:14:46.266 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:14:46.266 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:14:46.266 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:14:46.266 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:14:46.266 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:14:46.266 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:14:46.266 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:14:46.266 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:14:46.277 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:46.277 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:46.278 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:46.278 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:14:46.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:14:46.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:14:46.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:46.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:46.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:46.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:14:46.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:46.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:46.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:46.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:14:46.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:14:46.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:14:46.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:14:46.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:46.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:14:46.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:46.368 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 38 2026-02-18T08:14:46.368 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 38 2026-02-18T08:14:46.368 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:14:46.368 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=153, tag_tid=9, entry_tid=1], [object_number=152, tag_tid=9, entry_tid=0], [object_number=154, tag_tid=8, entry_tid=58722], [object_number=155, tag_tid=8, entry_tid=58719]]], state=connected] 2026-02-18T08:14:46.368 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=155, tag_tid=8, entry_tid=58231], [object_number=154, tag_tid=8, entry_tid=58230], [object_number=153, tag_tid=8, entry_tid=58229], [object_number=152, tag_tid=8, entry_tid=58228]]], state=connected] 2026-02-18T08:14:46.371 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:46.372 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:46.373 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:14:46.373 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:46.373 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:46.373 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:46.373 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:14:46.373 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:46.373 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:46.373 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:46.373 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:14:46.373 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:14:46.374 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:14:46.374 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:14:46.374 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:46.375 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:14:46.375 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:46.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 38 2026-02-18T08:14:46.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 38 2026-02-18T08:14:46.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:14:46.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=153, tag_tid=9, entry_tid=1], [object_number=152, tag_tid=9, entry_tid=0], [object_number=154, tag_tid=8, entry_tid=58722], [object_number=155, tag_tid=8, entry_tid=58719]]], state=connected] 2026-02-18T08:14:46.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=155, tag_tid=8, entry_tid=58231], [object_number=154, tag_tid=8, entry_tid=58230], [object_number=153, tag_tid=8, entry_tid=58229], [object_number=152, tag_tid=8, entry_tid=58228]]], state=connected] 2026-02-18T08:14:46.406 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:46.408 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=155, tag_tid=8, entry_tid=58231' 2026-02-18T08:14:46.408 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=153, tag_tid=9, entry_tid=1' -a 'object_number=153, tag_tid=9, entry_tid=1' = 'object_number=155, tag_tid=8, entry_tid=58231' 2026-02-18T08:14:46.408 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=155, tag_tid=8, entry_tid=58231' '!=' 'object_number=155, tag_tid=8, entry_tid=57959' 2026-02-18T08:14:46.408 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:14:46.408 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=155, tag_tid=8, entry_tid=58231' = 'object_number=155, tag_tid=8, entry_tid=57959' 2026-02-18T08:14:46.408 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=155, tag_tid=8, entry_tid=58231' 2026-02-18T08:14:46.408 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:46.409 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:14:46.409 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:46.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=9 2026-02-18T08:14:46.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=155, tag_tid=8, entry_tid=58231' 2026-02-18T08:14:46.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:14:46.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:46.413 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=8 2026-02-18T08:14:46.413 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:46.413 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:14:46.414 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:46.415 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:14:46.415 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=155, tag_tid=8, entry_tid=58231' 2026-02-18T08:14:46.416 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:14:46.416 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:46.418 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=58231 2026-02-18T08:14:46.418 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 9 = 8 -a 1 -le 58231 2026-02-18T08:14:46.418 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:14:46.418 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:14:46.418 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:14:46.619 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:14:46.620 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:14:46.620 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:14:46.620 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:14:46.620 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:14:46.620 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:14:46.620 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:14:46.620 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:14:46.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:14:46.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:14:46.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:14:46.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:14:46.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:14:46.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:14:46.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:14:46.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:14:46.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:14:46.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:14:46.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:14:46.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:14:46.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:14:46.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:14:46.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:14:46.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:14:46.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:14:46.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:14:46.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:14:46.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:14:46.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:14:46.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:14:46.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:14:46.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:14:46.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:14:46.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:14:46.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:14:46.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:14:46.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:14:46.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:14:46.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:14:46.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:14:46.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:14:46.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:14:46.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:14:46.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:14:46.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:14:46.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:14:46.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:14:46.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:14:46.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:14:46.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:46.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:46.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:46.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:14:46.696 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:14:46.697 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:14:46.697 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:46.697 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:46.697 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:46.697 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:14:46.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:46.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:46.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:46.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:14:46.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:14:46.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:14:46.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:14:46.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:46.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:14:46.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:46.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 38 2026-02-18T08:14:46.728 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 38 2026-02-18T08:14:46.728 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:14:46.728 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=153, tag_tid=9, entry_tid=1], [object_number=152, tag_tid=9, entry_tid=0], [object_number=154, tag_tid=8, entry_tid=58722], [object_number=155, tag_tid=8, entry_tid=58719]]], state=connected] 2026-02-18T08:14:46.728 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=152, tag_tid=8, entry_tid=58516], [object_number=155, tag_tid=8, entry_tid=58515], [object_number=154, tag_tid=8, entry_tid=58514], [object_number=153, tag_tid=8, entry_tid=58513]]], state=connected] 2026-02-18T08:14:46.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:46.732 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:46.733 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:14:46.733 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:46.733 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:46.733 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:46.733 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:14:46.734 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:46.734 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:46.734 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:46.734 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:14:46.734 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:14:46.734 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:14:46.735 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:14:46.735 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:46.735 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:14:46.736 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:46.769 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 38 2026-02-18T08:14:46.770 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 38 2026-02-18T08:14:46.770 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:14:46.770 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=153, tag_tid=9, entry_tid=1], [object_number=152, tag_tid=9, entry_tid=0], [object_number=154, tag_tid=8, entry_tid=58722], [object_number=155, tag_tid=8, entry_tid=58719]]], state=connected] 2026-02-18T08:14:46.770 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=152, tag_tid=8, entry_tid=58516], [object_number=155, tag_tid=8, entry_tid=58515], [object_number=154, tag_tid=8, entry_tid=58514], [object_number=153, tag_tid=8, entry_tid=58513]]], state=connected] 2026-02-18T08:14:46.772 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:46.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=152, tag_tid=8, entry_tid=58516' 2026-02-18T08:14:46.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=153, tag_tid=9, entry_tid=1' -a 'object_number=153, tag_tid=9, entry_tid=1' = 'object_number=152, tag_tid=8, entry_tid=58516' 2026-02-18T08:14:46.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=152, tag_tid=8, entry_tid=58516' '!=' 'object_number=155, tag_tid=8, entry_tid=58231' 2026-02-18T08:14:46.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:14:46.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=152, tag_tid=8, entry_tid=58516' = 'object_number=155, tag_tid=8, entry_tid=58231' 2026-02-18T08:14:46.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=152, tag_tid=8, entry_tid=58516' 2026-02-18T08:14:46.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:46.776 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:14:46.776 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:46.776 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=9 2026-02-18T08:14:46.777 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=152, tag_tid=8, entry_tid=58516' 2026-02-18T08:14:46.777 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:14:46.777 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:46.778 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=8 2026-02-18T08:14:46.779 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:46.780 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:14:46.780 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:46.781 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:14:46.782 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=152, tag_tid=8, entry_tid=58516' 2026-02-18T08:14:46.782 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:14:46.782 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:46.783 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=58516 2026-02-18T08:14:46.784 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 9 = 8 -a 1 -le 58516 2026-02-18T08:14:46.784 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:14:46.784 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:14:46.784 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:14:46.986 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:14:46.986 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:14:46.986 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:14:46.986 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:14:46.986 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:14:46.986 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:14:46.986 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:14:46.987 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:14:46.987 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:14:46.987 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:14:46.987 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:14:46.987 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:14:46.987 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:14:46.987 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:14:46.987 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:14:46.987 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:14:46.987 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:14:46.990 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:14:46.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:14:46.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:14:46.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:14:46.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:14:46.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:14:46.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:14:46.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:14:46.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:14:46.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:14:46.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:14:46.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:14:46.992 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:14:46.992 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:14:46.992 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:14:46.992 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:14:46.992 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:14:46.992 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:14:46.992 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:14:46.992 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:14:46.992 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:14:46.992 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:14:46.992 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:14:46.992 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:14:46.993 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:14:46.993 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:14:46.993 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:14:46.993 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:14:46.993 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:14:46.993 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:14:46.993 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:14:46.993 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:14:46.999 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:47.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:47.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:47.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:14:47.068 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:14:47.069 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:14:47.069 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:47.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:47.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:47.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:14:47.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:47.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:47.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:47.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:14:47.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:14:47.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:14:47.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:14:47.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:47.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:14:47.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:47.100 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 38 2026-02-18T08:14:47.100 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 38 2026-02-18T08:14:47.100 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:14:47.100 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=153, tag_tid=9, entry_tid=1], [object_number=152, tag_tid=9, entry_tid=0], [object_number=154, tag_tid=8, entry_tid=58722], [object_number=155, tag_tid=8, entry_tid=58719]]], state=connected] 2026-02-18T08:14:47.101 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=154, tag_tid=8, entry_tid=58722], [object_number=153, tag_tid=8, entry_tid=58721], [object_number=152, tag_tid=8, entry_tid=58720], [object_number=155, tag_tid=8, entry_tid=58719]]], state=connected] 2026-02-18T08:14:47.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:47.105 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:47.106 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:14:47.106 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:47.106 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:47.106 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:47.106 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:14:47.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:47.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:47.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:47.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:14:47.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:14:47.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:14:47.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:14:47.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:47.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:14:47.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:47.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 38 2026-02-18T08:14:47.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 38 2026-02-18T08:14:47.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:14:47.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=153, tag_tid=9, entry_tid=1], [object_number=152, tag_tid=9, entry_tid=0], [object_number=154, tag_tid=8, entry_tid=58722], [object_number=155, tag_tid=8, entry_tid=58719]]], state=connected] 2026-02-18T08:14:47.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=154, tag_tid=8, entry_tid=58722], [object_number=153, tag_tid=8, entry_tid=58721], [object_number=152, tag_tid=8, entry_tid=58720], [object_number=155, tag_tid=8, entry_tid=58719]]], state=connected] 2026-02-18T08:14:47.137 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:47.139 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=154, tag_tid=8, entry_tid=58722' 2026-02-18T08:14:47.139 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=153, tag_tid=9, entry_tid=1' -a 'object_number=153, tag_tid=9, entry_tid=1' = 'object_number=154, tag_tid=8, entry_tid=58722' 2026-02-18T08:14:47.139 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=154, tag_tid=8, entry_tid=58722' '!=' 'object_number=152, tag_tid=8, entry_tid=58516' 2026-02-18T08:14:47.139 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:14:47.139 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=154, tag_tid=8, entry_tid=58722' = 'object_number=152, tag_tid=8, entry_tid=58516' 2026-02-18T08:14:47.139 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=154, tag_tid=8, entry_tid=58722' 2026-02-18T08:14:47.140 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:47.140 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:14:47.140 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:47.141 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=9 2026-02-18T08:14:47.142 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=154, tag_tid=8, entry_tid=58722' 2026-02-18T08:14:47.142 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:14:47.142 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:47.143 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=8 2026-02-18T08:14:47.144 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:47.144 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:14:47.144 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:47.145 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:14:47.146 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=154, tag_tid=8, entry_tid=58722' 2026-02-18T08:14:47.146 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:14:47.146 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:14:47.147 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=58722 2026-02-18T08:14:47.147 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 9 = 8 -a 1 -le 58722 2026-02-18T08:14:47.148 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:14:47.148 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:14:47.148 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:14:47.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:14:47.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:14:47.350 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:14:47.350 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:14:47.350 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:14:47.350 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:14:47.350 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:14:47.350 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:14:47.350 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:14:47.350 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:14:47.350 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:14:47.350 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:14:47.351 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:14:47.351 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:14:47.351 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:14:47.354 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:14:47.354 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:14:47.354 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:14:47.354 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:14:47.354 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:14:47.354 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:14:47.355 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:14:47.355 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:14:47.355 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:14:47.355 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:14:47.355 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:14:47.355 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:14:47.355 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:14:47.355 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:14:47.355 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:14:47.355 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:14:47.355 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:14:47.356 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:14:47.356 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:14:47.356 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:14:47.356 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:14:47.356 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:14:47.356 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:14:47.356 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:14:47.356 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:14:47.356 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:14:47.356 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:14:47.357 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:14:47.357 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:14:47.357 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:14:47.357 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:14:47.357 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:14:47.357 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:14:47.357 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:14:47.364 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:47.365 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:47.365 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:47.365 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:14:47.431 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:14:47.431 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:14:47.431 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:47.431 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:47.431 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:47.431 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:14:47.431 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:47.431 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:47.432 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:47.432 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:14:47.432 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:14:47.432 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:14:47.432 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:14:47.433 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:47.433 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:14:47.433 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:47.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 38 2026-02-18T08:14:47.470 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 38 2026-02-18T08:14:47.470 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:14:47.470 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=153, tag_tid=9, entry_tid=1], [object_number=152, tag_tid=9, entry_tid=0], [object_number=154, tag_tid=8, entry_tid=58722], [object_number=155, tag_tid=8, entry_tid=58719]]], state=connected] 2026-02-18T08:14:47.470 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=154, tag_tid=8, entry_tid=58722], [object_number=153, tag_tid=8, entry_tid=58721], [object_number=152, tag_tid=8, entry_tid=58720], [object_number=155, tag_tid=8, entry_tid=58719]]], state=connected] 2026-02-18T08:14:47.473 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:47.475 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:47.476 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:14:47.476 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:47.477 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:47.477 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:47.477 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:14:47.477 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:47.477 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:47.477 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:47.477 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:14:47.477 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:14:47.477 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:14:47.478 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:14:47.478 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:47.478 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:14:47.479 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:47.506 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 38 2026-02-18T08:14:47.506 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 38 2026-02-18T08:14:47.506 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:14:47.506 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=153, tag_tid=9, entry_tid=1], [object_number=152, tag_tid=9, entry_tid=0], [object_number=154, tag_tid=8, entry_tid=58722], [object_number=155, tag_tid=8, entry_tid=58719]]], state=connected] 2026-02-18T08:14:47.506 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=154, tag_tid=8, entry_tid=58722], [object_number=153, tag_tid=8, entry_tid=58721], [object_number=152, tag_tid=8, entry_tid=58720], [object_number=155, tag_tid=8, entry_tid=58719]]], state=connected] 2026-02-18T08:14:47.508 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:47.510 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=154, tag_tid=8, entry_tid=58722' 2026-02-18T08:14:47.510 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=153, tag_tid=9, entry_tid=1' -a 'object_number=153, tag_tid=9, entry_tid=1' = 'object_number=154, tag_tid=8, entry_tid=58722' 2026-02-18T08:14:47.510 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=154, tag_tid=8, entry_tid=58722' '!=' 'object_number=154, tag_tid=8, entry_tid=58722' 2026-02-18T08:14:47.510 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:14:47.510 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.4 2026-02-18T08:14:47.911 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:14:47.911 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:14:47.911 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:14:47.911 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:14:47.912 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:14:47.912 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:14:47.912 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:14:47.912 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:14:47.912 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:14:47.912 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:14:47.912 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:14:47.912 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:14:47.912 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:14:47.912 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:14:47.912 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:14:47.912 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:14:47.913 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:14:47.914 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:14:47.914 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:14:47.914 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:14:47.914 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:14:47.914 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:14:47.914 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:14:47.914 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:14:47.914 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:14:47.914 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:14:47.914 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:14:47.915 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:14:47.915 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:14:47.915 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:14:47.915 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:14:47.915 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:14:47.915 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:14:47.915 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:14:47.916 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:14:47.916 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:14:47.916 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:14:47.916 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:14:47.916 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:14:47.916 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:14:47.916 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:14:47.916 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:14:47.916 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:14:47.916 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:14:47.916 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:14:47.916 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:14:47.916 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:14:47.916 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:14:47.916 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:14:47.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:47.928 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:47.928 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:47.928 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:14:47.993 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:14:47.994 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:14:47.994 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:47.994 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:47.994 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:47.994 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:14:47.994 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:47.994 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:47.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:47.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:14:47.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:14:47.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:14:47.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:14:47.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:47.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:14:47.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:48.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 38 2026-02-18T08:14:48.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 38 2026-02-18T08:14:48.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:14:48.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=153, tag_tid=9, entry_tid=1], [object_number=152, tag_tid=9, entry_tid=0], [object_number=154, tag_tid=8, entry_tid=58722], [object_number=155, tag_tid=8, entry_tid=58719]]], state=connected] 2026-02-18T08:14:48.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=154, tag_tid=8, entry_tid=58722], [object_number=153, tag_tid=8, entry_tid=58721], [object_number=152, tag_tid=8, entry_tid=58720], [object_number=155, tag_tid=8, entry_tid=58719]]], state=connected] 2026-02-18T08:14:48.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:48.028 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:48.029 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:14:48.029 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:48.029 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:48.029 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:48.029 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:14:48.029 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:48.029 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:48.029 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:48.029 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:14:48.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:14:48.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:14:48.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:14:48.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:48.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:14:48.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:48.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 38 2026-02-18T08:14:48.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 38 2026-02-18T08:14:48.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:14:48.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=153, tag_tid=9, entry_tid=1], [object_number=152, tag_tid=9, entry_tid=0], [object_number=154, tag_tid=8, entry_tid=58722], [object_number=155, tag_tid=8, entry_tid=58719]]], state=connected] 2026-02-18T08:14:48.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=154, tag_tid=8, entry_tid=58722], [object_number=153, tag_tid=8, entry_tid=58721], [object_number=152, tag_tid=8, entry_tid=58720], [object_number=155, tag_tid=8, entry_tid=58719]]], state=connected] 2026-02-18T08:14:48.069 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:48.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=154, tag_tid=8, entry_tid=58722' 2026-02-18T08:14:48.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=153, tag_tid=9, entry_tid=1' -a 'object_number=153, tag_tid=9, entry_tid=1' = 'object_number=154, tag_tid=8, entry_tid=58722' 2026-02-18T08:14:48.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=154, tag_tid=8, entry_tid=58722' '!=' 'object_number=154, tag_tid=8, entry_tid=58722' 2026-02-18T08:14:48.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:14:48.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.8 2026-02-18T08:14:48.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:14:48.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:14:48.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:14:48.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:14:48.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:14:48.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:14:48.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:14:48.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:14:48.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:14:48.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:14:48.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:14:48.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:14:48.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:14:48.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:14:48.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:14:48.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:14:48.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:14:48.875 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:14:48.875 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:14:48.875 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:14:48.875 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:14:48.875 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:14:48.875 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:14:48.875 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:14:48.875 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:14:48.875 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:14:48.875 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:14:48.875 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:14:48.875 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:14:48.876 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:14:48.876 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:14:48.876 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:14:48.876 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:14:48.877 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:14:48.877 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:14:48.877 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:14:48.877 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:14:48.877 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:14:48.877 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:14:48.877 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:14:48.877 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:14:48.877 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:14:48.877 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:14:48.877 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:14:48.877 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:14:48.878 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:14:48.878 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:14:48.878 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:14:48.878 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:14:48.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:48.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:48.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:48.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:14:48.954 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:14:48.955 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:14:48.955 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:48.955 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:48.955 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:48.955 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:14:48.955 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:48.955 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:48.955 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:48.955 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:14:48.955 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:14:48.955 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:14:48.955 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:14:48.956 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:48.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:14:48.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:48.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 38 2026-02-18T08:14:48.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 38 2026-02-18T08:14:48.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:14:48.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=153, tag_tid=9, entry_tid=1], [object_number=152, tag_tid=9, entry_tid=0], [object_number=154, tag_tid=8, entry_tid=58722], [object_number=155, tag_tid=8, entry_tid=58719]]], state=connected] 2026-02-18T08:14:48.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=154, tag_tid=8, entry_tid=58722], [object_number=153, tag_tid=8, entry_tid=58721], [object_number=152, tag_tid=8, entry_tid=58720], [object_number=155, tag_tid=8, entry_tid=58719]]], state=connected] 2026-02-18T08:14:48.994 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:48.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:48.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:14:48.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:48.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:48.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:48.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:14:48.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:48.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:48.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:48.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:14:48.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:14:48.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:14:48.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:14:48.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:48.999 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:14:48.999 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:49.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 38 2026-02-18T08:14:49.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 38 2026-02-18T08:14:49.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:14:49.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=153, tag_tid=9, entry_tid=1], [object_number=152, tag_tid=9, entry_tid=0], [object_number=154, tag_tid=8, entry_tid=58722], [object_number=155, tag_tid=8, entry_tid=58719]]], state=connected] 2026-02-18T08:14:49.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=154, tag_tid=8, entry_tid=58722], [object_number=153, tag_tid=8, entry_tid=58721], [object_number=152, tag_tid=8, entry_tid=58720], [object_number=155, tag_tid=8, entry_tid=58719]]], state=connected] 2026-02-18T08:14:49.036 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:49.037 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=154, tag_tid=8, entry_tid=58722' 2026-02-18T08:14:49.037 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=153, tag_tid=9, entry_tid=1' -a 'object_number=153, tag_tid=9, entry_tid=1' = 'object_number=154, tag_tid=8, entry_tid=58722' 2026-02-18T08:14:49.038 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=154, tag_tid=8, entry_tid=58722' '!=' 'object_number=154, tag_tid=8, entry_tid=58722' 2026-02-18T08:14:49.038 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:14:49.038 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1.6 2026-02-18T08:14:50.639 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:14:50.639 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:14:50.639 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:14:50.640 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:14:50.640 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:14:50.640 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:14:50.640 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:14:50.640 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:14:50.640 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:14:50.640 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:14:50.640 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:14:50.640 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:14:50.640 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:14:50.640 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:14:50.640 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:14:50.642 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:14:50.642 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:14:50.643 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:14:50.643 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:14:50.643 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:14:50.643 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:14:50.643 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:14:50.643 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:14:50.643 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:14:50.643 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:14:50.643 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:14:50.643 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:14:50.643 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:14:50.643 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:14:50.643 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:14:50.643 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:14:50.643 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:14:50.643 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:14:50.643 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:14:50.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:14:50.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:14:50.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:14:50.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:14:50.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:14:50.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:14:50.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:14:50.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:14:50.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:14:50.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:14:50.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:14:50.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:14:50.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:14:50.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:14:50.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:14:50.653 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:50.653 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:50.654 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:50.654 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:14:50.719 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:14:50.720 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:14:50.720 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:50.720 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:50.720 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:50.720 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:14:50.720 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:50.720 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:50.720 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:50.720 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:14:50.720 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:14:50.721 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:14:50.721 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:14:50.722 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:50.722 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:14:50.722 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:50.751 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 38 2026-02-18T08:14:50.751 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 38 2026-02-18T08:14:50.751 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:14:50.751 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=153, tag_tid=9, entry_tid=1], [object_number=152, tag_tid=9, entry_tid=0], [object_number=154, tag_tid=8, entry_tid=58722], [object_number=155, tag_tid=8, entry_tid=58719]]], state=connected] 2026-02-18T08:14:50.751 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=154, tag_tid=8, entry_tid=58722], [object_number=153, tag_tid=8, entry_tid=58721], [object_number=152, tag_tid=8, entry_tid=58720], [object_number=155, tag_tid=8, entry_tid=58719]]], state=connected] 2026-02-18T08:14:50.753 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:50.755 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:50.756 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:14:50.756 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:50.756 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:50.756 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:50.756 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:14:50.756 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:50.756 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:50.756 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:50.756 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:14:50.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:14:50.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:14:50.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:14:50.758 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:50.759 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:14:50.759 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:50.786 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 38 2026-02-18T08:14:50.787 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 38 2026-02-18T08:14:50.787 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:14:50.787 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=153, tag_tid=9, entry_tid=1], [object_number=152, tag_tid=9, entry_tid=0], [object_number=154, tag_tid=8, entry_tid=58722], [object_number=155, tag_tid=8, entry_tid=58719]]], state=connected] 2026-02-18T08:14:50.787 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=154, tag_tid=8, entry_tid=58722], [object_number=153, tag_tid=8, entry_tid=58721], [object_number=152, tag_tid=8, entry_tid=58720], [object_number=155, tag_tid=8, entry_tid=58719]]], state=connected] 2026-02-18T08:14:50.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:50.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=154, tag_tid=8, entry_tid=58722' 2026-02-18T08:14:50.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=153, tag_tid=9, entry_tid=1' -a 'object_number=153, tag_tid=9, entry_tid=1' = 'object_number=154, tag_tid=8, entry_tid=58722' 2026-02-18T08:14:50.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=154, tag_tid=8, entry_tid=58722' '!=' 'object_number=154, tag_tid=8, entry_tid=58722' 2026-02-18T08:14:50.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:14:50.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 2 2026-02-18T08:14:52.793 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:14:52.793 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:14:52.793 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:14:52.793 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:14:52.793 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:14:52.793 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:14:52.793 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:14:52.793 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:14:52.793 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:14:52.793 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:14:52.794 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:14:52.794 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:14:52.794 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:14:52.794 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:14:52.794 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:14:52.794 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:14:52.794 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:14:52.796 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:14:52.796 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:14:52.796 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:14:52.796 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:14:52.796 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:14:52.796 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:14:52.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:14:52.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:14:52.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:14:52.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:14:52.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:14:52.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:14:52.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:14:52.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:14:52.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:14:52.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:14:52.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:14:52.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:14:52.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:14:52.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:14:52.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:14:52.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:14:52.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:14:52.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:14:52.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:14:52.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:14:52.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:14:52.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:14:52.798 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:14:52.798 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:14:52.798 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:14:52.798 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:14:52.809 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:52.809 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:52.809 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:14:52.809 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:14:52.879 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:14:52.879 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:14:52.879 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:52.879 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:52.879 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:52.879 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:14:52.879 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:52.879 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:52.880 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:52.880 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:14:52.880 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:14:52.880 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:14:52.880 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:14:52.881 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:52.882 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:14:52.882 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:52.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 38 2026-02-18T08:14:52.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 38 2026-02-18T08:14:52.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:14:52.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=153, tag_tid=9, entry_tid=1], [object_number=152, tag_tid=9, entry_tid=0], [object_number=154, tag_tid=8, entry_tid=58722], [object_number=155, tag_tid=8, entry_tid=58719]]], state=connected] 2026-02-18T08:14:52.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=153, tag_tid=9, entry_tid=1], [object_number=152, tag_tid=9, entry_tid=0], [object_number=154, tag_tid=8, entry_tid=58722], [object_number=155, tag_tid=8, entry_tid=58719]]], state=connected] 2026-02-18T08:14:52.923 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:52.924 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:52.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:14:52.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:52.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:52.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:52.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:14:52.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:14:52.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:14:52.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:14:52.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:14:52.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:14:52.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:14:52.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:14:52.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:52.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:14:52.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:52.961 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 38 2026-02-18T08:14:52.961 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 38 2026-02-18T08:14:52.961 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:14:52.962 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=153, tag_tid=9, entry_tid=1], [object_number=152, tag_tid=9, entry_tid=0], [object_number=154, tag_tid=8, entry_tid=58722], [object_number=155, tag_tid=8, entry_tid=58719]]], state=connected] 2026-02-18T08:14:52.962 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=153, tag_tid=9, entry_tid=1], [object_number=152, tag_tid=9, entry_tid=0], [object_number=154, tag_tid=8, entry_tid=58722], [object_number=155, tag_tid=8, entry_tid=58719]]], state=connected] 2026-02-18T08:14:52.965 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:14:52.967 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:52.967 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=153, tag_tid=9, entry_tid=1' -a 'object_number=153, tag_tid=9, entry_tid=1' = 'object_number=153, tag_tid=9, entry_tid=1' 2026-02-18T08:14:52.967 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:14:52.967 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_snap_present cluster1 mirror test snap3 2026-02-18T08:14:52.968 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:14:52.968 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:14:52.968 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:14:52.968 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap3 2026-02-18T08:14:52.968 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-18T08:14:52.968 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-18T08:14:52.968 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1 2026-02-18T08:14:53.968 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster1 info mirror/test@snap3 2026-02-18T08:14:53.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:rbd image 'test': 2026-02-18T08:14:53.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: size 512 MiB in 128 objects 2026-02-18T08:14:53.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: order 22 (4 MiB objects) 2026-02-18T08:14:53.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: snapshot_count: 5 2026-02-18T08:14:53.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: id: 10a74aa8ebc0 2026-02-18T08:14:53.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: block_name_prefix: rbd_data.10a74aa8ebc0 2026-02-18T08:14:53.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: format: 2 2026-02-18T08:14:53.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: features: layering, exclusive-lock, journaling 2026-02-18T08:14:53.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: op_features: 2026-02-18T08:14:53.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: flags: 2026-02-18T08:14:53.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: create_timestamp: Wed Feb 18 08:10:48 2026 2026-02-18T08:14:53.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: access_timestamp: Wed Feb 18 08:13:13 2026 2026-02-18T08:14:53.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: modify_timestamp: Wed Feb 18 08:13:51 2026 2026-02-18T08:14:53.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: protected: False 2026-02-18T08:14:53.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: journal: 10a74aa8ebc0 2026-02-18T08:14:53.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring state: enabled 2026-02-18T08:14:53.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring mode: journal 2026-02-18T08:14:53.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring global id: 0b548823-d751-43df-a076-4efac2da4d3d 2026-02-18T08:14:53.999 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring primary: false 2026-02-18T08:14:54.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:14:54.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n snap2-clean ']' 2026-02-18T08:14:54.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ compare_image_snaps mirror test snap2-clean 2026-02-18T08:14:54.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:14:54.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:14:54.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap2-clean 2026-02-18T08:14:54.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local ret=0 2026-02-18T08:14:54.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local rmt_export=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.export 2026-02-18T08:14:54.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loc_export=/tmp/tmp.vybh1m9yJF/cluster1-mirror-test.export 2026-02-18T08:14:54.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.export /tmp/tmp.vybh1m9yJF/cluster1-mirror-test.export 2026-02-18T08:14:54.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 -p mirror export test@snap2-clean /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.export 2026-02-18T08:14:54.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-18T08:14:54.507 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster1 -p mirror export test@snap2-clean /tmp/tmp.vybh1m9yJF/cluster1-mirror-test.export 2026-02-18T08:14:55.143 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-18T08:14:55.157 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmp /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.export /tmp/tmp.vybh1m9yJF/cluster1-mirror-test.export 2026-02-18T08:14:55.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.export /tmp/tmp.vybh1m9yJF/cluster1-mirror-test.export 2026-02-18T08:14:55.465 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:14:55.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ compare_image_snaps mirror test snap3 2026-02-18T08:14:55.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:14:55.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:14:55.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap3 2026-02-18T08:14:55.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local ret=0 2026-02-18T08:14:55.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local rmt_export=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.export 2026-02-18T08:14:55.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loc_export=/tmp/tmp.vybh1m9yJF/cluster1-mirror-test.export 2026-02-18T08:14:55.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.export /tmp/tmp.vybh1m9yJF/cluster1-mirror-test.export 2026-02-18T08:14:55.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 -p mirror export test@snap3 /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.export 2026-02-18T08:14:55.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-18T08:14:55.949 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster1 -p mirror export test@snap3 /tmp/tmp.vybh1m9yJF/cluster1-mirror-test.export 2026-02-18T08:14:56.566 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-18T08:14:56.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmp /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.export /tmp/tmp.vybh1m9yJF/cluster1-mirror-test.export 2026-02-18T08:14:56.779 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.export /tmp/tmp.vybh1m9yJF/cluster1-mirror-test.export 2026-02-18T08:14:56.922 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:14:56.922 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ clean_snap_name=snap3-clean 2026-02-18T08:14:56.922 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_snap cluster2 mirror test snap3-clean 2026-02-18T08:14:56.922 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:14:56.922 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:14:56.922 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:14:56.922 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap3-clean 2026-02-18T08:14:56.922 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 -p mirror snap create test@snap3-clean --debug-rbd=20 --debug-journaler=20 2026-02-18T08:14:57.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 10` 2026-02-18T08:14:57.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ stress_write_image cluster2 mirror test 2026-02-18T08:14:57.700 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:14:57.700 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:14:57.700 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:14:57.700 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-18T08:14:57.702 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local duration=8 2026-02-18T08:14:57.702 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set +e 2026-02-18T08:14:57.702 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ timeout 8s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror test --debug-rbd=20 --debug-journaler=20 2026-02-18T08:14:58.129 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:100 IOs, 20889600 bytes 2026-02-18T08:14:58.821 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:200 IOs, 71605248 bytes 2026-02-18T08:14:59.524 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:300 IOs, 118567936 bytes 2026-02-18T08:15:00.253 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:400 IOs, 174823424 bytes 2026-02-18T08:15:01.097 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:500 IOs, 228816896 bytes 2026-02-18T08:15:02.362 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:600 IOs, 295046144 bytes 2026-02-18T08:15:03.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:700 IOs, 327223296 bytes 2026-02-18T08:15:03.959 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:800 IOs, 355356672 bytes 2026-02-18T08:15:04.513 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:900 IOs, 399242240 bytes 2026-02-18T08:15:05.712 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ error_code=124 2026-02-18T08:15:05.712 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set -e 2026-02-18T08:15:05.712 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 124 -eq 124 ']' 2026-02-18T08:15:05.712 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:15:05.712 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+replaying 2026-02-18T08:15:05.712 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:15:05.713 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:15:05.713 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:15:05.713 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local state_pattern=up+replaying 2026-02-18T08:15:05.713 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local description_pattern= 2026-02-18T08:15:05.713 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local service_pattern= 2026-02-18T08:15:05.713 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:15:05.713 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1 2026-02-18T08:15:06.713 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test_status_in_pool_dir cluster1 mirror test up+replaying '' '' 2026-02-18T08:15:06.713 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:15:06.713 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:15:06.713 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:15:06.713 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local state_pattern=up+replaying 2026-02-18T08:15:06.713 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local description_pattern= 2026-02-18T08:15:06.714 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local service_pattern= 2026-02-18T08:15:06.714 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local status 2026-02-18T08:15:06.714 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ CEPH_ARGS= 2026-02-18T08:15:06.714 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-18T08:15:06.752 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ status='test: 2026-02-18T08:15:06.752 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: global_id: 0b548823-d751-43df-a076-4efac2da4d3d 2026-02-18T08:15:06.753 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: state: up+replaying 2026-02-18T08:15:06.753 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: description: replaying, {"bytes_per_second":10154678.6,"entries_behind_primary":764,"entries_per_second":638.65,"non_primary_position":{"entry_tid":57959,"object_number":155,"tag_tid":8},"primary_position":{"entry_tid":1,"object_number":153,"tag_tid":9},"seconds_until_synced":1} 2026-02-18T08:15:06.753 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: service: mirror.0 on trial163 2026-02-18T08:15:06.753 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: last_update: 2026-02-18 08:14:46 2026-02-18T08:15:06.753 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: peer_sites: 2026-02-18T08:15:06.753 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: name: cluster2 2026-02-18T08:15:06.753 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: state: up+stopped 2026-02-18T08:15:06.753 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: description: local image is primary 2026-02-18T08:15:06.753 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: last_update: 2026-02-18 08:14:46' 2026-02-18T08:15:06.753 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ grep '^ state: .*up+replaying' 2026-02-18T08:15:06.754 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: state: up+replaying 2026-02-18T08:15:06.754 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ grep '^ description: .*' 2026-02-18T08:15:06.755 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: description: replaying, {"bytes_per_second":10154678.6,"entries_behind_primary":764,"entries_per_second":638.65,"non_primary_position":{"entry_tid":57959,"object_number":155,"tag_tid":8},"primary_position":{"entry_tid":1,"object_number":153,"tag_tid":9},"seconds_until_synced":1} 2026-02-18T08:15:06.756 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n '' ']' 2026-02-18T08:15:06.756 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ echo up+replaying 2026-02-18T08:15:06.756 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ grep '^up+' 2026-02-18T08:15:06.756 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:up+replaying 2026-02-18T08:15:06.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ grep 'service: *mirror..* on ' 2026-02-18T08:15:06.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: service: mirror.0 on trial163 2026-02-18T08:15:06.758 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local last_update 2026-02-18T08:15:06.758 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-18T08:15:06.760 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_update='2026-02-18 08:14:46' 2026-02-18T08:15:06.760 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+replaying '2026-02-18 08:14:46' 2026-02-18T08:15:06.760 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:15:06.760 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:15:06.760 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:15:06.760 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local state_pattern=up+replaying 2026-02-18T08:15:06.761 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'prev_last_update=2026-02-18 08:14:46' 2026-02-18T08:15:06.761 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local status 2026-02-18T08:15:06.761 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ CEPH_ARGS= 2026-02-18T08:15:06.761 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-18T08:15:06.804 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ status='OKOKOK142074263mirror.0trial16320.2.0-670-gadc19233trueOK42194268mirror.3trial16320.2.0-670-gadc19233falseOK43304333mirror.2trial16320.2.0-670-gadc19233falseOK43534356mirror.1trial16320.2.0-670-gadc19233falseOKtest0b548823-d751-43df-a076-4efac2da4d3dup+replayingreplaying, {"bytes_per_second":10154678.6,"entries_behind_primary":764,"entries_per_second":638.65,"non_primary_position":{"entry_tid":57959,"object_number":155,"tag_tid":8},"primary_position":{"entry_tid":1,"object_number":153,"tag_tid":9},"seconds_until_synced":1}42074263mirror.0trial1632026-02-18 08:14:46cluster2c5e4f85e-ec31-430b-a71d-56e000d0de0bup+stoppedlocal image is primary2026-02-18 08:14:46' 2026-02-18T08:15:06.804 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local last_update state 2026-02-18T08:15:06.805 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-18T08:15:06.807 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_update='2026-02-18 08:14:46' 2026-02-18T08:15:06.807 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-18T08:15:06.810 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ state=up+replaying 2026-02-18T08:15:06.811 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ echo up+replaying 2026-02-18T08:15:06.811 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ grep up+replaying 2026-02-18T08:15:06.812 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:up+replaying 2026-02-18T08:15:06.813 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:15:06.813 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:15:06.813 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ snap_name=snap4 2026-02-18T08:15:06.813 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_snap cluster2 mirror test snap4 2026-02-18T08:15:06.813 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:15:06.813 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:15:06.813 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:15:06.814 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap4 2026-02-18T08:15:06.814 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 -p mirror snap create test@snap4 --debug-rbd=20 --debug-journaler=20 2026-02-18T08:15:37.735 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_replay_started cluster1 mirror test 2026-02-18T08:15:37.735 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:15:37.736 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:15:37.736 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:15:37.736 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_replay_state cluster1 mirror test started 2026-02-18T08:15:37.736 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:15:37.736 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:15:37.736 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:15:37.736 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local state=started 2026-02-18T08:15:37.736 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-18T08:15:37.736 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:15:37.736 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1 2026-02-18T08:15:38.736 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test_image_replay_state cluster1 mirror test started 2026-02-18T08:15:38.736 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:15:38.736 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:15:38.736 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:15:38.737 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local test_state=started 2026-02-18T08:15:38.737 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local status_result 2026-02-18T08:15:38.737 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_state=stopped 2026-02-18T08:15:38.737 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-18T08:15:38.737 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -i state 2026-02-18T08:15:38.737 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_instance=cluster1 2026-02-18T08:15:38.737 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-18T08:15:38.737 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:15:38.737 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance=cluster1 2026-02-18T08:15:38.737 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local loop_instance 2026-02-18T08:15:38.737 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:15:38.737 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sleep 0 2026-02-18T08:15:38.738 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:15:38.738 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ seq 0 3 2026-02-18T08:15:38.740 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:15:38.740 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-18T08:15:38.740 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:0 2026-02-18T08:15:38.740 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-18T08:15:38.740 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:15:38.741 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:15:38.741 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:15:38.741 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-18T08:15:38.741 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:15:38.741 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:15:38.741 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:15:38.741 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:15:38.741 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-18T08:15:38.741 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:15:38.741 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:15:38.741 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:15:38.741 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:15:38.741 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:0 2026-02-18T08:15:38.741 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-18T08:15:38.741 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-18T08:15:38.741 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:15:38.741 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:0 2026-02-18T08:15:38.741 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T08:15:38.741 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-18T08:15:38.741 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:15:38.741 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:15:38.742 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-18T08:15:38.742 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-18T08:15:38.742 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-18T08:15:38.742 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=0 2026-02-18T08:15:38.742 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=0 2026-02-18T08:15:38.742 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:15:38.751 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:15:38.751 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:15:38.752 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:15:38.752 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-18T08:15:38.808 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ return 0 2026-02-18T08:15:38.809 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ status_result=' "state": "Replaying"' 2026-02-18T08:15:38.809 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ echo ' "state": "Replaying"' 2026-02-18T08:15:38.809 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ grep -i Replaying 2026-02-18T08:15:38.810 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: "state": "Replaying" 2026-02-18T08:15:38.812 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ current_state=started 2026-02-18T08:15:38.812 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test started = started 2026-02-18T08:15:38.812 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:15:38.812 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror test 2026-02-18T08:15:38.812 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_cluster=cluster1 2026-02-18T08:15:38.812 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:15:38.812 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_pool=mirror 2026-02-18T08:15:38.812 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local remote_pool=mirror 2026-02-18T08:15:38.812 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:15:38.812 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' journal = journal ']' 2026-02-18T08:15:38.813 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror mirror test 2026-02-18T08:15:38.813 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_cluster=cluster1 2026-02-18T08:15:38.813 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:15:38.813 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_pool=mirror 2026-02-18T08:15:38.813 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local remote_pool=mirror 2026-02-18T08:15:38.813 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:15:38.813 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-02-18T08:15:38.813 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-02-18T08:15:38.813 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:15:38.813 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:15:38.813 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:15:39.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:15:39.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:15:39.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:15:39.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:15:39.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:15:39.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:15:39.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:15:39.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:15:39.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:15:39.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:15:39.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:15:39.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:15:39.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:15:39.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:15:39.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:15:39.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:15:39.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:15:39.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:15:39.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:15:39.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:15:39.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:15:39.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:15:39.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:15:39.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:15:39.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:15:39.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:15:39.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:15:39.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:15:39.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:15:39.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:15:39.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:15:39.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:15:39.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:15:39.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:15:39.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:15:39.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:15:39.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:15:39.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:15:39.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:15:39.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:15:39.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:15:39.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:15:39.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:15:39.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:15:39.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:15:39.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:15:39.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:15:39.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:15:39.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:15:39.028 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:15:39.028 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:15:39.028 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:15:39.028 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:15:39.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:15:39.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:15:39.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:15:39.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:15:39.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:15:39.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:15:39.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:15:39.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:15:39.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:15:39.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:15:39.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:15:39.092 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:15:39.092 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:15:39.092 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:15:39.093 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:15:39.093 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:15:39.120 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 44 2026-02-18T08:15:39.120 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 45 2026-02-18T08:15:39.120 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:15:39.120 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=181, tag_tid=12, entry_tid=1], [object_number=180, tag_tid=12, entry_tid=0], [object_number=183, tag_tid=11, entry_tid=29711], [object_number=182, tag_tid=11, entry_tid=29710]]], state=connected] 2026-02-18T08:15:39.121 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=179, tag_tid=11, entry_tid=23567], [object_number=178, tag_tid=11, entry_tid=23566], [object_number=177, tag_tid=11, entry_tid=23565], [object_number=176, tag_tid=11, entry_tid=23564]]], state=connected] 2026-02-18T08:15:39.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:15:39.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=181, tag_tid=12, entry_tid=1' 2026-02-18T08:15:39.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:15:39.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:15:39.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:15:39.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:15:39.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:15:39.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:15:39.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:15:39.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:15:39.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:15:39.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:15:39.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:15:39.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:15:39.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:15:39.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:15:39.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:15:39.155 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 44 2026-02-18T08:15:39.155 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 45 2026-02-18T08:15:39.155 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:15:39.155 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=181, tag_tid=12, entry_tid=1], [object_number=180, tag_tid=12, entry_tid=0], [object_number=183, tag_tid=11, entry_tid=29711], [object_number=182, tag_tid=11, entry_tid=29710]]], state=connected] 2026-02-18T08:15:39.155 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=179, tag_tid=11, entry_tid=23567], [object_number=178, tag_tid=11, entry_tid=23566], [object_number=177, tag_tid=11, entry_tid=23565], [object_number=176, tag_tid=11, entry_tid=23564]]], state=connected] 2026-02-18T08:15:39.157 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:15:39.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=179, tag_tid=11, entry_tid=23567' 2026-02-18T08:15:39.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=181, tag_tid=12, entry_tid=1' -a 'object_number=181, tag_tid=12, entry_tid=1' = 'object_number=179, tag_tid=11, entry_tid=23567' 2026-02-18T08:15:39.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=179, tag_tid=11, entry_tid=23567' '!=' '' 2026-02-18T08:15:39.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:15:39.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=179, tag_tid=11, entry_tid=23567' = '' 2026-02-18T08:15:39.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=179, tag_tid=11, entry_tid=23567' 2026-02-18T08:15:39.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=181, tag_tid=12, entry_tid=1' 2026-02-18T08:15:39.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:15:39.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:15:39.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=12 2026-02-18T08:15:39.161 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=179, tag_tid=11, entry_tid=23567' 2026-02-18T08:15:39.161 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:15:39.161 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:15:39.163 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=11 2026-02-18T08:15:39.164 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=181, tag_tid=12, entry_tid=1' 2026-02-18T08:15:39.164 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:15:39.164 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:15:39.165 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:15:39.165 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=179, tag_tid=11, entry_tid=23567' 2026-02-18T08:15:39.166 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:15:39.166 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:15:39.167 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=23567 2026-02-18T08:15:39.167 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 12 = 11 -a 1 -le 23567 2026-02-18T08:15:39.167 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:15:39.167 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:15:39.167 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:15:39.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:15:39.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:15:39.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:15:39.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:15:39.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:15:39.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:15:39.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:15:39.370 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:15:39.370 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:15:39.370 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:15:39.370 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:15:39.370 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:15:39.370 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:15:39.370 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:15:39.370 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:15:39.370 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:15:39.371 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:15:39.372 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:15:39.372 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:15:39.372 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:15:39.372 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:15:39.372 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:15:39.372 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:15:39.372 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:15:39.372 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:15:39.372 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:15:39.372 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:15:39.372 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:15:39.372 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:15:39.372 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:15:39.372 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:15:39.372 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:15:39.372 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:15:39.372 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:15:39.372 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:15:39.372 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:15:39.372 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:15:39.372 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:15:39.373 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:15:39.373 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:15:39.373 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:15:39.373 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:15:39.373 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:15:39.373 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:15:39.373 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:15:39.373 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:15:39.373 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:15:39.373 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:15:39.373 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:15:39.383 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:15:39.384 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:15:39.384 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:15:39.384 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:15:39.443 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:15:39.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:15:39.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:15:39.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:15:39.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:15:39.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:15:39.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:15:39.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:15:39.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:15:39.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:15:39.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:15:39.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:15:39.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:15:39.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:15:39.446 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:15:39.446 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:15:39.482 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 44 2026-02-18T08:15:39.482 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 45 2026-02-18T08:15:39.482 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:15:39.482 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=181, tag_tid=12, entry_tid=1], [object_number=180, tag_tid=12, entry_tid=0], [object_number=183, tag_tid=11, entry_tid=29711], [object_number=182, tag_tid=11, entry_tid=29710]]], state=connected] 2026-02-18T08:15:39.482 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=179, tag_tid=11, entry_tid=23787], [object_number=178, tag_tid=11, entry_tid=23786], [object_number=177, tag_tid=11, entry_tid=23785], [object_number=176, tag_tid=11, entry_tid=23784]]], state=connected] 2026-02-18T08:15:39.486 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:15:39.487 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=181, tag_tid=12, entry_tid=1' 2026-02-18T08:15:39.488 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:15:39.488 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:15:39.488 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:15:39.488 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:15:39.488 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:15:39.488 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:15:39.488 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:15:39.488 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:15:39.488 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:15:39.488 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:15:39.489 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:15:39.489 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:15:39.490 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:15:39.490 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:15:39.490 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:15:39.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 44 2026-02-18T08:15:39.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 45 2026-02-18T08:15:39.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:15:39.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=181, tag_tid=12, entry_tid=1], [object_number=180, tag_tid=12, entry_tid=0], [object_number=183, tag_tid=11, entry_tid=29711], [object_number=182, tag_tid=11, entry_tid=29710]]], state=connected] 2026-02-18T08:15:39.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=179, tag_tid=11, entry_tid=23787], [object_number=178, tag_tid=11, entry_tid=23786], [object_number=177, tag_tid=11, entry_tid=23785], [object_number=176, tag_tid=11, entry_tid=23784]]], state=connected] 2026-02-18T08:15:39.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:15:39.523 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=179, tag_tid=11, entry_tid=23787' 2026-02-18T08:15:39.523 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=181, tag_tid=12, entry_tid=1' -a 'object_number=181, tag_tid=12, entry_tid=1' = 'object_number=179, tag_tid=11, entry_tid=23787' 2026-02-18T08:15:39.523 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=179, tag_tid=11, entry_tid=23787' '!=' 'object_number=179, tag_tid=11, entry_tid=23567' 2026-02-18T08:15:39.523 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:15:39.523 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=179, tag_tid=11, entry_tid=23787' = 'object_number=179, tag_tid=11, entry_tid=23567' 2026-02-18T08:15:39.523 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=179, tag_tid=11, entry_tid=23787' 2026-02-18T08:15:39.523 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=181, tag_tid=12, entry_tid=1' 2026-02-18T08:15:39.523 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:15:39.523 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:15:39.525 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=12 2026-02-18T08:15:39.526 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=179, tag_tid=11, entry_tid=23787' 2026-02-18T08:15:39.526 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:15:39.526 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:15:39.527 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=11 2026-02-18T08:15:39.528 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=181, tag_tid=12, entry_tid=1' 2026-02-18T08:15:39.528 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:15:39.528 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:15:39.530 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:15:39.530 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=179, tag_tid=11, entry_tid=23787' 2026-02-18T08:15:39.530 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:15:39.530 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:15:39.532 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=23787 2026-02-18T08:15:39.532 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 12 = 11 -a 1 -le 23787 2026-02-18T08:15:39.532 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:15:39.532 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:15:39.532 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:15:39.733 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:15:39.733 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:15:39.734 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:15:39.734 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:15:39.734 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:15:39.734 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:15:39.734 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:15:39.734 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:15:39.734 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:15:39.734 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:15:39.734 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:15:39.734 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:15:39.734 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:15:39.734 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:15:39.734 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:15:39.736 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:15:39.737 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:15:39.737 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:15:39.737 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:15:39.737 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:15:39.737 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:15:39.737 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:15:39.737 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:15:39.737 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:15:39.737 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:15:39.737 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:15:39.737 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:15:39.737 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:15:39.737 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:15:39.737 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:15:39.737 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:15:39.737 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:15:39.737 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:15:39.737 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:15:39.737 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:15:39.738 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:15:39.738 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:15:39.738 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:15:39.738 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:15:39.738 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:15:39.738 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:15:39.738 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:15:39.738 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:15:39.738 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:15:39.738 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:15:39.738 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:15:39.738 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:15:39.738 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:15:39.738 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:15:39.749 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:15:39.749 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:15:39.749 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:15:39.749 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:15:39.819 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:15:39.820 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:15:39.820 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:15:39.821 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:15:39.821 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:15:39.821 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:15:39.821 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:15:39.821 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:15:39.821 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:15:39.821 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:15:39.821 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:15:39.821 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:15:39.821 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:15:39.822 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:15:39.822 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:15:39.822 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:15:39.851 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 44 2026-02-18T08:15:39.851 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 45 2026-02-18T08:15:39.852 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:15:39.852 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=181, tag_tid=12, entry_tid=1], [object_number=180, tag_tid=12, entry_tid=0], [object_number=183, tag_tid=11, entry_tid=29711], [object_number=182, tag_tid=11, entry_tid=29710]]], state=connected] 2026-02-18T08:15:39.852 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=176, tag_tid=11, entry_tid=24008], [object_number=179, tag_tid=11, entry_tid=24007], [object_number=178, tag_tid=11, entry_tid=24006], [object_number=177, tag_tid=11, entry_tid=24005]]], state=connected] 2026-02-18T08:15:39.855 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:15:39.856 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=181, tag_tid=12, entry_tid=1' 2026-02-18T08:15:39.857 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:15:39.857 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:15:39.857 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:15:39.857 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:15:39.857 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:15:39.857 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:15:39.857 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:15:39.857 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:15:39.857 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:15:39.858 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:15:39.858 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:15:39.858 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:15:39.859 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:15:39.859 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:15:39.859 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:15:39.896 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 44 2026-02-18T08:15:39.896 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 45 2026-02-18T08:15:39.896 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:15:39.896 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=181, tag_tid=12, entry_tid=1], [object_number=180, tag_tid=12, entry_tid=0], [object_number=183, tag_tid=11, entry_tid=29711], [object_number=182, tag_tid=11, entry_tid=29710]]], state=connected] 2026-02-18T08:15:39.897 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=176, tag_tid=11, entry_tid=24008], [object_number=179, tag_tid=11, entry_tid=24007], [object_number=178, tag_tid=11, entry_tid=24006], [object_number=177, tag_tid=11, entry_tid=24005]]], state=connected] 2026-02-18T08:15:39.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:15:39.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=176, tag_tid=11, entry_tid=24008' 2026-02-18T08:15:39.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=181, tag_tid=12, entry_tid=1' -a 'object_number=181, tag_tid=12, entry_tid=1' = 'object_number=176, tag_tid=11, entry_tid=24008' 2026-02-18T08:15:39.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=176, tag_tid=11, entry_tid=24008' '!=' 'object_number=179, tag_tid=11, entry_tid=23787' 2026-02-18T08:15:39.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:15:39.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=176, tag_tid=11, entry_tid=24008' = 'object_number=179, tag_tid=11, entry_tid=23787' 2026-02-18T08:15:39.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=176, tag_tid=11, entry_tid=24008' 2026-02-18T08:15:39.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=181, tag_tid=12, entry_tid=1' 2026-02-18T08:15:39.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:15:39.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:15:39.904 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=12 2026-02-18T08:15:39.905 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=176, tag_tid=11, entry_tid=24008' 2026-02-18T08:15:39.905 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:15:39.905 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:15:39.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=11 2026-02-18T08:15:39.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=181, tag_tid=12, entry_tid=1' 2026-02-18T08:15:39.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:15:39.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:15:39.909 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:15:39.910 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=176, tag_tid=11, entry_tid=24008' 2026-02-18T08:15:39.910 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:15:39.910 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:15:39.911 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=24008 2026-02-18T08:15:39.911 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 12 = 11 -a 1 -le 24008 2026-02-18T08:15:39.911 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:15:39.911 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:15:39.912 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:15:40.113 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:15:40.113 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:15:40.113 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:15:40.113 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:15:40.113 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:15:40.113 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:15:40.113 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:15:40.113 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:15:40.113 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:15:40.113 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:15:40.114 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:15:40.114 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:15:40.114 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:15:40.114 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:15:40.114 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:15:40.114 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:15:40.114 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:15:40.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:15:40.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:15:40.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:15:40.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:15:40.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:15:40.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:15:40.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:15:40.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:15:40.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:15:40.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:15:40.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:15:40.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:15:40.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:15:40.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:15:40.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:15:40.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:15:40.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:15:40.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:15:40.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:15:40.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:15:40.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:15:40.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:15:40.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:15:40.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:15:40.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:15:40.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:15:40.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:15:40.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:15:40.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:15:40.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:15:40.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:15:40.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:15:40.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:15:40.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:15:40.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:15:40.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:15:40.196 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:15:40.197 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:15:40.198 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:15:40.198 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:15:40.198 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:15:40.198 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:15:40.198 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:15:40.198 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:15:40.198 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:15:40.198 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:15:40.198 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:15:40.198 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:15:40.198 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:15:40.199 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:15:40.200 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:15:40.200 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:15:40.234 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 44 2026-02-18T08:15:40.234 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 45 2026-02-18T08:15:40.234 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:15:40.234 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=181, tag_tid=12, entry_tid=1], [object_number=180, tag_tid=12, entry_tid=0], [object_number=183, tag_tid=11, entry_tid=29711], [object_number=182, tag_tid=11, entry_tid=29710]]], state=connected] 2026-02-18T08:15:40.234 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=177, tag_tid=11, entry_tid=24237], [object_number=176, tag_tid=11, entry_tid=24236], [object_number=179, tag_tid=11, entry_tid=24235], [object_number=178, tag_tid=11, entry_tid=24234]]], state=connected] 2026-02-18T08:15:40.238 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:15:40.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=181, tag_tid=12, entry_tid=1' 2026-02-18T08:15:40.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:15:40.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:15:40.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:15:40.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:15:40.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:15:40.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:15:40.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:15:40.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:15:40.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:15:40.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:15:40.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:15:40.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:15:40.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:15:40.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:15:40.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:15:40.276 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 44 2026-02-18T08:15:40.276 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 45 2026-02-18T08:15:40.276 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:15:40.276 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=181, tag_tid=12, entry_tid=1], [object_number=180, tag_tid=12, entry_tid=0], [object_number=183, tag_tid=11, entry_tid=29711], [object_number=182, tag_tid=11, entry_tid=29710]]], state=connected] 2026-02-18T08:15:40.276 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=177, tag_tid=11, entry_tid=24237], [object_number=176, tag_tid=11, entry_tid=24236], [object_number=179, tag_tid=11, entry_tid=24235], [object_number=178, tag_tid=11, entry_tid=24234]]], state=connected] 2026-02-18T08:15:40.280 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:15:40.281 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=177, tag_tid=11, entry_tid=24237' 2026-02-18T08:15:40.281 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=181, tag_tid=12, entry_tid=1' -a 'object_number=181, tag_tid=12, entry_tid=1' = 'object_number=177, tag_tid=11, entry_tid=24237' 2026-02-18T08:15:40.281 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=177, tag_tid=11, entry_tid=24237' '!=' 'object_number=176, tag_tid=11, entry_tid=24008' 2026-02-18T08:15:40.281 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:15:40.281 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=177, tag_tid=11, entry_tid=24237' = 'object_number=176, tag_tid=11, entry_tid=24008' 2026-02-18T08:15:40.281 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=177, tag_tid=11, entry_tid=24237' 2026-02-18T08:15:40.282 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=181, tag_tid=12, entry_tid=1' 2026-02-18T08:15:40.282 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:15:40.282 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:15:40.283 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=12 2026-02-18T08:15:40.283 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=177, tag_tid=11, entry_tid=24237' 2026-02-18T08:15:40.284 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:15:40.284 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:15:40.285 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=11 2026-02-18T08:15:40.286 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=181, tag_tid=12, entry_tid=1' 2026-02-18T08:15:40.286 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:15:40.286 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:15:40.287 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:15:40.287 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=177, tag_tid=11, entry_tid=24237' 2026-02-18T08:15:40.288 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:15:40.288 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:15:40.289 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=24237 2026-02-18T08:15:40.289 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 12 = 11 -a 1 -le 24237 2026-02-18T08:15:40.289 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:15:40.289 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:15:40.289 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:15:40.373 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.0 with skip (0.8610283926340341) > weight (0.25) 2026-02-18T08:15:40.373 INFO:tasks.rbd_mirror_thrash:kill cluster2.client.mirror.1 2026-02-18T08:15:40.373 INFO:tasks.rbd_mirror.cluster2.client.mirror.1:Sent signal 15 2026-02-18T08:15:40.373 INFO:tasks.rbd_mirror_thrash:waiting for 7.342048120718999 secs before reviving daemons 2026-02-18T08:15:40.374 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.trial163.stderr:2026-02-18T08:15:40.365+0000 7fa49f92e640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.1 (PID: 15003) UID: 1000 2026-02-18T08:15:40.491 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:15:40.491 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:15:40.491 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:15:40.491 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:15:40.491 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:15:40.491 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:15:40.491 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:15:40.491 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:15:40.491 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:15:40.491 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:15:40.492 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:15:40.492 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:15:40.492 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:15:40.492 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:15:40.492 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:15:40.493 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:15:40.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:15:40.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:15:40.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:15:40.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:15:40.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:15:40.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:15:40.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:15:40.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:15:40.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:15:40.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:15:40.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:15:40.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:15:40.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:15:40.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:15:40.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:15:40.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:15:40.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:15:40.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:15:40.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:15:40.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:15:40.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:15:40.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:15:40.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:15:40.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:15:40.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:15:40.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:15:40.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:15:40.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:15:40.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:15:40.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:15:40.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:15:40.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:15:40.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:15:40.505 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:15:40.506 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:15:40.506 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:15:40.506 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:15:40.578 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:15:40.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:15:40.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:15:40.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:15:40.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:15:40.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:15:40.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:15:40.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:15:40.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:15:40.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:15:40.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:15:40.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:15:40.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:15:40.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:15:40.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:15:40.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:15:40.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 44 2026-02-18T08:15:40.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 45 2026-02-18T08:15:40.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:15:40.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=181, tag_tid=12, entry_tid=1], [object_number=180, tag_tid=12, entry_tid=0], [object_number=183, tag_tid=11, entry_tid=29711], [object_number=182, tag_tid=11, entry_tid=29710]]], state=connected] 2026-02-18T08:15:40.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=179, tag_tid=11, entry_tid=24479], [object_number=178, tag_tid=11, entry_tid=24478], [object_number=177, tag_tid=11, entry_tid=24477], [object_number=176, tag_tid=11, entry_tid=24476]]], state=connected] 2026-02-18T08:15:40.611 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:15:40.613 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=181, tag_tid=12, entry_tid=1' 2026-02-18T08:15:40.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:15:40.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:15:40.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:15:40.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:15:40.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:15:40.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:15:40.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:15:40.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:15:40.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:15:40.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:15:40.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:15:40.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:15:40.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:15:40.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:15:40.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:15:40.642 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 44 2026-02-18T08:15:40.642 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 45 2026-02-18T08:15:40.643 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:15:40.643 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=181, tag_tid=12, entry_tid=1], [object_number=180, tag_tid=12, entry_tid=0], [object_number=183, tag_tid=11, entry_tid=29711], [object_number=182, tag_tid=11, entry_tid=29710]]], state=connected] 2026-02-18T08:15:40.643 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=179, tag_tid=11, entry_tid=24479], [object_number=178, tag_tid=11, entry_tid=24478], [object_number=177, tag_tid=11, entry_tid=24477], [object_number=176, tag_tid=11, entry_tid=24476]]], state=connected] 2026-02-18T08:15:40.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:15:40.646 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=179, tag_tid=11, entry_tid=24479' 2026-02-18T08:15:40.647 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=181, tag_tid=12, entry_tid=1' -a 'object_number=181, tag_tid=12, entry_tid=1' = 'object_number=179, tag_tid=11, entry_tid=24479' 2026-02-18T08:15:40.647 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=179, tag_tid=11, entry_tid=24479' '!=' 'object_number=177, tag_tid=11, entry_tid=24237' 2026-02-18T08:15:40.647 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:15:40.647 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=179, tag_tid=11, entry_tid=24479' = 'object_number=177, tag_tid=11, entry_tid=24237' 2026-02-18T08:15:40.647 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=179, tag_tid=11, entry_tid=24479' 2026-02-18T08:15:40.647 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=181, tag_tid=12, entry_tid=1' 2026-02-18T08:15:40.647 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:15:40.647 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:15:40.649 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=12 2026-02-18T08:15:40.649 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=179, tag_tid=11, entry_tid=24479' 2026-02-18T08:15:40.649 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:15:40.649 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:15:40.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=11 2026-02-18T08:15:40.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=181, tag_tid=12, entry_tid=1' 2026-02-18T08:15:40.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:15:40.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:15:40.653 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:15:40.654 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=179, tag_tid=11, entry_tid=24479' 2026-02-18T08:15:40.654 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:15:40.654 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:15:40.655 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=24479 2026-02-18T08:15:40.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 12 = 11 -a 1 -le 24479 2026-02-18T08:15:40.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:15:40.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:15:40.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:15:40.857 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:15:40.857 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:15:40.857 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:15:40.858 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:15:40.858 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:15:40.858 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:15:40.858 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:15:40.858 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:15:40.858 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:15:40.858 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:15:40.858 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:15:40.858 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:15:40.858 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:15:40.858 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:15:40.858 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:15:40.860 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:15:40.861 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:15:40.861 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:15:40.861 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:15:40.861 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:15:40.861 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:15:40.861 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:15:40.861 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:15:40.861 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:15:40.861 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:15:40.861 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:15:40.861 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:15:40.861 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:15:40.861 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:15:40.861 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:15:40.861 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:15:40.861 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:15:40.862 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:15:40.862 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:15:40.862 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:15:40.862 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:15:40.862 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:15:40.862 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:15:40.862 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:15:40.862 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:15:40.862 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:15:40.862 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:15:40.862 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:15:40.862 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:15:40.862 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:15:40.862 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:15:40.862 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:15:40.862 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:15:40.862 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:15:40.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:15:40.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:15:40.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:15:40.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:15:40.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:15:40.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:15:40.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:15:40.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:15:40.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:15:40.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:15:40.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:15:40.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:15:40.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:15:40.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:15:40.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:15:40.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:15:40.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:15:40.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:15:40.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:15:40.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:15:40.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 44 2026-02-18T08:15:40.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 45 2026-02-18T08:15:40.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:15:40.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=181, tag_tid=12, entry_tid=1], [object_number=180, tag_tid=12, entry_tid=0], [object_number=183, tag_tid=11, entry_tid=29711], [object_number=182, tag_tid=11, entry_tid=29710]]], state=connected] 2026-02-18T08:15:40.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=177, tag_tid=11, entry_tid=24709], [object_number=176, tag_tid=11, entry_tid=24708], [object_number=179, tag_tid=11, entry_tid=24707], [object_number=178, tag_tid=11, entry_tid=24706]]], state=connected] 2026-02-18T08:15:40.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:15:40.985 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=181, tag_tid=12, entry_tid=1' 2026-02-18T08:15:40.985 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:15:40.986 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:15:40.986 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:15:40.986 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:15:40.986 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:15:40.986 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:15:40.986 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:15:40.986 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:15:40.986 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:15:40.986 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:15:40.986 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:15:40.986 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:15:40.988 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:15:40.988 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:15:40.988 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:15:41.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 44 2026-02-18T08:15:41.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 45 2026-02-18T08:15:41.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:15:41.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=181, tag_tid=12, entry_tid=1], [object_number=180, tag_tid=12, entry_tid=0], [object_number=183, tag_tid=11, entry_tid=29711], [object_number=182, tag_tid=11, entry_tid=29710]]], state=connected] 2026-02-18T08:15:41.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=177, tag_tid=11, entry_tid=24709], [object_number=176, tag_tid=11, entry_tid=24708], [object_number=179, tag_tid=11, entry_tid=24707], [object_number=178, tag_tid=11, entry_tid=24706]]], state=connected] 2026-02-18T08:15:41.028 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:15:41.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=177, tag_tid=11, entry_tid=24709' 2026-02-18T08:15:41.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=181, tag_tid=12, entry_tid=1' -a 'object_number=181, tag_tid=12, entry_tid=1' = 'object_number=177, tag_tid=11, entry_tid=24709' 2026-02-18T08:15:41.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=177, tag_tid=11, entry_tid=24709' '!=' 'object_number=179, tag_tid=11, entry_tid=24479' 2026-02-18T08:15:41.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:15:41.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=177, tag_tid=11, entry_tid=24709' = 'object_number=179, tag_tid=11, entry_tid=24479' 2026-02-18T08:15:41.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=177, tag_tid=11, entry_tid=24709' 2026-02-18T08:15:41.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=181, tag_tid=12, entry_tid=1' 2026-02-18T08:15:41.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:15:41.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:15:41.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=12 2026-02-18T08:15:41.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=177, tag_tid=11, entry_tid=24709' 2026-02-18T08:15:41.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:15:41.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:15:41.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=11 2026-02-18T08:15:41.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=181, tag_tid=12, entry_tid=1' 2026-02-18T08:15:41.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:15:41.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:15:41.035 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:15:41.036 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=177, tag_tid=11, entry_tid=24709' 2026-02-18T08:15:41.036 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:15:41.036 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:15:41.038 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=24709 2026-02-18T08:15:41.038 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 12 = 11 -a 1 -le 24709 2026-02-18T08:15:41.038 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:15:41.038 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:15:41.038 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:15:41.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:15:41.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:15:41.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:15:41.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:15:41.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:15:41.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:15:41.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:15:41.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:15:41.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:15:41.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:15:41.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:15:41.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:15:41.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:15:41.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:15:41.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:15:41.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:15:41.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:15:41.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:15:41.244 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:15:41.244 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:15:41.244 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:15:41.244 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:15:41.244 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:15:41.244 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:15:41.244 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:15:41.244 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:15:41.244 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:15:41.244 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:15:41.244 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:15:41.244 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:15:41.244 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:15:41.244 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:15:41.244 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:15:41.244 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:15:41.244 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:15:41.244 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:15:41.244 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:15:41.244 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:15:41.244 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:15:41.244 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:15:41.244 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:15:41.244 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:15:41.244 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:15:41.244 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:15:41.244 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:15:41.244 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:15:41.244 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:15:41.244 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:15:41.244 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:15:41.255 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:15:41.255 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:15:41.255 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:15:41.255 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:15:41.329 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:15:41.329 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:15:41.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:15:41.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:15:41.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:15:41.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:15:41.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:15:41.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:15:41.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:15:41.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:15:41.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:15:41.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:15:41.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:15:41.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:15:41.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:15:41.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:15:41.360 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 44 2026-02-18T08:15:41.360 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 45 2026-02-18T08:15:41.360 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:15:41.360 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=181, tag_tid=12, entry_tid=1], [object_number=180, tag_tid=12, entry_tid=0], [object_number=183, tag_tid=11, entry_tid=29711], [object_number=182, tag_tid=11, entry_tid=29710]]], state=connected] 2026-02-18T08:15:41.360 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=177, tag_tid=11, entry_tid=24969], [object_number=176, tag_tid=11, entry_tid=24968], [object_number=179, tag_tid=11, entry_tid=24967], [object_number=178, tag_tid=11, entry_tid=24966]]], state=connected] 2026-02-18T08:15:41.362 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:15:41.364 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=181, tag_tid=12, entry_tid=1' 2026-02-18T08:15:41.364 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:15:41.365 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:15:41.365 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:15:41.365 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:15:41.365 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:15:41.365 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:15:41.365 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:15:41.365 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:15:41.365 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:15:41.366 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:15:41.366 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:15:41.366 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:15:41.366 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:15:41.367 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:15:41.367 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:15:41.394 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 44 2026-02-18T08:15:41.394 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 45 2026-02-18T08:15:41.394 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:15:41.394 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=181, tag_tid=12, entry_tid=1], [object_number=180, tag_tid=12, entry_tid=0], [object_number=183, tag_tid=11, entry_tid=29711], [object_number=182, tag_tid=11, entry_tid=29710]]], state=connected] 2026-02-18T08:15:41.394 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=177, tag_tid=11, entry_tid=24969], [object_number=176, tag_tid=11, entry_tid=24968], [object_number=179, tag_tid=11, entry_tid=24967], [object_number=178, tag_tid=11, entry_tid=24966]]], state=connected] 2026-02-18T08:15:41.397 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:15:41.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=177, tag_tid=11, entry_tid=24969' 2026-02-18T08:15:41.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=181, tag_tid=12, entry_tid=1' -a 'object_number=181, tag_tid=12, entry_tid=1' = 'object_number=177, tag_tid=11, entry_tid=24969' 2026-02-18T08:15:41.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=177, tag_tid=11, entry_tid=24969' '!=' 'object_number=177, tag_tid=11, entry_tid=24709' 2026-02-18T08:15:41.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:15:41.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=177, tag_tid=11, entry_tid=24969' = 'object_number=177, tag_tid=11, entry_tid=24709' 2026-02-18T08:15:41.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=177, tag_tid=11, entry_tid=24969' 2026-02-18T08:15:41.399 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=181, tag_tid=12, entry_tid=1' 2026-02-18T08:15:41.399 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:15:41.399 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:15:41.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=12 2026-02-18T08:15:41.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=177, tag_tid=11, entry_tid=24969' 2026-02-18T08:15:41.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:15:41.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:15:41.403 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=11 2026-02-18T08:15:41.403 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=181, tag_tid=12, entry_tid=1' 2026-02-18T08:15:41.403 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:15:41.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:15:41.405 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:15:41.406 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=177, tag_tid=11, entry_tid=24969' 2026-02-18T08:15:41.406 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:15:41.406 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:15:41.407 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=24969 2026-02-18T08:15:41.407 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 12 = 11 -a 1 -le 24969 2026-02-18T08:15:41.407 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:15:41.408 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:15:41.408 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:15:41.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:15:41.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:15:41.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:15:41.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:15:41.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:15:41.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:15:41.610 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:15:41.610 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:15:41.610 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:15:41.610 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:15:41.610 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:15:41.610 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:15:41.610 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:15:41.610 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:15:41.610 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:15:41.612 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:15:41.612 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:15:41.612 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:15:41.613 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:15:41.613 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:15:41.613 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:15:41.613 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:15:41.613 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:15:41.613 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:15:41.613 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:15:41.613 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:15:41.613 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:15:41.613 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:15:41.613 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:15:41.613 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:15:41.613 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:15:41.613 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:15:41.613 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:15:41.613 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:15:41.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:15:41.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:15:41.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:15:41.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:15:41.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:15:41.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:15:41.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:15:41.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:15:41.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:15:41.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:15:41.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:15:41.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:15:41.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:15:41.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:15:41.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:15:41.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:15:41.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:15:41.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:15:41.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:15:41.686 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:15:41.687 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:15:41.687 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:15:41.687 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:15:41.687 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:15:41.687 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:15:41.687 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:15:41.687 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:15:41.687 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:15:41.687 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:15:41.687 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:15:41.687 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:15:41.687 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:15:41.688 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:15:41.688 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:15:41.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:15:41.725 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 44 2026-02-18T08:15:41.725 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 45 2026-02-18T08:15:41.725 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:15:41.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=181, tag_tid=12, entry_tid=1], [object_number=180, tag_tid=12, entry_tid=0], [object_number=183, tag_tid=11, entry_tid=29711], [object_number=182, tag_tid=11, entry_tid=29710]]], state=connected] 2026-02-18T08:15:41.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=177, tag_tid=11, entry_tid=25197], [object_number=176, tag_tid=11, entry_tid=25196], [object_number=179, tag_tid=11, entry_tid=25195], [object_number=178, tag_tid=11, entry_tid=25194]]], state=connected] 2026-02-18T08:15:41.728 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:15:41.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=181, tag_tid=12, entry_tid=1' 2026-02-18T08:15:41.731 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:15:41.731 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:15:41.731 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:15:41.731 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:15:41.731 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:15:41.731 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:15:41.731 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:15:41.732 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:15:41.732 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:15:41.732 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:15:41.732 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:15:41.732 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:15:41.733 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:15:41.733 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:15:41.733 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:15:41.786 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 44 2026-02-18T08:15:41.786 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 45 2026-02-18T08:15:41.786 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:15:41.787 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=181, tag_tid=12, entry_tid=1], [object_number=180, tag_tid=12, entry_tid=0], [object_number=183, tag_tid=11, entry_tid=29711], [object_number=182, tag_tid=11, entry_tid=29710]]], state=connected] 2026-02-18T08:15:41.787 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=177, tag_tid=11, entry_tid=25197], [object_number=176, tag_tid=11, entry_tid=25196], [object_number=179, tag_tid=11, entry_tid=25195], [object_number=178, tag_tid=11, entry_tid=25194]]], state=connected] 2026-02-18T08:15:41.787 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:15:41.788 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=177, tag_tid=11, entry_tid=25197' 2026-02-18T08:15:41.788 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=181, tag_tid=12, entry_tid=1' -a 'object_number=181, tag_tid=12, entry_tid=1' = 'object_number=177, tag_tid=11, entry_tid=25197' 2026-02-18T08:15:41.788 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=177, tag_tid=11, entry_tid=25197' '!=' 'object_number=177, tag_tid=11, entry_tid=24969' 2026-02-18T08:15:41.788 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:15:41.788 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=177, tag_tid=11, entry_tid=25197' = 'object_number=177, tag_tid=11, entry_tid=24969' 2026-02-18T08:15:41.788 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=177, tag_tid=11, entry_tid=25197' 2026-02-18T08:15:41.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=181, tag_tid=12, entry_tid=1' 2026-02-18T08:15:41.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:15:41.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:15:41.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=12 2026-02-18T08:15:41.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=177, tag_tid=11, entry_tid=25197' 2026-02-18T08:15:41.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:15:41.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:15:41.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=11 2026-02-18T08:15:41.793 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=181, tag_tid=12, entry_tid=1' 2026-02-18T08:15:41.793 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:15:41.793 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:15:41.794 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:15:41.795 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=177, tag_tid=11, entry_tid=25197' 2026-02-18T08:15:41.795 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:15:41.795 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:15:41.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=25197 2026-02-18T08:15:41.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 12 = 11 -a 1 -le 25197 2026-02-18T08:15:41.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:15:41.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:15:41.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:15:41.999 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:15:41.999 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:15:41.999 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:15:41.999 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:15:41.999 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:15:41.999 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:15:41.999 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:15:41.999 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:15:42.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:15:42.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:15:42.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:15:42.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:15:42.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:15:42.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:15:42.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:15:42.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:15:42.003 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:15:42.003 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:15:42.003 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:15:42.003 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:15:42.003 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:15:42.003 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:15:42.003 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:15:42.003 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:15:42.003 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:15:42.003 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:15:42.003 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:15:42.003 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:15:42.003 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:15:42.003 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:15:42.003 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:15:42.003 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:15:42.004 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:15:42.004 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:15:42.004 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:15:42.004 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:15:42.004 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:15:42.004 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:15:42.004 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:15:42.004 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:15:42.004 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:15:42.004 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:15:42.004 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:15:42.004 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:15:42.004 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:15:42.004 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:15:42.004 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:15:42.004 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:15:42.004 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:15:42.012 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:15:42.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:15:42.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:15:42.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:15:42.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:15:42.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:15:42.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:15:42.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:15:42.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:15:42.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:15:42.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:15:42.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:15:42.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:15:42.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:15:42.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:15:42.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:15:42.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:15:42.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:15:42.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:15:42.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:15:42.118 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 44 2026-02-18T08:15:42.119 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 45 2026-02-18T08:15:42.119 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:15:42.119 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=181, tag_tid=12, entry_tid=1], [object_number=180, tag_tid=12, entry_tid=0], [object_number=183, tag_tid=11, entry_tid=29711], [object_number=182, tag_tid=11, entry_tid=29710]]], state=connected] 2026-02-18T08:15:42.119 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=178, tag_tid=11, entry_tid=25442], [object_number=177, tag_tid=11, entry_tid=25441], [object_number=176, tag_tid=11, entry_tid=25440], [object_number=179, tag_tid=11, entry_tid=25439]]], state=connected] 2026-02-18T08:15:42.122 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:15:42.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=181, tag_tid=12, entry_tid=1' 2026-02-18T08:15:42.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:15:42.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:15:42.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:15:42.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:15:42.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:15:42.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:15:42.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:15:42.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:15:42.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:15:42.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:15:42.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:15:42.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:15:42.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:15:42.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:15:42.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:15:42.158 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 44 2026-02-18T08:15:42.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 45 2026-02-18T08:15:42.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:15:42.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=181, tag_tid=12, entry_tid=1], [object_number=180, tag_tid=12, entry_tid=0], [object_number=183, tag_tid=11, entry_tid=29711], [object_number=182, tag_tid=11, entry_tid=29710]]], state=connected] 2026-02-18T08:15:42.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=178, tag_tid=11, entry_tid=25442], [object_number=177, tag_tid=11, entry_tid=25441], [object_number=176, tag_tid=11, entry_tid=25440], [object_number=179, tag_tid=11, entry_tid=25439]]], state=connected] 2026-02-18T08:15:42.162 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:15:42.164 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=178, tag_tid=11, entry_tid=25442' 2026-02-18T08:15:42.164 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=181, tag_tid=12, entry_tid=1' -a 'object_number=181, tag_tid=12, entry_tid=1' = 'object_number=178, tag_tid=11, entry_tid=25442' 2026-02-18T08:15:42.164 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=178, tag_tid=11, entry_tid=25442' '!=' 'object_number=177, tag_tid=11, entry_tid=25197' 2026-02-18T08:15:42.164 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:15:42.164 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=178, tag_tid=11, entry_tid=25442' = 'object_number=177, tag_tid=11, entry_tid=25197' 2026-02-18T08:15:42.164 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=178, tag_tid=11, entry_tid=25442' 2026-02-18T08:15:42.165 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=181, tag_tid=12, entry_tid=1' 2026-02-18T08:15:42.165 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:15:42.165 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:15:42.166 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=12 2026-02-18T08:15:42.167 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=178, tag_tid=11, entry_tid=25442' 2026-02-18T08:15:42.167 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:15:42.167 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:15:42.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=11 2026-02-18T08:15:42.169 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=181, tag_tid=12, entry_tid=1' 2026-02-18T08:15:42.169 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:15:42.169 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:15:42.170 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:15:42.171 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=178, tag_tid=11, entry_tid=25442' 2026-02-18T08:15:42.171 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:15:42.171 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:15:42.173 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=25442 2026-02-18T08:15:42.173 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 12 = 11 -a 1 -le 25442 2026-02-18T08:15:42.173 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:15:42.173 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:15:42.173 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:15:42.374 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:15:42.374 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:15:42.374 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:15:42.374 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:15:42.374 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:15:42.375 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:15:42.375 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:15:42.375 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:15:42.375 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:15:42.375 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:15:42.375 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:15:42.375 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:15:42.375 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:15:42.375 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:15:42.375 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:15:42.376 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:15:42.376 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:15:42.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:15:42.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:15:42.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:15:42.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:15:42.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:15:42.379 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:15:42.379 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:15:42.379 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:15:42.379 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:15:42.379 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:15:42.379 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:15:42.379 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:15:42.379 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:15:42.379 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:15:42.379 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:15:42.379 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:15:42.379 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:15:42.379 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:15:42.379 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:15:42.379 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:15:42.379 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:15:42.379 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:15:42.380 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:15:42.380 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:15:42.380 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:15:42.380 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:15:42.380 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:15:42.380 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:15:42.380 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:15:42.380 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:15:42.380 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:15:42.380 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:15:42.389 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:15:42.389 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:15:42.390 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:15:42.390 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:15:42.460 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:15:42.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:15:42.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:15:42.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:15:42.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:15:42.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:15:42.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:15:42.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:15:42.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:15:42.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:15:42.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:15:42.462 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:15:42.462 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:15:42.463 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:15:42.463 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:15:42.463 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:15:42.493 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 44 2026-02-18T08:15:42.493 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 45 2026-02-18T08:15:42.493 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:15:42.493 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=181, tag_tid=12, entry_tid=1], [object_number=180, tag_tid=12, entry_tid=0], [object_number=183, tag_tid=11, entry_tid=29711], [object_number=182, tag_tid=11, entry_tid=29710]]], state=connected] 2026-02-18T08:15:42.493 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=178, tag_tid=11, entry_tid=25678], [object_number=177, tag_tid=11, entry_tid=25677], [object_number=176, tag_tid=11, entry_tid=25676], [object_number=179, tag_tid=11, entry_tid=25675]]], state=connected] 2026-02-18T08:15:42.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:15:42.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=181, tag_tid=12, entry_tid=1' 2026-02-18T08:15:42.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:15:42.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:15:42.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:15:42.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:15:42.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:15:42.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:15:42.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:15:42.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:15:42.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:15:42.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:15:42.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:15:42.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:15:42.500 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:15:42.501 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:15:42.501 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:15:42.536 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 44 2026-02-18T08:15:42.536 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 45 2026-02-18T08:15:42.536 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:15:42.536 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=181, tag_tid=12, entry_tid=1], [object_number=180, tag_tid=12, entry_tid=0], [object_number=183, tag_tid=11, entry_tid=29711], [object_number=182, tag_tid=11, entry_tid=29710]]], state=connected] 2026-02-18T08:15:42.536 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=178, tag_tid=11, entry_tid=25678], [object_number=177, tag_tid=11, entry_tid=25677], [object_number=176, tag_tid=11, entry_tid=25676], [object_number=179, tag_tid=11, entry_tid=25675]]], state=connected] 2026-02-18T08:15:42.540 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:15:42.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=178, tag_tid=11, entry_tid=25678' 2026-02-18T08:15:42.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=181, tag_tid=12, entry_tid=1' -a 'object_number=181, tag_tid=12, entry_tid=1' = 'object_number=178, tag_tid=11, entry_tid=25678' 2026-02-18T08:15:42.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=178, tag_tid=11, entry_tid=25678' '!=' 'object_number=178, tag_tid=11, entry_tid=25442' 2026-02-18T08:15:42.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:15:42.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=178, tag_tid=11, entry_tid=25678' = 'object_number=178, tag_tid=11, entry_tid=25442' 2026-02-18T08:15:42.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=178, tag_tid=11, entry_tid=25678' 2026-02-18T08:15:42.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=181, tag_tid=12, entry_tid=1' 2026-02-18T08:15:42.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:15:42.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:15:42.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=12 2026-02-18T08:15:42.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=178, tag_tid=11, entry_tid=25678' 2026-02-18T08:15:42.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:15:42.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:15:42.545 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=11 2026-02-18T08:15:42.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=181, tag_tid=12, entry_tid=1' 2026-02-18T08:15:42.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:15:42.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:15:42.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:15:42.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=178, tag_tid=11, entry_tid=25678' 2026-02-18T08:15:42.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:15:42.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:15:42.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=25678 2026-02-18T08:15:42.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 12 = 11 -a 1 -le 25678 2026-02-18T08:15:42.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:15:42.551 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:15:42.551 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:15:42.752 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:15:42.752 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:15:42.752 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:15:42.753 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:15:42.753 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:15:42.753 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:15:42.753 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:15:42.753 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:15:42.753 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:15:42.753 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:15:42.753 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:15:42.753 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:15:42.753 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:15:42.753 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:15:42.753 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:15:42.753 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:15:42.756 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:15:42.756 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:15:42.756 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:15:42.756 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:15:42.756 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:15:42.756 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:15:42.756 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:15:42.756 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:15:42.756 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:15:42.756 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:15:42.756 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:15:42.756 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:15:42.756 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:15:42.756 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:15:42.756 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:15:42.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:15:42.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:15:42.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:15:42.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:15:42.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:15:42.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:15:42.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:15:42.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:15:42.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:15:42.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:15:42.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:15:42.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:15:42.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:15:42.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:15:42.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:15:42.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:15:42.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:15:42.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:15:42.768 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:15:42.768 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:15:42.768 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:15:42.768 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:15:42.828 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:15:42.830 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:15:42.830 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:15:42.830 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:15:42.830 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:15:42.830 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:15:42.830 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:15:42.830 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:15:42.830 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:15:42.830 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:15:42.830 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:15:42.830 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:15:42.830 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:15:42.831 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:15:42.831 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:15:42.832 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:15:42.860 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 44 2026-02-18T08:15:42.860 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 45 2026-02-18T08:15:42.861 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:15:42.861 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=181, tag_tid=12, entry_tid=1], [object_number=180, tag_tid=12, entry_tid=0], [object_number=183, tag_tid=11, entry_tid=29711], [object_number=182, tag_tid=11, entry_tid=29710]]], state=connected] 2026-02-18T08:15:42.861 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=178, tag_tid=11, entry_tid=25914], [object_number=177, tag_tid=11, entry_tid=25913], [object_number=176, tag_tid=11, entry_tid=25912], [object_number=179, tag_tid=11, entry_tid=25911]]], state=connected] 2026-02-18T08:15:42.863 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:15:42.864 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=181, tag_tid=12, entry_tid=1' 2026-02-18T08:15:42.865 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:15:42.866 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:15:42.866 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:15:42.866 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:15:42.866 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:15:42.866 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:15:42.866 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:15:42.866 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:15:42.866 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:15:42.866 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:15:42.867 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:15:42.867 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:15:42.867 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:15:42.868 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:15:42.868 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:15:42.895 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 44 2026-02-18T08:15:42.895 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 45 2026-02-18T08:15:42.895 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:15:42.895 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=181, tag_tid=12, entry_tid=1], [object_number=180, tag_tid=12, entry_tid=0], [object_number=183, tag_tid=11, entry_tid=29711], [object_number=182, tag_tid=11, entry_tid=29710]]], state=connected] 2026-02-18T08:15:42.895 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=178, tag_tid=11, entry_tid=25914], [object_number=177, tag_tid=11, entry_tid=25913], [object_number=176, tag_tid=11, entry_tid=25912], [object_number=179, tag_tid=11, entry_tid=25911]]], state=connected] 2026-02-18T08:15:42.897 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:15:42.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=178, tag_tid=11, entry_tid=25914' 2026-02-18T08:15:42.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=181, tag_tid=12, entry_tid=1' -a 'object_number=181, tag_tid=12, entry_tid=1' = 'object_number=178, tag_tid=11, entry_tid=25914' 2026-02-18T08:15:42.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=178, tag_tid=11, entry_tid=25914' '!=' 'object_number=178, tag_tid=11, entry_tid=25678' 2026-02-18T08:15:42.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:15:42.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=178, tag_tid=11, entry_tid=25914' = 'object_number=178, tag_tid=11, entry_tid=25678' 2026-02-18T08:15:42.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=178, tag_tid=11, entry_tid=25914' 2026-02-18T08:15:42.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=181, tag_tid=12, entry_tid=1' 2026-02-18T08:15:42.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:15:42.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:15:42.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=12 2026-02-18T08:15:42.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=178, tag_tid=11, entry_tid=25914' 2026-02-18T08:15:42.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:15:42.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:15:42.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=11 2026-02-18T08:15:42.904 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=181, tag_tid=12, entry_tid=1' 2026-02-18T08:15:42.904 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:15:42.904 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:15:42.905 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:15:42.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=178, tag_tid=11, entry_tid=25914' 2026-02-18T08:15:42.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:15:42.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:15:42.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=25914 2026-02-18T08:15:42.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 12 = 11 -a 1 -le 25914 2026-02-18T08:15:42.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:15:42.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:15:42.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:15:43.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:15:43.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:15:43.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:15:43.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:15:43.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:15:43.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:15:43.110 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:15:43.110 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:15:43.110 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:15:43.110 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:15:43.110 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:15:43.110 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:15:43.110 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:15:43.110 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:15:43.110 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:15:43.110 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:15:43.110 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:15:43.113 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:15:43.113 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:15:43.113 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:15:43.113 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:15:43.113 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:15:43.113 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:15:43.113 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:15:43.113 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:15:43.113 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:15:43.113 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:15:43.113 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:15:43.114 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:15:43.114 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:15:43.114 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:15:43.114 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:15:43.114 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:15:43.114 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:15:43.114 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:15:43.114 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:15:43.114 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:15:43.114 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:15:43.114 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:15:43.114 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:15:43.114 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:15:43.114 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:15:43.114 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:15:43.114 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:15:43.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:15:43.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:15:43.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:15:43.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:15:43.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:15:43.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:15:43.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:15:43.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:15:43.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:15:43.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:15:43.194 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:15:43.194 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:15:43.194 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:15:43.195 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:15:43.195 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:15:43.195 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:15:43.195 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:15:43.195 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:15:43.195 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:15:43.195 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:15:43.195 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:15:43.195 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:15:43.196 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:15:43.196 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:15:43.197 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:15:43.223 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 44 2026-02-18T08:15:43.223 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 45 2026-02-18T08:15:43.223 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:15:43.224 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=181, tag_tid=12, entry_tid=1], [object_number=180, tag_tid=12, entry_tid=0], [object_number=183, tag_tid=11, entry_tid=29711], [object_number=182, tag_tid=11, entry_tid=29710]]], state=connected] 2026-02-18T08:15:43.224 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=176, tag_tid=11, entry_tid=26136], [object_number=179, tag_tid=11, entry_tid=26135], [object_number=178, tag_tid=11, entry_tid=26134], [object_number=177, tag_tid=11, entry_tid=26133]]], state=connected] 2026-02-18T08:15:43.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:15:43.227 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=181, tag_tid=12, entry_tid=1' 2026-02-18T08:15:43.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:15:43.229 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:15:43.229 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:15:43.229 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:15:43.229 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:15:43.229 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:15:43.229 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:15:43.229 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:15:43.229 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:15:43.229 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:15:43.229 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:15:43.229 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:15:43.230 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:15:43.231 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:15:43.231 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:15:43.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 44 2026-02-18T08:15:43.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 45 2026-02-18T08:15:43.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:15:43.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=181, tag_tid=12, entry_tid=1], [object_number=180, tag_tid=12, entry_tid=0], [object_number=183, tag_tid=11, entry_tid=29711], [object_number=182, tag_tid=11, entry_tid=29710]]], state=connected] 2026-02-18T08:15:43.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=176, tag_tid=11, entry_tid=26136], [object_number=179, tag_tid=11, entry_tid=26135], [object_number=178, tag_tid=11, entry_tid=26134], [object_number=177, tag_tid=11, entry_tid=26133]]], state=connected] 2026-02-18T08:15:43.269 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:15:43.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=176, tag_tid=11, entry_tid=26136' 2026-02-18T08:15:43.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=181, tag_tid=12, entry_tid=1' -a 'object_number=181, tag_tid=12, entry_tid=1' = 'object_number=176, tag_tid=11, entry_tid=26136' 2026-02-18T08:15:43.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=176, tag_tid=11, entry_tid=26136' '!=' 'object_number=178, tag_tid=11, entry_tid=25914' 2026-02-18T08:15:43.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:15:43.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=176, tag_tid=11, entry_tid=26136' = 'object_number=178, tag_tid=11, entry_tid=25914' 2026-02-18T08:15:43.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=176, tag_tid=11, entry_tid=26136' 2026-02-18T08:15:43.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=181, tag_tid=12, entry_tid=1' 2026-02-18T08:15:43.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:15:43.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:15:43.273 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=12 2026-02-18T08:15:43.273 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=176, tag_tid=11, entry_tid=26136' 2026-02-18T08:15:43.274 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:15:43.274 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:15:43.275 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=11 2026-02-18T08:15:43.275 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=181, tag_tid=12, entry_tid=1' 2026-02-18T08:15:43.276 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:15:43.276 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:15:43.277 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:15:43.278 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=176, tag_tid=11, entry_tid=26136' 2026-02-18T08:15:43.278 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:15:43.278 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:15:43.280 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=26136 2026-02-18T08:15:43.280 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 12 = 11 -a 1 -le 26136 2026-02-18T08:15:43.280 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:15:43.280 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:15:43.280 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:15:43.482 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:15:43.482 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:15:43.482 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:15:43.482 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:15:43.482 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:15:43.482 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:15:43.482 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:15:43.482 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:15:43.482 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:15:43.482 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:15:43.482 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:15:43.482 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:15:43.483 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:15:43.483 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:15:43.483 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:15:43.485 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:15:43.485 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:15:43.485 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:15:43.485 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:15:43.485 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:15:43.485 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:15:43.485 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:15:43.485 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:15:43.485 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:15:43.486 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:15:43.486 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:15:43.486 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:15:43.486 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:15:43.486 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:15:43.486 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:15:43.486 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:15:43.486 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:15:43.486 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:15:43.486 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:15:43.486 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:15:43.486 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:15:43.486 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:15:43.486 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:15:43.486 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:15:43.486 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:15:43.486 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:15:43.486 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:15:43.487 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:15:43.487 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:15:43.487 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:15:43.487 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:15:43.487 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:15:43.487 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:15:43.487 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:15:43.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:15:43.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:15:43.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:15:43.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:15:43.559 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:15:43.560 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:15:43.560 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:15:43.560 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:15:43.560 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:15:43.560 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:15:43.560 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:15:43.560 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:15:43.560 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:15:43.561 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:15:43.561 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:15:43.561 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:15:43.561 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:15:43.561 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:15:43.562 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:15:43.562 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:15:43.595 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 44 2026-02-18T08:15:43.595 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 45 2026-02-18T08:15:43.595 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:15:43.595 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=181, tag_tid=12, entry_tid=1], [object_number=180, tag_tid=12, entry_tid=0], [object_number=183, tag_tid=11, entry_tid=29711], [object_number=182, tag_tid=11, entry_tid=29710]]], state=connected] 2026-02-18T08:15:43.595 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=176, tag_tid=11, entry_tid=26360], [object_number=179, tag_tid=11, entry_tid=26359], [object_number=178, tag_tid=11, entry_tid=26358], [object_number=177, tag_tid=11, entry_tid=26357]]], state=connected] 2026-02-18T08:15:43.599 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:15:43.600 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=181, tag_tid=12, entry_tid=1' 2026-02-18T08:15:43.601 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:15:43.601 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:15:43.601 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:15:43.602 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:15:43.602 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:15:43.602 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:15:43.602 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:15:43.602 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:15:43.602 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:15:43.602 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:15:43.602 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:15:43.603 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:15:43.603 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:15:43.603 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:15:43.603 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:15:43.641 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 44 2026-02-18T08:15:43.641 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 45 2026-02-18T08:15:43.641 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:15:43.641 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=181, tag_tid=12, entry_tid=1], [object_number=180, tag_tid=12, entry_tid=0], [object_number=183, tag_tid=11, entry_tid=29711], [object_number=182, tag_tid=11, entry_tid=29710]]], state=connected] 2026-02-18T08:15:43.641 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=176, tag_tid=11, entry_tid=26360], [object_number=179, tag_tid=11, entry_tid=26359], [object_number=178, tag_tid=11, entry_tid=26358], [object_number=177, tag_tid=11, entry_tid=26357]]], state=connected] 2026-02-18T08:15:43.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:15:43.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=176, tag_tid=11, entry_tid=26360' 2026-02-18T08:15:43.646 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=181, tag_tid=12, entry_tid=1' -a 'object_number=181, tag_tid=12, entry_tid=1' = 'object_number=176, tag_tid=11, entry_tid=26360' 2026-02-18T08:15:43.646 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=176, tag_tid=11, entry_tid=26360' '!=' 'object_number=176, tag_tid=11, entry_tid=26136' 2026-02-18T08:15:43.646 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:15:43.646 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=176, tag_tid=11, entry_tid=26360' = 'object_number=176, tag_tid=11, entry_tid=26136' 2026-02-18T08:15:43.646 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=176, tag_tid=11, entry_tid=26360' 2026-02-18T08:15:43.646 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=181, tag_tid=12, entry_tid=1' 2026-02-18T08:15:43.646 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:15:43.647 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:15:43.647 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=12 2026-02-18T08:15:43.648 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=176, tag_tid=11, entry_tid=26360' 2026-02-18T08:15:43.648 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:15:43.648 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:15:43.649 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=11 2026-02-18T08:15:43.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=181, tag_tid=12, entry_tid=1' 2026-02-18T08:15:43.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:15:43.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:15:43.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:15:43.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=176, tag_tid=11, entry_tid=26360' 2026-02-18T08:15:43.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:15:43.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:15:43.653 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=26360 2026-02-18T08:15:43.653 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 12 = 11 -a 1 -le 26360 2026-02-18T08:15:43.653 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:15:43.653 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:15:43.654 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:15:43.855 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:15:43.855 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:15:43.855 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:15:43.855 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:15:43.855 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:15:43.855 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:15:43.855 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:15:43.855 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:15:43.855 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:15:43.855 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:15:43.855 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:15:43.855 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:15:43.855 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:15:43.855 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:15:43.855 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:15:43.856 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:15:43.856 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:15:43.858 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:15:43.858 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:15:43.858 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:15:43.858 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:15:43.858 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:15:43.858 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:15:43.858 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:15:43.858 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:15:43.858 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:15:43.858 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:15:43.858 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:15:43.858 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:15:43.858 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:15:43.858 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:15:43.858 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:15:43.858 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:15:43.858 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:15:43.858 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:15:43.858 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:15:43.858 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:15:43.859 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:15:43.859 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:15:43.859 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:15:43.859 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:15:43.859 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:15:43.859 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:15:43.859 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:15:43.859 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:15:43.859 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:15:43.859 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:15:43.859 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:15:43.859 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:15:43.869 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:15:43.869 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:15:43.870 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:15:43.870 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:15:43.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:15:43.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:15:43.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:15:43.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:15:43.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:15:43.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:15:43.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:15:43.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:15:43.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:15:43.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:15:43.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:15:43.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:15:43.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:15:43.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:15:43.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:15:43.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:15:43.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 44 2026-02-18T08:15:43.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 45 2026-02-18T08:15:43.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:15:43.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=181, tag_tid=12, entry_tid=1], [object_number=180, tag_tid=12, entry_tid=0], [object_number=183, tag_tid=11, entry_tid=29711], [object_number=182, tag_tid=11, entry_tid=29710]]], state=connected] 2026-02-18T08:15:43.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=178, tag_tid=11, entry_tid=26582], [object_number=177, tag_tid=11, entry_tid=26581], [object_number=176, tag_tid=11, entry_tid=26580], [object_number=179, tag_tid=11, entry_tid=26579]]], state=connected] 2026-02-18T08:15:43.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:15:43.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=181, tag_tid=12, entry_tid=1' 2026-02-18T08:15:43.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:15:43.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:15:43.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:15:43.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:15:43.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:15:43.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:15:43.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:15:43.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:15:43.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:15:43.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:15:43.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:15:43.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:15:43.982 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:15:43.982 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:15:43.982 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:15:44.010 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 44 2026-02-18T08:15:44.010 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 45 2026-02-18T08:15:44.010 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:15:44.010 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=181, tag_tid=12, entry_tid=1], [object_number=180, tag_tid=12, entry_tid=0], [object_number=183, tag_tid=11, entry_tid=29711], [object_number=182, tag_tid=11, entry_tid=29710]]], state=connected] 2026-02-18T08:15:44.010 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=178, tag_tid=11, entry_tid=26582], [object_number=177, tag_tid=11, entry_tid=26581], [object_number=176, tag_tid=11, entry_tid=26580], [object_number=179, tag_tid=11, entry_tid=26579]]], state=connected] 2026-02-18T08:15:44.012 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:15:44.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=178, tag_tid=11, entry_tid=26582' 2026-02-18T08:15:44.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=181, tag_tid=12, entry_tid=1' -a 'object_number=181, tag_tid=12, entry_tid=1' = 'object_number=178, tag_tid=11, entry_tid=26582' 2026-02-18T08:15:44.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=178, tag_tid=11, entry_tid=26582' '!=' 'object_number=176, tag_tid=11, entry_tid=26360' 2026-02-18T08:15:44.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:15:44.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=178, tag_tid=11, entry_tid=26582' = 'object_number=176, tag_tid=11, entry_tid=26360' 2026-02-18T08:15:44.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=178, tag_tid=11, entry_tid=26582' 2026-02-18T08:15:44.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=181, tag_tid=12, entry_tid=1' 2026-02-18T08:15:44.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:15:44.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:15:44.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=12 2026-02-18T08:15:44.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:15:44.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=178, tag_tid=11, entry_tid=26582' 2026-02-18T08:15:44.018 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:15:44.019 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=11 2026-02-18T08:15:44.019 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=181, tag_tid=12, entry_tid=1' 2026-02-18T08:15:44.019 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:15:44.019 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:15:44.021 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:15:44.021 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=178, tag_tid=11, entry_tid=26582' 2026-02-18T08:15:44.021 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:15:44.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:15:44.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=26582 2026-02-18T08:15:44.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 12 = 11 -a 1 -le 26582 2026-02-18T08:15:44.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:15:44.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:15:44.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:15:44.225 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:15:44.225 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:15:44.225 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:15:44.225 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:15:44.225 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:15:44.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:15:44.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:15:44.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:15:44.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:15:44.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:15:44.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:15:44.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:15:44.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:15:44.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:15:44.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:15:44.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:15:44.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:15:44.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:15:44.229 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:15:44.229 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:15:44.229 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:15:44.229 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:15:44.229 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:15:44.229 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:15:44.229 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:15:44.229 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:15:44.229 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:15:44.229 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:15:44.229 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:15:44.229 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:15:44.229 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:15:44.229 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:15:44.229 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:15:44.229 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:15:44.229 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:15:44.230 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:15:44.230 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:15:44.230 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:15:44.230 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:15:44.230 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:15:44.230 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:15:44.230 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:15:44.230 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:15:44.230 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:15:44.230 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:15:44.230 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:15:44.230 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:15:44.230 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:15:44.230 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:15:44.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:15:44.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:15:44.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:15:44.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:15:44.315 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:15:44.316 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:15:44.316 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:15:44.316 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:15:44.316 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:15:44.316 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:15:44.316 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:15:44.316 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:15:44.316 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:15:44.316 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:15:44.316 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:15:44.317 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:15:44.317 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:15:44.318 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:15:44.318 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:15:44.318 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:15:44.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 45 2026-02-18T08:15:44.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 45 2026-02-18T08:15:44.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:15:44.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=181, tag_tid=12, entry_tid=1], [object_number=180, tag_tid=12, entry_tid=0], [object_number=183, tag_tid=11, entry_tid=29711], [object_number=182, tag_tid=11, entry_tid=29710]]], state=connected] 2026-02-18T08:15:44.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=182, tag_tid=11, entry_tid=26722], [object_number=181, tag_tid=11, entry_tid=26721], [object_number=180, tag_tid=11, entry_tid=26720], [object_number=183, tag_tid=11, entry_tid=26719]]], state=connected] 2026-02-18T08:15:44.348 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:15:44.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=181, tag_tid=12, entry_tid=1' 2026-02-18T08:15:44.350 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:15:44.350 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:15:44.350 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:15:44.350 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:15:44.350 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:15:44.350 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:15:44.350 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:15:44.350 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:15:44.350 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:15:44.350 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:15:44.351 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:15:44.351 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:15:44.352 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:15:44.352 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:15:44.352 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:15:44.379 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 45 2026-02-18T08:15:44.379 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 45 2026-02-18T08:15:44.379 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:15:44.379 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=181, tag_tid=12, entry_tid=1], [object_number=180, tag_tid=12, entry_tid=0], [object_number=183, tag_tid=11, entry_tid=29711], [object_number=182, tag_tid=11, entry_tid=29710]]], state=connected] 2026-02-18T08:15:44.379 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=182, tag_tid=11, entry_tid=26722], [object_number=181, tag_tid=11, entry_tid=26721], [object_number=180, tag_tid=11, entry_tid=26720], [object_number=183, tag_tid=11, entry_tid=26719]]], state=connected] 2026-02-18T08:15:44.382 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:15:44.383 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=182, tag_tid=11, entry_tid=26722' 2026-02-18T08:15:44.383 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=181, tag_tid=12, entry_tid=1' -a 'object_number=181, tag_tid=12, entry_tid=1' = 'object_number=182, tag_tid=11, entry_tid=26722' 2026-02-18T08:15:44.383 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=182, tag_tid=11, entry_tid=26722' '!=' 'object_number=178, tag_tid=11, entry_tid=26582' 2026-02-18T08:15:44.383 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:15:44.384 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=182, tag_tid=11, entry_tid=26722' = 'object_number=178, tag_tid=11, entry_tid=26582' 2026-02-18T08:15:44.384 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=182, tag_tid=11, entry_tid=26722' 2026-02-18T08:15:44.384 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=181, tag_tid=12, entry_tid=1' 2026-02-18T08:15:44.384 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:15:44.384 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:15:44.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=12 2026-02-18T08:15:44.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=182, tag_tid=11, entry_tid=26722' 2026-02-18T08:15:44.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:15:44.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:15:44.388 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=11 2026-02-18T08:15:44.388 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=181, tag_tid=12, entry_tid=1' 2026-02-18T08:15:44.389 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:15:44.389 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:15:44.390 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:15:44.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=182, tag_tid=11, entry_tid=26722' 2026-02-18T08:15:44.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:15:44.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:15:44.393 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=26722 2026-02-18T08:15:44.393 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 12 = 11 -a 1 -le 26722 2026-02-18T08:15:44.393 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:15:44.393 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:15:44.393 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:15:44.594 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:15:44.594 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:15:44.595 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:15:44.595 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:15:44.595 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:15:44.595 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:15:44.595 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:15:44.595 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:15:44.595 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:15:44.595 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:15:44.595 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:15:44.595 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:15:44.595 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:15:44.595 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:15:44.595 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:15:44.597 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:15:44.597 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:15:44.598 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:15:44.598 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:15:44.598 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:15:44.598 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:15:44.598 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:15:44.598 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:15:44.598 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:15:44.598 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:15:44.598 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:15:44.598 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:15:44.598 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:15:44.598 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:15:44.598 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:15:44.598 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:15:44.598 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:15:44.598 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:15:44.598 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:15:44.598 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:15:44.598 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:15:44.598 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:15:44.598 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:15:44.598 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:15:44.598 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:15:44.599 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:15:44.599 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:15:44.599 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:15:44.599 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:15:44.599 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:15:44.599 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:15:44.599 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:15:44.599 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:15:44.599 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:15:44.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:15:44.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:15:44.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:15:44.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:15:44.670 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:15:44.671 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:15:44.671 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:15:44.671 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:15:44.671 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:15:44.671 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:15:44.671 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:15:44.671 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:15:44.671 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:15:44.671 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:15:44.672 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:15:44.672 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:15:44.672 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:15:44.673 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:15:44.673 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:15:44.673 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:15:44.700 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 45 2026-02-18T08:15:44.700 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 45 2026-02-18T08:15:44.700 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:15:44.700 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=181, tag_tid=12, entry_tid=1], [object_number=180, tag_tid=12, entry_tid=0], [object_number=183, tag_tid=11, entry_tid=29711], [object_number=182, tag_tid=11, entry_tid=29710]]], state=connected] 2026-02-18T08:15:44.700 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=180, tag_tid=11, entry_tid=26928], [object_number=183, tag_tid=11, entry_tid=26927], [object_number=182, tag_tid=11, entry_tid=26926], [object_number=181, tag_tid=11, entry_tid=26925]]], state=connected] 2026-02-18T08:15:44.703 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:15:44.704 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=181, tag_tid=12, entry_tid=1' 2026-02-18T08:15:44.705 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:15:44.705 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:15:44.705 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:15:44.705 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:15:44.705 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:15:44.705 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:15:44.705 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:15:44.705 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:15:44.705 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:15:44.705 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:15:44.705 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:15:44.705 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:15:44.707 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:15:44.707 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:15:44.707 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:15:44.741 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 45 2026-02-18T08:15:44.742 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 45 2026-02-18T08:15:44.742 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:15:44.742 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=181, tag_tid=12, entry_tid=1], [object_number=180, tag_tid=12, entry_tid=0], [object_number=183, tag_tid=11, entry_tid=29711], [object_number=182, tag_tid=11, entry_tid=29710]]], state=connected] 2026-02-18T08:15:44.742 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=180, tag_tid=11, entry_tid=26928], [object_number=183, tag_tid=11, entry_tid=26927], [object_number=182, tag_tid=11, entry_tid=26926], [object_number=181, tag_tid=11, entry_tid=26925]]], state=connected] 2026-02-18T08:15:44.745 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:15:44.747 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=180, tag_tid=11, entry_tid=26928' 2026-02-18T08:15:44.747 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=181, tag_tid=12, entry_tid=1' -a 'object_number=181, tag_tid=12, entry_tid=1' = 'object_number=180, tag_tid=11, entry_tid=26928' 2026-02-18T08:15:44.747 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=180, tag_tid=11, entry_tid=26928' '!=' 'object_number=182, tag_tid=11, entry_tid=26722' 2026-02-18T08:15:44.747 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:15:44.747 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=180, tag_tid=11, entry_tid=26928' = 'object_number=182, tag_tid=11, entry_tid=26722' 2026-02-18T08:15:44.747 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=180, tag_tid=11, entry_tid=26928' 2026-02-18T08:15:44.748 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=181, tag_tid=12, entry_tid=1' 2026-02-18T08:15:44.748 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:15:44.748 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:15:44.749 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=12 2026-02-18T08:15:44.749 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=180, tag_tid=11, entry_tid=26928' 2026-02-18T08:15:44.750 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:15:44.750 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:15:44.751 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=11 2026-02-18T08:15:44.751 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=181, tag_tid=12, entry_tid=1' 2026-02-18T08:15:44.752 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:15:44.752 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:15:44.753 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:15:44.754 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=180, tag_tid=11, entry_tid=26928' 2026-02-18T08:15:44.754 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:15:44.754 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:15:44.756 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=26928 2026-02-18T08:15:44.756 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 12 = 11 -a 1 -le 26928 2026-02-18T08:15:44.756 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:15:44.756 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:15:44.756 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:15:44.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:15:44.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:15:44.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:15:44.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:15:44.958 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:15:44.958 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:15:44.958 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:15:44.958 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:15:44.958 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:15:44.958 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:15:44.958 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:15:44.958 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:15:44.958 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:15:44.958 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:15:44.958 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:15:44.958 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:15:44.959 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:15:44.961 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:15:44.961 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:15:44.961 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:15:44.961 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:15:44.961 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:15:44.961 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:15:44.962 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:15:44.962 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:15:44.962 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:15:44.962 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:15:44.962 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:15:44.962 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:15:44.962 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:15:44.962 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:15:44.962 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:15:44.962 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:15:44.962 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:15:44.962 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:15:44.962 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:15:44.962 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:15:44.962 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:15:44.963 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:15:44.963 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:15:44.963 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:15:44.963 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:15:44.963 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:15:44.963 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:15:44.963 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:15:44.963 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:15:44.963 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:15:44.963 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:15:44.963 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:15:44.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:15:44.974 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:15:44.974 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:15:44.974 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:15:45.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:15:45.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:15:45.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:15:45.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:15:45.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:15:45.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:15:45.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:15:45.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:15:45.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:15:45.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:15:45.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:15:45.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:15:45.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:15:45.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:15:45.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:15:45.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:15:45.074 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 45 2026-02-18T08:15:45.074 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 45 2026-02-18T08:15:45.074 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:15:45.074 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=181, tag_tid=12, entry_tid=1], [object_number=180, tag_tid=12, entry_tid=0], [object_number=183, tag_tid=11, entry_tid=29711], [object_number=182, tag_tid=11, entry_tid=29710]]], state=connected] 2026-02-18T08:15:45.074 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=180, tag_tid=11, entry_tid=27168], [object_number=183, tag_tid=11, entry_tid=27167], [object_number=182, tag_tid=11, entry_tid=27166], [object_number=181, tag_tid=11, entry_tid=27165]]], state=connected] 2026-02-18T08:15:45.076 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:15:45.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=181, tag_tid=12, entry_tid=1' 2026-02-18T08:15:45.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:15:45.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:15:45.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:15:45.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:15:45.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:15:45.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:15:45.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:15:45.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:15:45.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:15:45.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:15:45.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:15:45.080 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:15:45.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:15:45.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:15:45.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:15:45.118 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 45 2026-02-18T08:15:45.118 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 45 2026-02-18T08:15:45.118 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:15:45.118 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=181, tag_tid=12, entry_tid=1], [object_number=180, tag_tid=12, entry_tid=0], [object_number=183, tag_tid=11, entry_tid=29711], [object_number=182, tag_tid=11, entry_tid=29710]]], state=connected] 2026-02-18T08:15:45.118 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=180, tag_tid=11, entry_tid=27168], [object_number=183, tag_tid=11, entry_tid=27167], [object_number=182, tag_tid=11, entry_tid=27166], [object_number=181, tag_tid=11, entry_tid=27165]]], state=connected] 2026-02-18T08:15:45.121 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:15:45.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=180, tag_tid=11, entry_tid=27168' 2026-02-18T08:15:45.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=181, tag_tid=12, entry_tid=1' -a 'object_number=181, tag_tid=12, entry_tid=1' = 'object_number=180, tag_tid=11, entry_tid=27168' 2026-02-18T08:15:45.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=180, tag_tid=11, entry_tid=27168' '!=' 'object_number=180, tag_tid=11, entry_tid=26928' 2026-02-18T08:15:45.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:15:45.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=180, tag_tid=11, entry_tid=27168' = 'object_number=180, tag_tid=11, entry_tid=26928' 2026-02-18T08:15:45.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=180, tag_tid=11, entry_tid=27168' 2026-02-18T08:15:45.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=181, tag_tid=12, entry_tid=1' 2026-02-18T08:15:45.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:15:45.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:15:45.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=12 2026-02-18T08:15:45.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=180, tag_tid=11, entry_tid=27168' 2026-02-18T08:15:45.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:15:45.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:15:45.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=11 2026-02-18T08:15:45.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=181, tag_tid=12, entry_tid=1' 2026-02-18T08:15:45.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:15:45.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:15:45.130 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:15:45.131 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=180, tag_tid=11, entry_tid=27168' 2026-02-18T08:15:45.131 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:15:45.131 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:15:45.132 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=27168 2026-02-18T08:15:45.132 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 12 = 11 -a 1 -le 27168 2026-02-18T08:15:45.132 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:15:45.132 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:15:45.132 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:15:45.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:15:45.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:15:45.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:15:45.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:15:45.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:15:45.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:15:45.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:15:45.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:15:45.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:15:45.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:15:45.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:15:45.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:15:45.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:15:45.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:15:45.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:15:45.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:15:45.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:15:45.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:15:45.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:15:45.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:15:45.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:15:45.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:15:45.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:15:45.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:15:45.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:15:45.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:15:45.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:15:45.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:15:45.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:15:45.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:15:45.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:15:45.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:15:45.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:15:45.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:15:45.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:15:45.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:15:45.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:15:45.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:15:45.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:15:45.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:15:45.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:15:45.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:15:45.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:15:45.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:15:45.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:15:45.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:15:45.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:15:45.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:15:45.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:15:45.348 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:15:45.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:15:45.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:15:45.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:15:45.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:15:45.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:15:45.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:15:45.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:15:45.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:15:45.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:15:45.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:15:45.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:15:45.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:15:45.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:15:45.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:15:45.412 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:15:45.412 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:15:45.413 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:15:45.413 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:15:45.413 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:15:45.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 45 2026-02-18T08:15:45.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 45 2026-02-18T08:15:45.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:15:45.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=181, tag_tid=12, entry_tid=1], [object_number=180, tag_tid=12, entry_tid=0], [object_number=183, tag_tid=11, entry_tid=29711], [object_number=182, tag_tid=11, entry_tid=29710]]], state=connected] 2026-02-18T08:15:45.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=182, tag_tid=11, entry_tid=27402], [object_number=181, tag_tid=11, entry_tid=27401], [object_number=180, tag_tid=11, entry_tid=27400], [object_number=183, tag_tid=11, entry_tid=27399]]], state=connected] 2026-02-18T08:15:45.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:15:45.446 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=181, tag_tid=12, entry_tid=1' 2026-02-18T08:15:45.446 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:15:45.446 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:15:45.446 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:15:45.446 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:15:45.446 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:15:45.446 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:15:45.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:15:45.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:15:45.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:15:45.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:15:45.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:15:45.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:15:45.449 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:15:45.449 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:15:45.449 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:15:45.476 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 45 2026-02-18T08:15:45.476 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 45 2026-02-18T08:15:45.476 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:15:45.476 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=181, tag_tid=12, entry_tid=1], [object_number=180, tag_tid=12, entry_tid=0], [object_number=183, tag_tid=11, entry_tid=29711], [object_number=182, tag_tid=11, entry_tid=29710]]], state=connected] 2026-02-18T08:15:45.476 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=182, tag_tid=11, entry_tid=27402], [object_number=181, tag_tid=11, entry_tid=27401], [object_number=180, tag_tid=11, entry_tid=27400], [object_number=183, tag_tid=11, entry_tid=27399]]], state=connected] 2026-02-18T08:15:45.479 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:15:45.481 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=182, tag_tid=11, entry_tid=27402' 2026-02-18T08:15:45.481 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=181, tag_tid=12, entry_tid=1' -a 'object_number=181, tag_tid=12, entry_tid=1' = 'object_number=182, tag_tid=11, entry_tid=27402' 2026-02-18T08:15:45.481 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=182, tag_tid=11, entry_tid=27402' '!=' 'object_number=180, tag_tid=11, entry_tid=27168' 2026-02-18T08:15:45.481 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:15:45.481 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=182, tag_tid=11, entry_tid=27402' = 'object_number=180, tag_tid=11, entry_tid=27168' 2026-02-18T08:15:45.481 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=182, tag_tid=11, entry_tid=27402' 2026-02-18T08:15:45.482 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=181, tag_tid=12, entry_tid=1' 2026-02-18T08:15:45.482 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:15:45.482 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:15:45.483 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=12 2026-02-18T08:15:45.484 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=182, tag_tid=11, entry_tid=27402' 2026-02-18T08:15:45.484 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:15:45.484 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:15:45.485 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=11 2026-02-18T08:15:45.485 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=181, tag_tid=12, entry_tid=1' 2026-02-18T08:15:45.486 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:15:45.486 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:15:45.487 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:15:45.488 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=182, tag_tid=11, entry_tid=27402' 2026-02-18T08:15:45.488 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:15:45.488 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:15:45.490 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=27402 2026-02-18T08:15:45.490 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 12 = 11 -a 1 -le 27402 2026-02-18T08:15:45.490 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:15:45.490 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:15:45.490 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:15:45.691 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:15:45.692 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:15:45.692 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:15:45.692 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:15:45.692 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:15:45.692 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:15:45.692 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:15:45.692 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:15:45.692 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:15:45.692 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:15:45.692 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:15:45.692 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:15:45.692 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:15:45.692 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:15:45.692 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:15:45.693 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:15:45.693 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:15:45.693 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:15:45.693 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:15:45.693 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:15:45.693 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:15:45.693 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:15:45.694 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:15:45.694 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:15:45.694 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:15:45.694 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:15:45.694 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:15:45.694 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:15:45.694 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:15:45.694 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:15:45.694 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:15:45.694 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:15:45.694 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:15:45.695 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:15:45.695 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:15:45.695 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:15:45.695 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:15:45.695 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:15:45.695 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:15:45.695 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:15:45.695 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:15:45.695 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:15:45.695 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:15:45.695 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:15:45.695 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:15:45.695 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:15:45.695 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:15:45.696 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:15:45.696 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:15:45.705 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:15:45.706 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:15:45.706 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:15:45.706 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:15:45.774 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:15:45.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:15:45.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:15:45.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:15:45.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:15:45.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:15:45.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:15:45.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:15:45.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:15:45.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:15:45.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:15:45.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:15:45.776 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:15:45.777 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:15:45.777 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:15:45.777 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:15:45.804 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 45 2026-02-18T08:15:45.804 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 45 2026-02-18T08:15:45.804 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:15:45.804 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=181, tag_tid=12, entry_tid=1], [object_number=180, tag_tid=12, entry_tid=0], [object_number=183, tag_tid=11, entry_tid=29711], [object_number=182, tag_tid=11, entry_tid=29710]]], state=connected] 2026-02-18T08:15:45.804 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=180, tag_tid=11, entry_tid=27632], [object_number=183, tag_tid=11, entry_tid=27631], [object_number=182, tag_tid=11, entry_tid=27630], [object_number=181, tag_tid=11, entry_tid=27629]]], state=connected] 2026-02-18T08:15:45.807 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:15:45.809 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=181, tag_tid=12, entry_tid=1' 2026-02-18T08:15:45.809 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:15:45.809 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:15:45.809 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:15:45.810 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:15:45.810 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:15:45.810 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:15:45.810 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:15:45.810 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:15:45.810 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:15:45.810 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:15:45.811 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:15:45.811 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:15:45.812 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:15:45.812 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:15:45.812 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:15:45.839 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 45 2026-02-18T08:15:45.839 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 45 2026-02-18T08:15:45.839 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:15:45.839 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=181, tag_tid=12, entry_tid=1], [object_number=180, tag_tid=12, entry_tid=0], [object_number=183, tag_tid=11, entry_tid=29711], [object_number=182, tag_tid=11, entry_tid=29710]]], state=connected] 2026-02-18T08:15:45.839 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=180, tag_tid=11, entry_tid=27632], [object_number=183, tag_tid=11, entry_tid=27631], [object_number=182, tag_tid=11, entry_tid=27630], [object_number=181, tag_tid=11, entry_tid=27629]]], state=connected] 2026-02-18T08:15:45.841 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:15:45.843 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=180, tag_tid=11, entry_tid=27632' 2026-02-18T08:15:45.843 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=181, tag_tid=12, entry_tid=1' -a 'object_number=181, tag_tid=12, entry_tid=1' = 'object_number=180, tag_tid=11, entry_tid=27632' 2026-02-18T08:15:45.844 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=180, tag_tid=11, entry_tid=27632' '!=' 'object_number=182, tag_tid=11, entry_tid=27402' 2026-02-18T08:15:45.844 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:15:45.844 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=180, tag_tid=11, entry_tid=27632' = 'object_number=182, tag_tid=11, entry_tid=27402' 2026-02-18T08:15:45.844 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=180, tag_tid=11, entry_tid=27632' 2026-02-18T08:15:45.844 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=181, tag_tid=12, entry_tid=1' 2026-02-18T08:15:45.844 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:15:45.845 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:15:45.846 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=12 2026-02-18T08:15:45.846 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=180, tag_tid=11, entry_tid=27632' 2026-02-18T08:15:45.846 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:15:45.846 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:15:45.848 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=11 2026-02-18T08:15:45.848 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=181, tag_tid=12, entry_tid=1' 2026-02-18T08:15:45.848 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:15:45.848 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:15:45.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:15:45.850 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=180, tag_tid=11, entry_tid=27632' 2026-02-18T08:15:45.850 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:15:45.850 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:15:45.851 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=27632 2026-02-18T08:15:45.852 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 12 = 11 -a 1 -le 27632 2026-02-18T08:15:45.852 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:15:45.852 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:15:45.852 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:15:46.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:15:46.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:15:46.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:15:46.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:15:46.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:15:46.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:15:46.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:15:46.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:15:46.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:15:46.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:15:46.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:15:46.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:15:46.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:15:46.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:15:46.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:15:46.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:15:46.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:15:46.056 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:15:46.056 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:15:46.056 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:15:46.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:15:46.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:15:46.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:15:46.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:15:46.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:15:46.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:15:46.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:15:46.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:15:46.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:15:46.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:15:46.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:15:46.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:15:46.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:15:46.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:15:46.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:15:46.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:15:46.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:15:46.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:15:46.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:15:46.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:15:46.058 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:15:46.058 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:15:46.058 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:15:46.058 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:15:46.058 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:15:46.058 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:15:46.058 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:15:46.058 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:15:46.058 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:15:46.069 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:15:46.069 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:15:46.069 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:15:46.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:15:46.130 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:15:46.130 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:15:46.131 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:15:46.131 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:15:46.131 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:15:46.131 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:15:46.131 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:15:46.131 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:15:46.131 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:15:46.131 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:15:46.131 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:15:46.131 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:15:46.131 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:15:46.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:15:46.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:15:46.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:15:46.169 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 45 2026-02-18T08:15:46.169 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 45 2026-02-18T08:15:46.169 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:15:46.170 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=181, tag_tid=12, entry_tid=1], [object_number=180, tag_tid=12, entry_tid=0], [object_number=183, tag_tid=11, entry_tid=29711], [object_number=182, tag_tid=11, entry_tid=29710]]], state=connected] 2026-02-18T08:15:46.170 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=183, tag_tid=11, entry_tid=27859], [object_number=182, tag_tid=11, entry_tid=27858], [object_number=181, tag_tid=11, entry_tid=27857], [object_number=180, tag_tid=11, entry_tid=27856]]], state=connected] 2026-02-18T08:15:46.173 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:15:46.175 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=181, tag_tid=12, entry_tid=1' 2026-02-18T08:15:46.175 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:15:46.175 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:15:46.176 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:15:46.176 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:15:46.176 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:15:46.176 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:15:46.176 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:15:46.176 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:15:46.176 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:15:46.176 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:15:46.176 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:15:46.177 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:15:46.178 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:15:46.178 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:15:46.178 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:15:46.206 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 45 2026-02-18T08:15:46.206 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 45 2026-02-18T08:15:46.206 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:15:46.206 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=181, tag_tid=12, entry_tid=1], [object_number=180, tag_tid=12, entry_tid=0], [object_number=183, tag_tid=11, entry_tid=29711], [object_number=182, tag_tid=11, entry_tid=29710]]], state=connected] 2026-02-18T08:15:46.206 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=183, tag_tid=11, entry_tid=27859], [object_number=182, tag_tid=11, entry_tid=27858], [object_number=181, tag_tid=11, entry_tid=27857], [object_number=180, tag_tid=11, entry_tid=27856]]], state=connected] 2026-02-18T08:15:46.210 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:15:46.212 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=183, tag_tid=11, entry_tid=27859' 2026-02-18T08:15:46.212 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=181, tag_tid=12, entry_tid=1' -a 'object_number=181, tag_tid=12, entry_tid=1' = 'object_number=183, tag_tid=11, entry_tid=27859' 2026-02-18T08:15:46.212 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=183, tag_tid=11, entry_tid=27859' '!=' 'object_number=180, tag_tid=11, entry_tid=27632' 2026-02-18T08:15:46.212 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:15:46.212 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=183, tag_tid=11, entry_tid=27859' = 'object_number=180, tag_tid=11, entry_tid=27632' 2026-02-18T08:15:46.212 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=183, tag_tid=11, entry_tid=27859' 2026-02-18T08:15:46.212 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=181, tag_tid=12, entry_tid=1' 2026-02-18T08:15:46.212 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:15:46.212 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:15:46.214 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=12 2026-02-18T08:15:46.215 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=183, tag_tid=11, entry_tid=27859' 2026-02-18T08:15:46.215 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:15:46.215 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:15:46.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=11 2026-02-18T08:15:46.217 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=181, tag_tid=12, entry_tid=1' 2026-02-18T08:15:46.217 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:15:46.217 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:15:46.218 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:15:46.219 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=183, tag_tid=11, entry_tid=27859' 2026-02-18T08:15:46.219 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:15:46.219 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:15:46.221 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=27859 2026-02-18T08:15:46.221 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 12 = 11 -a 1 -le 27859 2026-02-18T08:15:46.221 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:15:46.221 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:15:46.221 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:15:46.422 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:15:46.422 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:15:46.422 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:15:46.422 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:15:46.422 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:15:46.422 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:15:46.422 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:15:46.423 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:15:46.423 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:15:46.423 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:15:46.423 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:15:46.423 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:15:46.423 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:15:46.423 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:15:46.423 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:15:46.423 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:15:46.423 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:15:46.425 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:15:46.425 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:15:46.425 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:15:46.425 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:15:46.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:15:46.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:15:46.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:15:46.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:15:46.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:15:46.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:15:46.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:15:46.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:15:46.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:15:46.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:15:46.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:15:46.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:15:46.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:15:46.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:15:46.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:15:46.427 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:15:46.427 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:15:46.427 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:15:46.427 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:15:46.427 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:15:46.427 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:15:46.427 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:15:46.427 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:15:46.427 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:15:46.427 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:15:46.427 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:15:46.427 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:15:46.427 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:15:46.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:15:46.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:15:46.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:15:46.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:15:46.500 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:15:46.501 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:15:46.501 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:15:46.501 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:15:46.501 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:15:46.501 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:15:46.501 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:15:46.501 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:15:46.501 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:15:46.501 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:15:46.502 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:15:46.502 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:15:46.502 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:15:46.503 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:15:46.503 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:15:46.503 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:15:46.532 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 45 2026-02-18T08:15:46.532 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 45 2026-02-18T08:15:46.532 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:15:46.532 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=181, tag_tid=12, entry_tid=1], [object_number=180, tag_tid=12, entry_tid=0], [object_number=183, tag_tid=11, entry_tid=29711], [object_number=182, tag_tid=11, entry_tid=29710]]], state=connected] 2026-02-18T08:15:46.532 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=181, tag_tid=11, entry_tid=28093], [object_number=180, tag_tid=11, entry_tid=28092], [object_number=183, tag_tid=11, entry_tid=28091], [object_number=182, tag_tid=11, entry_tid=28090]]], state=connected] 2026-02-18T08:15:46.535 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:15:46.536 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=181, tag_tid=12, entry_tid=1' 2026-02-18T08:15:46.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:15:46.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:15:46.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:15:46.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:15:46.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:15:46.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:15:46.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:15:46.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:15:46.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:15:46.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:15:46.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:15:46.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:15:46.539 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:15:46.539 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:15:46.540 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:15:46.781 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 45 2026-02-18T08:15:46.782 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 45 2026-02-18T08:15:46.782 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:15:46.782 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=181, tag_tid=12, entry_tid=1], [object_number=180, tag_tid=12, entry_tid=0], [object_number=183, tag_tid=11, entry_tid=29711], [object_number=182, tag_tid=11, entry_tid=29710]]], state=connected] 2026-02-18T08:15:46.782 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=181, tag_tid=11, entry_tid=28093], [object_number=180, tag_tid=11, entry_tid=28092], [object_number=183, tag_tid=11, entry_tid=28091], [object_number=182, tag_tid=11, entry_tid=28090]]], state=connected] 2026-02-18T08:15:46.786 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:15:46.787 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=181, tag_tid=11, entry_tid=28093' 2026-02-18T08:15:46.787 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=181, tag_tid=12, entry_tid=1' -a 'object_number=181, tag_tid=12, entry_tid=1' = 'object_number=181, tag_tid=11, entry_tid=28093' 2026-02-18T08:15:46.787 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=181, tag_tid=11, entry_tid=28093' '!=' 'object_number=183, tag_tid=11, entry_tid=27859' 2026-02-18T08:15:46.787 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:15:46.787 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=181, tag_tid=11, entry_tid=28093' = 'object_number=183, tag_tid=11, entry_tid=27859' 2026-02-18T08:15:46.787 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=181, tag_tid=11, entry_tid=28093' 2026-02-18T08:15:46.788 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=181, tag_tid=12, entry_tid=1' 2026-02-18T08:15:46.788 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:15:46.788 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:15:46.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=12 2026-02-18T08:15:46.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=181, tag_tid=11, entry_tid=28093' 2026-02-18T08:15:46.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:15:46.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:15:46.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=11 2026-02-18T08:15:46.793 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=181, tag_tid=12, entry_tid=1' 2026-02-18T08:15:46.793 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:15:46.793 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:15:46.794 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:15:46.795 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=181, tag_tid=11, entry_tid=28093' 2026-02-18T08:15:46.795 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:15:46.795 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:15:46.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=28093 2026-02-18T08:15:46.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 12 = 11 -a 1 -le 28093 2026-02-18T08:15:46.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:15:46.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:15:46.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:15:46.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:15:46.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:15:46.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:15:46.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:15:46.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:15:46.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:15:46.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:15:46.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:15:46.999 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:15:46.999 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:15:46.999 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:15:46.999 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:15:46.999 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:15:46.999 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:15:46.999 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:15:46.999 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:15:46.999 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:15:47.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:15:47.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:15:47.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:15:47.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:15:47.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:15:47.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:15:47.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:15:47.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:15:47.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:15:47.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:15:47.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:15:47.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:15:47.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:15:47.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:15:47.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:15:47.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:15:47.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:15:47.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:15:47.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:15:47.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:15:47.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:15:47.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:15:47.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:15:47.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:15:47.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:15:47.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:15:47.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:15:47.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:15:47.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:15:47.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:15:47.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:15:47.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:15:47.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:15:47.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:15:47.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:15:47.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:15:47.076 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:15:47.076 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:15:47.076 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:15:47.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:15:47.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:15:47.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:15:47.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:15:47.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:15:47.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:15:47.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:15:47.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:15:47.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:15:47.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:15:47.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:15:47.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:15:47.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:15:47.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 45 2026-02-18T08:15:47.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 45 2026-02-18T08:15:47.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:15:47.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=181, tag_tid=12, entry_tid=1], [object_number=180, tag_tid=12, entry_tid=0], [object_number=183, tag_tid=11, entry_tid=29711], [object_number=182, tag_tid=11, entry_tid=29710]]], state=connected] 2026-02-18T08:15:47.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=181, tag_tid=11, entry_tid=28477], [object_number=180, tag_tid=11, entry_tid=28476], [object_number=183, tag_tid=11, entry_tid=28475], [object_number=182, tag_tid=11, entry_tid=28474]]], state=connected] 2026-02-18T08:15:47.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:15:47.111 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=181, tag_tid=12, entry_tid=1' 2026-02-18T08:15:47.112 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:15:47.112 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:15:47.112 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:15:47.112 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:15:47.112 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:15:47.112 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:15:47.112 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:15:47.112 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:15:47.112 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:15:47.113 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:15:47.113 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:15:47.113 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:15:47.114 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:15:47.114 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:15:47.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:15:47.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 45 2026-02-18T08:15:47.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 45 2026-02-18T08:15:47.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:15:47.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=181, tag_tid=12, entry_tid=1], [object_number=180, tag_tid=12, entry_tid=0], [object_number=183, tag_tid=11, entry_tid=29711], [object_number=182, tag_tid=11, entry_tid=29710]]], state=connected] 2026-02-18T08:15:47.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=181, tag_tid=11, entry_tid=28477], [object_number=180, tag_tid=11, entry_tid=28476], [object_number=183, tag_tid=11, entry_tid=28475], [object_number=182, tag_tid=11, entry_tid=28474]]], state=connected] 2026-02-18T08:15:47.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:15:47.156 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=181, tag_tid=11, entry_tid=28477' 2026-02-18T08:15:47.157 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=181, tag_tid=12, entry_tid=1' -a 'object_number=181, tag_tid=12, entry_tid=1' = 'object_number=181, tag_tid=11, entry_tid=28477' 2026-02-18T08:15:47.157 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=181, tag_tid=11, entry_tid=28477' '!=' 'object_number=181, tag_tid=11, entry_tid=28093' 2026-02-18T08:15:47.157 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:15:47.157 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=181, tag_tid=11, entry_tid=28477' = 'object_number=181, tag_tid=11, entry_tid=28093' 2026-02-18T08:15:47.157 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=181, tag_tid=11, entry_tid=28477' 2026-02-18T08:15:47.157 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=181, tag_tid=12, entry_tid=1' 2026-02-18T08:15:47.158 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:15:47.158 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:15:47.158 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=12 2026-02-18T08:15:47.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=181, tag_tid=11, entry_tid=28477' 2026-02-18T08:15:47.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:15:47.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:15:47.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=11 2026-02-18T08:15:47.161 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=181, tag_tid=12, entry_tid=1' 2026-02-18T08:15:47.161 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:15:47.161 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:15:47.162 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:15:47.163 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=181, tag_tid=11, entry_tid=28477' 2026-02-18T08:15:47.163 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:15:47.163 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:15:47.165 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=28477 2026-02-18T08:15:47.165 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 12 = 11 -a 1 -le 28477 2026-02-18T08:15:47.165 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:15:47.165 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:15:47.165 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:15:47.366 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:15:47.367 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:15:47.367 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:15:47.367 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:15:47.367 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:15:47.367 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:15:47.367 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:15:47.367 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:15:47.367 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:15:47.367 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:15:47.367 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:15:47.367 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:15:47.367 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:15:47.367 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:15:47.367 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:15:47.368 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:15:47.368 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:15:47.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:15:47.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:15:47.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:15:47.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:15:47.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:15:47.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:15:47.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:15:47.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:15:47.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:15:47.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:15:47.370 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:15:47.370 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:15:47.370 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:15:47.370 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:15:47.370 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:15:47.370 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:15:47.370 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:15:47.370 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:15:47.370 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:15:47.370 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:15:47.370 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:15:47.370 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:15:47.370 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:15:47.370 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:15:47.370 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:15:47.370 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:15:47.370 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:15:47.370 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:15:47.370 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:15:47.370 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:15:47.370 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:15:47.370 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:15:47.380 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:15:47.381 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:15:47.381 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:15:47.381 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:15:47.532 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:15:47.532 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:15:47.532 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:15:47.532 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:15:47.532 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:15:47.532 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:15:47.532 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:15:47.532 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:15:47.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:15:47.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:15:47.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:15:47.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:15:47.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:15:47.534 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:15:47.535 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:15:47.535 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:15:47.561 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 45 2026-02-18T08:15:47.561 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 45 2026-02-18T08:15:47.561 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:15:47.561 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=181, tag_tid=12, entry_tid=1], [object_number=180, tag_tid=12, entry_tid=0], [object_number=183, tag_tid=11, entry_tid=29711], [object_number=182, tag_tid=11, entry_tid=29710]]], state=connected] 2026-02-18T08:15:47.561 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=180, tag_tid=11, entry_tid=28632], [object_number=183, tag_tid=11, entry_tid=28631], [object_number=182, tag_tid=11, entry_tid=28630], [object_number=181, tag_tid=11, entry_tid=28629]]], state=connected] 2026-02-18T08:15:47.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:15:47.565 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=181, tag_tid=12, entry_tid=1' 2026-02-18T08:15:47.566 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:15:47.566 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:15:47.566 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:15:47.567 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:15:47.567 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:15:47.567 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:15:47.567 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:15:47.567 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:15:47.567 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:15:47.567 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:15:47.568 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:15:47.568 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:15:47.568 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:15:47.569 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:15:47.569 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:15:47.595 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 45 2026-02-18T08:15:47.595 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 45 2026-02-18T08:15:47.596 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:15:47.596 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=181, tag_tid=12, entry_tid=1], [object_number=180, tag_tid=12, entry_tid=0], [object_number=183, tag_tid=11, entry_tid=29711], [object_number=182, tag_tid=11, entry_tid=29710]]], state=connected] 2026-02-18T08:15:47.596 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=180, tag_tid=11, entry_tid=28632], [object_number=183, tag_tid=11, entry_tid=28631], [object_number=182, tag_tid=11, entry_tid=28630], [object_number=181, tag_tid=11, entry_tid=28629]]], state=connected] 2026-02-18T08:15:47.598 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:15:47.600 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=180, tag_tid=11, entry_tid=28632' 2026-02-18T08:15:47.600 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=181, tag_tid=12, entry_tid=1' -a 'object_number=181, tag_tid=12, entry_tid=1' = 'object_number=180, tag_tid=11, entry_tid=28632' 2026-02-18T08:15:47.600 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=180, tag_tid=11, entry_tid=28632' '!=' 'object_number=181, tag_tid=11, entry_tid=28477' 2026-02-18T08:15:47.600 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:15:47.600 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=180, tag_tid=11, entry_tid=28632' = 'object_number=181, tag_tid=11, entry_tid=28477' 2026-02-18T08:15:47.600 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=180, tag_tid=11, entry_tid=28632' 2026-02-18T08:15:47.601 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=181, tag_tid=12, entry_tid=1' 2026-02-18T08:15:47.601 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:15:47.601 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:15:47.602 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=12 2026-02-18T08:15:47.603 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=180, tag_tid=11, entry_tid=28632' 2026-02-18T08:15:47.603 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:15:47.603 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:15:47.604 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=11 2026-02-18T08:15:47.605 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=181, tag_tid=12, entry_tid=1' 2026-02-18T08:15:47.605 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:15:47.605 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:15:47.607 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:15:47.607 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=180, tag_tid=11, entry_tid=28632' 2026-02-18T08:15:47.607 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:15:47.607 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:15:47.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=28632 2026-02-18T08:15:47.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 12 = 11 -a 1 -le 28632 2026-02-18T08:15:47.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:15:47.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:15:47.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:15:47.716 INFO:tasks.rbd_mirror_thrash:waiting for cluster2.client.mirror.1 2026-02-18T08:15:47.716 INFO:teuthology.orchestra.run:waiting for 600 2026-02-18T08:15:47.716 INFO:tasks.rbd_mirror_thrash:reviving cluster2.client.mirror.1 2026-02-18T08:15:47.717 INFO:tasks.rbd_mirror.cluster2.client.mirror.1:Restarting daemon 2026-02-18T08:15:47.717 DEBUG:teuthology.orchestra.run.trial163:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.1 2026-02-18T08:15:47.720 INFO:tasks.rbd_mirror.cluster2.client.mirror.1:Started 2026-02-18T08:15:47.720 INFO:tasks.rbd_mirror_thrash:waiting for 97.31678291117456 secs before thrashing 2026-02-18T08:15:47.811 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:15:47.811 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:15:47.811 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:15:47.811 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:15:47.811 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:15:47.811 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:15:47.811 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:15:47.811 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:15:47.812 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:15:47.812 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:15:47.812 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:15:47.812 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:15:47.812 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:15:47.812 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:15:47.812 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:15:47.814 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:15:47.814 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:15:47.814 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:15:47.814 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:15:47.814 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:15:47.814 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:15:47.814 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:15:47.814 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:15:47.814 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:15:47.814 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:15:47.814 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:15:47.814 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:15:47.814 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:15:47.814 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:15:47.814 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:15:47.814 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:15:47.814 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:15:47.814 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:15:47.814 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:15:47.815 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:15:47.815 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:15:47.815 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:15:47.815 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:15:47.815 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:15:47.815 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:15:47.815 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:15:47.815 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:15:47.815 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:15:47.815 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:15:47.815 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:15:47.815 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:15:47.815 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:15:47.815 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:15:47.815 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:15:47.824 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:15:47.825 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:15:47.825 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:15:47.825 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:15:47.884 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:15:47.885 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:15:47.885 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:15:47.885 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:15:47.885 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:15:47.885 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:15:47.885 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:15:47.885 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:15:47.885 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:15:47.885 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:15:47.885 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:15:47.885 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:15:47.885 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:15:47.887 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:15:47.887 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:15:47.887 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:15:47.915 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 45 2026-02-18T08:15:47.915 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 45 2026-02-18T08:15:47.915 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:15:47.915 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=181, tag_tid=12, entry_tid=1], [object_number=180, tag_tid=12, entry_tid=0], [object_number=183, tag_tid=11, entry_tid=29711], [object_number=182, tag_tid=11, entry_tid=29710]]], state=connected] 2026-02-18T08:15:47.915 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=180, tag_tid=11, entry_tid=28896], [object_number=183, tag_tid=11, entry_tid=28895], [object_number=182, tag_tid=11, entry_tid=28894], [object_number=181, tag_tid=11, entry_tid=28893]]], state=connected] 2026-02-18T08:15:47.917 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:15:47.918 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=181, tag_tid=12, entry_tid=1' 2026-02-18T08:15:47.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:15:47.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:15:47.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:15:47.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:15:47.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:15:47.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:15:47.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:15:47.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:15:47.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:15:47.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:15:47.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:15:47.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:15:47.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:15:47.922 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:15:47.922 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:15:47.948 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 45 2026-02-18T08:15:47.948 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 45 2026-02-18T08:15:47.949 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:15:47.949 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=181, tag_tid=12, entry_tid=1], [object_number=180, tag_tid=12, entry_tid=0], [object_number=183, tag_tid=11, entry_tid=29711], [object_number=182, tag_tid=11, entry_tid=29710]]], state=connected] 2026-02-18T08:15:47.949 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=180, tag_tid=11, entry_tid=28896], [object_number=183, tag_tid=11, entry_tid=28895], [object_number=182, tag_tid=11, entry_tid=28894], [object_number=181, tag_tid=11, entry_tid=28893]]], state=connected] 2026-02-18T08:15:47.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:15:47.953 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=180, tag_tid=11, entry_tid=28896' 2026-02-18T08:15:47.953 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=181, tag_tid=12, entry_tid=1' -a 'object_number=181, tag_tid=12, entry_tid=1' = 'object_number=180, tag_tid=11, entry_tid=28896' 2026-02-18T08:15:47.953 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=180, tag_tid=11, entry_tid=28896' '!=' 'object_number=180, tag_tid=11, entry_tid=28632' 2026-02-18T08:15:47.953 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:15:47.953 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=180, tag_tid=11, entry_tid=28896' = 'object_number=180, tag_tid=11, entry_tid=28632' 2026-02-18T08:15:47.953 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=180, tag_tid=11, entry_tid=28896' 2026-02-18T08:15:47.953 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=181, tag_tid=12, entry_tid=1' 2026-02-18T08:15:47.954 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:15:47.954 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:15:47.955 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=12 2026-02-18T08:15:47.955 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=180, tag_tid=11, entry_tid=28896' 2026-02-18T08:15:47.955 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:15:47.955 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:15:47.956 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=11 2026-02-18T08:15:47.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=181, tag_tid=12, entry_tid=1' 2026-02-18T08:15:47.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:15:47.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:15:47.959 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:15:47.959 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=180, tag_tid=11, entry_tid=28896' 2026-02-18T08:15:47.959 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:15:47.959 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:15:47.961 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=28896 2026-02-18T08:15:47.961 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 12 = 11 -a 1 -le 28896 2026-02-18T08:15:47.961 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:15:47.961 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:15:47.961 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:15:48.163 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:15:48.163 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:15:48.163 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:15:48.163 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:15:48.163 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:15:48.163 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:15:48.163 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:15:48.163 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:15:48.163 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:15:48.163 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:15:48.163 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:15:48.163 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:15:48.163 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:15:48.163 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:15:48.163 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:15:48.164 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:15:48.164 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:15:48.166 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:15:48.166 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:15:48.166 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:15:48.166 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:15:48.166 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:15:48.166 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:15:48.166 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:15:48.166 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:15:48.166 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:15:48.166 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:15:48.167 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:15:48.167 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:15:48.167 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:15:48.167 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:15:48.167 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:15:48.167 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:15:48.167 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:15:48.167 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:15:48.167 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:15:48.167 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:15:48.167 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:15:48.167 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:15:48.167 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:15:48.167 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:15:48.167 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:15:48.167 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:15:48.167 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:15:48.167 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:15:48.167 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:15:48.167 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:15:48.167 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:15:48.167 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:15:48.178 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:15:48.178 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:15:48.178 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:15:48.179 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:15:48.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:15:48.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:15:48.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:15:48.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:15:48.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:15:48.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:15:48.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:15:48.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:15:48.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:15:48.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:15:48.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:15:48.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:15:48.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:15:48.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:15:48.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:15:48.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:15:48.270 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 45 2026-02-18T08:15:48.270 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 45 2026-02-18T08:15:48.270 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:15:48.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=181, tag_tid=12, entry_tid=1], [object_number=180, tag_tid=12, entry_tid=0], [object_number=183, tag_tid=11, entry_tid=29711], [object_number=182, tag_tid=11, entry_tid=29710]]], state=connected] 2026-02-18T08:15:48.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=181, tag_tid=11, entry_tid=29109], [object_number=180, tag_tid=11, entry_tid=29108], [object_number=183, tag_tid=11, entry_tid=29107], [object_number=182, tag_tid=11, entry_tid=29106]]], state=connected] 2026-02-18T08:15:48.273 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:15:48.275 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=181, tag_tid=12, entry_tid=1' 2026-02-18T08:15:48.276 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:15:48.276 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:15:48.276 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:15:48.276 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:15:48.276 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:15:48.276 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:15:48.276 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:15:48.276 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:15:48.276 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:15:48.276 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:15:48.277 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:15:48.277 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:15:48.277 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:15:48.278 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:15:48.278 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:15:48.304 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 45 2026-02-18T08:15:48.304 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 45 2026-02-18T08:15:48.304 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:15:48.304 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=181, tag_tid=12, entry_tid=1], [object_number=180, tag_tid=12, entry_tid=0], [object_number=183, tag_tid=11, entry_tid=29711], [object_number=182, tag_tid=11, entry_tid=29710]]], state=connected] 2026-02-18T08:15:48.304 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=181, tag_tid=11, entry_tid=29109], [object_number=180, tag_tid=11, entry_tid=29108], [object_number=183, tag_tid=11, entry_tid=29107], [object_number=182, tag_tid=11, entry_tid=29106]]], state=connected] 2026-02-18T08:15:48.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:15:48.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=181, tag_tid=11, entry_tid=29109' 2026-02-18T08:15:48.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=181, tag_tid=12, entry_tid=1' -a 'object_number=181, tag_tid=12, entry_tid=1' = 'object_number=181, tag_tid=11, entry_tid=29109' 2026-02-18T08:15:48.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=181, tag_tid=11, entry_tid=29109' '!=' 'object_number=180, tag_tid=11, entry_tid=28896' 2026-02-18T08:15:48.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:15:48.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=181, tag_tid=11, entry_tid=29109' = 'object_number=180, tag_tid=11, entry_tid=28896' 2026-02-18T08:15:48.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=181, tag_tid=11, entry_tid=29109' 2026-02-18T08:15:48.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=181, tag_tid=12, entry_tid=1' 2026-02-18T08:15:48.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:15:48.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:15:48.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=12 2026-02-18T08:15:48.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=181, tag_tid=11, entry_tid=29109' 2026-02-18T08:15:48.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:15:48.312 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:15:48.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=11 2026-02-18T08:15:48.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=181, tag_tid=12, entry_tid=1' 2026-02-18T08:15:48.314 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:15:48.314 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:15:48.315 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:15:48.315 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=181, tag_tid=11, entry_tid=29109' 2026-02-18T08:15:48.316 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:15:48.316 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:15:48.317 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=29109 2026-02-18T08:15:48.317 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 12 = 11 -a 1 -le 29109 2026-02-18T08:15:48.317 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:15:48.317 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:15:48.318 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:15:48.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:15:48.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:15:48.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:15:48.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:15:48.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:15:48.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:15:48.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:15:48.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:15:48.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:15:48.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:15:48.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:15:48.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:15:48.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:15:48.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:15:48.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:15:48.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:15:48.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:15:48.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:15:48.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:15:48.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:15:48.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:15:48.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:15:48.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:15:48.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:15:48.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:15:48.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:15:48.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:15:48.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:15:48.523 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:15:48.523 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:15:48.523 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:15:48.523 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:15:48.523 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:15:48.523 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:15:48.523 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:15:48.523 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:15:48.523 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:15:48.523 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:15:48.523 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:15:48.523 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:15:48.523 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:15:48.523 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:15:48.523 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:15:48.523 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:15:48.523 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:15:48.523 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:15:48.523 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:15:48.523 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:15:48.523 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:15:48.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:15:48.534 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:15:48.534 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:15:48.534 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:15:48.595 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:15:48.596 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:15:48.596 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:15:48.596 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:15:48.596 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:15:48.596 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:15:48.596 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:15:48.596 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:15:48.596 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:15:48.596 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:15:48.596 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:15:48.597 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:15:48.597 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:15:48.598 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:15:48.598 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:15:48.598 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:15:48.628 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 45 2026-02-18T08:15:48.628 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 45 2026-02-18T08:15:48.628 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:15:48.628 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=181, tag_tid=12, entry_tid=1], [object_number=180, tag_tid=12, entry_tid=0], [object_number=183, tag_tid=11, entry_tid=29711], [object_number=182, tag_tid=11, entry_tid=29710]]], state=connected] 2026-02-18T08:15:48.628 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=181, tag_tid=11, entry_tid=29333], [object_number=180, tag_tid=11, entry_tid=29332], [object_number=183, tag_tid=11, entry_tid=29331], [object_number=182, tag_tid=11, entry_tid=29330]]], state=connected] 2026-02-18T08:15:48.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:15:48.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=181, tag_tid=12, entry_tid=1' 2026-02-18T08:15:48.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:15:48.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:15:48.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:15:48.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:15:48.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:15:48.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:15:48.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:15:48.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:15:48.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:15:48.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:15:48.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:15:48.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:15:48.635 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:15:48.635 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:15:48.635 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:15:48.664 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 45 2026-02-18T08:15:48.664 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 45 2026-02-18T08:15:48.664 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:15:48.664 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=181, tag_tid=12, entry_tid=1], [object_number=180, tag_tid=12, entry_tid=0], [object_number=183, tag_tid=11, entry_tid=29711], [object_number=182, tag_tid=11, entry_tid=29710]]], state=connected] 2026-02-18T08:15:48.664 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=181, tag_tid=11, entry_tid=29333], [object_number=180, tag_tid=11, entry_tid=29332], [object_number=183, tag_tid=11, entry_tid=29331], [object_number=182, tag_tid=11, entry_tid=29330]]], state=connected] 2026-02-18T08:15:48.667 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:15:48.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=181, tag_tid=11, entry_tid=29333' 2026-02-18T08:15:48.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=181, tag_tid=12, entry_tid=1' -a 'object_number=181, tag_tid=12, entry_tid=1' = 'object_number=181, tag_tid=11, entry_tid=29333' 2026-02-18T08:15:48.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=181, tag_tid=11, entry_tid=29333' '!=' 'object_number=181, tag_tid=11, entry_tid=29109' 2026-02-18T08:15:48.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:15:48.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=181, tag_tid=11, entry_tid=29333' = 'object_number=181, tag_tid=11, entry_tid=29109' 2026-02-18T08:15:48.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=181, tag_tid=11, entry_tid=29333' 2026-02-18T08:15:48.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=181, tag_tid=12, entry_tid=1' 2026-02-18T08:15:48.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:15:48.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:15:48.670 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=12 2026-02-18T08:15:48.671 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=181, tag_tid=11, entry_tid=29333' 2026-02-18T08:15:48.671 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:15:48.671 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:15:48.673 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=11 2026-02-18T08:15:48.673 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=181, tag_tid=12, entry_tid=1' 2026-02-18T08:15:48.673 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:15:48.674 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:15:48.675 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:15:48.676 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=181, tag_tid=11, entry_tid=29333' 2026-02-18T08:15:48.676 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:15:48.676 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:15:48.678 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=29333 2026-02-18T08:15:48.678 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 12 = 11 -a 1 -le 29333 2026-02-18T08:15:48.678 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:15:48.678 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:15:48.678 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:15:48.879 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:15:48.879 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:15:48.879 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:15:48.879 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:15:48.879 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:15:48.879 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:15:48.879 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:15:48.879 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:15:48.879 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:15:48.879 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:15:48.879 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:15:48.880 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:15:48.880 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:15:48.880 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:15:48.880 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:15:48.880 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:15:48.880 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:15:48.882 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:15:48.882 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:15:48.882 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:15:48.883 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:15:48.883 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:15:48.883 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:15:48.883 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:15:48.883 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:15:48.883 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:15:48.883 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:15:48.883 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:15:48.883 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:15:48.883 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:15:48.883 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:15:48.883 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:15:48.883 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:15:48.883 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:15:48.884 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:15:48.884 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:15:48.884 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:15:48.884 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:15:48.884 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:15:48.884 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:15:48.884 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:15:48.884 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:15:48.884 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:15:48.884 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:15:48.884 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:15:48.884 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:15:48.884 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:15:48.884 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:15:48.884 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:15:48.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:15:48.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:15:48.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:15:48.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:15:48.962 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:15:48.963 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:15:48.963 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:15:48.963 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:15:48.963 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:15:48.963 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:15:48.963 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:15:48.963 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:15:48.963 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:15:48.963 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:15:48.963 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:15:48.963 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:15:48.963 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:15:48.964 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:15:48.964 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:15:48.964 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:15:48.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 45 2026-02-18T08:15:48.992 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 45 2026-02-18T08:15:48.992 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:15:48.992 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=181, tag_tid=12, entry_tid=1], [object_number=180, tag_tid=12, entry_tid=0], [object_number=183, tag_tid=11, entry_tid=29711], [object_number=182, tag_tid=11, entry_tid=29710]]], state=connected] 2026-02-18T08:15:48.992 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=183, tag_tid=11, entry_tid=29571], [object_number=182, tag_tid=11, entry_tid=29570], [object_number=181, tag_tid=11, entry_tid=29569], [object_number=180, tag_tid=11, entry_tid=29568]]], state=connected] 2026-02-18T08:15:48.994 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:15:48.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=181, tag_tid=12, entry_tid=1' 2026-02-18T08:15:48.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:15:48.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:15:48.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:15:48.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:15:48.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:15:48.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:15:48.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:15:48.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:15:48.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:15:48.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:15:48.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:15:48.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:15:48.999 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:15:48.999 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:15:48.999 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:15:49.035 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 45 2026-02-18T08:15:49.035 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 45 2026-02-18T08:15:49.035 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:15:49.035 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=181, tag_tid=12, entry_tid=1], [object_number=180, tag_tid=12, entry_tid=0], [object_number=183, tag_tid=11, entry_tid=29711], [object_number=182, tag_tid=11, entry_tid=29710]]], state=connected] 2026-02-18T08:15:49.035 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=183, tag_tid=11, entry_tid=29571], [object_number=182, tag_tid=11, entry_tid=29570], [object_number=181, tag_tid=11, entry_tid=29569], [object_number=180, tag_tid=11, entry_tid=29568]]], state=connected] 2026-02-18T08:15:49.038 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:15:49.039 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=183, tag_tid=11, entry_tid=29571' 2026-02-18T08:15:49.040 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=181, tag_tid=12, entry_tid=1' -a 'object_number=181, tag_tid=12, entry_tid=1' = 'object_number=183, tag_tid=11, entry_tid=29571' 2026-02-18T08:15:49.040 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=183, tag_tid=11, entry_tid=29571' '!=' 'object_number=181, tag_tid=11, entry_tid=29333' 2026-02-18T08:15:49.040 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:15:49.040 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=183, tag_tid=11, entry_tid=29571' = 'object_number=181, tag_tid=11, entry_tid=29333' 2026-02-18T08:15:49.040 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=183, tag_tid=11, entry_tid=29571' 2026-02-18T08:15:49.040 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=181, tag_tid=12, entry_tid=1' 2026-02-18T08:15:49.040 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:15:49.040 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:15:49.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=12 2026-02-18T08:15:49.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=183, tag_tid=11, entry_tid=29571' 2026-02-18T08:15:49.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:15:49.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:15:49.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=11 2026-02-18T08:15:49.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=181, tag_tid=12, entry_tid=1' 2026-02-18T08:15:49.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:15:49.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:15:49.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:15:49.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=183, tag_tid=11, entry_tid=29571' 2026-02-18T08:15:49.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:15:49.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:15:49.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=29571 2026-02-18T08:15:49.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 12 = 11 -a 1 -le 29571 2026-02-18T08:15:49.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:15:49.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:15:49.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:15:49.250 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:15:49.250 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:15:49.251 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:15:49.251 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:15:49.251 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:15:49.251 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:15:49.251 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:15:49.251 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:15:49.251 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:15:49.251 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:15:49.251 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:15:49.251 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:15:49.251 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:15:49.251 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:15:49.251 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:15:49.251 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:15:49.251 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:15:49.253 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:15:49.253 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:15:49.253 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:15:49.253 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:15:49.253 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:15:49.253 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:15:49.254 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:15:49.254 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:15:49.254 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:15:49.254 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:15:49.254 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:15:49.254 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:15:49.254 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:15:49.254 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:15:49.254 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:15:49.254 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:15:49.254 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:15:49.254 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:15:49.254 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:15:49.254 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:15:49.254 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:15:49.254 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:15:49.254 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:15:49.254 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:15:49.254 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:15:49.255 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:15:49.255 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:15:49.255 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:15:49.255 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:15:49.255 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:15:49.255 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:15:49.255 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:15:49.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:15:49.266 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:15:49.266 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:15:49.266 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:15:49.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:15:49.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:15:49.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:15:49.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:15:49.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:15:49.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:15:49.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:15:49.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:15:49.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:15:49.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:15:49.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:15:49.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:15:49.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:15:49.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:15:49.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:15:49.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:15:49.352 INFO:tasks.rbd_mirror_thrash:kill cluster1.client.mirror.0 2026-02-18T08:15:49.352 INFO:tasks.rbd_mirror.cluster1.client.mirror.0:Sent signal 15 2026-02-18T08:15:49.352 INFO:tasks.rbd_mirror_thrash:waiting for 1.4028504548934084 secs before reviving daemons 2026-02-18T08:15:49.352 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.trial163.stderr:2026-02-18T08:15:49.345+0000 7fda7febd640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.0 (PID: 14900) UID: 1000 2026-02-18T08:15:49.368 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 45 2026-02-18T08:15:49.368 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 45 2026-02-18T08:15:49.368 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:15:49.368 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=181, tag_tid=12, entry_tid=1], [object_number=180, tag_tid=12, entry_tid=0], [object_number=183, tag_tid=11, entry_tid=29711], [object_number=182, tag_tid=11, entry_tid=29710]]], state=connected] 2026-02-18T08:15:49.368 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=181, tag_tid=11, entry_tid=29713], [object_number=180, tag_tid=11, entry_tid=29712], [object_number=183, tag_tid=11, entry_tid=29711], [object_number=182, tag_tid=11, entry_tid=29710]]], state=connected] 2026-02-18T08:15:49.371 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:15:49.373 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=181, tag_tid=12, entry_tid=1' 2026-02-18T08:15:49.374 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:15:49.374 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:15:49.374 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:15:49.374 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:15:49.374 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:15:49.374 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:15:49.374 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:15:49.374 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:15:49.374 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:15:49.374 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:15:49.375 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:15:49.375 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:15:49.376 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:15:49.376 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:15:49.376 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:15:49.413 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 45 2026-02-18T08:15:49.413 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 45 2026-02-18T08:15:49.413 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:15:49.413 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=181, tag_tid=12, entry_tid=1], [object_number=180, tag_tid=12, entry_tid=0], [object_number=183, tag_tid=11, entry_tid=29711], [object_number=182, tag_tid=11, entry_tid=29710]]], state=connected] 2026-02-18T08:15:49.413 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=181, tag_tid=11, entry_tid=29713], [object_number=180, tag_tid=11, entry_tid=29712], [object_number=183, tag_tid=11, entry_tid=29711], [object_number=182, tag_tid=11, entry_tid=29710]]], state=connected] 2026-02-18T08:15:49.417 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:15:49.419 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=181, tag_tid=11, entry_tid=29713' 2026-02-18T08:15:49.419 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=181, tag_tid=12, entry_tid=1' -a 'object_number=181, tag_tid=12, entry_tid=1' = 'object_number=181, tag_tid=11, entry_tid=29713' 2026-02-18T08:15:49.419 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=181, tag_tid=11, entry_tid=29713' '!=' 'object_number=183, tag_tid=11, entry_tid=29571' 2026-02-18T08:15:49.419 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:15:49.419 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=181, tag_tid=11, entry_tid=29713' = 'object_number=183, tag_tid=11, entry_tid=29571' 2026-02-18T08:15:49.419 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=181, tag_tid=11, entry_tid=29713' 2026-02-18T08:15:49.419 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=181, tag_tid=12, entry_tid=1' 2026-02-18T08:15:49.420 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:15:49.420 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:15:49.420 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=12 2026-02-18T08:15:49.421 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=181, tag_tid=11, entry_tid=29713' 2026-02-18T08:15:49.421 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:15:49.421 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:15:49.422 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=11 2026-02-18T08:15:49.423 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=181, tag_tid=12, entry_tid=1' 2026-02-18T08:15:49.423 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:15:49.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:15:49.425 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:15:49.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=181, tag_tid=11, entry_tid=29713' 2026-02-18T08:15:49.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:15:49.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:15:49.428 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=29713 2026-02-18T08:15:49.428 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 12 = 11 -a 1 -le 29713 2026-02-18T08:15:49.428 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:15:49.428 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:15:49.428 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:15:49.629 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:15:49.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:15:49.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:15:49.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:15:49.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:15:49.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:15:49.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:15:49.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:15:49.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:15:49.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:15:49.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:15:49.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:15:49.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:15:49.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:15:49.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:15:49.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:15:49.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:15:49.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:15:49.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:15:49.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:15:49.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:15:49.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:15:49.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:15:49.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:15:49.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:15:49.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:15:49.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:15:49.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:15:49.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:15:49.635 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:15:49.635 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:15:49.635 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:15:49.635 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:15:49.635 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:15:49.635 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:15:49.635 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:15:49.635 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:15:49.635 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:15:49.635 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:15:49.635 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:15:49.635 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:15:49.636 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:15:49.636 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:15:49.636 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:15:49.636 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:15:49.636 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:15:49.636 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:15:49.636 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:15:49.636 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:15:49.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:15:49.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:15:49.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:15:49.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:15:49.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: exception getting command descriptions: [Errno 2] No such file or directory 2026-02-18T08:15:49.701 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:15:49.701 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:15:49.702 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:15:49.702 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:15:49.702 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:15:49.702 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:15:49.702 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:15:49.702 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:15:49.702 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:15:49.702 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:15:49.702 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:15:49.702 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:15:49.702 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:15:49.703 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:15:49.703 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:15:49.703 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:15:49.703 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:15:49.703 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:15:49.703 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:15:49.703 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:15:49.703 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:15:49.703 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:15:49.703 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:15:49.703 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:15:49.703 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:15:49.704 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:15:49.704 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:15:49.704 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:15:49.704 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:15:49.704 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:15:49.704 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:15:49.704 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:15:49.712 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:15:49.713 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:15:49.713 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:15:49.713 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:15:49.762 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:15:49.762 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:15:49.762 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:15:49.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:15:49.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:15:49.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:15:49.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:15:49.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:15:49.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:15:49.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:15:49.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:15:49.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:15:49.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:15:49.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:15:49.766 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:15:49.766 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:15:49.766 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:15:49.766 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:15:49.766 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:15:49.766 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:15:49.766 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:15:49.766 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:15:49.766 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:15:49.766 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:15:49.766 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:15:49.766 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:15:49.767 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:15:49.767 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:15:49.767 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:15:49.767 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:15:49.767 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:15:49.767 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:15:49.767 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:15:49.767 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:15:49.767 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:15:49.776 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:15:49.776 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:15:49.776 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:15:49.777 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:15:49.825 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:15:49.825 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:15:49.825 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:15:49.828 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:15:49.828 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:15:49.828 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:15:49.828 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:15:49.828 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:15:49.828 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:15:49.828 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:15:49.828 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:15:49.828 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:15:49.829 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:15:49.829 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:15:49.829 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:15:49.829 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:15:49.829 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:15:49.829 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:15:49.829 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:15:49.829 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:15:49.829 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:15:49.829 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:15:49.829 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:15:49.829 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:15:49.830 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:15:49.830 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:15:49.830 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:15:49.830 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:15:49.830 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:15:49.830 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:15:49.830 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:15:49.830 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:15:49.830 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:15:49.830 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:15:49.830 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:15:49.839 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:15:49.839 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:15:49.839 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:15:49.839 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:15:49.887 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:15:49.887 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:15:49.887 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:15:49.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:15:49.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1 2026-02-18T08:15:50.756 INFO:tasks.rbd_mirror_thrash:waiting for cluster1.client.mirror.0 2026-02-18T08:15:50.756 INFO:teuthology.orchestra.run:waiting for 600 2026-02-18T08:15:50.756 INFO:tasks.rbd_mirror_thrash:reviving cluster1.client.mirror.0 2026-02-18T08:15:50.756 INFO:tasks.rbd_mirror.cluster1.client.mirror.0:Restarting daemon 2026-02-18T08:15:50.757 DEBUG:teuthology.orchestra.run.trial163:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.0 2026-02-18T08:15:50.759 INFO:tasks.rbd_mirror.cluster1.client.mirror.0:Started 2026-02-18T08:15:50.759 INFO:tasks.rbd_mirror_thrash:waiting for 111.25348607228409 secs before thrashing 2026-02-18T08:15:50.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:15:50.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:15:50.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:15:50.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:15:50.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:15:50.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:15:50.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:15:50.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:15:50.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:15:50.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:15:50.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:15:50.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:15:50.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:15:50.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:15:50.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:15:50.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:15:50.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:15:50.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:15:50.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:15:50.894 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:15:50.894 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:15:50.894 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:15:50.894 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:15:50.894 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:15:50.894 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:15:50.894 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:15:50.894 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:15:50.894 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:15:50.894 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:15:50.894 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:15:50.894 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:15:50.895 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:15:50.895 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:15:50.895 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:15:50.904 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:15:50.904 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:15:50.904 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:15:50.904 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:15:50.954 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:15:50.954 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:15:50.954 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:15:50.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:15:50.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:15:50.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:15:50.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:15:50.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:15:50.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:15:50.958 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:15:50.958 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:15:50.958 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:15:50.958 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:15:50.958 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:15:50.958 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:15:50.958 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:15:50.958 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:15:50.958 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:15:50.958 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:15:50.958 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:15:50.958 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:15:50.959 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:15:50.959 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:15:50.959 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:15:50.959 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:15:50.959 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:15:50.959 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:15:50.959 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:15:50.959 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:15:50.959 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:15:50.959 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:15:50.959 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:15:50.959 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:15:50.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:15:50.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:15:50.968 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:15:50.969 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:15:50.969 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:15:50.969 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:15:51.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:15:51.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:15:51.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:15:51.027 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:15:51.027 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:15:51.028 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:15:51.028 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:15:51.028 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:15:51.028 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:15:51.028 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:15:51.028 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:15:51.028 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:15:51.028 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:15:51.028 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:15:51.028 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:15:51.028 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:15:51.028 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:15:51.029 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:15:51.029 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:15:51.029 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:15:51.029 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:15:51.029 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:15:51.029 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:15:51.029 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:15:51.029 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:15:51.029 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:15:51.029 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:15:51.029 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:15:51.029 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:15:51.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:15:51.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:15:51.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:15:51.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:15:51.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:15:51.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:15:51.039 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:15:51.039 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:15:51.039 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:15:51.039 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:15:51.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:15:51.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:15:51.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:15:51.092 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:15:51.092 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:15:51.092 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:15:51.092 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:15:51.092 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:15:51.092 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:15:51.093 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:15:51.093 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:15:51.093 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:15:51.093 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:15:51.093 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:15:51.093 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:15:51.093 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:15:51.093 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:15:51.093 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:15:51.093 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:15:51.093 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:15:51.093 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:15:51.094 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:15:51.094 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:15:51.094 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:15:51.094 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:15:51.094 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:15:51.094 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:15:51.094 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:15:51.094 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:15:51.094 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:15:51.094 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:15:51.094 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:15:51.094 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:15:51.095 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:15:51.095 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:15:51.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:15:51.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:15:51.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:15:51.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:15:51.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:15:51.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:15:51.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:15:51.155 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:15:51.155 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 2 2026-02-18T08:15:53.155 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:15:53.156 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:15:53.157 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:15:53.157 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:15:53.157 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:15:53.157 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:15:53.157 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:15:53.157 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:15:53.157 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:15:53.157 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:15:53.157 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:15:53.157 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:15:53.157 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:15:53.157 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:15:53.157 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:15:53.157 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:15:53.157 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:15:53.157 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:15:53.158 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:15:53.158 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:15:53.158 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:15:53.158 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:15:53.158 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:15:53.158 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:15:53.158 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:15:53.158 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:15:53.158 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:15:53.158 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:15:53.158 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:15:53.158 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:15:53.158 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:15:53.158 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:15:53.158 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:15:53.158 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:15:53.170 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:15:53.170 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:15:53.170 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:15:53.170 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:15:53.221 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:15:53.221 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:15:53.221 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:15:53.222 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:15:53.222 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:15:53.222 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:15:53.222 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:15:53.222 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:15:53.222 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:15:53.223 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:15:53.223 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:15:53.223 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:15:53.223 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:15:53.223 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:15:53.223 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:15:53.223 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:15:53.223 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:15:53.223 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:15:53.223 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:15:53.223 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:15:53.223 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:15:53.223 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:15:53.223 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:15:53.223 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:15:53.223 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:15:53.223 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:15:53.223 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:15:53.223 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:15:53.223 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:15:53.223 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:15:53.223 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:15:53.223 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:15:53.223 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:15:53.223 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:15:53.223 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:15:53.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:15:53.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:15:53.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:15:53.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:15:53.292 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:15:53.292 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:15:53.293 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:15:53.294 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:15:53.294 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:15:53.294 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:15:53.294 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:15:53.294 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:15:53.294 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:15:53.294 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:15:53.294 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:15:53.294 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:15:53.294 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:15:53.294 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:15:53.294 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:15:53.294 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:15:53.294 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:15:53.294 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:15:53.294 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:15:53.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:15:53.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:15:53.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:15:53.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:15:53.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:15:53.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:15:53.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:15:53.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:15:53.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:15:53.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:15:53.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:15:53.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:15:53.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:15:53.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:15:53.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:15:53.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:15:53.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:15:53.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:15:53.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:15:53.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:15:53.367 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:15:53.367 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:15:53.367 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:15:53.368 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:15:53.368 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:15:53.368 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:15:53.368 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:15:53.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:15:53.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:15:53.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:15:53.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:15:53.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:15:53.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:15:53.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:15:53.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:15:53.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:15:53.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:15:53.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:15:53.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:15:53.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:15:53.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:15:53.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:15:53.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:15:53.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:15:53.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:15:53.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:15:53.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:15:53.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:15:53.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:15:53.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:15:53.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:15:53.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:15:53.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:15:53.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:15:53.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:15:53.380 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:15:53.381 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:15:53.381 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:15:53.381 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:15:53.437 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:15:53.437 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:15:53.437 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:15:53.437 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:15:53.437 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:15:53.437 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:15:53.437 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:15:53.437 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:15:53.437 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:15:53.437 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:15:53.437 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:15:53.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:15:53.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:15:53.439 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:15:53.440 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:15:53.440 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:15:53.476 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 45 2026-02-18T08:15:53.476 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 45 2026-02-18T08:15:53.476 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:15:53.477 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=181, tag_tid=12, entry_tid=1], [object_number=180, tag_tid=12, entry_tid=0], [object_number=183, tag_tid=11, entry_tid=29711], [object_number=182, tag_tid=11, entry_tid=29710]]], state=connected] 2026-02-18T08:15:53.477 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=181, tag_tid=11, entry_tid=29713], [object_number=180, tag_tid=11, entry_tid=29712], [object_number=183, tag_tid=11, entry_tid=29711], [object_number=182, tag_tid=11, entry_tid=29710]]], state=connected] 2026-02-18T08:15:53.480 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:15:53.482 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=181, tag_tid=12, entry_tid=1' 2026-02-18T08:15:53.483 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:15:53.483 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:15:53.483 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:15:53.483 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:15:53.483 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:15:53.483 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:15:53.483 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:15:53.483 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:15:53.483 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:15:53.483 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:15:53.483 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:15:53.483 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:15:53.485 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:15:53.485 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:15:53.486 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:15:53.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 45 2026-02-18T08:15:53.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 45 2026-02-18T08:15:53.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:15:53.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=181, tag_tid=12, entry_tid=1], [object_number=180, tag_tid=12, entry_tid=0], [object_number=183, tag_tid=11, entry_tid=29711], [object_number=182, tag_tid=11, entry_tid=29710]]], state=connected] 2026-02-18T08:15:53.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=181, tag_tid=11, entry_tid=29713], [object_number=180, tag_tid=11, entry_tid=29712], [object_number=183, tag_tid=11, entry_tid=29711], [object_number=182, tag_tid=11, entry_tid=29710]]], state=connected] 2026-02-18T08:15:53.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:15:53.523 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=181, tag_tid=11, entry_tid=29713' 2026-02-18T08:15:53.523 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=181, tag_tid=12, entry_tid=1' -a 'object_number=181, tag_tid=12, entry_tid=1' = 'object_number=181, tag_tid=11, entry_tid=29713' 2026-02-18T08:15:53.523 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=181, tag_tid=11, entry_tid=29713' '!=' 'object_number=181, tag_tid=11, entry_tid=29713' 2026-02-18T08:15:53.523 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:15:53.523 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.4 2026-02-18T08:15:53.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:15:53.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:15:53.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:15:53.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:15:53.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:15:53.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:15:53.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:15:53.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:15:53.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:15:53.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:15:53.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:15:53.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:15:53.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:15:53.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:15:53.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:15:53.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:15:53.929 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:15:53.929 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:15:53.929 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:15:53.929 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:15:53.929 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:15:53.930 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:15:53.930 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:15:53.930 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:15:53.930 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:15:53.930 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:15:53.930 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:15:53.930 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:15:53.930 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:15:53.930 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:15:53.930 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:15:53.930 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:15:53.930 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:15:53.931 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:15:53.931 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:15:53.931 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:15:53.931 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:15:53.931 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:15:53.931 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:15:53.931 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:15:53.931 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:15:53.931 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:15:53.931 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:15:53.931 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:15:53.931 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:15:53.932 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:15:53.932 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:15:53.932 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:15:53.932 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:15:53.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:15:53.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:15:53.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:15:53.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:15:54.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:15:54.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:15:54.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:15:54.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:15:54.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:15:54.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:15:54.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:15:54.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:15:54.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:15:54.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:15:54.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:15:54.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:15:54.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:15:54.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:15:54.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:15:54.003 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:15:54.003 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:15:54.003 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:15:54.003 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:15:54.003 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:15:54.003 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:15:54.003 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:15:54.003 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:15:54.003 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:15:54.003 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:15:54.003 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:15:54.003 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:15:54.003 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:15:54.003 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:15:54.003 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:15:54.003 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:15:54.003 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:15:54.003 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:15:54.003 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:15:54.003 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:15:54.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:15:54.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:15:54.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:15:54.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:15:54.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:15:54.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:15:54.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:15:54.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:15:54.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:15:54.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:15:54.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:15:54.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:15:54.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:15:54.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:15:54.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:15:54.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:15:54.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:15:54.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:15:54.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:15:54.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:15:54.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:15:54.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:15:54.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:15:54.074 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:15:54.074 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:15:54.074 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:15:54.074 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:15:54.074 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:15:54.074 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:15:54.074 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:15:54.074 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:15:54.074 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:15:54.074 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:15:54.074 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:15:54.074 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:15:54.074 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:15:54.074 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:15:54.074 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:15:54.074 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:15:54.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:15:54.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:15:54.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:15:54.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:15:54.141 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:15:54.141 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:15:54.141 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:15:54.142 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:15:54.142 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:15:54.142 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:15:54.142 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:15:54.142 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:15:54.143 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:15:54.143 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:15:54.143 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:15:54.143 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:15:54.143 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:15:54.143 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:15:54.143 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:15:54.143 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:15:54.143 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:15:54.143 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:15:54.143 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:15:54.143 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:15:54.143 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:15:54.143 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:15:54.143 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:15:54.143 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:15:54.143 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:15:54.143 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:15:54.143 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:15:54.143 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:15:54.143 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:15:54.143 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:15:54.144 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:15:54.144 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:15:54.144 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:15:54.144 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:15:54.144 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:15:54.155 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:15:54.155 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:15:54.155 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:15:54.155 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:15:54.223 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:15:54.225 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:15:54.225 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:15:54.225 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:15:54.225 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:15:54.225 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:15:54.225 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:15:54.225 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:15:54.225 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:15:54.225 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:15:54.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:15:54.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:15:54.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:15:54.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:15:54.227 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:15:54.227 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:15:54.255 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 45 2026-02-18T08:15:54.255 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 45 2026-02-18T08:15:54.255 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:15:54.255 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=181, tag_tid=12, entry_tid=1], [object_number=180, tag_tid=12, entry_tid=0], [object_number=183, tag_tid=11, entry_tid=29711], [object_number=182, tag_tid=11, entry_tid=29710]]], state=connected] 2026-02-18T08:15:54.255 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=181, tag_tid=11, entry_tid=29713], [object_number=180, tag_tid=11, entry_tid=29712], [object_number=183, tag_tid=11, entry_tid=29711], [object_number=182, tag_tid=11, entry_tid=29710]]], state=connected] 2026-02-18T08:15:54.257 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:15:54.259 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=181, tag_tid=12, entry_tid=1' 2026-02-18T08:15:54.260 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:15:54.260 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:15:54.261 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:15:54.261 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:15:54.261 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:15:54.261 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:15:54.261 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:15:54.261 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:15:54.261 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:15:54.261 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:15:54.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:15:54.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:15:54.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:15:54.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:15:54.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:15:54.290 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 45 2026-02-18T08:15:54.290 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 45 2026-02-18T08:15:54.290 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:15:54.290 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=181, tag_tid=12, entry_tid=1], [object_number=180, tag_tid=12, entry_tid=0], [object_number=183, tag_tid=11, entry_tid=29711], [object_number=182, tag_tid=11, entry_tid=29710]]], state=connected] 2026-02-18T08:15:54.290 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=181, tag_tid=11, entry_tid=29713], [object_number=180, tag_tid=11, entry_tid=29712], [object_number=183, tag_tid=11, entry_tid=29711], [object_number=182, tag_tid=11, entry_tid=29710]]], state=connected] 2026-02-18T08:15:54.293 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:15:54.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=181, tag_tid=11, entry_tid=29713' 2026-02-18T08:15:54.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=181, tag_tid=12, entry_tid=1' -a 'object_number=181, tag_tid=12, entry_tid=1' = 'object_number=181, tag_tid=11, entry_tid=29713' 2026-02-18T08:15:54.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=181, tag_tid=11, entry_tid=29713' '!=' 'object_number=181, tag_tid=11, entry_tid=29713' 2026-02-18T08:15:54.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:15:54.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.8 2026-02-18T08:15:55.097 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:15:55.097 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:15:55.097 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:15:55.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:15:55.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:15:55.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:15:55.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:15:55.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:15:55.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:15:55.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:15:55.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:15:55.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:15:55.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:15:55.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:15:55.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:15:55.101 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:15:55.102 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:15:55.102 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:15:55.102 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:15:55.102 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:15:55.102 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:15:55.102 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:15:55.102 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:15:55.102 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:15:55.102 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:15:55.102 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:15:55.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:15:55.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:15:55.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:15:55.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:15:55.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:15:55.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:15:55.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:15:55.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:15:55.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:15:55.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:15:55.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:15:55.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:15:55.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:15:55.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:15:55.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:15:55.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:15:55.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:15:55.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:15:55.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:15:55.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:15:55.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:15:55.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:15:55.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:15:55.113 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:15:55.113 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:15:55.113 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:15:55.113 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:15:55.170 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:15:55.171 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:15:55.171 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:15:55.174 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:15:55.174 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:15:55.174 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:15:55.174 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:15:55.174 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:15:55.174 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:15:55.174 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:15:55.174 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:15:55.175 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:15:55.175 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:15:55.175 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:15:55.175 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:15:55.175 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:15:55.175 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:15:55.175 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:15:55.175 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:15:55.175 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:15:55.175 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:15:55.175 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:15:55.175 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:15:55.176 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:15:55.176 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:15:55.176 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:15:55.176 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:15:55.176 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:15:55.176 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:15:55.176 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:15:55.176 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:15:55.176 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:15:55.176 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:15:55.176 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:15:55.176 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:15:55.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:15:55.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:15:55.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:15:55.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:15:55.234 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:15:55.234 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:15:55.234 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:15:55.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:15:55.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:15:55.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:15:55.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:15:55.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:15:55.238 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:15:55.238 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:15:55.238 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:15:55.238 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:15:55.238 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:15:55.238 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:15:55.238 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:15:55.238 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:15:55.238 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:15:55.238 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:15:55.238 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:15:55.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:15:55.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:15:55.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:15:55.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:15:55.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:15:55.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:15:55.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:15:55.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:15:55.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:15:55.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:15:55.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:15:55.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:15:55.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:15:55.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:15:55.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:15:55.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:15:55.247 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:15:55.248 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:15:55.248 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:15:55.248 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:15:55.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:15:55.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:15:55.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:15:55.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:15:55.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:15:55.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:15:55.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:15:55.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:15:55.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:15:55.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:15:55.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:15:55.310 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:15:55.310 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:15:55.310 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:15:55.310 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:15:55.310 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:15:55.310 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:15:55.310 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:15:55.310 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:15:55.310 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:15:55.310 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:15:55.310 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:15:55.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:15:55.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:15:55.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:15:55.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:15:55.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:15:55.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:15:55.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:15:55.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:15:55.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:15:55.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:15:55.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:15:55.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:15:55.312 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:15:55.319 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:15:55.319 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:15:55.320 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:15:55.320 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:15:55.377 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:15:55.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:15:55.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:15:55.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:15:55.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:15:55.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:15:55.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:15:55.379 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:15:55.379 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:15:55.379 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:15:55.379 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:15:55.379 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:15:55.379 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:15:55.381 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:15:55.381 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:15:55.381 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:15:55.408 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 45 2026-02-18T08:15:55.408 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 45 2026-02-18T08:15:55.408 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:15:55.408 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=181, tag_tid=12, entry_tid=1], [object_number=180, tag_tid=12, entry_tid=0], [object_number=183, tag_tid=11, entry_tid=29711], [object_number=182, tag_tid=11, entry_tid=29710]]], state=connected] 2026-02-18T08:15:55.408 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=181, tag_tid=11, entry_tid=29713], [object_number=180, tag_tid=11, entry_tid=29712], [object_number=183, tag_tid=11, entry_tid=29711], [object_number=182, tag_tid=11, entry_tid=29710]]], state=connected] 2026-02-18T08:15:55.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:15:55.412 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=181, tag_tid=12, entry_tid=1' 2026-02-18T08:15:55.413 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:15:55.413 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:15:55.413 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:15:55.414 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:15:55.414 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:15:55.414 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:15:55.414 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:15:55.414 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:15:55.414 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:15:55.414 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:15:55.415 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:15:55.415 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:15:55.415 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:15:55.416 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:15:55.416 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:15:55.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 45 2026-02-18T08:15:55.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 45 2026-02-18T08:15:55.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:15:55.443 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=181, tag_tid=12, entry_tid=1], [object_number=180, tag_tid=12, entry_tid=0], [object_number=183, tag_tid=11, entry_tid=29711], [object_number=182, tag_tid=11, entry_tid=29710]]], state=connected] 2026-02-18T08:15:55.443 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=181, tag_tid=11, entry_tid=29713], [object_number=180, tag_tid=11, entry_tid=29712], [object_number=183, tag_tid=11, entry_tid=29711], [object_number=182, tag_tid=11, entry_tid=29710]]], state=connected] 2026-02-18T08:15:55.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:15:55.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=181, tag_tid=11, entry_tid=29713' 2026-02-18T08:15:55.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=181, tag_tid=12, entry_tid=1' -a 'object_number=181, tag_tid=12, entry_tid=1' = 'object_number=181, tag_tid=11, entry_tid=29713' 2026-02-18T08:15:55.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=181, tag_tid=11, entry_tid=29713' '!=' 'object_number=181, tag_tid=11, entry_tid=29713' 2026-02-18T08:15:55.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:15:55.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1.6 2026-02-18T08:15:57.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:15:57.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:15:57.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:15:57.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:15:57.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:15:57.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:15:57.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:15:57.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:15:57.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:15:57.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:15:57.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:15:57.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:15:57.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:15:57.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:15:57.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:15:57.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:15:57.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:15:57.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:15:57.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:15:57.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:15:57.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:15:57.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:15:57.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:15:57.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:15:57.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:15:57.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:15:57.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:15:57.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:15:57.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:15:57.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:15:57.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:15:57.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:15:57.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:15:57.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:15:57.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:15:57.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:15:57.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:15:57.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:15:57.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:15:57.055 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:15:57.055 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:15:57.055 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:15:57.055 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:15:57.055 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:15:57.055 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:15:57.055 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:15:57.055 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:15:57.055 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:15:57.055 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:15:57.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:15:57.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:15:57.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:15:57.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:15:57.113 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:15:57.114 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:15:57.114 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:15:57.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:15:57.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:15:57.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:15:57.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:15:57.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:15:57.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:15:57.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:15:57.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:15:57.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:15:57.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:15:57.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:15:57.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:15:57.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:15:57.118 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:15:57.118 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:15:57.118 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:15:57.118 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:15:57.118 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:15:57.118 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:15:57.118 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:15:57.118 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:15:57.118 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:15:57.118 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:15:57.118 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:15:57.118 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:15:57.119 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:15:57.119 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:15:57.119 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:15:57.119 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:15:57.119 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:15:57.119 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:15:57.119 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:15:57.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:15:57.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:15:57.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:15:57.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:15:57.177 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:15:57.177 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:15:57.177 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:15:57.179 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:15:57.179 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:15:57.180 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:15:57.180 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:15:57.180 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:15:57.180 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:15:57.180 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:15:57.180 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:15:57.180 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:15:57.180 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:15:57.180 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:15:57.180 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:15:57.180 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:15:57.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:15:57.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:15:57.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:15:57.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:15:57.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:15:57.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:15:57.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:15:57.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:15:57.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:15:57.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:15:57.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:15:57.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:15:57.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:15:57.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:15:57.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:15:57.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:15:57.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:15:57.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:15:57.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:15:57.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:15:57.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:15:57.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:15:57.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:15:57.248 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:15:57.248 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:15:57.249 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:15:57.251 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:15:57.251 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:15:57.251 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:15:57.252 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:15:57.252 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:15:57.252 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:15:57.252 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:15:57.252 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:15:57.252 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:15:57.252 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:15:57.252 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:15:57.252 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:15:57.252 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:15:57.252 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:15:57.253 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:15:57.253 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:15:57.253 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:15:57.253 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:15:57.253 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:15:57.253 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:15:57.253 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:15:57.253 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:15:57.253 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:15:57.253 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:15:57.253 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:15:57.253 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:15:57.254 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:15:57.254 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:15:57.254 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:15:57.254 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:15:57.254 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:15:57.254 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:15:57.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:15:57.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:15:57.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:15:57.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:15:57.318 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:15:57.319 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:15:57.319 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:15:57.319 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:15:57.319 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:15:57.320 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:15:57.320 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:15:57.320 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:15:57.320 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:15:57.320 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:15:57.320 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:15:57.320 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:15:57.320 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:15:57.321 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:15:57.321 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:15:57.321 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:15:57.357 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 45 2026-02-18T08:15:57.357 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 45 2026-02-18T08:15:57.357 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:15:57.357 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=181, tag_tid=12, entry_tid=1], [object_number=180, tag_tid=12, entry_tid=0], [object_number=183, tag_tid=11, entry_tid=29711], [object_number=182, tag_tid=11, entry_tid=29710]]], state=connected] 2026-02-18T08:15:57.357 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=181, tag_tid=11, entry_tid=29713], [object_number=180, tag_tid=11, entry_tid=29712], [object_number=183, tag_tid=11, entry_tid=29711], [object_number=182, tag_tid=11, entry_tid=29710]]], state=connected] 2026-02-18T08:15:57.361 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:15:57.363 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=181, tag_tid=12, entry_tid=1' 2026-02-18T08:15:57.364 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:15:57.364 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:15:57.364 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:15:57.364 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:15:57.364 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:15:57.364 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:15:57.364 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:15:57.364 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:15:57.364 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:15:57.364 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:15:57.365 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:15:57.365 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:15:57.365 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:15:57.366 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:15:57.366 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:15:57.403 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 45 2026-02-18T08:15:57.403 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 45 2026-02-18T08:15:57.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:15:57.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=181, tag_tid=12, entry_tid=1], [object_number=180, tag_tid=12, entry_tid=0], [object_number=183, tag_tid=11, entry_tid=29711], [object_number=182, tag_tid=11, entry_tid=29710]]], state=connected] 2026-02-18T08:15:57.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=181, tag_tid=11, entry_tid=29713], [object_number=180, tag_tid=11, entry_tid=29712], [object_number=183, tag_tid=11, entry_tid=29711], [object_number=182, tag_tid=11, entry_tid=29710]]], state=connected] 2026-02-18T08:15:57.407 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:15:57.409 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=181, tag_tid=11, entry_tid=29713' 2026-02-18T08:15:57.409 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=181, tag_tid=12, entry_tid=1' -a 'object_number=181, tag_tid=12, entry_tid=1' = 'object_number=181, tag_tid=11, entry_tid=29713' 2026-02-18T08:15:57.409 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=181, tag_tid=11, entry_tid=29713' '!=' 'object_number=181, tag_tid=11, entry_tid=29713' 2026-02-18T08:15:57.409 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:15:57.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 2 2026-02-18T08:15:59.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:15:59.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:15:59.412 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:15:59.412 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:15:59.412 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:15:59.412 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:15:59.412 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:15:59.412 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:15:59.412 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:15:59.412 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:15:59.412 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:15:59.412 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:15:59.412 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:15:59.413 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:15:59.413 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:15:59.413 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:15:59.415 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:15:59.416 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:15:59.416 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:15:59.416 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:15:59.416 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:15:59.416 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:15:59.416 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:15:59.416 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:15:59.416 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:15:59.416 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:15:59.416 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:15:59.416 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:15:59.417 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:15:59.417 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:15:59.417 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:15:59.417 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:15:59.417 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:15:59.417 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:15:59.417 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:15:59.417 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:15:59.417 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:15:59.417 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:15:59.417 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:15:59.417 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:15:59.417 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:15:59.418 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:15:59.418 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:15:59.418 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:15:59.418 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:15:59.418 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:15:59.418 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:15:59.418 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:15:59.418 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:15:59.427 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:15:59.428 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:15:59.428 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:15:59.428 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:15:59.486 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:15:59.486 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:15:59.486 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:15:59.489 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:15:59.489 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:15:59.489 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:15:59.489 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:15:59.489 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:15:59.489 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:15:59.489 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:15:59.489 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:15:59.489 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:15:59.490 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:15:59.490 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:15:59.490 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:15:59.490 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:15:59.490 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:15:59.490 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:15:59.490 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:15:59.490 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:15:59.490 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:15:59.490 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:15:59.490 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:15:59.490 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:15:59.491 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:15:59.491 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:15:59.491 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:15:59.491 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:15:59.491 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:15:59.491 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:15:59.491 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:15:59.491 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:15:59.491 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:15:59.491 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:15:59.491 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:15:59.500 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:15:59.501 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:15:59.501 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:15:59.501 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:15:59.558 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:15:59.558 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:15:59.558 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:15:59.561 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:15:59.561 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:15:59.561 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:15:59.562 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:15:59.562 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:15:59.562 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:15:59.562 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:15:59.562 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:15:59.562 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:15:59.562 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:15:59.562 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:15:59.562 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:15:59.562 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:15:59.562 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:15:59.562 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:15:59.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:15:59.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:15:59.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:15:59.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:15:59.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:15:59.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:15:59.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:15:59.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:15:59.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:15:59.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:15:59.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:15:59.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:15:59.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:15:59.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:15:59.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:15:59.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:15:59.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:15:59.571 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:15:59.572 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:15:59.572 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:15:59.572 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:15:59.629 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:15:59.629 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:15:59.629 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:15:59.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:15:59.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:15:59.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:15:59.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:15:59.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:15:59.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:15:59.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:15:59.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:15:59.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:15:59.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:15:59.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:15:59.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:15:59.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:15:59.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:15:59.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:15:59.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:15:59.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:15:59.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:15:59.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:15:59.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:15:59.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:15:59.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:15:59.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:15:59.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:15:59.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:15:59.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:15:59.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:15:59.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:15:59.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:15:59.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:15:59.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:15:59.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:15:59.643 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:15:59.643 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:15:59.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:15:59.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:15:59.708 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:15:59.709 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:15:59.709 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:15:59.709 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:15:59.709 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:15:59.709 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:15:59.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:15:59.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:15:59.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:15:59.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:15:59.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:15:59.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:15:59.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:15:59.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:15:59.711 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:15:59.711 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:15:59.740 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 45 2026-02-18T08:15:59.740 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 45 2026-02-18T08:15:59.740 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:15:59.740 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=181, tag_tid=12, entry_tid=1], [object_number=180, tag_tid=12, entry_tid=0], [object_number=183, tag_tid=11, entry_tid=29711], [object_number=182, tag_tid=11, entry_tid=29710]]], state=connected] 2026-02-18T08:15:59.740 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=181, tag_tid=11, entry_tid=29713], [object_number=180, tag_tid=11, entry_tid=29712], [object_number=183, tag_tid=11, entry_tid=29711], [object_number=182, tag_tid=11, entry_tid=29710]]], state=connected] 2026-02-18T08:15:59.743 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:15:59.744 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=181, tag_tid=12, entry_tid=1' 2026-02-18T08:15:59.744 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:15:59.745 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:15:59.745 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:15:59.745 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:15:59.745 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:15:59.745 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:15:59.745 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:15:59.745 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:15:59.745 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:15:59.745 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:15:59.745 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:15:59.745 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:15:59.747 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:15:59.747 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:15:59.747 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:15:59.781 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 45 2026-02-18T08:15:59.781 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 45 2026-02-18T08:15:59.781 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:15:59.781 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=181, tag_tid=12, entry_tid=1], [object_number=180, tag_tid=12, entry_tid=0], [object_number=183, tag_tid=11, entry_tid=29711], [object_number=182, tag_tid=11, entry_tid=29710]]], state=connected] 2026-02-18T08:15:59.781 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=181, tag_tid=11, entry_tid=29713], [object_number=180, tag_tid=11, entry_tid=29712], [object_number=183, tag_tid=11, entry_tid=29711], [object_number=182, tag_tid=11, entry_tid=29710]]], state=connected] 2026-02-18T08:15:59.784 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:15:59.786 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=181, tag_tid=11, entry_tid=29713' 2026-02-18T08:15:59.786 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=181, tag_tid=12, entry_tid=1' -a 'object_number=181, tag_tid=12, entry_tid=1' = 'object_number=181, tag_tid=11, entry_tid=29713' 2026-02-18T08:15:59.786 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=181, tag_tid=11, entry_tid=29713' '!=' 'object_number=181, tag_tid=11, entry_tid=29713' 2026-02-18T08:15:59.786 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:15:59.786 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 2 2026-02-18T08:16:01.788 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:16:01.788 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:16:01.788 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:16:01.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:16:01.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:16:01.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:16:01.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:16:01.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:16:01.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:16:01.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:16:01.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:16:01.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:16:01.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:16:01.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:16:01.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:16:01.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:16:01.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:16:01.793 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:16:01.793 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:16:01.793 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:16:01.793 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:16:01.793 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:16:01.793 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:16:01.793 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:16:01.793 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:16:01.793 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:16:01.793 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:16:01.793 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:16:01.793 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:16:01.794 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:16:01.794 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:16:01.794 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:16:01.794 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:16:01.794 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:16:01.794 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:16:01.794 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:16:01.794 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:16:01.794 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:16:01.794 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:16:01.794 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:16:01.794 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:16:01.795 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:16:01.795 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:16:01.795 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:16:01.795 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:16:01.795 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:16:01.795 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:16:01.795 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:16:01.795 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:16:01.802 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:16:01.803 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:16:01.803 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:16:01.803 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:16:01.850 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:16:01.850 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:16:01.851 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:16:01.853 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:16:01.853 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:16:01.853 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:16:01.853 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:16:01.853 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:16:01.854 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:16:01.854 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:16:01.854 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:16:01.854 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:16:01.854 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:16:01.854 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:16:01.854 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:16:01.854 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:16:01.854 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:16:01.854 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:16:01.854 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:16:01.854 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:16:01.855 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:16:01.855 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:16:01.855 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:16:01.855 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:16:01.855 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:16:01.855 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:16:01.855 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:16:01.855 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:16:01.855 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:16:01.855 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:16:01.855 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:16:01.855 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:16:01.856 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:16:01.856 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:16:01.856 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:16:01.864 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:16:01.865 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:16:01.865 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:16:01.865 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:16:01.922 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:16:01.922 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:16:01.922 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:16:01.924 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:16:01.924 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:16:01.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:16:01.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:16:01.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:16:01.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:16:01.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:16:01.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:16:01.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:16:01.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:16:01.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:16:01.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:16:01.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:16:01.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:16:01.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:16:01.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:16:01.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:16:01.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:16:01.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:16:01.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:16:01.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:16:01.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:16:01.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:16:01.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:16:01.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:16:01.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:16:01.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:16:01.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:16:01.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:16:01.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:16:01.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:16:01.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:16:01.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:16:01.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:16:01.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:16:01.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:16:01.993 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:16:01.993 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:16:01.994 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:16:01.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:16:01.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:16:01.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:16:01.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:16:01.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:16:01.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:16:01.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:16:01.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:16:01.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:16:01.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:16:01.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:16:01.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:16:01.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:16:01.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:16:01.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:16:01.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:16:01.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:16:01.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:16:01.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:16:01.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:16:01.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:16:01.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:16:01.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:16:01.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:16:01.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:16:01.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:16:01.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:16:01.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:16:01.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:16:01.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:16:01.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:16:01.999 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:16:02.008 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:16:02.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:16:02.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:16:02.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:16:02.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:16:02.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:16:02.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:16:02.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:16:02.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:16:02.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:16:02.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:16:02.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:16:02.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:16:02.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:16:02.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:16:02.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:16:02.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:16:02.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:16:02.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:16:02.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:16:02.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 45 2026-02-18T08:16:02.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 45 2026-02-18T08:16:02.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:16:02.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=181, tag_tid=12, entry_tid=1], [object_number=180, tag_tid=12, entry_tid=0], [object_number=183, tag_tid=11, entry_tid=29711], [object_number=182, tag_tid=11, entry_tid=29710]]], state=connected] 2026-02-18T08:16:02.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=181, tag_tid=11, entry_tid=29713], [object_number=180, tag_tid=11, entry_tid=29712], [object_number=183, tag_tid=11, entry_tid=29711], [object_number=182, tag_tid=11, entry_tid=29710]]], state=connected] 2026-02-18T08:16:02.113 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:16:02.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=181, tag_tid=12, entry_tid=1' 2026-02-18T08:16:02.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:16:02.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:16:02.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:16:02.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:16:02.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:16:02.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:16:02.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:16:02.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:16:02.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:16:02.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:16:02.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:16:02.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:16:02.118 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:16:02.118 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:16:02.118 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:16:02.155 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 45 2026-02-18T08:16:02.156 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 45 2026-02-18T08:16:02.156 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:16:02.156 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=181, tag_tid=12, entry_tid=1], [object_number=180, tag_tid=12, entry_tid=0], [object_number=183, tag_tid=11, entry_tid=29711], [object_number=182, tag_tid=11, entry_tid=29710]]], state=connected] 2026-02-18T08:16:02.156 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=181, tag_tid=11, entry_tid=29713], [object_number=180, tag_tid=11, entry_tid=29712], [object_number=183, tag_tid=11, entry_tid=29711], [object_number=182, tag_tid=11, entry_tid=29710]]], state=connected] 2026-02-18T08:16:02.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:16:02.161 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=181, tag_tid=11, entry_tid=29713' 2026-02-18T08:16:02.161 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=181, tag_tid=12, entry_tid=1' -a 'object_number=181, tag_tid=12, entry_tid=1' = 'object_number=181, tag_tid=11, entry_tid=29713' 2026-02-18T08:16:02.161 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=181, tag_tid=11, entry_tid=29713' '!=' 'object_number=181, tag_tid=11, entry_tid=29713' 2026-02-18T08:16:02.162 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:16:02.162 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 4 2026-02-18T08:16:06.164 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:16:06.164 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:16:06.165 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:16:06.165 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:16:06.165 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:16:06.165 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:16:06.165 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:16:06.165 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:16:06.166 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:16:06.166 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:16:06.166 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:16:06.166 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:16:06.166 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:16:06.166 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:16:06.167 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:16:06.167 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:16:06.167 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:16:06.171 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:16:06.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:16:06.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:16:06.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:16:06.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:16:06.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:16:06.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:16:06.173 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:16:06.173 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:16:06.173 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:16:06.173 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:16:06.173 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:16:06.173 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:16:06.173 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:16:06.174 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:16:06.174 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:16:06.174 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:16:06.174 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:16:06.174 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:16:06.174 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:16:06.174 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:16:06.175 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:16:06.175 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:16:06.175 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:16:06.175 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:16:06.175 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:16:06.175 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:16:06.175 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:16:06.176 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:16:06.176 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:16:06.176 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:16:06.176 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:16:06.178 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:16:06.179 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:16:06.179 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:16:06.179 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:16:06.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:16:06.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:16:06.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:16:06.238 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:16:06.238 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:16:06.238 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:16:06.238 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:16:06.238 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:16:06.238 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:16:06.238 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:16:06.238 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:16:06.238 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:16:06.238 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:16:06.238 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:16:06.238 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:16:06.238 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:16:06.238 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:16:06.238 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:16:06.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:16:06.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:16:06.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:16:06.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:16:06.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:16:06.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:16:06.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:16:06.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:16:06.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:16:06.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:16:06.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:16:06.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:16:06.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:16:06.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:16:06.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:16:06.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:16:06.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:16:06.248 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:16:06.248 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:16:06.249 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:16:06.249 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:16:06.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:16:06.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:16:06.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:16:06.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:16:06.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:16:06.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:16:06.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:16:06.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:16:06.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:16:06.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:16:06.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:16:06.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:16:06.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:16:06.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:16:06.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:16:06.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:16:06.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:16:06.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:16:06.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:16:06.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:16:06.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:16:06.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:16:06.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:16:06.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:16:06.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:16:06.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:16:06.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:16:06.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:16:06.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:16:06.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:16:06.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:16:06.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:16:06.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:16:06.310 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:16:06.310 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:16:06.319 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:16:06.319 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:16:06.319 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:16:06.319 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:16:06.377 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:16:06.377 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:16:06.377 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:16:06.381 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:16:06.381 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:16:06.381 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:16:06.381 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:16:06.381 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:16:06.381 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:16:06.381 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:16:06.381 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:16:06.381 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:16:06.381 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:16:06.382 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:16:06.382 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:16:06.382 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:16:06.382 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:16:06.382 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:16:06.382 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:16:06.382 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:16:06.382 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:16:06.382 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:16:06.382 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:16:06.382 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:16:06.383 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:16:06.383 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:16:06.383 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:16:06.383 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:16:06.383 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:16:06.383 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:16:06.383 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:16:06.383 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:16:06.383 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:16:06.383 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:16:06.383 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:16:06.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:16:06.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:16:06.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:16:06.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:16:06.452 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:16:06.452 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:16:06.453 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:16:06.453 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:16:06.453 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:16:06.453 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:16:06.453 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:16:06.453 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:16:06.453 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:16:06.453 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:16:06.453 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:16:06.453 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:16:06.454 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:16:06.454 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:16:06.455 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:16:06.455 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:16:06.483 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 45 2026-02-18T08:16:06.484 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 45 2026-02-18T08:16:06.484 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:16:06.484 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=181, tag_tid=12, entry_tid=1], [object_number=180, tag_tid=12, entry_tid=0], [object_number=183, tag_tid=11, entry_tid=29711], [object_number=182, tag_tid=11, entry_tid=29710]]], state=connected] 2026-02-18T08:16:06.484 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=181, tag_tid=12, entry_tid=1], [object_number=180, tag_tid=12, entry_tid=0], [object_number=183, tag_tid=11, entry_tid=29711], [object_number=182, tag_tid=11, entry_tid=29710]]], state=connected] 2026-02-18T08:16:06.486 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:16:06.488 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=181, tag_tid=12, entry_tid=1' 2026-02-18T08:16:06.489 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:16:06.489 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:16:06.490 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:16:06.490 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:16:06.490 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:16:06.490 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:16:06.490 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:16:06.490 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:16:06.490 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:16:06.490 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:16:06.490 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:16:06.490 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:16:06.491 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:16:06.491 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:16:06.491 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:16:06.525 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 45 2026-02-18T08:16:06.526 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 45 2026-02-18T08:16:06.526 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:16:06.526 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=181, tag_tid=12, entry_tid=1], [object_number=180, tag_tid=12, entry_tid=0], [object_number=183, tag_tid=11, entry_tid=29711], [object_number=182, tag_tid=11, entry_tid=29710]]], state=connected] 2026-02-18T08:16:06.526 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=181, tag_tid=12, entry_tid=1], [object_number=180, tag_tid=12, entry_tid=0], [object_number=183, tag_tid=11, entry_tid=29711], [object_number=182, tag_tid=11, entry_tid=29710]]], state=connected] 2026-02-18T08:16:06.529 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:16:06.531 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=181, tag_tid=12, entry_tid=1' 2026-02-18T08:16:06.531 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=181, tag_tid=12, entry_tid=1' -a 'object_number=181, tag_tid=12, entry_tid=1' = 'object_number=181, tag_tid=12, entry_tid=1' 2026-02-18T08:16:06.531 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:16:06.531 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_snap_present cluster1 mirror test snap4 2026-02-18T08:16:06.531 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:16:06.531 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:16:06.532 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:16:06.532 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap4 2026-02-18T08:16:06.532 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-18T08:16:06.532 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-18T08:16:06.532 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1 2026-02-18T08:16:07.532 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster1 info mirror/test@snap4 2026-02-18T08:16:07.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:rbd image 'test': 2026-02-18T08:16:07.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: size 512 MiB in 128 objects 2026-02-18T08:16:07.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: order 22 (4 MiB objects) 2026-02-18T08:16:07.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: snapshot_count: 7 2026-02-18T08:16:07.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: id: 10a74aa8ebc0 2026-02-18T08:16:07.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: block_name_prefix: rbd_data.10a74aa8ebc0 2026-02-18T08:16:07.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: format: 2 2026-02-18T08:16:07.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: features: layering, exclusive-lock, journaling 2026-02-18T08:16:07.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: op_features: 2026-02-18T08:16:07.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: flags: 2026-02-18T08:16:07.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: create_timestamp: Wed Feb 18 08:10:48 2026 2026-02-18T08:16:07.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: access_timestamp: Wed Feb 18 08:14:54 2026 2026-02-18T08:16:07.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: modify_timestamp: Wed Feb 18 08:15:02 2026 2026-02-18T08:16:07.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: protected: False 2026-02-18T08:16:07.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: journal: 10a74aa8ebc0 2026-02-18T08:16:07.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring state: enabled 2026-02-18T08:16:07.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring mode: journal 2026-02-18T08:16:07.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring global id: 0b548823-d751-43df-a076-4efac2da4d3d 2026-02-18T08:16:07.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring primary: false 2026-02-18T08:16:07.566 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:16:07.566 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n snap3-clean ']' 2026-02-18T08:16:07.566 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ compare_image_snaps mirror test snap3-clean 2026-02-18T08:16:07.566 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:16:07.567 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:16:07.567 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap3-clean 2026-02-18T08:16:07.567 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local ret=0 2026-02-18T08:16:07.567 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local rmt_export=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.export 2026-02-18T08:16:07.567 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loc_export=/tmp/tmp.vybh1m9yJF/cluster1-mirror-test.export 2026-02-18T08:16:07.567 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.export /tmp/tmp.vybh1m9yJF/cluster1-mirror-test.export 2026-02-18T08:16:07.567 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 -p mirror export test@snap3-clean /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.export 2026-02-18T08:16:08.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-18T08:16:08.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster1 -p mirror export test@snap3-clean /tmp/tmp.vybh1m9yJF/cluster1-mirror-test.export 2026-02-18T08:16:08.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-18T08:16:08.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmp /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.export /tmp/tmp.vybh1m9yJF/cluster1-mirror-test.export 2026-02-18T08:16:08.831 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.export /tmp/tmp.vybh1m9yJF/cluster1-mirror-test.export 2026-02-18T08:16:08.971 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:16:08.971 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ compare_image_snaps mirror test snap4 2026-02-18T08:16:08.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:16:08.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:16:08.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap4 2026-02-18T08:16:08.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local ret=0 2026-02-18T08:16:08.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local rmt_export=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.export 2026-02-18T08:16:08.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loc_export=/tmp/tmp.vybh1m9yJF/cluster1-mirror-test.export 2026-02-18T08:16:08.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.export /tmp/tmp.vybh1m9yJF/cluster1-mirror-test.export 2026-02-18T08:16:08.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 -p mirror export test@snap4 /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.export 2026-02-18T08:16:09.455 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-18T08:16:09.467 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster1 -p mirror export test@snap4 /tmp/tmp.vybh1m9yJF/cluster1-mirror-test.export 2026-02-18T08:16:10.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-18T08:16:10.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmp /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.export /tmp/tmp.vybh1m9yJF/cluster1-mirror-test.export 2026-02-18T08:16:10.322 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.export /tmp/tmp.vybh1m9yJF/cluster1-mirror-test.export 2026-02-18T08:16:10.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:16:10.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ clean_snap_name=snap4-clean 2026-02-18T08:16:10.448 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_snap cluster2 mirror test snap4-clean 2026-02-18T08:16:10.448 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:16:10.448 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:16:10.448 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:16:10.448 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap4-clean 2026-02-18T08:16:10.448 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 -p mirror snap create test@snap4-clean --debug-rbd=20 --debug-journaler=20 2026-02-18T08:16:10.755 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 10` 2026-02-18T08:16:10.756 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ stress_write_image cluster2 mirror test 2026-02-18T08:16:10.756 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:16:10.756 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:16:10.756 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:16:10.756 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-18T08:16:10.758 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local duration=12 2026-02-18T08:16:10.758 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set +e 2026-02-18T08:16:10.758 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ timeout 12s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror test --debug-rbd=20 --debug-journaler=20 2026-02-18T08:16:11.388 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:100 IOs, 35178496 bytes 2026-02-18T08:16:11.856 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:200 IOs, 52372480 bytes 2026-02-18T08:16:12.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:300 IOs, 75026432 bytes 2026-02-18T08:16:12.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:400 IOs, 79900672 bytes 2026-02-18T08:16:12.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:500 IOs, 98507776 bytes 2026-02-18T08:16:13.355 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:600 IOs, 160301056 bytes 2026-02-18T08:16:14.261 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:700 IOs, 220798976 bytes 2026-02-18T08:16:14.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:800 IOs, 268097536 bytes 2026-02-18T08:16:15.010 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:900 IOs, 268711936 bytes 2026-02-18T08:16:15.370 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1000 IOs, 315078656 bytes 2026-02-18T08:16:16.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1100 IOs, 375687168 bytes 2026-02-18T08:16:16.531 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1200 IOs, 404258816 bytes 2026-02-18T08:16:17.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1300 IOs, 442570752 bytes 2026-02-18T08:16:17.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1400 IOs, 459776000 bytes 2026-02-18T08:16:18.167 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1500 IOs, 511489024 bytes 2026-02-18T08:16:19.204 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1600 IOs, 589348864 bytes 2026-02-18T08:16:20.292 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1700 IOs, 651335680 bytes 2026-02-18T08:16:20.642 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1800 IOs, 690070528 bytes 2026-02-18T08:16:21.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1900 IOs, 760825856 bytes 2026-02-18T08:16:22.640 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:2000 IOs, 833517568 bytes 2026-02-18T08:16:22.768 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ error_code=124 2026-02-18T08:16:22.768 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set -e 2026-02-18T08:16:22.768 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 124 -eq 124 ']' 2026-02-18T08:16:22.768 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:16:22.768 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+replaying 2026-02-18T08:16:22.768 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:16:22.769 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:16:22.769 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:16:22.769 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local state_pattern=up+replaying 2026-02-18T08:16:22.769 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local description_pattern= 2026-02-18T08:16:22.769 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local service_pattern= 2026-02-18T08:16:22.769 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:16:22.769 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1 2026-02-18T08:16:23.769 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test_status_in_pool_dir cluster1 mirror test up+replaying '' '' 2026-02-18T08:16:23.769 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:16:23.769 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:16:23.770 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:16:23.770 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local state_pattern=up+replaying 2026-02-18T08:16:23.770 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local description_pattern= 2026-02-18T08:16:23.770 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local service_pattern= 2026-02-18T08:16:23.770 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local status 2026-02-18T08:16:23.770 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ CEPH_ARGS= 2026-02-18T08:16:23.770 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-18T08:16:23.807 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ status='test: 2026-02-18T08:16:23.807 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: global_id: 0b548823-d751-43df-a076-4efac2da4d3d 2026-02-18T08:16:23.807 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: state: up+replaying 2026-02-18T08:16:23.807 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: description: replaying, {"bytes_per_second":4.75,"entries_behind_primary":1,"entries_per_second":0.1,"non_primary_position":{"entry_tid":1,"object_number":181,"tag_tid":12},"primary_position":{"entry_tid":1,"object_number":181,"tag_tid":13},"seconds_until_synced":10} 2026-02-18T08:16:23.807 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: service: mirror.3 on trial163 2026-02-18T08:16:23.808 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: last_update: 2026-02-18 08:16:16 2026-02-18T08:16:23.808 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: peer_sites: 2026-02-18T08:16:23.808 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: name: cluster2 2026-02-18T08:16:23.808 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: state: up+stopped 2026-02-18T08:16:23.808 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: description: local image is primary 2026-02-18T08:16:23.808 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: last_update: 2026-02-18 08:16:16' 2026-02-18T08:16:23.808 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ grep '^ state: .*up+replaying' 2026-02-18T08:16:23.809 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: state: up+replaying 2026-02-18T08:16:23.809 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ grep '^ description: .*' 2026-02-18T08:16:23.809 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: description: replaying, {"bytes_per_second":4.75,"entries_behind_primary":1,"entries_per_second":0.1,"non_primary_position":{"entry_tid":1,"object_number":181,"tag_tid":12},"primary_position":{"entry_tid":1,"object_number":181,"tag_tid":13},"seconds_until_synced":10} 2026-02-18T08:16:23.810 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n '' ']' 2026-02-18T08:16:23.810 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ echo up+replaying 2026-02-18T08:16:23.810 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ grep '^up+' 2026-02-18T08:16:23.811 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:up+replaying 2026-02-18T08:16:23.811 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ grep 'service: *mirror..* on ' 2026-02-18T08:16:23.812 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: service: mirror.3 on trial163 2026-02-18T08:16:23.812 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local last_update 2026-02-18T08:16:23.812 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-18T08:16:23.815 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_update='2026-02-18 08:16:16' 2026-02-18T08:16:23.815 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+replaying '2026-02-18 08:16:16' 2026-02-18T08:16:23.815 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:16:23.815 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:16:23.815 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:16:23.815 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local state_pattern=up+replaying 2026-02-18T08:16:23.815 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'prev_last_update=2026-02-18 08:16:16' 2026-02-18T08:16:23.815 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local status 2026-02-18T08:16:23.815 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ CEPH_ARGS= 2026-02-18T08:16:23.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-18T08:16:23.852 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ status='OKOKOK142194268mirror.3trial16320.2.0-670-gadc19233falseOK43304333mirror.2trial16320.2.0-670-gadc19233falseOK43534356mirror.1trial16320.2.0-670-gadc19233trueOK43834386mirror.0trial16320.2.0-670-gadc19233falseOKtest0b548823-d751-43df-a076-4efac2da4d3dup+replayingreplaying, {"bytes_per_second":4.75,"entries_behind_primary":1,"entries_per_second":0.1,"non_primary_position":{"entry_tid":1,"object_number":181,"tag_tid":12},"primary_position":{"entry_tid":1,"object_number":181,"tag_tid":13},"seconds_until_synced":10}42194268mirror.3trial1632026-02-18 08:16:16cluster2c5e4f85e-ec31-430b-a71d-56e000d0de0bup+stoppedlocal image is primary2026-02-18 08:16:16' 2026-02-18T08:16:23.852 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local last_update state 2026-02-18T08:16:23.852 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-18T08:16:23.855 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_update='2026-02-18 08:16:16' 2026-02-18T08:16:23.856 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-18T08:16:23.858 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ state=up+replaying 2026-02-18T08:16:23.859 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ echo up+replaying 2026-02-18T08:16:23.859 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ grep up+replaying 2026-02-18T08:16:23.860 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:up+replaying 2026-02-18T08:16:23.861 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:16:23.861 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:16:23.861 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ snap_name=snap5 2026-02-18T08:16:23.861 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_snap cluster2 mirror test snap5 2026-02-18T08:16:23.861 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:16:23.861 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:16:23.861 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:16:23.861 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap5 2026-02-18T08:16:23.862 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 -p mirror snap create test@snap5 --debug-rbd=20 --debug-journaler=20 2026-02-18T08:16:53.784 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_replay_started cluster1 mirror test 2026-02-18T08:16:53.785 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:16:53.785 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:16:53.785 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:16:53.785 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_replay_state cluster1 mirror test started 2026-02-18T08:16:53.785 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:16:53.785 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:16:53.785 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:16:53.785 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local state=started 2026-02-18T08:16:53.785 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-18T08:16:53.786 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:16:53.786 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1 2026-02-18T08:16:54.786 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test_image_replay_state cluster1 mirror test started 2026-02-18T08:16:54.786 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:16:54.786 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:16:54.787 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:16:54.787 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local test_state=started 2026-02-18T08:16:54.787 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local status_result 2026-02-18T08:16:54.787 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_state=stopped 2026-02-18T08:16:54.787 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-18T08:16:54.787 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -i state 2026-02-18T08:16:54.787 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_instance=cluster1 2026-02-18T08:16:54.787 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-18T08:16:54.787 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:16:54.787 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance=cluster1 2026-02-18T08:16:54.788 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local loop_instance 2026-02-18T08:16:54.788 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:16:54.788 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sleep 0 2026-02-18T08:16:54.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:16:54.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ seq 0 3 2026-02-18T08:16:54.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:16:54.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-18T08:16:54.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:0 2026-02-18T08:16:54.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-18T08:16:54.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:16:54.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:16:54.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:16:54.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-18T08:16:54.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:16:54.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:16:54.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:16:54.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:16:54.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-18T08:16:54.793 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:16:54.793 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:16:54.793 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:16:54.793 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:16:54.793 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:0 2026-02-18T08:16:54.793 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-18T08:16:54.793 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-18T08:16:54.793 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:16:54.793 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:0 2026-02-18T08:16:54.793 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T08:16:54.794 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-18T08:16:54.794 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:16:54.794 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:16:54.794 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-18T08:16:54.794 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-18T08:16:54.794 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-18T08:16:54.794 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=0 2026-02-18T08:16:54.794 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=0 2026-02-18T08:16:54.794 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:16:54.802 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:16:54.802 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:16:54.802 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:16:54.802 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-18T08:16:54.852 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:16:54.852 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status : get status for rbd mirror 2026-02-18T08:16:54.852 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:16:54.855 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:16:54.855 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test 2026-02-18T08:16:54.855 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:1 2026-02-18T08:16:54.855 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-18T08:16:54.855 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:16:54.855 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:16:54.855 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:16:54.855 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-18T08:16:54.856 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:16:54.856 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:16:54.856 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:16:54.856 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:16:54.856 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-18T08:16:54.856 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:16:54.856 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:16:54.856 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:16:54.856 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:16:54.856 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:1 2026-02-18T08:16:54.856 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-18T08:16:54.856 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-18T08:16:54.856 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:16:54.857 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:1 2026-02-18T08:16:54.857 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T08:16:54.857 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-18T08:16:54.857 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:16:54.857 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:16:54.857 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-18T08:16:54.857 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-18T08:16:54.857 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-18T08:16:54.857 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=1 2026-02-18T08:16:54.857 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=1 2026-02-18T08:16:54.857 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:16:54.865 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:16:54.865 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:16:54.865 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:16:54.865 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test 2026-02-18T08:16:54.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:16:54.922 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status : get status for rbd mirror 2026-02-18T08:16:54.922 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:16:54.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:16:54.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/test 2026-02-18T08:16:54.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:2 2026-02-18T08:16:54.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-18T08:16:54.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:16:54.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:16:54.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:16:54.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-18T08:16:54.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:16:54.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:16:54.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:16:54.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:16:54.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-18T08:16:54.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:16:54.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:16:54.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:16:54.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:16:54.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:2 2026-02-18T08:16:54.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-18T08:16:54.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-18T08:16:54.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:16:54.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:2 2026-02-18T08:16:54.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T08:16:54.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-18T08:16:54.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:16:54.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:16:54.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-18T08:16:54.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-18T08:16:54.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-18T08:16:54.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=2 2026-02-18T08:16:54.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=2 2026-02-18T08:16:54.928 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:16:54.936 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:16:54.936 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:16:54.936 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:16:54.936 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/test 2026-02-18T08:16:54.985 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:16:54.985 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status : get status for rbd mirror 2026-02-18T08:16:54.985 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:16:54.988 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:16:54.988 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/test 2026-02-18T08:16:54.988 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:3 2026-02-18T08:16:54.989 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-18T08:16:54.989 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:16:54.989 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:16:54.989 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:16:54.989 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-18T08:16:54.989 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:16:54.989 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:16:54.989 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:16:54.989 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:16:54.989 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-18T08:16:54.990 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:16:54.990 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:16:54.990 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:16:54.990 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:16:54.990 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:3 2026-02-18T08:16:54.990 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-18T08:16:54.990 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-18T08:16:54.990 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:16:54.990 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:3 2026-02-18T08:16:54.990 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T08:16:54.990 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-18T08:16:54.990 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:16:54.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:16:54.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-18T08:16:54.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-18T08:16:54.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-18T08:16:54.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=3 2026-02-18T08:16:54.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=3 2026-02-18T08:16:54.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:16:54.999 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:16:55.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:16:55.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:16:55.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/test 2026-02-18T08:16:55.055 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ return 0 2026-02-18T08:16:55.055 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ status_result=' "state": "Replaying"' 2026-02-18T08:16:55.056 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ echo ' "state": "Replaying"' 2026-02-18T08:16:55.056 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ grep -i Replaying 2026-02-18T08:16:55.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: "state": "Replaying" 2026-02-18T08:16:55.058 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ current_state=started 2026-02-18T08:16:55.058 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test started = started 2026-02-18T08:16:55.058 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:16:55.058 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror test 2026-02-18T08:16:55.058 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_cluster=cluster1 2026-02-18T08:16:55.058 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:16:55.058 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_pool=mirror 2026-02-18T08:16:55.058 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local remote_pool=mirror 2026-02-18T08:16:55.058 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:16:55.058 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' journal = journal ']' 2026-02-18T08:16:55.058 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror mirror test 2026-02-18T08:16:55.058 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_cluster=cluster1 2026-02-18T08:16:55.058 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:16:55.058 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_pool=mirror 2026-02-18T08:16:55.058 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local remote_pool=mirror 2026-02-18T08:16:55.059 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:16:55.059 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-02-18T08:16:55.059 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-02-18T08:16:55.059 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:16:55.059 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:16:55.059 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:16:55.260 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:16:55.260 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:16:55.260 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:16:55.260 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:16:55.260 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:16:55.261 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:16:55.261 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:16:55.261 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:16:55.261 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:16:55.261 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:16:55.261 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:16:55.261 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:16:55.261 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:16:55.261 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:16:55.261 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:16:55.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:16:55.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:16:55.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:16:55.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:16:55.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:16:55.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:16:55.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:16:55.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:16:55.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:16:55.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:16:55.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:16:55.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:16:55.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:16:55.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:16:55.266 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:16:55.266 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:16:55.266 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:16:55.266 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:16:55.266 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:16:55.266 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:16:55.266 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:16:55.266 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:16:55.266 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:16:55.266 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:16:55.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:16:55.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:16:55.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:16:55.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:16:55.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:16:55.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:16:55.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:16:55.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:16:55.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:16:55.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:16:55.273 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:16:55.273 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:16:55.273 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:16:55.274 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:16:55.321 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:16:55.321 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:16:55.321 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:16:55.324 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:16:55.324 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:16:55.324 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:16:55.324 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:16:55.324 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:16:55.324 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:16:55.324 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:16:55.324 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:16:55.324 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:16:55.325 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:16:55.325 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:16:55.325 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:16:55.325 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:16:55.325 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:16:55.325 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:16:55.325 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:16:55.325 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:16:55.325 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:16:55.325 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:16:55.325 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:16:55.326 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:16:55.326 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:16:55.326 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:16:55.326 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:16:55.326 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:16:55.326 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:16:55.326 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:16:55.326 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:16:55.326 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:16:55.327 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:16:55.327 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:16:55.327 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:16:55.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:16:55.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:16:55.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:16:55.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:16:55.381 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:16:55.381 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:16:55.381 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:16:55.383 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:16:55.383 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:16:55.383 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:16:55.384 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:16:55.384 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:16:55.384 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:16:55.384 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:16:55.384 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:16:55.384 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:16:55.384 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:16:55.384 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:16:55.384 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:16:55.384 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:16:55.384 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:16:55.385 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:16:55.385 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:16:55.385 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:16:55.385 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:16:55.385 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:16:55.385 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:16:55.385 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:16:55.385 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:16:55.385 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:16:55.385 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:16:55.385 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:16:55.385 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:16:55.385 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:16:55.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:16:55.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:16:55.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:16:55.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:16:55.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:16:55.394 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:16:55.394 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:16:55.394 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:16:55.395 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:16:55.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:16:55.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:16:55.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:16:55.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:16:55.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:16:55.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:16:55.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:16:55.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:16:55.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:16:55.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:16:55.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:16:55.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:16:55.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:16:55.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:16:55.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:16:55.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:16:55.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:16:55.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:16:55.446 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:16:55.446 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:16:55.446 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:16:55.446 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:16:55.446 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:16:55.446 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:16:55.446 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:16:55.446 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:16:55.446 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:16:55.446 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:16:55.446 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:16:55.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:16:55.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:16:55.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:16:55.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:16:55.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:16:55.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:16:55.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:16:55.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:16:55.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:16:55.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:16:55.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:16:55.523 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:16:55.523 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:16:55.523 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:16:55.523 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:16:55.523 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:16:55.523 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:16:55.523 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:16:55.523 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:16:55.523 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:16:55.523 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:16:55.523 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:16:55.523 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:16:55.525 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:16:55.525 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:16:55.525 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:16:55.552 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 51 2026-02-18T08:16:55.552 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 58 2026-02-18T08:16:55.552 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:16:55.552 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=233, tag_tid=15, entry_tid=1], [object_number=232, tag_tid=15, entry_tid=0], [object_number=234, tag_tid=14, entry_tid=52758], [object_number=235, tag_tid=14, entry_tid=52755]]], state=connected] 2026-02-18T08:16:55.552 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=206, tag_tid=14, entry_tid=24054], [object_number=205, tag_tid=14, entry_tid=24053], [object_number=204, tag_tid=14, entry_tid=24052], [object_number=207, tag_tid=14, entry_tid=24051]]], state=connected] 2026-02-18T08:16:55.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:16:55.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=233, tag_tid=15, entry_tid=1' 2026-02-18T08:16:55.557 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:16:55.557 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:16:55.557 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:16:55.557 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:16:55.557 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:16:55.557 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:16:55.557 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:16:55.557 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:16:55.557 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:16:55.557 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:16:55.557 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:16:55.557 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:16:55.559 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:16:55.559 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:16:55.559 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:16:55.587 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 51 2026-02-18T08:16:55.587 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 58 2026-02-18T08:16:55.587 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:16:55.587 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=233, tag_tid=15, entry_tid=1], [object_number=232, tag_tid=15, entry_tid=0], [object_number=234, tag_tid=14, entry_tid=52758], [object_number=235, tag_tid=14, entry_tid=52755]]], state=connected] 2026-02-18T08:16:55.587 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=206, tag_tid=14, entry_tid=24054], [object_number=205, tag_tid=14, entry_tid=24053], [object_number=204, tag_tid=14, entry_tid=24052], [object_number=207, tag_tid=14, entry_tid=24051]]], state=connected] 2026-02-18T08:16:55.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:16:55.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=206, tag_tid=14, entry_tid=24054' 2026-02-18T08:16:55.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=233, tag_tid=15, entry_tid=1' -a 'object_number=233, tag_tid=15, entry_tid=1' = 'object_number=206, tag_tid=14, entry_tid=24054' 2026-02-18T08:16:55.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=206, tag_tid=14, entry_tid=24054' '!=' '' 2026-02-18T08:16:55.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:16:55.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=206, tag_tid=14, entry_tid=24054' = '' 2026-02-18T08:16:55.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=206, tag_tid=14, entry_tid=24054' 2026-02-18T08:16:55.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=233, tag_tid=15, entry_tid=1' 2026-02-18T08:16:55.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:16:55.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:16:55.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=15 2026-02-18T08:16:55.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=206, tag_tid=14, entry_tid=24054' 2026-02-18T08:16:55.594 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:16:55.594 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:16:55.595 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=14 2026-02-18T08:16:55.596 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=233, tag_tid=15, entry_tid=1' 2026-02-18T08:16:55.596 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:16:55.596 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:16:55.597 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:16:55.598 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=206, tag_tid=14, entry_tid=24054' 2026-02-18T08:16:55.598 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:16:55.598 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:16:55.600 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=24054 2026-02-18T08:16:55.600 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 15 = 14 -a 1 -le 24054 2026-02-18T08:16:55.600 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:16:55.600 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:16:55.600 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:16:55.802 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:16:55.802 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:16:55.803 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:16:55.803 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:16:55.803 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:16:55.803 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:16:55.803 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:16:55.803 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:16:55.803 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:16:55.803 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:16:55.804 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:16:55.804 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:16:55.804 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:16:55.804 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:16:55.804 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:16:55.804 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:16:55.804 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:16:55.807 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:16:55.807 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:16:55.807 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:16:55.808 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:16:55.808 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:16:55.808 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:16:55.808 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:16:55.808 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:16:55.808 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:16:55.808 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:16:55.808 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:16:55.808 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:16:55.808 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:16:55.809 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:16:55.809 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:16:55.809 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:16:55.809 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:16:55.809 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:16:55.809 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:16:55.809 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:16:55.809 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:16:55.809 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:16:55.809 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:16:55.809 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:16:55.809 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:16:55.809 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:16:55.810 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:16:55.810 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:16:55.810 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:16:55.810 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:16:55.810 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:16:55.810 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:16:55.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:16:55.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:16:55.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:16:55.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:16:55.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:16:55.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:16:55.874 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:16:55.876 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:16:55.876 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:16:55.877 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:16:55.877 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:16:55.877 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:16:55.877 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:16:55.877 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:16:55.877 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:16:55.877 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:16:55.877 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:16:55.877 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:16:55.877 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:16:55.877 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:16:55.878 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:16:55.878 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:16:55.878 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:16:55.878 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:16:55.878 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:16:55.878 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:16:55.878 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:16:55.878 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:16:55.878 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:16:55.878 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:16:55.878 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:16:55.878 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:16:55.879 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:16:55.879 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:16:55.879 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:16:55.879 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:16:55.879 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:16:55.879 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:16:55.879 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:16:55.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:16:55.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:16:55.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:16:55.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:16:55.946 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:16:55.946 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:16:55.946 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:16:55.948 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:16:55.949 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:16:55.949 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:16:55.949 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:16:55.949 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:16:55.949 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:16:55.949 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:16:55.949 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:16:55.949 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:16:55.949 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:16:55.949 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:16:55.950 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:16:55.950 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:16:55.950 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:16:55.950 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:16:55.950 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:16:55.950 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:16:55.950 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:16:55.950 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:16:55.950 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:16:55.950 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:16:55.950 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:16:55.950 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:16:55.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:16:55.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:16:55.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:16:55.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:16:55.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:16:55.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:16:55.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:16:55.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:16:55.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:16:55.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:16:55.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:16:55.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:16:55.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:16:56.010 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:16:56.010 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:16:56.010 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:16:56.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:16:56.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:16:56.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:16:56.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:16:56.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:16:56.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:16:56.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:16:56.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:16:56.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:16:56.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:16:56.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:16:56.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:16:56.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:16:56.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:16:56.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:16:56.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:16:56.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:16:56.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:16:56.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:16:56.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:16:56.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:16:56.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:16:56.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:16:56.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:16:56.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:16:56.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:16:56.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:16:56.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:16:56.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:16:56.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:16:56.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:16:56.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:16:56.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:16:56.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:16:56.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:16:56.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:16:56.092 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:16:56.093 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:16:56.093 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:16:56.093 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:16:56.093 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:16:56.093 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:16:56.093 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:16:56.093 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:16:56.093 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:16:56.093 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:16:56.093 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:16:56.094 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:16:56.094 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:16:56.095 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:16:56.095 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:16:56.095 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:16:56.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 51 2026-02-18T08:16:56.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 58 2026-02-18T08:16:56.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:16:56.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=233, tag_tid=15, entry_tid=1], [object_number=232, tag_tid=15, entry_tid=0], [object_number=234, tag_tid=14, entry_tid=52758], [object_number=235, tag_tid=14, entry_tid=52755]]], state=connected] 2026-02-18T08:16:56.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=207, tag_tid=14, entry_tid=24395], [object_number=206, tag_tid=14, entry_tid=24394], [object_number=205, tag_tid=14, entry_tid=24393], [object_number=204, tag_tid=14, entry_tid=24392]]], state=connected] 2026-02-18T08:16:56.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:16:56.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=233, tag_tid=15, entry_tid=1' 2026-02-18T08:16:56.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:16:56.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:16:56.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:16:56.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:16:56.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:16:56.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:16:56.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:16:56.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:16:56.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:16:56.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:16:56.129 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:16:56.129 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:16:56.129 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:16:56.130 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:16:56.130 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:16:56.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 51 2026-02-18T08:16:56.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 58 2026-02-18T08:16:56.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:16:56.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=233, tag_tid=15, entry_tid=1], [object_number=232, tag_tid=15, entry_tid=0], [object_number=234, tag_tid=14, entry_tid=52758], [object_number=235, tag_tid=14, entry_tid=52755]]], state=connected] 2026-02-18T08:16:56.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=207, tag_tid=14, entry_tid=24395], [object_number=206, tag_tid=14, entry_tid=24394], [object_number=205, tag_tid=14, entry_tid=24393], [object_number=204, tag_tid=14, entry_tid=24392]]], state=connected] 2026-02-18T08:16:56.361 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:16:56.363 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=207, tag_tid=14, entry_tid=24395' 2026-02-18T08:16:56.363 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=233, tag_tid=15, entry_tid=1' -a 'object_number=233, tag_tid=15, entry_tid=1' = 'object_number=207, tag_tid=14, entry_tid=24395' 2026-02-18T08:16:56.363 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=207, tag_tid=14, entry_tid=24395' '!=' 'object_number=206, tag_tid=14, entry_tid=24054' 2026-02-18T08:16:56.364 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:16:56.364 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=207, tag_tid=14, entry_tid=24395' = 'object_number=206, tag_tid=14, entry_tid=24054' 2026-02-18T08:16:56.364 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=207, tag_tid=14, entry_tid=24395' 2026-02-18T08:16:56.364 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=233, tag_tid=15, entry_tid=1' 2026-02-18T08:16:56.364 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:16:56.365 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:16:56.365 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=15 2026-02-18T08:16:56.366 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=207, tag_tid=14, entry_tid=24395' 2026-02-18T08:16:56.366 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:16:56.366 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:16:56.367 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=14 2026-02-18T08:16:56.368 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=233, tag_tid=15, entry_tid=1' 2026-02-18T08:16:56.368 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:16:56.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:16:56.370 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:16:56.370 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=207, tag_tid=14, entry_tid=24395' 2026-02-18T08:16:56.371 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:16:56.371 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:16:56.372 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=24395 2026-02-18T08:16:56.372 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 15 = 14 -a 1 -le 24395 2026-02-18T08:16:56.372 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:16:56.372 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:16:56.373 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:16:56.575 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:16:56.575 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:16:56.575 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:16:56.575 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:16:56.575 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:16:56.575 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:16:56.576 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:16:56.576 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:16:56.576 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:16:56.576 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:16:56.576 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:16:56.576 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:16:56.576 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:16:56.576 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:16:56.576 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:16:56.576 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:16:56.577 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:16:56.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:16:56.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:16:56.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:16:56.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:16:56.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:16:56.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:16:56.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:16:56.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:16:56.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:16:56.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:16:56.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:16:56.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:16:56.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:16:56.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:16:56.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:16:56.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:16:56.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:16:56.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:16:56.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:16:56.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:16:56.582 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:16:56.582 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:16:56.582 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:16:56.582 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:16:56.582 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:16:56.582 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:16:56.582 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:16:56.582 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:16:56.582 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:16:56.582 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:16:56.582 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:16:56.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:16:56.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:16:56.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:16:56.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:16:56.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:16:56.639 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:16:56.639 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:16:56.639 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:16:56.642 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:16:56.642 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:16:56.642 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:16:56.643 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:16:56.643 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:16:56.643 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:16:56.643 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:16:56.643 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:16:56.643 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:16:56.643 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:16:56.643 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:16:56.643 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:16:56.643 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:16:56.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:16:56.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:16:56.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:16:56.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:16:56.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:16:56.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:16:56.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:16:56.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:16:56.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:16:56.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:16:56.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:16:56.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:16:56.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:16:56.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:16:56.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:16:56.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:16:56.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:16:56.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:16:56.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:16:56.653 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:16:56.653 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:16:56.653 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:16:56.654 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:16:56.703 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:16:56.703 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:16:56.703 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:16:56.705 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:16:56.706 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:16:56.706 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:16:56.706 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:16:56.706 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:16:56.706 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:16:56.706 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:16:56.706 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:16:56.706 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:16:56.706 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:16:56.707 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:16:56.707 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:16:56.707 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:16:56.707 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:16:56.707 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:16:56.707 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:16:56.707 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:16:56.707 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:16:56.707 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:16:56.707 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:16:56.708 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:16:56.708 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:16:56.708 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:16:56.708 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:16:56.708 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:16:56.708 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:16:56.708 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:16:56.708 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:16:56.708 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:16:56.708 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:16:56.708 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:16:56.709 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:16:56.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:16:56.717 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:16:56.717 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:16:56.718 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:16:56.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:16:56.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:16:56.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:16:56.768 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:16:56.768 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:16:56.769 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:16:56.769 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:16:56.769 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:16:56.769 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:16:56.769 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:16:56.769 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:16:56.769 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:16:56.769 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:16:56.769 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:16:56.769 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:16:56.770 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:16:56.770 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:16:56.770 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:16:56.770 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:16:56.770 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:16:56.770 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:16:56.770 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:16:56.770 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:16:56.770 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:16:56.770 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:16:56.770 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:16:56.770 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:16:56.771 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:16:56.771 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:16:56.771 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:16:56.771 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:16:56.771 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:16:56.771 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:16:56.771 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:16:56.771 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:16:56.780 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:16:56.780 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:16:56.780 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:16:56.780 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:16:56.838 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:16:56.839 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:16:56.839 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:16:56.840 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:16:56.840 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:16:56.840 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:16:56.840 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:16:56.840 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:16:56.840 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:16:56.840 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:16:56.840 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:16:56.840 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:16:56.840 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:16:56.841 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:16:56.841 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:16:56.841 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:16:56.867 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 51 2026-02-18T08:16:56.867 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 58 2026-02-18T08:16:56.868 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:16:56.868 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=233, tag_tid=15, entry_tid=1], [object_number=232, tag_tid=15, entry_tid=0], [object_number=234, tag_tid=14, entry_tid=52758], [object_number=235, tag_tid=14, entry_tid=52755]]], state=connected] 2026-02-18T08:16:56.868 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=204, tag_tid=14, entry_tid=24896], [object_number=207, tag_tid=14, entry_tid=24895], [object_number=206, tag_tid=14, entry_tid=24894], [object_number=205, tag_tid=14, entry_tid=24893]]], state=connected] 2026-02-18T08:16:56.870 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:16:56.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=233, tag_tid=15, entry_tid=1' 2026-02-18T08:16:56.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:16:56.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:16:56.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:16:56.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:16:56.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:16:56.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:16:56.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:16:56.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:16:56.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:16:56.874 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:16:56.874 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:16:56.874 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:16:56.875 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:16:56.875 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:16:56.875 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:16:56.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 51 2026-02-18T08:16:56.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 58 2026-02-18T08:16:56.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:16:56.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=233, tag_tid=15, entry_tid=1], [object_number=232, tag_tid=15, entry_tid=0], [object_number=234, tag_tid=14, entry_tid=52758], [object_number=235, tag_tid=14, entry_tid=52755]]], state=connected] 2026-02-18T08:16:56.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=204, tag_tid=14, entry_tid=24896], [object_number=207, tag_tid=14, entry_tid=24895], [object_number=206, tag_tid=14, entry_tid=24894], [object_number=205, tag_tid=14, entry_tid=24893]]], state=connected] 2026-02-18T08:16:56.905 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:16:56.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=204, tag_tid=14, entry_tid=24896' 2026-02-18T08:16:56.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=233, tag_tid=15, entry_tid=1' -a 'object_number=233, tag_tid=15, entry_tid=1' = 'object_number=204, tag_tid=14, entry_tid=24896' 2026-02-18T08:16:56.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=204, tag_tid=14, entry_tid=24896' '!=' 'object_number=207, tag_tid=14, entry_tid=24395' 2026-02-18T08:16:56.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:16:56.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=204, tag_tid=14, entry_tid=24896' = 'object_number=207, tag_tid=14, entry_tid=24395' 2026-02-18T08:16:56.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=204, tag_tid=14, entry_tid=24896' 2026-02-18T08:16:56.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=233, tag_tid=15, entry_tid=1' 2026-02-18T08:16:56.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:16:56.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:16:56.909 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=15 2026-02-18T08:16:56.910 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=204, tag_tid=14, entry_tid=24896' 2026-02-18T08:16:56.910 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:16:56.910 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:16:56.911 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=14 2026-02-18T08:16:56.912 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=233, tag_tid=15, entry_tid=1' 2026-02-18T08:16:56.912 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:16:56.912 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:16:56.914 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:16:56.914 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=204, tag_tid=14, entry_tid=24896' 2026-02-18T08:16:56.915 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:16:56.915 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:16:56.916 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=24896 2026-02-18T08:16:56.916 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 15 = 14 -a 1 -le 24896 2026-02-18T08:16:56.917 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:16:56.917 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:16:56.917 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:16:57.118 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:16:57.118 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:16:57.119 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:16:57.119 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:16:57.119 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:16:57.119 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:16:57.119 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:16:57.119 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:16:57.119 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:16:57.119 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:16:57.120 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:16:57.120 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:16:57.120 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:16:57.120 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:16:57.120 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:16:57.120 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:16:57.120 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:16:57.122 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:16:57.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:16:57.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:16:57.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:16:57.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:16:57.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:16:57.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:16:57.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:16:57.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:16:57.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:16:57.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:16:57.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:16:57.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:16:57.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:16:57.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:16:57.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:16:57.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:16:57.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:16:57.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:16:57.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:16:57.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:16:57.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:16:57.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:16:57.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:16:57.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:16:57.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:16:57.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:16:57.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:16:57.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:16:57.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:16:57.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:16:57.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:16:57.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:16:57.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:16:57.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:16:57.134 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:16:57.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:16:57.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:16:57.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:16:57.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:16:57.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:16:57.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:16:57.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:16:57.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:16:57.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:16:57.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:16:57.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:16:57.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:16:57.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:16:57.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:16:57.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:16:57.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:16:57.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:16:57.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:16:57.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:16:57.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:16:57.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:16:57.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:16:57.187 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:16:57.187 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:16:57.187 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:16:57.187 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:16:57.187 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:16:57.187 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:16:57.187 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:16:57.187 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:16:57.187 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:16:57.187 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:16:57.187 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:16:57.187 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:16:57.188 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:16:57.196 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:16:57.196 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:16:57.196 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:16:57.196 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:16:57.245 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:16:57.245 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:16:57.245 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:16:57.248 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:16:57.248 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:16:57.248 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:16:57.248 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:16:57.248 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:16:57.248 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:16:57.248 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:16:57.248 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:16:57.248 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:16:57.249 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:16:57.249 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:16:57.249 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:16:57.249 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:16:57.249 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:16:57.249 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:16:57.249 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:16:57.249 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:16:57.249 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:16:57.249 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:16:57.249 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:16:57.250 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:16:57.250 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:16:57.250 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:16:57.250 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:16:57.250 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:16:57.250 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:16:57.250 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:16:57.250 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:16:57.250 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:16:57.250 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:16:57.250 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:16:57.250 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:16:57.259 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:16:57.260 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:16:57.260 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:16:57.260 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:16:57.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:16:57.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:16:57.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:16:57.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:16:57.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:16:57.312 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:16:57.312 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:16:57.312 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:16:57.312 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:16:57.312 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:16:57.312 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:16:57.312 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:16:57.312 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:16:57.312 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:16:57.312 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:16:57.312 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:16:57.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:16:57.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:16:57.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:16:57.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:16:57.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:16:57.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:16:57.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:16:57.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:16:57.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:16:57.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:16:57.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:16:57.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:16:57.314 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:16:57.314 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:16:57.314 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:16:57.314 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:16:57.314 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:16:57.314 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:16:57.314 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:16:57.321 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:16:57.322 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:16:57.322 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:16:57.322 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:16:57.383 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:16:57.385 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:16:57.385 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:16:57.385 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:16:57.385 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:16:57.385 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:16:57.385 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:16:57.385 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:16:57.385 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:16:57.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:16:57.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:16:57.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:16:57.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:16:57.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:16:57.387 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:16:57.387 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:16:57.413 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 51 2026-02-18T08:16:57.413 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 58 2026-02-18T08:16:57.413 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:16:57.413 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=233, tag_tid=15, entry_tid=1], [object_number=232, tag_tid=15, entry_tid=0], [object_number=234, tag_tid=14, entry_tid=52758], [object_number=235, tag_tid=14, entry_tid=52755]]], state=connected] 2026-02-18T08:16:57.414 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=204, tag_tid=14, entry_tid=25268], [object_number=207, tag_tid=14, entry_tid=25267], [object_number=206, tag_tid=14, entry_tid=25266], [object_number=205, tag_tid=14, entry_tid=25265]]], state=connected] 2026-02-18T08:16:57.416 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:16:57.417 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=233, tag_tid=15, entry_tid=1' 2026-02-18T08:16:57.418 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:16:57.418 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:16:57.419 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:16:57.419 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:16:57.419 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:16:57.419 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:16:57.419 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:16:57.419 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:16:57.419 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:16:57.419 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:16:57.419 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:16:57.420 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:16:57.420 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:16:57.421 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:16:57.421 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:16:57.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 51 2026-02-18T08:16:57.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 58 2026-02-18T08:16:57.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:16:57.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=233, tag_tid=15, entry_tid=1], [object_number=232, tag_tid=15, entry_tid=0], [object_number=234, tag_tid=14, entry_tid=52758], [object_number=235, tag_tid=14, entry_tid=52755]]], state=connected] 2026-02-18T08:16:57.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=204, tag_tid=14, entry_tid=25268], [object_number=207, tag_tid=14, entry_tid=25267], [object_number=206, tag_tid=14, entry_tid=25266], [object_number=205, tag_tid=14, entry_tid=25265]]], state=connected] 2026-02-18T08:16:57.449 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:16:57.451 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=204, tag_tid=14, entry_tid=25268' 2026-02-18T08:16:57.451 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=233, tag_tid=15, entry_tid=1' -a 'object_number=233, tag_tid=15, entry_tid=1' = 'object_number=204, tag_tid=14, entry_tid=25268' 2026-02-18T08:16:57.451 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=204, tag_tid=14, entry_tid=25268' '!=' 'object_number=204, tag_tid=14, entry_tid=24896' 2026-02-18T08:16:57.451 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:16:57.451 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=204, tag_tid=14, entry_tid=25268' = 'object_number=204, tag_tid=14, entry_tid=24896' 2026-02-18T08:16:57.451 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=204, tag_tid=14, entry_tid=25268' 2026-02-18T08:16:57.452 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=233, tag_tid=15, entry_tid=1' 2026-02-18T08:16:57.452 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:16:57.452 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:16:57.453 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=15 2026-02-18T08:16:57.454 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=204, tag_tid=14, entry_tid=25268' 2026-02-18T08:16:57.454 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:16:57.454 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:16:57.455 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=14 2026-02-18T08:16:57.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=233, tag_tid=15, entry_tid=1' 2026-02-18T08:16:57.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:16:57.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:16:57.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:16:57.458 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=204, tag_tid=14, entry_tid=25268' 2026-02-18T08:16:57.458 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:16:57.458 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:16:57.460 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=25268 2026-02-18T08:16:57.460 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 15 = 14 -a 1 -le 25268 2026-02-18T08:16:57.460 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:16:57.460 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:16:57.460 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:16:57.662 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:16:57.662 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:16:57.662 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:16:57.662 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:16:57.662 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:16:57.663 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:16:57.663 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:16:57.663 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:16:57.663 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:16:57.663 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:16:57.663 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:16:57.663 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:16:57.663 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:16:57.663 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:16:57.663 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:16:57.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:16:57.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:16:57.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:16:57.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:16:57.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:16:57.667 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:16:57.667 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:16:57.667 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:16:57.667 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:16:57.667 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:16:57.667 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:16:57.667 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:16:57.667 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:16:57.667 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:16:57.667 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:16:57.667 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:16:57.667 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:16:57.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:16:57.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:16:57.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:16:57.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:16:57.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:16:57.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:16:57.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:16:57.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:16:57.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:16:57.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:16:57.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:16:57.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:16:57.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:16:57.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:16:57.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:16:57.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:16:57.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:16:57.676 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:16:57.677 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:16:57.677 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:16:57.677 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:16:57.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:16:57.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:16:57.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:16:57.729 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:16:57.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:16:57.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:16:57.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:16:57.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:16:57.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:16:57.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:16:57.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:16:57.731 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:16:57.731 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:16:57.731 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:16:57.731 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:16:57.731 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:16:57.731 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:16:57.731 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:16:57.731 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:16:57.731 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:16:57.731 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:16:57.732 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:16:57.732 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:16:57.732 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:16:57.732 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:16:57.732 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:16:57.732 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:16:57.732 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:16:57.732 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:16:57.732 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:16:57.732 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:16:57.732 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:16:57.733 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:16:57.733 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:16:57.733 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:16:57.740 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:16:57.740 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:16:57.741 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:16:57.741 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:16:57.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:16:57.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:16:57.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:16:57.795 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:16:57.795 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:16:57.795 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:16:57.795 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:16:57.795 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:16:57.796 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:16:57.796 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:16:57.796 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:16:57.796 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:16:57.796 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:16:57.796 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:16:57.796 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:16:57.796 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:16:57.796 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:16:57.796 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:16:57.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:16:57.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:16:57.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:16:57.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:16:57.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:16:57.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:16:57.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:16:57.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:16:57.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:16:57.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:16:57.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:16:57.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:16:57.798 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:16:57.798 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:16:57.798 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:16:57.798 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:16:57.798 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:16:57.805 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:16:57.805 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:16:57.805 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:16:57.805 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:16:57.863 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:16:57.863 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:16:57.863 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:16:57.866 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:16:57.866 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:16:57.866 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:16:57.866 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:16:57.866 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:16:57.866 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:16:57.866 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:16:57.866 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:16:57.867 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:16:57.867 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:16:57.867 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:16:57.867 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:16:57.867 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:16:57.867 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:16:57.867 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:16:57.867 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:16:57.867 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:16:57.867 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:16:57.868 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:16:57.868 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:16:57.868 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:16:57.868 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:16:57.868 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:16:57.868 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:16:57.868 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:16:57.868 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:16:57.868 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:16:57.868 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:16:57.868 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:16:57.869 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:16:57.869 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:16:57.869 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:16:57.876 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:16:57.876 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:16:57.877 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:16:57.877 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:16:57.952 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:16:57.954 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:16:57.954 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:16:57.954 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:16:57.954 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:16:57.954 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:16:57.954 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:16:57.954 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:16:57.954 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:16:57.955 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:16:57.955 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:16:57.955 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:16:57.955 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:16:57.956 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:16:57.956 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:16:57.956 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:16:57.989 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 52 2026-02-18T08:16:57.989 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 58 2026-02-18T08:16:57.989 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:16:57.989 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=233, tag_tid=15, entry_tid=1], [object_number=232, tag_tid=15, entry_tid=0], [object_number=234, tag_tid=14, entry_tid=52758], [object_number=235, tag_tid=14, entry_tid=52755]]], state=connected] 2026-02-18T08:16:57.989 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=209, tag_tid=14, entry_tid=25501], [object_number=208, tag_tid=14, entry_tid=25500], [object_number=211, tag_tid=14, entry_tid=25499], [object_number=210, tag_tid=14, entry_tid=25498]]], state=connected] 2026-02-18T08:16:57.993 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:16:57.994 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=233, tag_tid=15, entry_tid=1' 2026-02-18T08:16:57.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:16:57.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:16:57.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:16:57.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:16:57.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:16:57.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:16:57.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:16:57.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:16:57.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:16:57.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:16:57.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:16:57.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:16:57.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:16:57.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:16:57.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:16:58.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 52 2026-02-18T08:16:58.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 58 2026-02-18T08:16:58.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:16:58.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=233, tag_tid=15, entry_tid=1], [object_number=232, tag_tid=15, entry_tid=0], [object_number=234, tag_tid=14, entry_tid=52758], [object_number=235, tag_tid=14, entry_tid=52755]]], state=connected] 2026-02-18T08:16:58.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=209, tag_tid=14, entry_tid=25501], [object_number=208, tag_tid=14, entry_tid=25500], [object_number=211, tag_tid=14, entry_tid=25499], [object_number=210, tag_tid=14, entry_tid=25498]]], state=connected] 2026-02-18T08:16:58.037 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:16:58.038 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=209, tag_tid=14, entry_tid=25501' 2026-02-18T08:16:58.039 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=233, tag_tid=15, entry_tid=1' -a 'object_number=233, tag_tid=15, entry_tid=1' = 'object_number=209, tag_tid=14, entry_tid=25501' 2026-02-18T08:16:58.039 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=209, tag_tid=14, entry_tid=25501' '!=' 'object_number=204, tag_tid=14, entry_tid=25268' 2026-02-18T08:16:58.039 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:16:58.039 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=209, tag_tid=14, entry_tid=25501' = 'object_number=204, tag_tid=14, entry_tid=25268' 2026-02-18T08:16:58.039 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=209, tag_tid=14, entry_tid=25501' 2026-02-18T08:16:58.039 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=233, tag_tid=15, entry_tid=1' 2026-02-18T08:16:58.039 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:16:58.039 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:16:58.040 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=15 2026-02-18T08:16:58.041 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=209, tag_tid=14, entry_tid=25501' 2026-02-18T08:16:58.041 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:16:58.041 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:16:58.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=14 2026-02-18T08:16:58.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=233, tag_tid=15, entry_tid=1' 2026-02-18T08:16:58.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:16:58.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:16:58.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:16:58.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=209, tag_tid=14, entry_tid=25501' 2026-02-18T08:16:58.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:16:58.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:16:58.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=25501 2026-02-18T08:16:58.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 15 = 14 -a 1 -le 25501 2026-02-18T08:16:58.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:16:58.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:16:58.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:16:58.249 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:16:58.249 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:16:58.250 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:16:58.250 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:16:58.250 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:16:58.250 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:16:58.250 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:16:58.250 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:16:58.250 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:16:58.250 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:16:58.250 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:16:58.251 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:16:58.251 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:16:58.251 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:16:58.251 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:16:58.251 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:16:58.251 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:16:58.254 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:16:58.254 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:16:58.254 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:16:58.254 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:16:58.255 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:16:58.255 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:16:58.255 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:16:58.255 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:16:58.255 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:16:58.255 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:16:58.255 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:16:58.255 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:16:58.255 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:16:58.255 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:16:58.255 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:16:58.256 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:16:58.256 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:16:58.256 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:16:58.256 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:16:58.256 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:16:58.256 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:16:58.256 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:16:58.256 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:16:58.256 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:16:58.256 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:16:58.256 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:16:58.257 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:16:58.257 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:16:58.257 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:16:58.257 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:16:58.257 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:16:58.257 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:16:58.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:16:58.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:16:58.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:16:58.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:16:58.323 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:16:58.323 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:16:58.323 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:16:58.326 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:16:58.327 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:16:58.327 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:16:58.327 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:16:58.327 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:16:58.327 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:16:58.327 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:16:58.327 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:16:58.327 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:16:58.327 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:16:58.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:16:58.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:16:58.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:16:58.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:16:58.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:16:58.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:16:58.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:16:58.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:16:58.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:16:58.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:16:58.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:16:58.329 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:16:58.329 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:16:58.329 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:16:58.329 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:16:58.329 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:16:58.329 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:16:58.329 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:16:58.329 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:16:58.329 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:16:58.329 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:16:58.329 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:16:58.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:16:58.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:16:58.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:16:58.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:16:58.395 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:16:58.395 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:16:58.395 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:16:58.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:16:58.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:16:58.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:16:58.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:16:58.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:16:58.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:16:58.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:16:58.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:16:58.399 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:16:58.399 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:16:58.399 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:16:58.399 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:16:58.399 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:16:58.399 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:16:58.399 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:16:58.399 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:16:58.399 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:16:58.399 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:16:58.399 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:16:58.399 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:16:58.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:16:58.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:16:58.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:16:58.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:16:58.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:16:58.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:16:58.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:16:58.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:16:58.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:16:58.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:16:58.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:16:58.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:16:58.409 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:16:58.409 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:16:58.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:16:58.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:16:58.472 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:16:58.472 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:16:58.472 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:16:58.474 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:16:58.475 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:16:58.475 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:16:58.475 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:16:58.475 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:16:58.475 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:16:58.475 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:16:58.475 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:16:58.475 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:16:58.475 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:16:58.475 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:16:58.476 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:16:58.476 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:16:58.476 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:16:58.476 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:16:58.476 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:16:58.476 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:16:58.476 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:16:58.476 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:16:58.476 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:16:58.476 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:16:58.477 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:16:58.477 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:16:58.477 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:16:58.477 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:16:58.477 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:16:58.477 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:16:58.477 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:16:58.477 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:16:58.477 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:16:58.477 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:16:58.478 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:16:58.485 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:16:58.485 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:16:58.486 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:16:58.486 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:16:58.553 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:16:58.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:16:58.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:16:58.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:16:58.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:16:58.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:16:58.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:16:58.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:16:58.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:16:58.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:16:58.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:16:58.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:16:58.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:16:58.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:16:58.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:16:58.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:16:58.582 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 52 2026-02-18T08:16:58.582 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 58 2026-02-18T08:16:58.582 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:16:58.582 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=233, tag_tid=15, entry_tid=1], [object_number=232, tag_tid=15, entry_tid=0], [object_number=234, tag_tid=14, entry_tid=52758], [object_number=235, tag_tid=14, entry_tid=52755]]], state=connected] 2026-02-18T08:16:58.582 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=210, tag_tid=14, entry_tid=25890], [object_number=209, tag_tid=14, entry_tid=25889], [object_number=208, tag_tid=14, entry_tid=25888], [object_number=211, tag_tid=14, entry_tid=25887]]], state=connected] 2026-02-18T08:16:58.584 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:16:58.586 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=233, tag_tid=15, entry_tid=1' 2026-02-18T08:16:58.587 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:16:58.587 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:16:58.587 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:16:58.587 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:16:58.587 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:16:58.587 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:16:58.587 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:16:58.587 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:16:58.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:16:58.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:16:58.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:16:58.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:16:58.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:16:58.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:16:58.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:16:58.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 52 2026-02-18T08:16:58.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 58 2026-02-18T08:16:58.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:16:58.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=233, tag_tid=15, entry_tid=1], [object_number=232, tag_tid=15, entry_tid=0], [object_number=234, tag_tid=14, entry_tid=52758], [object_number=235, tag_tid=14, entry_tid=52755]]], state=connected] 2026-02-18T08:16:58.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=210, tag_tid=14, entry_tid=25890], [object_number=209, tag_tid=14, entry_tid=25889], [object_number=208, tag_tid=14, entry_tid=25888], [object_number=211, tag_tid=14, entry_tid=25887]]], state=connected] 2026-02-18T08:16:58.628 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:16:58.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=210, tag_tid=14, entry_tid=25890' 2026-02-18T08:16:58.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=233, tag_tid=15, entry_tid=1' -a 'object_number=233, tag_tid=15, entry_tid=1' = 'object_number=210, tag_tid=14, entry_tid=25890' 2026-02-18T08:16:58.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=210, tag_tid=14, entry_tid=25890' '!=' 'object_number=209, tag_tid=14, entry_tid=25501' 2026-02-18T08:16:58.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:16:58.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=210, tag_tid=14, entry_tid=25890' = 'object_number=209, tag_tid=14, entry_tid=25501' 2026-02-18T08:16:58.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=210, tag_tid=14, entry_tid=25890' 2026-02-18T08:16:58.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=233, tag_tid=15, entry_tid=1' 2026-02-18T08:16:58.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:16:58.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:16:58.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=15 2026-02-18T08:16:58.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=210, tag_tid=14, entry_tid=25890' 2026-02-18T08:16:58.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:16:58.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:16:58.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=14 2026-02-18T08:16:58.635 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=233, tag_tid=15, entry_tid=1' 2026-02-18T08:16:58.635 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:16:58.635 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:16:58.636 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:16:58.637 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=210, tag_tid=14, entry_tid=25890' 2026-02-18T08:16:58.637 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:16:58.637 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:16:58.638 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=25890 2026-02-18T08:16:58.638 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 15 = 14 -a 1 -le 25890 2026-02-18T08:16:58.638 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:16:58.638 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:16:58.639 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:16:58.840 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:16:58.840 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:16:58.840 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:16:58.840 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:16:58.840 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:16:58.840 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:16:58.841 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:16:58.841 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:16:58.841 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:16:58.841 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:16:58.841 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:16:58.841 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:16:58.841 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:16:58.841 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:16:58.841 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:16:58.844 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:16:58.844 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:16:58.844 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:16:58.844 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:16:58.844 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:16:58.845 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:16:58.845 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:16:58.845 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:16:58.845 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:16:58.845 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:16:58.845 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:16:58.845 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:16:58.845 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:16:58.845 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:16:58.845 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:16:58.845 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:16:58.845 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:16:58.846 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:16:58.846 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:16:58.846 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:16:58.846 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:16:58.846 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:16:58.846 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:16:58.846 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:16:58.846 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:16:58.846 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:16:58.846 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:16:58.846 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:16:58.846 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:16:58.846 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:16:58.847 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:16:58.847 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:16:58.847 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:16:58.847 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:16:58.854 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:16:58.854 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:16:58.854 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:16:58.854 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:16:58.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:16:58.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:16:58.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:16:58.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:16:58.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:16:58.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:16:58.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:16:58.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:16:58.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:16:58.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:16:58.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:16:58.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:16:58.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:16:58.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:16:58.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:16:58.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:16:58.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:16:58.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:16:58.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:16:58.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:16:58.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:16:58.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:16:58.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:16:58.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:16:58.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:16:58.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:16:58.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:16:58.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:16:58.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:16:58.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:16:58.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:16:58.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:16:58.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:16:58.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:16:58.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:16:58.917 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:16:58.918 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:16:58.918 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:16:58.918 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:16:58.965 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:16:58.966 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:16:58.966 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:16:58.968 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:16:58.969 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:16:58.969 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:16:58.969 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:16:58.969 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:16:58.969 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:16:58.969 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:16:58.969 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:16:58.969 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:16:58.969 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:16:58.969 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:16:58.970 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:16:58.970 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:16:58.970 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:16:58.970 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:16:58.970 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:16:58.970 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:16:58.970 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:16:58.970 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:16:58.970 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:16:58.970 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:16:58.970 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:16:58.970 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:16:58.971 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:16:58.971 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:16:58.971 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:16:58.971 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:16:58.971 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:16:58.971 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:16:58.971 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:16:58.971 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:16:58.971 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:16:58.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:16:58.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:16:58.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:16:58.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:16:59.027 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:16:59.027 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:16:59.028 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:16:59.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:16:59.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:16:59.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:16:59.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:16:59.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:16:59.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:16:59.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:16:59.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:16:59.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:16:59.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:16:59.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:16:59.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:16:59.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:16:59.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:16:59.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:16:59.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:16:59.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:16:59.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:16:59.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:16:59.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:16:59.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:16:59.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:16:59.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:16:59.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:16:59.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:16:59.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:16:59.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:16:59.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:16:59.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:16:59.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:16:59.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:16:59.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:16:59.040 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:16:59.041 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:16:59.041 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:16:59.041 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:16:59.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:16:59.101 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:16:59.101 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:16:59.101 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:16:59.101 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:16:59.101 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:16:59.102 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:16:59.102 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:16:59.102 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:16:59.102 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:16:59.102 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:16:59.102 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:16:59.102 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:16:59.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:16:59.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:16:59.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:16:59.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 52 2026-02-18T08:16:59.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 58 2026-02-18T08:16:59.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:16:59.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=233, tag_tid=15, entry_tid=1], [object_number=232, tag_tid=15, entry_tid=0], [object_number=234, tag_tid=14, entry_tid=52758], [object_number=235, tag_tid=14, entry_tid=52755]]], state=connected] 2026-02-18T08:16:59.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=208, tag_tid=14, entry_tid=26268], [object_number=211, tag_tid=14, entry_tid=26267], [object_number=210, tag_tid=14, entry_tid=26266], [object_number=209, tag_tid=14, entry_tid=26265]]], state=connected] 2026-02-18T08:16:59.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:16:59.137 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=233, tag_tid=15, entry_tid=1' 2026-02-18T08:16:59.138 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:16:59.138 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:16:59.139 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:16:59.139 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:16:59.139 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:16:59.139 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:16:59.139 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:16:59.139 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:16:59.139 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:16:59.139 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:16:59.140 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:16:59.141 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:16:59.141 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:16:59.141 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:16:59.141 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:16:59.174 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 52 2026-02-18T08:16:59.174 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 58 2026-02-18T08:16:59.174 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:16:59.174 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=233, tag_tid=15, entry_tid=1], [object_number=232, tag_tid=15, entry_tid=0], [object_number=234, tag_tid=14, entry_tid=52758], [object_number=235, tag_tid=14, entry_tid=52755]]], state=connected] 2026-02-18T08:16:59.175 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=208, tag_tid=14, entry_tid=26268], [object_number=211, tag_tid=14, entry_tid=26267], [object_number=210, tag_tid=14, entry_tid=26266], [object_number=209, tag_tid=14, entry_tid=26265]]], state=connected] 2026-02-18T08:16:59.177 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:16:59.179 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=208, tag_tid=14, entry_tid=26268' 2026-02-18T08:16:59.179 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=233, tag_tid=15, entry_tid=1' -a 'object_number=233, tag_tid=15, entry_tid=1' = 'object_number=208, tag_tid=14, entry_tid=26268' 2026-02-18T08:16:59.179 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=208, tag_tid=14, entry_tid=26268' '!=' 'object_number=210, tag_tid=14, entry_tid=25890' 2026-02-18T08:16:59.180 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:16:59.180 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=208, tag_tid=14, entry_tid=26268' = 'object_number=210, tag_tid=14, entry_tid=25890' 2026-02-18T08:16:59.180 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=208, tag_tid=14, entry_tid=26268' 2026-02-18T08:16:59.180 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=233, tag_tid=15, entry_tid=1' 2026-02-18T08:16:59.180 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:16:59.180 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:16:59.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=15 2026-02-18T08:16:59.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=208, tag_tid=14, entry_tid=26268' 2026-02-18T08:16:59.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:16:59.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:16:59.183 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=14 2026-02-18T08:16:59.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=233, tag_tid=15, entry_tid=1' 2026-02-18T08:16:59.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:16:59.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:16:59.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:16:59.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=208, tag_tid=14, entry_tid=26268' 2026-02-18T08:16:59.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:16:59.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:16:59.188 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=26268 2026-02-18T08:16:59.188 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 15 = 14 -a 1 -le 26268 2026-02-18T08:16:59.188 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:16:59.188 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:16:59.188 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:16:59.390 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:16:59.390 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:16:59.390 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:16:59.390 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:16:59.390 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:16:59.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:16:59.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:16:59.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:16:59.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:16:59.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:16:59.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:16:59.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:16:59.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:16:59.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:16:59.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:16:59.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:16:59.394 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:16:59.395 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:16:59.395 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:16:59.395 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:16:59.395 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:16:59.395 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:16:59.395 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:16:59.395 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:16:59.395 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:16:59.395 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:16:59.395 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:16:59.396 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:16:59.396 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:16:59.396 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:16:59.396 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:16:59.396 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:16:59.396 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:16:59.396 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:16:59.396 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:16:59.396 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:16:59.396 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:16:59.396 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:16:59.397 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:16:59.397 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:16:59.397 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:16:59.397 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:16:59.397 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:16:59.397 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:16:59.397 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:16:59.397 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:16:59.397 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:16:59.397 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:16:59.397 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:16:59.405 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:16:59.405 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:16:59.405 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:16:59.405 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:16:59.455 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:16:59.455 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:16:59.455 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:16:59.458 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:16:59.458 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:16:59.458 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:16:59.458 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:16:59.458 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:16:59.458 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:16:59.458 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:16:59.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:16:59.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:16:59.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:16:59.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:16:59.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:16:59.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:16:59.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:16:59.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:16:59.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:16:59.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:16:59.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:16:59.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:16:59.460 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:16:59.460 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:16:59.460 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:16:59.460 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:16:59.460 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:16:59.460 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:16:59.460 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:16:59.460 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:16:59.460 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:16:59.460 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:16:59.460 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:16:59.460 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:16:59.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:16:59.468 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:16:59.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:16:59.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:16:59.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:16:59.526 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:16:59.526 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:16:59.526 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:16:59.529 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:16:59.529 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:16:59.530 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:16:59.530 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:16:59.530 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:16:59.530 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:16:59.530 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:16:59.530 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:16:59.530 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:16:59.530 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:16:59.530 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:16:59.530 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:16:59.530 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:16:59.531 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:16:59.531 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:16:59.531 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:16:59.531 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:16:59.531 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:16:59.531 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:16:59.531 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:16:59.531 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:16:59.531 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:16:59.531 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:16:59.531 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:16:59.532 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:16:59.532 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:16:59.532 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:16:59.532 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:16:59.532 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:16:59.532 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:16:59.532 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:16:59.532 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:16:59.540 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:16:59.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:16:59.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:16:59.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:16:59.599 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:16:59.599 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:16:59.599 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:16:59.602 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:16:59.602 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:16:59.602 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:16:59.602 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:16:59.602 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:16:59.602 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:16:59.602 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:16:59.602 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:16:59.602 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:16:59.602 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:16:59.603 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:16:59.603 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:16:59.603 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:16:59.603 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:16:59.603 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:16:59.603 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:16:59.603 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:16:59.603 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:16:59.603 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:16:59.603 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:16:59.603 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:16:59.603 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:16:59.604 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:16:59.604 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:16:59.604 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:16:59.604 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:16:59.604 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:16:59.604 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:16:59.604 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:16:59.604 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:16:59.604 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:16:59.604 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:16:59.613 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:16:59.613 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:16:59.613 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:16:59.613 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:16:59.672 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:16:59.673 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:16:59.673 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:16:59.674 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:16:59.674 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:16:59.674 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:16:59.674 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:16:59.674 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:16:59.674 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:16:59.674 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:16:59.674 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:16:59.674 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:16:59.674 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:16:59.675 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:16:59.675 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:16:59.675 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:16:59.703 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 52 2026-02-18T08:16:59.703 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 58 2026-02-18T08:16:59.704 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:16:59.704 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=233, tag_tid=15, entry_tid=1], [object_number=232, tag_tid=15, entry_tid=0], [object_number=234, tag_tid=14, entry_tid=52758], [object_number=235, tag_tid=14, entry_tid=52755]]], state=connected] 2026-02-18T08:16:59.704 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=211, tag_tid=14, entry_tid=26623], [object_number=210, tag_tid=14, entry_tid=26622], [object_number=209, tag_tid=14, entry_tid=26621], [object_number=208, tag_tid=14, entry_tid=26620]]], state=connected] 2026-02-18T08:16:59.706 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:16:59.708 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=233, tag_tid=15, entry_tid=1' 2026-02-18T08:16:59.709 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:16:59.709 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:16:59.709 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:16:59.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:16:59.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:16:59.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:16:59.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:16:59.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:16:59.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:16:59.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:16:59.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:16:59.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:16:59.711 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:16:59.711 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:16:59.711 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:16:59.737 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 52 2026-02-18T08:16:59.737 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 58 2026-02-18T08:16:59.737 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:16:59.737 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=233, tag_tid=15, entry_tid=1], [object_number=232, tag_tid=15, entry_tid=0], [object_number=234, tag_tid=14, entry_tid=52758], [object_number=235, tag_tid=14, entry_tid=52755]]], state=connected] 2026-02-18T08:16:59.738 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=211, tag_tid=14, entry_tid=26623], [object_number=210, tag_tid=14, entry_tid=26622], [object_number=209, tag_tid=14, entry_tid=26621], [object_number=208, tag_tid=14, entry_tid=26620]]], state=connected] 2026-02-18T08:16:59.740 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:16:59.741 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=211, tag_tid=14, entry_tid=26623' 2026-02-18T08:16:59.741 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=233, tag_tid=15, entry_tid=1' -a 'object_number=233, tag_tid=15, entry_tid=1' = 'object_number=211, tag_tid=14, entry_tid=26623' 2026-02-18T08:16:59.741 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=211, tag_tid=14, entry_tid=26623' '!=' 'object_number=208, tag_tid=14, entry_tid=26268' 2026-02-18T08:16:59.742 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:16:59.742 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=211, tag_tid=14, entry_tid=26623' = 'object_number=208, tag_tid=14, entry_tid=26268' 2026-02-18T08:16:59.742 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=211, tag_tid=14, entry_tid=26623' 2026-02-18T08:16:59.743 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=233, tag_tid=15, entry_tid=1' 2026-02-18T08:16:59.743 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:16:59.743 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:16:59.743 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=15 2026-02-18T08:16:59.744 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=211, tag_tid=14, entry_tid=26623' 2026-02-18T08:16:59.744 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:16:59.744 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:16:59.746 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=14 2026-02-18T08:16:59.747 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=233, tag_tid=15, entry_tid=1' 2026-02-18T08:16:59.747 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:16:59.747 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:16:59.748 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:16:59.749 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=211, tag_tid=14, entry_tid=26623' 2026-02-18T08:16:59.749 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:16:59.749 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:16:59.750 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=26623 2026-02-18T08:16:59.751 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 15 = 14 -a 1 -le 26623 2026-02-18T08:16:59.751 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:16:59.751 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:16:59.751 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:16:59.952 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:16:59.953 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:16:59.953 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:16:59.953 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:16:59.953 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:16:59.953 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:16:59.953 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:16:59.953 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:16:59.953 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:16:59.953 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:16:59.953 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:16:59.954 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:16:59.954 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:16:59.954 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:16:59.954 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:16:59.956 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:16:59.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:16:59.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:16:59.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:16:59.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:16:59.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:16:59.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:16:59.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:16:59.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:16:59.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:16:59.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:16:59.958 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:16:59.958 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:16:59.958 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:16:59.958 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:16:59.958 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:16:59.958 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:16:59.958 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:16:59.958 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:16:59.958 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:16:59.958 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:16:59.958 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:16:59.958 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:16:59.958 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:16:59.959 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:16:59.959 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:16:59.959 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:16:59.959 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:16:59.959 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:16:59.959 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:16:59.959 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:16:59.959 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:16:59.959 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:16:59.959 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:16:59.968 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:16:59.968 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:16:59.969 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:16:59.969 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:17:00.018 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:17:00.018 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:17:00.018 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:17:00.021 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:17:00.021 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:17:00.021 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:17:00.021 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:00.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:17:00.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:17:00.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:17:00.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:17:00.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:17:00.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:17:00.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:17:00.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:17:00.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:17:00.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:00.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:17:00.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:17:00.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:17:00.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:17:00.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:17:00.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:17:00.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:17:00.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:17:00.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:17:00.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:17:00.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:00.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:17:00.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:17:00.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:17:00.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:17:00.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:17:00.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:17:00.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:17:00.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:17:00.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:17:00.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:17:00.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:17:00.080 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:17:00.080 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:17:00.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:17:00.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:17:00.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:17:00.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:17:00.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:00.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:17:00.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:17:00.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:17:00.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:17:00.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:17:00.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:17:00.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:17:00.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:17:00.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:17:00.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:00.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:17:00.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:17:00.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:17:00.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:17:00.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:17:00.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:17:00.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:17:00.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:17:00.086 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:17:00.086 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:17:00.086 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:00.086 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:17:00.086 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:17:00.086 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:17:00.086 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:17:00.086 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:17:00.086 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:17:00.086 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:17:00.093 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:17:00.094 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:17:00.094 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:17:00.094 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:17:00.142 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:17:00.142 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:17:00.142 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:17:00.145 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:17:00.145 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:17:00.145 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:17:00.145 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:00.145 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:17:00.145 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:17:00.145 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:17:00.145 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:17:00.145 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:17:00.145 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:17:00.146 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:17:00.146 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:17:00.146 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:17:00.146 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:00.146 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:17:00.146 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:17:00.146 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:17:00.146 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:17:00.146 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:17:00.146 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:17:00.146 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:17:00.146 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:17:00.147 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:17:00.147 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:17:00.147 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:00.147 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:17:00.147 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:17:00.147 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:17:00.147 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:17:00.147 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:17:00.147 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:17:00.147 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:17:00.155 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:17:00.156 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:17:00.156 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:17:00.156 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:17:00.214 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:17:00.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:17:00.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:17:00.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:17:00.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:17:00.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:17:00.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:17:00.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:17:00.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:17:00.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:17:00.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:17:00.217 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:17:00.217 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:17:00.217 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:00.218 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:17:00.218 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:00.257 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 52 2026-02-18T08:17:00.257 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 58 2026-02-18T08:17:00.257 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:17:00.257 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=233, tag_tid=15, entry_tid=1], [object_number=232, tag_tid=15, entry_tid=0], [object_number=234, tag_tid=14, entry_tid=52758], [object_number=235, tag_tid=14, entry_tid=52755]]], state=connected] 2026-02-18T08:17:00.257 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=209, tag_tid=14, entry_tid=26977], [object_number=208, tag_tid=14, entry_tid=26976], [object_number=211, tag_tid=14, entry_tid=26975], [object_number=210, tag_tid=14, entry_tid=26974]]], state=connected] 2026-02-18T08:17:00.261 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:00.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=233, tag_tid=15, entry_tid=1' 2026-02-18T08:17:00.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:17:00.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:17:00.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:17:00.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:17:00.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:17:00.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:17:00.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:17:00.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:17:00.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:17:00.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:17:00.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:17:00.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:17:00.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:00.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:17:00.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:00.299 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 52 2026-02-18T08:17:00.299 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 58 2026-02-18T08:17:00.299 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:17:00.300 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=233, tag_tid=15, entry_tid=1], [object_number=232, tag_tid=15, entry_tid=0], [object_number=234, tag_tid=14, entry_tid=52758], [object_number=235, tag_tid=14, entry_tid=52755]]], state=connected] 2026-02-18T08:17:00.300 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=209, tag_tid=14, entry_tid=26977], [object_number=208, tag_tid=14, entry_tid=26976], [object_number=211, tag_tid=14, entry_tid=26975], [object_number=210, tag_tid=14, entry_tid=26974]]], state=connected] 2026-02-18T08:17:00.302 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:00.304 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=209, tag_tid=14, entry_tid=26977' 2026-02-18T08:17:00.304 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=233, tag_tid=15, entry_tid=1' -a 'object_number=233, tag_tid=15, entry_tid=1' = 'object_number=209, tag_tid=14, entry_tid=26977' 2026-02-18T08:17:00.304 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=209, tag_tid=14, entry_tid=26977' '!=' 'object_number=211, tag_tid=14, entry_tid=26623' 2026-02-18T08:17:00.304 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:17:00.304 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=209, tag_tid=14, entry_tid=26977' = 'object_number=211, tag_tid=14, entry_tid=26623' 2026-02-18T08:17:00.304 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=209, tag_tid=14, entry_tid=26977' 2026-02-18T08:17:00.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=233, tag_tid=15, entry_tid=1' 2026-02-18T08:17:00.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:17:00.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:17:00.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=15 2026-02-18T08:17:00.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=209, tag_tid=14, entry_tid=26977' 2026-02-18T08:17:00.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:17:00.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:17:00.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=14 2026-02-18T08:17:00.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=233, tag_tid=15, entry_tid=1' 2026-02-18T08:17:00.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:17:00.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:17:00.310 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:17:00.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=209, tag_tid=14, entry_tid=26977' 2026-02-18T08:17:00.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:17:00.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:17:00.312 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=26977 2026-02-18T08:17:00.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 15 = 14 -a 1 -le 26977 2026-02-18T08:17:00.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:17:00.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:17:00.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:17:00.514 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:17:00.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:17:00.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:17:00.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:17:00.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:17:00.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:17:00.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:17:00.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:17:00.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:17:00.516 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:00.516 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:17:00.516 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:17:00.516 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:17:00.516 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:17:00.516 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:17:00.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:17:00.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:17:00.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:17:00.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:17:00.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:17:00.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:00.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:17:00.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:17:00.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:17:00.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:17:00.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:17:00.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:17:00.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:17:00.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:17:00.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:17:00.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:00.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:17:00.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:17:00.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:17:00.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:17:00.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:17:00.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:17:00.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:17:00.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:17:00.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:17:00.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:17:00.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:00.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:17:00.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:17:00.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:17:00.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:17:00.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:17:00.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:17:00.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:17:00.529 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:17:00.529 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:17:00.529 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:17:00.529 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:17:00.577 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:17:00.578 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:17:00.578 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:17:00.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:17:00.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:17:00.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:17:00.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:00.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:17:00.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:17:00.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:17:00.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:17:00.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:17:00.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:17:00.582 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:17:00.582 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:17:00.582 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:17:00.582 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:00.582 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:17:00.582 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:17:00.582 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:17:00.582 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:17:00.582 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:17:00.582 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:17:00.582 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:17:00.582 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:17:00.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:17:00.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:17:00.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:00.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:17:00.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:17:00.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:17:00.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:17:00.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:17:00.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:17:00.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:17:00.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:17:00.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:17:00.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:17:00.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:17:00.640 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:17:00.640 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:17:00.640 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:17:00.643 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:17:00.643 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:17:00.643 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:17:00.643 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:00.643 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:17:00.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:17:00.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:17:00.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:17:00.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:17:00.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:17:00.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:17:00.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:17:00.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:17:00.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:00.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:17:00.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:17:00.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:17:00.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:17:00.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:17:00.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:17:00.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:17:00.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:17:00.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:17:00.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:17:00.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:00.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:17:00.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:17:00.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:17:00.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:17:00.646 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:17:00.646 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:17:00.646 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:17:00.653 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:17:00.654 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:17:00.654 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:17:00.654 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:17:00.702 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:17:00.702 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:17:00.702 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:17:00.705 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:17:00.705 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:17:00.705 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:17:00.705 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:00.705 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:17:00.705 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:17:00.705 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:17:00.705 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:17:00.705 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:17:00.706 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:17:00.706 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:17:00.706 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:17:00.706 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:17:00.706 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:00.706 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:17:00.706 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:17:00.706 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:17:00.706 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:17:00.706 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:17:00.706 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:17:00.706 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:17:00.707 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:17:00.707 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:17:00.707 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:17:00.707 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:00.707 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:17:00.707 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:17:00.707 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:17:00.707 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:17:00.707 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:17:00.707 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:17:00.707 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:17:00.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:17:00.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:17:00.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:17:00.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:17:00.778 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:17:00.779 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:17:00.779 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:17:00.779 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:17:00.779 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:17:00.780 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:17:00.780 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:17:00.780 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:17:00.780 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:17:00.780 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:17:00.780 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:17:00.780 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:17:00.780 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:17:00.781 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:00.781 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:17:00.781 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:00.809 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 52 2026-02-18T08:17:00.809 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 58 2026-02-18T08:17:00.810 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:17:00.810 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=233, tag_tid=15, entry_tid=1], [object_number=232, tag_tid=15, entry_tid=0], [object_number=234, tag_tid=14, entry_tid=52758], [object_number=235, tag_tid=14, entry_tid=52755]]], state=connected] 2026-02-18T08:17:00.810 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=211, tag_tid=14, entry_tid=27367], [object_number=210, tag_tid=14, entry_tid=27366], [object_number=209, tag_tid=14, entry_tid=27365], [object_number=208, tag_tid=14, entry_tid=27364]]], state=connected] 2026-02-18T08:17:00.812 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:00.813 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=233, tag_tid=15, entry_tid=1' 2026-02-18T08:17:00.815 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:17:00.815 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:17:00.815 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:17:00.815 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:17:00.815 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:17:00.815 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:17:00.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:17:00.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:17:00.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:17:00.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:17:00.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:17:00.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:17:00.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:00.817 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:17:00.817 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:00.844 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 52 2026-02-18T08:17:00.844 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 58 2026-02-18T08:17:00.844 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:17:00.844 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=233, tag_tid=15, entry_tid=1], [object_number=232, tag_tid=15, entry_tid=0], [object_number=234, tag_tid=14, entry_tid=52758], [object_number=235, tag_tid=14, entry_tid=52755]]], state=connected] 2026-02-18T08:17:00.844 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=211, tag_tid=14, entry_tid=27367], [object_number=210, tag_tid=14, entry_tid=27366], [object_number=209, tag_tid=14, entry_tid=27365], [object_number=208, tag_tid=14, entry_tid=27364]]], state=connected] 2026-02-18T08:17:00.846 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:00.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=211, tag_tid=14, entry_tid=27367' 2026-02-18T08:17:00.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=233, tag_tid=15, entry_tid=1' -a 'object_number=233, tag_tid=15, entry_tid=1' = 'object_number=211, tag_tid=14, entry_tid=27367' 2026-02-18T08:17:00.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=211, tag_tid=14, entry_tid=27367' '!=' 'object_number=209, tag_tid=14, entry_tid=26977' 2026-02-18T08:17:00.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:17:00.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=211, tag_tid=14, entry_tid=27367' = 'object_number=209, tag_tid=14, entry_tid=26977' 2026-02-18T08:17:00.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=211, tag_tid=14, entry_tid=27367' 2026-02-18T08:17:00.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=233, tag_tid=15, entry_tid=1' 2026-02-18T08:17:00.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:17:00.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:17:00.850 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=15 2026-02-18T08:17:00.851 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=211, tag_tid=14, entry_tid=27367' 2026-02-18T08:17:00.851 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:17:00.851 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:17:00.852 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=14 2026-02-18T08:17:00.853 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=233, tag_tid=15, entry_tid=1' 2026-02-18T08:17:00.853 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:17:00.853 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:17:00.854 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:17:00.855 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=211, tag_tid=14, entry_tid=27367' 2026-02-18T08:17:00.855 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:17:00.855 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:17:00.856 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=27367 2026-02-18T08:17:00.856 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 15 = 14 -a 1 -le 27367 2026-02-18T08:17:00.856 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:17:00.857 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:17:00.857 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:17:01.058 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:17:01.058 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:17:01.059 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:17:01.059 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:17:01.059 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:17:01.059 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:17:01.059 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:17:01.059 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:17:01.059 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:17:01.059 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:01.059 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:17:01.060 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:17:01.060 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:17:01.060 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:17:01.060 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:17:01.060 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:17:01.062 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:17:01.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:17:01.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:17:01.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:17:01.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:01.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:17:01.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:17:01.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:17:01.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:17:01.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:17:01.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:17:01.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:17:01.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:17:01.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:17:01.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:01.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:17:01.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:17:01.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:17:01.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:17:01.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:17:01.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:17:01.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:17:01.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:17:01.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:17:01.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:17:01.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:01.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:17:01.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:17:01.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:17:01.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:17:01.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:17:01.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:17:01.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:17:01.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:17:01.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:17:01.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:17:01.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:17:01.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:17:01.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:17:01.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:17:01.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:17:01.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:17:01.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:17:01.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:01.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:17:01.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:17:01.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:17:01.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:17:01.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:17:01.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:17:01.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:17:01.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:17:01.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:17:01.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:01.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:17:01.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:17:01.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:17:01.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:17:01.129 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:17:01.129 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:17:01.129 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:17:01.129 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:17:01.129 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:17:01.129 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:17:01.129 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:01.129 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:17:01.129 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:17:01.129 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:17:01.129 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:17:01.129 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:17:01.130 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:17:01.130 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:17:01.138 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:17:01.138 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:17:01.139 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:17:01.139 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:17:01.187 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:17:01.188 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:17:01.188 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:17:01.190 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:17:01.190 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:17:01.190 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:17:01.190 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:01.190 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:17:01.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:17:01.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:17:01.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:17:01.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:17:01.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:17:01.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:17:01.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:17:01.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:17:01.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:01.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:17:01.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:17:01.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:17:01.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:17:01.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:17:01.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:17:01.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:17:01.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:17:01.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:17:01.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:17:01.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:01.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:17:01.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:17:01.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:17:01.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:17:01.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:17:01.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:17:01.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:17:01.202 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:17:01.202 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:17:01.202 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:17:01.202 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:17:01.249 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:17:01.250 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:17:01.250 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:17:01.252 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:17:01.252 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:17:01.252 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:17:01.252 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:01.252 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:17:01.252 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:17:01.252 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:17:01.252 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:17:01.253 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:17:01.253 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:17:01.253 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:17:01.253 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:17:01.253 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:17:01.253 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:01.253 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:17:01.253 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:17:01.253 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:17:01.253 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:17:01.253 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:17:01.253 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:17:01.253 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:17:01.253 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:17:01.253 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:17:01.254 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:17:01.254 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:01.254 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:17:01.254 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:17:01.254 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:17:01.254 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:17:01.254 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:17:01.254 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:17:01.254 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:17:01.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:17:01.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:17:01.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:17:01.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:17:01.323 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:17:01.323 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:17:01.324 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:17:01.324 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:17:01.324 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:17:01.324 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:17:01.324 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:17:01.324 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:17:01.324 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:17:01.324 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:17:01.324 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:17:01.325 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:17:01.325 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:17:01.325 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:01.326 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:17:01.326 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:01.363 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 52 2026-02-18T08:17:01.364 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 58 2026-02-18T08:17:01.364 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:17:01.364 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=233, tag_tid=15, entry_tid=1], [object_number=232, tag_tid=15, entry_tid=0], [object_number=234, tag_tid=14, entry_tid=52758], [object_number=235, tag_tid=14, entry_tid=52755]]], state=connected] 2026-02-18T08:17:01.364 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=211, tag_tid=14, entry_tid=27715], [object_number=210, tag_tid=14, entry_tid=27714], [object_number=209, tag_tid=14, entry_tid=27713], [object_number=208, tag_tid=14, entry_tid=27712]]], state=connected] 2026-02-18T08:17:01.366 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:01.367 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=233, tag_tid=15, entry_tid=1' 2026-02-18T08:17:01.368 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:17:01.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:17:01.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:17:01.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:17:01.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:17:01.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:17:01.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:17:01.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:17:01.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:17:01.370 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:17:01.370 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:17:01.370 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:17:01.371 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:01.371 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:17:01.371 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:01.396 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 52 2026-02-18T08:17:01.397 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 58 2026-02-18T08:17:01.397 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:17:01.397 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=233, tag_tid=15, entry_tid=1], [object_number=232, tag_tid=15, entry_tid=0], [object_number=234, tag_tid=14, entry_tid=52758], [object_number=235, tag_tid=14, entry_tid=52755]]], state=connected] 2026-02-18T08:17:01.397 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=211, tag_tid=14, entry_tid=27715], [object_number=210, tag_tid=14, entry_tid=27714], [object_number=209, tag_tid=14, entry_tid=27713], [object_number=208, tag_tid=14, entry_tid=27712]]], state=connected] 2026-02-18T08:17:01.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:01.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=211, tag_tid=14, entry_tid=27715' 2026-02-18T08:17:01.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=233, tag_tid=15, entry_tid=1' -a 'object_number=233, tag_tid=15, entry_tid=1' = 'object_number=211, tag_tid=14, entry_tid=27715' 2026-02-18T08:17:01.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=211, tag_tid=14, entry_tid=27715' '!=' 'object_number=211, tag_tid=14, entry_tid=27367' 2026-02-18T08:17:01.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:17:01.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=211, tag_tid=14, entry_tid=27715' = 'object_number=211, tag_tid=14, entry_tid=27367' 2026-02-18T08:17:01.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=211, tag_tid=14, entry_tid=27715' 2026-02-18T08:17:01.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=233, tag_tid=15, entry_tid=1' 2026-02-18T08:17:01.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:17:01.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:17:01.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=15 2026-02-18T08:17:01.403 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=211, tag_tid=14, entry_tid=27715' 2026-02-18T08:17:01.403 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:17:01.403 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:17:01.405 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=14 2026-02-18T08:17:01.406 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=233, tag_tid=15, entry_tid=1' 2026-02-18T08:17:01.406 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:17:01.406 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:17:01.407 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:17:01.408 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=211, tag_tid=14, entry_tid=27715' 2026-02-18T08:17:01.408 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:17:01.408 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:17:01.409 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=27715 2026-02-18T08:17:01.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 15 = 14 -a 1 -le 27715 2026-02-18T08:17:01.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:17:01.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:17:01.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:17:01.611 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:17:01.611 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:17:01.612 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:17:01.612 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:17:01.612 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:17:01.612 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:17:01.612 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:17:01.612 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:17:01.612 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:17:01.613 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:01.613 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:17:01.613 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:17:01.613 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:17:01.613 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:17:01.613 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:17:01.613 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:17:01.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:17:01.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:17:01.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:17:01.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:17:01.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:01.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:17:01.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:17:01.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:17:01.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:17:01.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:17:01.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:17:01.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:17:01.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:17:01.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:17:01.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:01.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:17:01.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:17:01.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:17:01.618 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:17:01.618 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:17:01.618 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:17:01.618 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:17:01.618 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:17:01.618 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:17:01.618 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:17:01.618 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:01.618 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:17:01.618 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:17:01.618 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:17:01.618 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:17:01.618 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:17:01.619 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:17:01.619 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:17:01.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:17:01.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:17:01.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:17:01.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:17:01.676 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:17:01.676 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:17:01.676 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:17:01.678 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:17:01.679 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:17:01.679 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:17:01.679 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:01.679 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:17:01.679 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:17:01.679 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:17:01.679 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:17:01.679 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:17:01.679 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:17:01.680 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:17:01.680 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:17:01.680 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:17:01.680 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:01.680 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:17:01.680 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:17:01.680 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:17:01.680 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:17:01.680 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:17:01.680 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:17:01.680 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:17:01.680 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:17:01.681 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:17:01.681 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:17:01.681 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:01.681 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:17:01.681 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:17:01.681 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:17:01.681 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:17:01.681 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:17:01.681 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:17:01.681 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:17:01.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:17:01.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:17:01.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:17:01.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:17:01.738 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:17:01.738 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:17:01.739 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:17:01.741 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:17:01.741 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:17:01.742 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:17:01.742 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:01.742 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:17:01.742 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:17:01.742 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:17:01.742 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:17:01.742 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:17:01.742 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:17:01.742 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:17:01.742 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:17:01.742 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:17:01.743 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:01.743 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:17:01.743 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:17:01.743 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:17:01.743 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:17:01.743 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:17:01.743 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:17:01.743 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:17:01.743 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:17:01.743 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:17:01.743 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:17:01.743 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:01.744 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:17:01.744 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:17:01.744 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:17:01.744 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:17:01.744 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:17:01.744 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:17:01.744 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:17:01.752 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:17:01.753 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:17:01.753 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:17:01.753 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:17:01.812 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:17:01.812 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:17:01.812 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:17:01.815 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:17:01.815 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:17:01.815 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:17:01.815 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:01.815 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:17:01.815 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:17:01.815 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:17:01.815 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:17:01.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:17:01.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:17:01.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:17:01.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:17:01.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:17:01.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:01.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:17:01.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:17:01.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:17:01.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:17:01.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:17:01.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:17:01.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:17:01.817 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:17:01.817 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:17:01.817 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:17:01.817 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:01.817 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:17:01.817 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:17:01.817 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:17:01.817 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:17:01.817 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:17:01.817 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:17:01.818 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:17:01.826 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:17:01.826 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:17:01.826 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:17:01.826 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:17:01.895 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:17:01.895 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:17:01.895 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:17:01.895 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:17:01.895 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:17:01.895 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:17:01.895 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:17:01.896 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:17:01.896 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:17:01.896 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:17:01.896 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:17:01.896 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:17:01.897 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:17:01.897 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:01.897 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:17:01.897 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:01.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 52 2026-02-18T08:17:01.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 58 2026-02-18T08:17:01.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:17:01.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=233, tag_tid=15, entry_tid=1], [object_number=232, tag_tid=15, entry_tid=0], [object_number=234, tag_tid=14, entry_tid=52758], [object_number=235, tag_tid=14, entry_tid=52755]]], state=connected] 2026-02-18T08:17:01.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=210, tag_tid=14, entry_tid=28034], [object_number=209, tag_tid=14, entry_tid=28033], [object_number=208, tag_tid=14, entry_tid=28032], [object_number=211, tag_tid=14, entry_tid=28031]]], state=connected] 2026-02-18T08:17:01.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:01.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=233, tag_tid=15, entry_tid=1' 2026-02-18T08:17:01.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:17:01.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:17:01.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:17:01.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:17:01.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:17:01.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:17:01.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:17:01.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:17:01.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:17:01.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:17:01.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:17:01.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:17:01.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:01.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:17:01.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:01.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 52 2026-02-18T08:17:01.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 58 2026-02-18T08:17:01.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:17:01.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=233, tag_tid=15, entry_tid=1], [object_number=232, tag_tid=15, entry_tid=0], [object_number=234, tag_tid=14, entry_tid=52758], [object_number=235, tag_tid=14, entry_tid=52755]]], state=connected] 2026-02-18T08:17:01.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=210, tag_tid=14, entry_tid=28034], [object_number=209, tag_tid=14, entry_tid=28033], [object_number=208, tag_tid=14, entry_tid=28032], [object_number=211, tag_tid=14, entry_tid=28031]]], state=connected] 2026-02-18T08:17:01.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:01.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=210, tag_tid=14, entry_tid=28034' 2026-02-18T08:17:01.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=233, tag_tid=15, entry_tid=1' -a 'object_number=233, tag_tid=15, entry_tid=1' = 'object_number=210, tag_tid=14, entry_tid=28034' 2026-02-18T08:17:01.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=210, tag_tid=14, entry_tid=28034' '!=' 'object_number=211, tag_tid=14, entry_tid=27715' 2026-02-18T08:17:01.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:17:01.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=210, tag_tid=14, entry_tid=28034' = 'object_number=211, tag_tid=14, entry_tid=27715' 2026-02-18T08:17:01.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=210, tag_tid=14, entry_tid=28034' 2026-02-18T08:17:01.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=233, tag_tid=15, entry_tid=1' 2026-02-18T08:17:01.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:17:01.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:17:01.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=15 2026-02-18T08:17:01.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=210, tag_tid=14, entry_tid=28034' 2026-02-18T08:17:01.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:17:01.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:17:01.982 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=14 2026-02-18T08:17:01.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=233, tag_tid=15, entry_tid=1' 2026-02-18T08:17:01.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:17:01.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:17:01.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:17:01.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=210, tag_tid=14, entry_tid=28034' 2026-02-18T08:17:01.985 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:17:01.985 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:17:01.987 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=28034 2026-02-18T08:17:01.987 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 15 = 14 -a 1 -le 28034 2026-02-18T08:17:01.987 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:17:01.987 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:17:01.987 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:17:02.189 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:17:02.189 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:17:02.189 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:17:02.189 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:17:02.189 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:17:02.189 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:17:02.190 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:17:02.190 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:17:02.190 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:17:02.190 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:02.190 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:17:02.190 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:17:02.190 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:17:02.190 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:17:02.190 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:17:02.190 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:17:02.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:17:02.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:17:02.194 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:17:02.194 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:17:02.194 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:02.194 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:17:02.194 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:17:02.194 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:17:02.194 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:17:02.194 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:17:02.194 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:17:02.194 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:17:02.195 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:17:02.195 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:17:02.195 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:02.195 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:17:02.195 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:17:02.195 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:17:02.195 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:17:02.195 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:17:02.195 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:17:02.196 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:17:02.196 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:17:02.196 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:17:02.196 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:17:02.196 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:02.196 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:17:02.196 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:17:02.196 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:17:02.196 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:17:02.196 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:17:02.197 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:17:02.197 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:17:02.202 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:17:02.202 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:17:02.202 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:17:02.202 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:17:02.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:17:02.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:17:02.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:17:02.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:17:02.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:17:02.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:17:02.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:02.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:17:02.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:17:02.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:17:02.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:17:02.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:17:02.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:17:02.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:17:02.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:17:02.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:17:02.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:02.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:17:02.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:17:02.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:17:02.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:17:02.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:17:02.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:17:02.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:17:02.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:17:02.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:17:02.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:17:02.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:02.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:17:02.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:17:02.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:17:02.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:17:02.266 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:17:02.266 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:17:02.266 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:17:02.276 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:17:02.276 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:17:02.276 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:17:02.276 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:17:02.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:17:02.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:17:02.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:17:02.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:17:02.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:17:02.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:17:02.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:02.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:17:02.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:17:02.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:17:02.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:17:02.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:17:02.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:17:02.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:17:02.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:17:02.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:17:02.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:02.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:17:02.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:17:02.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:17:02.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:17:02.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:17:02.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:17:02.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:17:02.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:17:02.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:17:02.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:17:02.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:02.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:17:02.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:17:02.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:17:02.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:17:02.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:17:02.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:17:02.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:17:02.348 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:17:02.348 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:17:02.348 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:17:02.348 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:17:02.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:17:02.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:17:02.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:17:02.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:17:02.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:17:02.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:17:02.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:02.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:17:02.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:17:02.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:17:02.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:17:02.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:17:02.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:17:02.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:17:02.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:17:02.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:17:02.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:02.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:17:02.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:17:02.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:17:02.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:17:02.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:17:02.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:17:02.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:17:02.403 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:17:02.403 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:17:02.403 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:17:02.403 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:02.403 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:17:02.403 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:17:02.403 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:17:02.403 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:17:02.403 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:17:02.403 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:17:02.403 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:17:02.412 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:17:02.412 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:17:02.413 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:17:02.413 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:17:02.480 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:17:02.481 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:17:02.481 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:17:02.481 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:17:02.481 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:17:02.481 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:17:02.481 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:17:02.481 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:17:02.481 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:17:02.481 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:17:02.481 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:17:02.482 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:17:02.482 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:17:02.482 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:02.483 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:17:02.483 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:02.511 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 52 2026-02-18T08:17:02.512 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 58 2026-02-18T08:17:02.512 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:17:02.512 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=233, tag_tid=15, entry_tid=1], [object_number=232, tag_tid=15, entry_tid=0], [object_number=234, tag_tid=14, entry_tid=52758], [object_number=235, tag_tid=14, entry_tid=52755]]], state=connected] 2026-02-18T08:17:02.512 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=209, tag_tid=14, entry_tid=28401], [object_number=208, tag_tid=14, entry_tid=28400], [object_number=211, tag_tid=14, entry_tid=28399], [object_number=210, tag_tid=14, entry_tid=28398]]], state=connected] 2026-02-18T08:17:02.514 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:02.516 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=233, tag_tid=15, entry_tid=1' 2026-02-18T08:17:02.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:17:02.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:17:02.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:17:02.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:17:02.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:17:02.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:17:02.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:17:02.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:17:02.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:17:02.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:17:02.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:17:02.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:17:02.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:02.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:17:02.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:02.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 52 2026-02-18T08:17:02.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 58 2026-02-18T08:17:02.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:17:02.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=233, tag_tid=15, entry_tid=1], [object_number=232, tag_tid=15, entry_tid=0], [object_number=234, tag_tid=14, entry_tid=52758], [object_number=235, tag_tid=14, entry_tid=52755]]], state=connected] 2026-02-18T08:17:02.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=209, tag_tid=14, entry_tid=28401], [object_number=208, tag_tid=14, entry_tid=28400], [object_number=211, tag_tid=14, entry_tid=28399], [object_number=210, tag_tid=14, entry_tid=28398]]], state=connected] 2026-02-18T08:17:02.551 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:02.553 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=209, tag_tid=14, entry_tid=28401' 2026-02-18T08:17:02.553 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=233, tag_tid=15, entry_tid=1' -a 'object_number=233, tag_tid=15, entry_tid=1' = 'object_number=209, tag_tid=14, entry_tid=28401' 2026-02-18T08:17:02.553 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=209, tag_tid=14, entry_tid=28401' '!=' 'object_number=210, tag_tid=14, entry_tid=28034' 2026-02-18T08:17:02.553 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:17:02.553 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=209, tag_tid=14, entry_tid=28401' = 'object_number=210, tag_tid=14, entry_tid=28034' 2026-02-18T08:17:02.553 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=209, tag_tid=14, entry_tid=28401' 2026-02-18T08:17:02.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=233, tag_tid=15, entry_tid=1' 2026-02-18T08:17:02.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:17:02.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:17:02.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=15 2026-02-18T08:17:02.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=209, tag_tid=14, entry_tid=28401' 2026-02-18T08:17:02.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:17:02.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:17:02.557 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=14 2026-02-18T08:17:02.558 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=233, tag_tid=15, entry_tid=1' 2026-02-18T08:17:02.558 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:17:02.558 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:17:02.559 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:17:02.560 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=209, tag_tid=14, entry_tid=28401' 2026-02-18T08:17:02.560 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:17:02.560 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:17:02.562 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=28401 2026-02-18T08:17:02.562 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 15 = 14 -a 1 -le 28401 2026-02-18T08:17:02.562 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:17:02.562 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:17:02.562 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:17:02.763 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:17:02.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:17:02.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:17:02.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:17:02.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:17:02.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:17:02.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:17:02.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:17:02.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:17:02.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:02.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:17:02.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:17:02.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:17:02.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:17:02.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:17:02.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:17:02.768 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:17:02.768 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:17:02.768 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:17:02.768 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:17:02.768 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:02.768 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:17:02.768 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:17:02.768 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:17:02.768 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:17:02.769 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:17:02.769 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:17:02.769 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:17:02.769 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:17:02.769 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:17:02.769 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:02.769 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:17:02.769 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:17:02.769 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:17:02.769 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:17:02.769 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:17:02.769 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:17:02.770 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:17:02.770 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:17:02.770 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:17:02.770 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:17:02.770 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:02.770 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:17:02.770 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:17:02.770 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:17:02.770 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:17:02.770 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:17:02.770 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:17:02.770 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:17:02.778 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:17:02.778 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:17:02.778 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:17:02.778 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:17:02.826 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:17:02.826 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:17:02.827 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:17:02.830 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:17:02.830 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:17:02.830 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:17:02.830 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:02.830 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:17:02.830 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:17:02.830 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:17:02.830 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:17:02.830 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:17:02.830 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:17:02.830 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:17:02.831 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:17:02.831 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:17:02.831 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:02.831 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:17:02.831 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:17:02.831 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:17:02.831 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:17:02.831 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:17:02.831 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:17:02.831 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:17:02.831 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:17:02.832 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:17:02.832 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:17:02.832 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:02.832 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:17:02.832 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:17:02.832 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:17:02.832 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:17:02.832 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:17:02.832 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:17:02.832 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:17:02.840 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:17:02.841 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:17:02.841 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:17:02.841 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:17:02.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:17:02.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:17:02.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:17:02.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:17:02.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:17:02.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:17:02.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:02.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:17:02.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:17:02.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:17:02.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:17:02.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:17:02.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:17:02.894 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:17:02.894 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:17:02.894 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:17:02.894 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:02.894 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:17:02.894 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:17:02.894 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:17:02.894 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:17:02.894 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:17:02.894 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:17:02.894 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:17:02.894 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:17:02.895 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:17:02.895 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:17:02.895 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:02.895 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:17:02.895 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:17:02.895 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:17:02.895 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:17:02.895 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:17:02.895 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:17:02.895 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:17:02.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:17:02.904 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:17:02.904 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:17:02.904 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:17:02.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:17:02.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:17:02.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:17:02.954 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:17:02.954 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:17:02.954 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:17:02.955 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:02.955 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:17:02.955 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:17:02.955 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:17:02.955 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:17:02.955 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:17:02.955 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:17:02.955 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:17:02.955 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:17:02.955 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:17:02.955 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:02.955 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:17:02.956 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:17:02.956 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:17:02.956 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:17:02.956 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:17:02.956 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:17:02.956 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:17:02.956 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:17:02.956 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:17:02.956 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:17:02.956 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:02.956 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:17:02.956 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:17:02.956 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:17:02.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:17:02.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:17:02.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:17:02.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:17:02.964 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:17:02.965 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:17:02.965 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:17:02.965 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:17:03.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:17:03.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:17:03.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:17:03.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:17:03.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:17:03.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:17:03.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:17:03.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:17:03.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:17:03.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:17:03.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:17:03.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:17:03.035 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:17:03.035 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:03.036 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:17:03.036 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:03.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 52 2026-02-18T08:17:03.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 58 2026-02-18T08:17:03.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:17:03.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=233, tag_tid=15, entry_tid=1], [object_number=232, tag_tid=15, entry_tid=0], [object_number=234, tag_tid=14, entry_tid=52758], [object_number=235, tag_tid=14, entry_tid=52755]]], state=connected] 2026-02-18T08:17:03.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=211, tag_tid=14, entry_tid=28751], [object_number=210, tag_tid=14, entry_tid=28750], [object_number=209, tag_tid=14, entry_tid=28749], [object_number=208, tag_tid=14, entry_tid=28748]]], state=connected] 2026-02-18T08:17:03.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:03.068 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=233, tag_tid=15, entry_tid=1' 2026-02-18T08:17:03.069 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:17:03.069 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:17:03.069 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:17:03.069 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:17:03.069 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:17:03.069 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:17:03.069 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:17:03.069 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:17:03.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:17:03.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:17:03.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:17:03.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:17:03.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:03.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:17:03.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:03.100 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 52 2026-02-18T08:17:03.100 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 58 2026-02-18T08:17:03.100 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:17:03.100 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=233, tag_tid=15, entry_tid=1], [object_number=232, tag_tid=15, entry_tid=0], [object_number=234, tag_tid=14, entry_tid=52758], [object_number=235, tag_tid=14, entry_tid=52755]]], state=connected] 2026-02-18T08:17:03.101 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=211, tag_tid=14, entry_tid=28751], [object_number=210, tag_tid=14, entry_tid=28750], [object_number=209, tag_tid=14, entry_tid=28749], [object_number=208, tag_tid=14, entry_tid=28748]]], state=connected] 2026-02-18T08:17:03.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:03.105 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=211, tag_tid=14, entry_tid=28751' 2026-02-18T08:17:03.105 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=233, tag_tid=15, entry_tid=1' -a 'object_number=233, tag_tid=15, entry_tid=1' = 'object_number=211, tag_tid=14, entry_tid=28751' 2026-02-18T08:17:03.105 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=211, tag_tid=14, entry_tid=28751' '!=' 'object_number=209, tag_tid=14, entry_tid=28401' 2026-02-18T08:17:03.105 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:17:03.105 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=211, tag_tid=14, entry_tid=28751' = 'object_number=209, tag_tid=14, entry_tid=28401' 2026-02-18T08:17:03.105 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=211, tag_tid=14, entry_tid=28751' 2026-02-18T08:17:03.106 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=233, tag_tid=15, entry_tid=1' 2026-02-18T08:17:03.106 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:17:03.106 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:17:03.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=15 2026-02-18T08:17:03.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=211, tag_tid=14, entry_tid=28751' 2026-02-18T08:17:03.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:17:03.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:17:03.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=14 2026-02-18T08:17:03.110 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=233, tag_tid=15, entry_tid=1' 2026-02-18T08:17:03.110 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:17:03.110 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:17:03.111 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:17:03.112 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=211, tag_tid=14, entry_tid=28751' 2026-02-18T08:17:03.112 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:17:03.113 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:17:03.114 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=28751 2026-02-18T08:17:03.114 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 15 = 14 -a 1 -le 28751 2026-02-18T08:17:03.114 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:17:03.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:17:03.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:17:03.316 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:17:03.316 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:17:03.316 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:17:03.317 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:17:03.317 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:17:03.317 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:17:03.317 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:17:03.317 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:17:03.317 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:17:03.317 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:03.317 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:17:03.317 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:17:03.317 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:17:03.318 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:17:03.318 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:17:03.318 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:17:03.321 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:17:03.321 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:17:03.321 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:17:03.321 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:17:03.321 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:03.321 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:17:03.321 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:17:03.321 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:17:03.321 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:17:03.321 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:17:03.322 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:17:03.322 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:17:03.322 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:17:03.322 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:17:03.322 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:03.322 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:17:03.322 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:17:03.322 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:17:03.322 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:17:03.322 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:17:03.322 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:17:03.322 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:17:03.323 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:17:03.323 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:17:03.323 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:17:03.323 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:03.323 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:17:03.323 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:17:03.323 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:17:03.323 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:17:03.323 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:17:03.323 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:17:03.323 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:17:03.329 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:17:03.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:17:03.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:17:03.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:17:03.389 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:17:03.390 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:17:03.390 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:17:03.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:17:03.393 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:17:03.393 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:17:03.393 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:03.393 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:17:03.393 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:17:03.393 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:17:03.393 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:17:03.393 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:17:03.393 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:17:03.393 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:17:03.394 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:17:03.394 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:17:03.394 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:03.394 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:17:03.394 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:17:03.394 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:17:03.394 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:17:03.394 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:17:03.394 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:17:03.394 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:17:03.394 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:17:03.394 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:17:03.394 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:17:03.395 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:03.395 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:17:03.395 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:17:03.395 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:17:03.395 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:17:03.395 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:17:03.395 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:17:03.395 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:17:03.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:17:03.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:17:03.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:17:03.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:17:03.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:17:03.462 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:17:03.462 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:17:03.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:17:03.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:17:03.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:17:03.465 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:03.465 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:17:03.465 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:17:03.465 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:17:03.465 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:17:03.465 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:17:03.465 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:17:03.465 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:17:03.465 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:17:03.465 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:17:03.465 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:03.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:17:03.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:17:03.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:17:03.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:17:03.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:17:03.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:17:03.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:17:03.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:17:03.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:17:03.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:17:03.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:03.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:17:03.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:17:03.467 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:17:03.467 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:17:03.467 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:17:03.467 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:17:03.467 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:17:03.476 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:17:03.476 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:17:03.476 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:17:03.477 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:17:03.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:17:03.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:17:03.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:17:03.536 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:17:03.536 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:17:03.536 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:17:03.536 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:03.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:17:03.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:17:03.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:17:03.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:17:03.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:17:03.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:17:03.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:17:03.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:17:03.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:17:03.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:03.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:17:03.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:17:03.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:17:03.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:17:03.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:17:03.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:17:03.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:17:03.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:17:03.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:17:03.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:17:03.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:03.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:17:03.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:17:03.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:17:03.539 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:17:03.539 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:17:03.539 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:17:03.539 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:17:03.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:17:03.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:17:03.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:17:03.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:17:03.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:17:03.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:17:03.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:17:03.610 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:17:03.610 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:17:03.610 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:17:03.610 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:17:03.610 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:17:03.610 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:17:03.610 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:17:03.610 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:17:03.610 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:17:03.610 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:17:03.611 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:03.611 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:17:03.612 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:03.640 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 52 2026-02-18T08:17:03.640 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 58 2026-02-18T08:17:03.640 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:17:03.640 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=233, tag_tid=15, entry_tid=1], [object_number=232, tag_tid=15, entry_tid=0], [object_number=234, tag_tid=14, entry_tid=52758], [object_number=235, tag_tid=14, entry_tid=52755]]], state=connected] 2026-02-18T08:17:03.640 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=211, tag_tid=14, entry_tid=29131], [object_number=210, tag_tid=14, entry_tid=29130], [object_number=209, tag_tid=14, entry_tid=29129], [object_number=208, tag_tid=14, entry_tid=29128]]], state=connected] 2026-02-18T08:17:03.643 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:03.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=233, tag_tid=15, entry_tid=1' 2026-02-18T08:17:03.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:17:03.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:17:03.646 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:17:03.646 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:17:03.646 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:17:03.646 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:17:03.646 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:17:03.646 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:17:03.646 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:17:03.646 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:17:03.647 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:17:03.647 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:17:03.647 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:03.648 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:17:03.648 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:03.676 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 52 2026-02-18T08:17:03.676 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 58 2026-02-18T08:17:03.676 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:17:03.676 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=233, tag_tid=15, entry_tid=1], [object_number=232, tag_tid=15, entry_tid=0], [object_number=234, tag_tid=14, entry_tid=52758], [object_number=235, tag_tid=14, entry_tid=52755]]], state=connected] 2026-02-18T08:17:03.676 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=211, tag_tid=14, entry_tid=29131], [object_number=210, tag_tid=14, entry_tid=29130], [object_number=209, tag_tid=14, entry_tid=29129], [object_number=208, tag_tid=14, entry_tid=29128]]], state=connected] 2026-02-18T08:17:03.678 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:03.680 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=211, tag_tid=14, entry_tid=29131' 2026-02-18T08:17:03.681 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=233, tag_tid=15, entry_tid=1' -a 'object_number=233, tag_tid=15, entry_tid=1' = 'object_number=211, tag_tid=14, entry_tid=29131' 2026-02-18T08:17:03.681 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=211, tag_tid=14, entry_tid=29131' '!=' 'object_number=211, tag_tid=14, entry_tid=28751' 2026-02-18T08:17:03.681 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:17:03.681 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=211, tag_tid=14, entry_tid=29131' = 'object_number=211, tag_tid=14, entry_tid=28751' 2026-02-18T08:17:03.681 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=211, tag_tid=14, entry_tid=29131' 2026-02-18T08:17:03.682 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=233, tag_tid=15, entry_tid=1' 2026-02-18T08:17:03.682 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:17:03.682 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:17:03.683 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=15 2026-02-18T08:17:03.684 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=211, tag_tid=14, entry_tid=29131' 2026-02-18T08:17:03.684 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:17:03.684 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:17:03.685 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=14 2026-02-18T08:17:03.686 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=233, tag_tid=15, entry_tid=1' 2026-02-18T08:17:03.686 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:17:03.686 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:17:03.687 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:17:03.688 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=211, tag_tid=14, entry_tid=29131' 2026-02-18T08:17:03.688 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:17:03.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:17:03.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=29131 2026-02-18T08:17:03.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 15 = 14 -a 1 -le 29131 2026-02-18T08:17:03.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:17:03.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:17:03.691 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:17:03.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:17:03.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:17:03.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:17:03.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:17:03.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:17:03.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:17:03.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:17:03.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:17:03.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:17:03.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:03.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:17:03.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:17:03.894 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:17:03.894 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:17:03.894 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:17:03.894 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:17:03.896 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:17:03.896 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:17:03.897 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:17:03.897 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:17:03.897 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:03.897 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:17:03.897 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:17:03.897 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:17:03.897 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:17:03.897 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:17:03.897 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:17:03.897 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:17:03.897 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:17:03.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:17:03.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:03.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:17:03.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:17:03.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:17:03.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:17:03.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:17:03.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:17:03.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:17:03.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:17:03.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:17:03.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:17:03.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:03.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:17:03.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:17:03.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:17:03.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:17:03.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:17:03.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:17:03.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:17:03.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:17:03.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:17:03.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:17:03.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:17:03.954 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:17:03.955 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:17:03.955 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:17:03.958 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:17:03.958 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:17:03.958 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:17:03.958 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:03.958 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:17:03.958 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:17:03.958 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:17:03.958 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:17:03.959 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:17:03.959 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:17:03.959 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:17:03.959 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:17:03.959 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:17:03.959 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:03.959 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:17:03.959 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:17:03.959 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:17:03.959 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:17:03.959 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:17:03.959 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:17:03.959 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:17:03.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:17:03.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:17:03.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:17:03.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:03.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:17:03.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:17:03.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:17:03.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:17:03.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:17:03.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:17:03.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:17:03.969 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:17:03.969 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:17:03.969 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:17:03.969 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:17:04.018 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:17:04.018 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:17:04.018 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:17:04.021 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:17:04.021 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:17:04.021 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:17:04.021 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:04.021 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:17:04.021 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:17:04.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:17:04.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:17:04.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:17:04.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:17:04.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:17:04.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:17:04.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:17:04.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:04.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:17:04.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:17:04.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:17:04.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:17:04.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:17:04.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:17:04.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:17:04.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:17:04.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:17:04.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:17:04.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:04.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:17:04.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:17:04.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:17:04.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:17:04.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:17:04.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:17:04.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:17:04.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:17:04.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:17:04.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:17:04.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:17:04.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:17:04.080 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:17:04.080 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:17:04.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:17:04.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:17:04.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:17:04.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:04.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:17:04.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:17:04.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:17:04.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:17:04.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:17:04.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:17:04.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:17:04.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:17:04.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:17:04.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:04.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:17:04.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:17:04.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:17:04.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:17:04.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:17:04.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:17:04.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:17:04.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:17:04.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:17:04.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:17:04.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:04.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:17:04.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:17:04.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:17:04.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:17:04.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:17:04.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:17:04.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:17:04.092 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:17:04.093 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:17:04.093 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:17:04.093 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:17:04.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:17:04.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:17:04.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:17:04.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:17:04.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:17:04.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:17:04.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:17:04.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:17:04.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:17:04.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:17:04.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:17:04.155 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:17:04.155 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:17:04.155 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:04.156 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:17:04.156 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:04.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 52 2026-02-18T08:17:04.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 58 2026-02-18T08:17:04.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:17:04.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=233, tag_tid=15, entry_tid=1], [object_number=232, tag_tid=15, entry_tid=0], [object_number=234, tag_tid=14, entry_tid=52758], [object_number=235, tag_tid=14, entry_tid=52755]]], state=connected] 2026-02-18T08:17:04.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=209, tag_tid=14, entry_tid=29501], [object_number=208, tag_tid=14, entry_tid=29500], [object_number=211, tag_tid=14, entry_tid=29499], [object_number=210, tag_tid=14, entry_tid=29498]]], state=connected] 2026-02-18T08:17:04.188 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:04.189 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=233, tag_tid=15, entry_tid=1' 2026-02-18T08:17:04.190 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:17:04.190 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:17:04.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:17:04.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:17:04.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:17:04.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:17:04.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:17:04.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:17:04.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:17:04.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:17:04.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:17:04.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:17:04.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:04.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:17:04.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:04.220 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 52 2026-02-18T08:17:04.220 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 58 2026-02-18T08:17:04.220 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:17:04.220 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=233, tag_tid=15, entry_tid=1], [object_number=232, tag_tid=15, entry_tid=0], [object_number=234, tag_tid=14, entry_tid=52758], [object_number=235, tag_tid=14, entry_tid=52755]]], state=connected] 2026-02-18T08:17:04.220 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=209, tag_tid=14, entry_tid=29501], [object_number=208, tag_tid=14, entry_tid=29500], [object_number=211, tag_tid=14, entry_tid=29499], [object_number=210, tag_tid=14, entry_tid=29498]]], state=connected] 2026-02-18T08:17:04.222 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:04.224 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=209, tag_tid=14, entry_tid=29501' 2026-02-18T08:17:04.224 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=233, tag_tid=15, entry_tid=1' -a 'object_number=233, tag_tid=15, entry_tid=1' = 'object_number=209, tag_tid=14, entry_tid=29501' 2026-02-18T08:17:04.224 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=209, tag_tid=14, entry_tid=29501' '!=' 'object_number=211, tag_tid=14, entry_tid=29131' 2026-02-18T08:17:04.225 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:17:04.225 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=209, tag_tid=14, entry_tid=29501' = 'object_number=211, tag_tid=14, entry_tid=29131' 2026-02-18T08:17:04.225 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=209, tag_tid=14, entry_tid=29501' 2026-02-18T08:17:04.225 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=233, tag_tid=15, entry_tid=1' 2026-02-18T08:17:04.225 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:17:04.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:17:04.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=15 2026-02-18T08:17:04.227 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=209, tag_tid=14, entry_tid=29501' 2026-02-18T08:17:04.227 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:17:04.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:17:04.229 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=14 2026-02-18T08:17:04.230 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=233, tag_tid=15, entry_tid=1' 2026-02-18T08:17:04.230 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:17:04.230 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:17:04.231 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:17:04.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=209, tag_tid=14, entry_tid=29501' 2026-02-18T08:17:04.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:17:04.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:17:04.234 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=29501 2026-02-18T08:17:04.234 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 15 = 14 -a 1 -le 29501 2026-02-18T08:17:04.234 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:17:04.234 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:17:04.234 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:17:04.435 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:17:04.435 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:17:04.435 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:17:04.435 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:17:04.435 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:17:04.435 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:17:04.435 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:17:04.435 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:17:04.435 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:17:04.435 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:04.435 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:17:04.435 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:17:04.435 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:17:04.435 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:17:04.435 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:17:04.436 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:17:04.436 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:17:04.437 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:17:04.437 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:17:04.437 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:17:04.437 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:04.437 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:17:04.437 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:17:04.437 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:17:04.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:17:04.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:17:04.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:17:04.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:17:04.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:17:04.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:17:04.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:04.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:17:04.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:17:04.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:17:04.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:17:04.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:17:04.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:17:04.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:17:04.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:17:04.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:17:04.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:17:04.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:04.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:17:04.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:17:04.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:17:04.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:17:04.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:17:04.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:17:04.439 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:17:04.449 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:17:04.449 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:17:04.449 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:17:04.449 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:17:04.510 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:17:04.510 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:17:04.510 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:17:04.513 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:17:04.513 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:17:04.513 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:17:04.513 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:04.513 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:17:04.513 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:17:04.514 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:17:04.514 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:17:04.514 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:17:04.514 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:17:04.514 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:17:04.514 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:17:04.514 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:17:04.514 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:04.514 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:17:04.514 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:17:04.514 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:17:04.514 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:17:04.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:17:04.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:17:04.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:17:04.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:17:04.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:17:04.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:17:04.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:04.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:17:04.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:17:04.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:17:04.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:17:04.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:17:04.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:17:04.516 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:17:04.524 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:17:04.525 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:17:04.525 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:17:04.525 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:17:04.575 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:17:04.576 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:17:04.576 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:17:04.578 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:17:04.578 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:17:04.578 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:17:04.578 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:04.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:17:04.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:17:04.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:17:04.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:17:04.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:17:04.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:17:04.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:17:04.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:17:04.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:17:04.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:04.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:17:04.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:17:04.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:17:04.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:17:04.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:17:04.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:17:04.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:17:04.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:17:04.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:17:04.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:17:04.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:04.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:17:04.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:17:04.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:17:04.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:17:04.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:17:04.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:17:04.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:17:04.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:17:04.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:17:04.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:17:04.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:17:04.639 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:17:04.639 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:17:04.639 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:17:04.641 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:17:04.642 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:17:04.642 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:17:04.642 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:04.642 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:17:04.642 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:17:04.642 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:17:04.642 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:17:04.642 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:17:04.642 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:17:04.642 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:17:04.643 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:17:04.643 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:17:04.643 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:04.643 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:17:04.643 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:17:04.643 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:17:04.643 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:17:04.643 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:17:04.643 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:17:04.643 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:17:04.643 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:17:04.643 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:17:04.643 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:17:04.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:04.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:17:04.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:17:04.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:17:04.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:17:04.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:17:04.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:17:04.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:17:04.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:17:04.653 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:17:04.653 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:17:04.653 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:17:04.711 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:17:04.713 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:17:04.713 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:17:04.713 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:17:04.713 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:17:04.713 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:17:04.713 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:17:04.713 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:17:04.713 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:17:04.713 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:17:04.713 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:17:04.714 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:17:04.714 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:17:04.714 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:04.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:17:04.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:04.751 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 53 2026-02-18T08:17:04.751 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 58 2026-02-18T08:17:04.751 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:17:04.751 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=233, tag_tid=15, entry_tid=1], [object_number=232, tag_tid=15, entry_tid=0], [object_number=234, tag_tid=14, entry_tid=52758], [object_number=235, tag_tid=14, entry_tid=52755]]], state=connected] 2026-02-18T08:17:04.751 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=213, tag_tid=14, entry_tid=29817], [object_number=212, tag_tid=14, entry_tid=29816], [object_number=215, tag_tid=14, entry_tid=29815], [object_number=214, tag_tid=14, entry_tid=29814]]], state=connected] 2026-02-18T08:17:04.755 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:04.756 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=233, tag_tid=15, entry_tid=1' 2026-02-18T08:17:04.758 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:17:04.758 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:17:04.758 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:17:04.758 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:17:04.758 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:17:04.758 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:17:04.758 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:17:04.758 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:17:04.758 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:17:04.758 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:17:04.759 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:17:04.759 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:17:04.759 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:04.760 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:17:04.760 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:04.788 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 53 2026-02-18T08:17:04.788 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 58 2026-02-18T08:17:04.788 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:17:04.788 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=233, tag_tid=15, entry_tid=1], [object_number=232, tag_tid=15, entry_tid=0], [object_number=234, tag_tid=14, entry_tid=52758], [object_number=235, tag_tid=14, entry_tid=52755]]], state=connected] 2026-02-18T08:17:04.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=213, tag_tid=14, entry_tid=29817], [object_number=212, tag_tid=14, entry_tid=29816], [object_number=215, tag_tid=14, entry_tid=29815], [object_number=214, tag_tid=14, entry_tid=29814]]], state=connected] 2026-02-18T08:17:04.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:04.793 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=213, tag_tid=14, entry_tid=29817' 2026-02-18T08:17:04.793 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=233, tag_tid=15, entry_tid=1' -a 'object_number=233, tag_tid=15, entry_tid=1' = 'object_number=213, tag_tid=14, entry_tid=29817' 2026-02-18T08:17:04.793 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=213, tag_tid=14, entry_tid=29817' '!=' 'object_number=209, tag_tid=14, entry_tid=29501' 2026-02-18T08:17:04.794 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:17:04.794 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=213, tag_tid=14, entry_tid=29817' = 'object_number=209, tag_tid=14, entry_tid=29501' 2026-02-18T08:17:04.794 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=213, tag_tid=14, entry_tid=29817' 2026-02-18T08:17:04.794 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=233, tag_tid=15, entry_tid=1' 2026-02-18T08:17:04.795 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:17:04.795 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:17:04.795 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=15 2026-02-18T08:17:04.796 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=213, tag_tid=14, entry_tid=29817' 2026-02-18T08:17:04.796 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:17:04.796 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:17:04.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=14 2026-02-18T08:17:04.798 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=233, tag_tid=15, entry_tid=1' 2026-02-18T08:17:04.798 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:17:04.798 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:17:04.800 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:17:04.801 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=213, tag_tid=14, entry_tid=29817' 2026-02-18T08:17:04.801 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:17:04.801 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:17:04.802 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=29817 2026-02-18T08:17:04.803 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 15 = 14 -a 1 -le 29817 2026-02-18T08:17:04.803 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:17:04.803 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:17:04.803 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:17:05.004 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:17:05.004 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:17:05.004 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:17:05.004 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:17:05.004 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:17:05.004 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:17:05.004 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:17:05.004 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:17:05.004 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:17:05.004 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:05.004 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:17:05.004 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:17:05.004 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:17:05.004 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:17:05.004 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:17:05.004 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:17:05.004 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:17:05.006 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:17:05.006 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:17:05.006 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:17:05.006 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:05.006 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:17:05.006 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:17:05.006 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:17:05.006 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:17:05.006 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:17:05.006 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:17:05.006 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:17:05.006 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:17:05.006 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:17:05.006 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:05.006 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:17:05.006 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:17:05.006 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:17:05.006 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:17:05.006 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:17:05.006 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:17:05.006 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:17:05.006 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:17:05.006 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:17:05.007 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:17:05.007 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:05.007 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:17:05.007 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:17:05.007 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:17:05.007 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:17:05.007 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:17:05.007 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:17:05.007 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:17:05.019 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:17:05.020 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:17:05.020 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:17:05.020 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:17:05.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:17:05.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:17:05.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:17:05.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:17:05.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:17:05.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:17:05.074 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:05.074 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:17:05.074 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:17:05.074 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:17:05.074 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:17:05.074 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:17:05.074 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:17:05.074 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:17:05.074 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:17:05.074 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:17:05.074 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:05.074 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:17:05.075 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:17:05.075 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:17:05.075 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:17:05.075 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:17:05.075 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:17:05.075 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:17:05.075 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:17:05.075 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:17:05.075 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:17:05.075 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:05.075 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:17:05.075 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:17:05.076 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:17:05.076 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:17:05.076 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:17:05.076 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:17:05.076 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:17:05.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:17:05.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:17:05.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:17:05.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:17:05.142 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:17:05.142 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:17:05.142 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:17:05.145 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:17:05.145 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:17:05.145 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:17:05.145 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:05.146 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:17:05.146 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:17:05.146 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:17:05.146 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:17:05.146 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:17:05.146 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:17:05.146 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:17:05.146 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:17:05.146 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:17:05.146 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:05.146 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:17:05.146 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:17:05.147 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:17:05.147 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:17:05.147 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:17:05.147 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:17:05.147 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:17:05.147 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:17:05.147 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:17:05.147 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:17:05.147 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:05.147 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:17:05.147 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:17:05.147 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:17:05.148 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:17:05.148 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:17:05.148 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:17:05.148 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:17:05.156 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:17:05.156 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:17:05.157 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:17:05.157 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:17:05.207 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:17:05.207 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:17:05.207 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:17:05.210 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:17:05.210 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:17:05.210 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:17:05.210 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:05.210 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:17:05.210 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:17:05.210 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:17:05.211 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:17:05.211 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:17:05.211 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:17:05.211 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:17:05.211 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:17:05.211 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:17:05.211 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:05.211 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:17:05.211 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:17:05.211 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:17:05.211 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:17:05.211 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:17:05.212 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:17:05.212 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:17:05.212 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:17:05.212 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:17:05.212 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:17:05.212 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:05.212 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:17:05.212 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:17:05.212 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:17:05.212 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:17:05.212 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:17:05.212 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:17:05.213 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:17:05.220 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:17:05.221 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:17:05.221 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:17:05.221 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:17:05.289 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:17:05.290 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:17:05.290 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:17:05.290 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:17:05.291 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:17:05.291 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:17:05.291 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:17:05.291 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:17:05.291 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:17:05.291 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:17:05.291 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:17:05.291 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:17:05.291 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:17:05.292 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:05.292 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:17:05.292 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:05.322 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 53 2026-02-18T08:17:05.322 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 58 2026-02-18T08:17:05.322 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:17:05.322 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=233, tag_tid=15, entry_tid=1], [object_number=232, tag_tid=15, entry_tid=0], [object_number=234, tag_tid=14, entry_tid=52758], [object_number=235, tag_tid=14, entry_tid=52755]]], state=connected] 2026-02-18T08:17:05.322 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=214, tag_tid=14, entry_tid=30214], [object_number=213, tag_tid=14, entry_tid=30213], [object_number=212, tag_tid=14, entry_tid=30212], [object_number=215, tag_tid=14, entry_tid=30211]]], state=connected] 2026-02-18T08:17:05.325 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:05.327 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=233, tag_tid=15, entry_tid=1' 2026-02-18T08:17:05.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:17:05.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:17:05.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:17:05.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:17:05.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:17:05.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:17:05.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:17:05.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:17:05.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:17:05.329 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:17:05.329 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:17:05.329 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:17:05.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:05.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:17:05.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:05.366 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 53 2026-02-18T08:17:05.366 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 58 2026-02-18T08:17:05.367 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:17:05.367 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=233, tag_tid=15, entry_tid=1], [object_number=232, tag_tid=15, entry_tid=0], [object_number=234, tag_tid=14, entry_tid=52758], [object_number=235, tag_tid=14, entry_tid=52755]]], state=connected] 2026-02-18T08:17:05.367 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=214, tag_tid=14, entry_tid=30214], [object_number=213, tag_tid=14, entry_tid=30213], [object_number=212, tag_tid=14, entry_tid=30212], [object_number=215, tag_tid=14, entry_tid=30211]]], state=connected] 2026-02-18T08:17:05.370 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:05.372 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=214, tag_tid=14, entry_tid=30214' 2026-02-18T08:17:05.372 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=233, tag_tid=15, entry_tid=1' -a 'object_number=233, tag_tid=15, entry_tid=1' = 'object_number=214, tag_tid=14, entry_tid=30214' 2026-02-18T08:17:05.372 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=214, tag_tid=14, entry_tid=30214' '!=' 'object_number=213, tag_tid=14, entry_tid=29817' 2026-02-18T08:17:05.372 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:17:05.372 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=214, tag_tid=14, entry_tid=30214' = 'object_number=213, tag_tid=14, entry_tid=29817' 2026-02-18T08:17:05.372 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=214, tag_tid=14, entry_tid=30214' 2026-02-18T08:17:05.373 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=233, tag_tid=15, entry_tid=1' 2026-02-18T08:17:05.373 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:17:05.373 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:17:05.374 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=15 2026-02-18T08:17:05.375 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=214, tag_tid=14, entry_tid=30214' 2026-02-18T08:17:05.375 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:17:05.375 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:17:05.376 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=14 2026-02-18T08:17:05.377 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=233, tag_tid=15, entry_tid=1' 2026-02-18T08:17:05.377 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:17:05.377 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:17:05.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:17:05.379 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=214, tag_tid=14, entry_tid=30214' 2026-02-18T08:17:05.379 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:17:05.380 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:17:05.381 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=30214 2026-02-18T08:17:05.381 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 15 = 14 -a 1 -le 30214 2026-02-18T08:17:05.381 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:17:05.381 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:17:05.382 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:17:05.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:17:05.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:17:05.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:17:05.584 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:17:05.584 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:17:05.584 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:17:05.584 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:17:05.584 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:17:05.584 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:17:05.584 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:05.584 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:17:05.584 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:17:05.585 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:17:05.585 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:17:05.585 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:17:05.585 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:17:05.587 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:17:05.587 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:17:05.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:17:05.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:17:05.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:05.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:17:05.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:17:05.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:17:05.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:17:05.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:17:05.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:17:05.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:17:05.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:17:05.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:17:05.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:05.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:17:05.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:17:05.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:17:05.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:17:05.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:17:05.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:17:05.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:17:05.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:17:05.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:17:05.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:17:05.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:05.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:17:05.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:17:05.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:17:05.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:17:05.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:17:05.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:17:05.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:17:05.596 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:17:05.597 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:17:05.597 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:17:05.597 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:17:05.647 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:17:05.647 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:17:05.648 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:17:05.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:17:05.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:17:05.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:17:05.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:05.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:17:05.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:17:05.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:17:05.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:17:05.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:17:05.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:17:05.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:17:05.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:17:05.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:17:05.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:05.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:17:05.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:17:05.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:17:05.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:17:05.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:17:05.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:17:05.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:17:05.653 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:17:05.653 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:17:05.653 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:17:05.653 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:05.653 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:17:05.653 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:17:05.653 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:17:05.653 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:17:05.653 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:17:05.653 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:17:05.653 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:17:05.661 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:17:05.662 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:17:05.662 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:17:05.662 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:17:05.711 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:17:05.711 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:17:05.711 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:17:05.714 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:17:05.714 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:17:05.714 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:17:05.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:05.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:17:05.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:17:05.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:17:05.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:17:05.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:17:05.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:17:05.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:17:05.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:17:05.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:17:05.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:05.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:17:05.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:17:05.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:17:05.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:17:05.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:17:05.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:17:05.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:17:05.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:17:05.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:17:05.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:17:05.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:05.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:17:05.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:17:05.717 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:17:05.717 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:17:05.717 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:17:05.717 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:17:05.717 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:17:05.725 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:17:05.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:17:05.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:17:05.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:17:05.783 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:17:05.783 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:17:05.783 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:17:05.786 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:17:05.786 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:17:05.786 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:17:05.786 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:05.786 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:17:05.786 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:17:05.786 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:17:05.786 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:17:05.786 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:17:05.786 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:17:05.787 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:17:05.787 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:17:05.787 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:17:05.787 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:05.787 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:17:05.787 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:17:05.787 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:17:05.787 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:17:05.787 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:17:05.787 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:17:05.787 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:17:05.787 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:17:05.788 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:17:05.788 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:17:05.788 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:05.788 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:17:05.788 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:17:05.788 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:17:05.788 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:17:05.788 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:17:05.788 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:17:05.788 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:17:05.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:17:05.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:17:05.798 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:17:05.798 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:17:05.859 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:17:05.860 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:17:05.860 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:17:05.860 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:17:05.860 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:17:05.860 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:17:05.860 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:17:05.860 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:17:05.861 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:17:05.861 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:17:05.861 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:17:05.861 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:17:05.861 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:17:05.861 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:05.862 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:17:05.862 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:05.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 53 2026-02-18T08:17:05.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 58 2026-02-18T08:17:05.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:17:05.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=233, tag_tid=15, entry_tid=1], [object_number=232, tag_tid=15, entry_tid=0], [object_number=234, tag_tid=14, entry_tid=52758], [object_number=235, tag_tid=14, entry_tid=52755]]], state=connected] 2026-02-18T08:17:05.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=214, tag_tid=14, entry_tid=30614], [object_number=213, tag_tid=14, entry_tid=30613], [object_number=212, tag_tid=14, entry_tid=30612], [object_number=215, tag_tid=14, entry_tid=30611]]], state=connected] 2026-02-18T08:17:05.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:05.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=233, tag_tid=15, entry_tid=1' 2026-02-18T08:17:05.894 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:17:05.895 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:17:05.895 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:17:05.895 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:17:05.895 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:17:05.895 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:17:05.895 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:17:05.895 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:17:05.895 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:17:05.895 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:17:05.896 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:17:05.896 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:17:05.896 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:05.897 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:17:05.897 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:05.924 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 53 2026-02-18T08:17:05.924 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 58 2026-02-18T08:17:05.924 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:17:05.924 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=233, tag_tid=15, entry_tid=1], [object_number=232, tag_tid=15, entry_tid=0], [object_number=234, tag_tid=14, entry_tid=52758], [object_number=235, tag_tid=14, entry_tid=52755]]], state=connected] 2026-02-18T08:17:05.924 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=214, tag_tid=14, entry_tid=30614], [object_number=213, tag_tid=14, entry_tid=30613], [object_number=212, tag_tid=14, entry_tid=30612], [object_number=215, tag_tid=14, entry_tid=30611]]], state=connected] 2026-02-18T08:17:05.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:05.928 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=214, tag_tid=14, entry_tid=30614' 2026-02-18T08:17:05.928 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=233, tag_tid=15, entry_tid=1' -a 'object_number=233, tag_tid=15, entry_tid=1' = 'object_number=214, tag_tid=14, entry_tid=30614' 2026-02-18T08:17:05.928 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=214, tag_tid=14, entry_tid=30614' '!=' 'object_number=214, tag_tid=14, entry_tid=30214' 2026-02-18T08:17:05.928 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:17:05.928 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=214, tag_tid=14, entry_tid=30614' = 'object_number=214, tag_tid=14, entry_tid=30214' 2026-02-18T08:17:05.928 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=214, tag_tid=14, entry_tid=30614' 2026-02-18T08:17:05.929 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=233, tag_tid=15, entry_tid=1' 2026-02-18T08:17:05.929 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:17:05.929 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:17:05.930 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=15 2026-02-18T08:17:05.931 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=214, tag_tid=14, entry_tid=30614' 2026-02-18T08:17:05.931 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:17:05.931 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:17:05.932 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=14 2026-02-18T08:17:05.933 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=233, tag_tid=15, entry_tid=1' 2026-02-18T08:17:05.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:17:05.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:17:05.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:17:05.936 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=214, tag_tid=14, entry_tid=30614' 2026-02-18T08:17:05.936 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:17:05.936 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:17:05.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=30614 2026-02-18T08:17:05.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 15 = 14 -a 1 -le 30614 2026-02-18T08:17:05.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:17:05.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:17:05.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:17:06.139 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:17:06.139 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:17:06.140 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:17:06.140 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:17:06.140 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:17:06.140 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:17:06.140 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:17:06.140 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:17:06.140 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:17:06.140 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:06.140 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:17:06.140 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:17:06.140 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:17:06.141 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:17:06.141 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:17:06.143 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:17:06.143 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:17:06.144 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:17:06.144 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:17:06.144 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:17:06.144 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:06.144 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:17:06.144 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:17:06.144 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:17:06.144 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:17:06.144 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:17:06.144 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:17:06.144 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:17:06.145 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:17:06.145 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:17:06.145 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:06.145 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:17:06.145 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:17:06.145 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:17:06.145 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:17:06.145 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:17:06.145 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:17:06.145 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:17:06.145 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:17:06.145 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:17:06.146 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:17:06.146 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:06.146 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:17:06.146 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:17:06.146 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:17:06.146 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:17:06.146 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:17:06.146 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:17:06.146 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:17:06.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:17:06.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:17:06.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:17:06.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:17:06.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:17:06.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:17:06.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:17:06.208 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:17:06.208 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:17:06.209 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:17:06.209 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:06.209 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:17:06.209 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:17:06.209 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:17:06.209 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:17:06.209 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:17:06.209 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:17:06.209 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:17:06.209 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:17:06.209 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:17:06.210 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:06.210 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:17:06.210 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:17:06.210 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:17:06.210 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:17:06.210 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:17:06.210 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:17:06.210 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:17:06.210 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:17:06.210 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:17:06.210 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:17:06.210 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:06.211 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:17:06.211 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:17:06.211 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:17:06.211 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:17:06.211 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:17:06.211 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:17:06.211 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:17:06.220 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:17:06.220 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:17:06.220 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:17:06.220 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:17:06.277 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:17:06.278 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:17:06.278 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:17:06.280 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:17:06.280 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:17:06.280 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:17:06.281 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:06.281 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:17:06.281 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:17:06.281 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:17:06.281 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:17:06.281 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:17:06.281 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:17:06.281 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:17:06.281 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:17:06.281 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:17:06.281 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:06.282 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:17:06.282 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:17:06.282 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:17:06.282 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:17:06.282 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:17:06.282 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:17:06.282 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:17:06.282 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:17:06.282 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:17:06.282 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:17:06.282 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:06.282 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:17:06.282 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:17:06.283 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:17:06.283 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:17:06.283 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:17:06.283 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:17:06.283 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:17:06.291 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:17:06.292 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:17:06.292 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:17:06.292 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:17:06.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:17:06.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:17:06.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:17:06.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:17:06.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:17:06.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:17:06.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:06.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:17:06.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:17:06.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:17:06.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:17:06.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:17:06.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:17:06.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:17:06.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:17:06.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:17:06.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:06.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:17:06.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:17:06.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:17:06.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:17:06.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:17:06.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:17:06.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:17:06.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:17:06.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:17:06.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:17:06.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:06.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:17:06.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:17:06.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:17:06.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:17:06.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:17:06.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:17:06.348 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:17:06.356 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:17:06.357 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:17:06.357 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:17:06.357 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:17:06.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:17:06.428 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:17:06.428 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:17:06.428 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:17:06.428 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:17:06.428 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:17:06.428 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:17:06.429 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:17:06.429 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:17:06.429 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:17:06.429 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:17:06.429 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:17:06.429 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:17:06.429 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:06.430 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:17:06.430 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:06.458 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 53 2026-02-18T08:17:06.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 58 2026-02-18T08:17:06.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:17:06.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=233, tag_tid=15, entry_tid=1], [object_number=232, tag_tid=15, entry_tid=0], [object_number=234, tag_tid=14, entry_tid=52758], [object_number=235, tag_tid=14, entry_tid=52755]]], state=connected] 2026-02-18T08:17:06.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=215, tag_tid=14, entry_tid=30979], [object_number=214, tag_tid=14, entry_tid=30978], [object_number=213, tag_tid=14, entry_tid=30977], [object_number=212, tag_tid=14, entry_tid=30976]]], state=connected] 2026-02-18T08:17:06.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:06.462 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=233, tag_tid=15, entry_tid=1' 2026-02-18T08:17:06.463 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:17:06.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:17:06.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:17:06.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:17:06.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:17:06.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:17:06.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:17:06.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:17:06.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:17:06.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:17:06.465 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:17:06.465 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:17:06.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:06.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:17:06.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:06.500 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 53 2026-02-18T08:17:06.500 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 58 2026-02-18T08:17:06.501 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:17:06.501 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=233, tag_tid=15, entry_tid=1], [object_number=232, tag_tid=15, entry_tid=0], [object_number=234, tag_tid=14, entry_tid=52758], [object_number=235, tag_tid=14, entry_tid=52755]]], state=connected] 2026-02-18T08:17:06.501 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=215, tag_tid=14, entry_tid=30979], [object_number=214, tag_tid=14, entry_tid=30978], [object_number=213, tag_tid=14, entry_tid=30977], [object_number=212, tag_tid=14, entry_tid=30976]]], state=connected] 2026-02-18T08:17:06.503 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:06.505 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=215, tag_tid=14, entry_tid=30979' 2026-02-18T08:17:06.505 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=233, tag_tid=15, entry_tid=1' -a 'object_number=233, tag_tid=15, entry_tid=1' = 'object_number=215, tag_tid=14, entry_tid=30979' 2026-02-18T08:17:06.506 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=215, tag_tid=14, entry_tid=30979' '!=' 'object_number=214, tag_tid=14, entry_tid=30614' 2026-02-18T08:17:06.506 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:17:06.506 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=215, tag_tid=14, entry_tid=30979' = 'object_number=214, tag_tid=14, entry_tid=30614' 2026-02-18T08:17:06.506 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=215, tag_tid=14, entry_tid=30979' 2026-02-18T08:17:06.506 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=233, tag_tid=15, entry_tid=1' 2026-02-18T08:17:06.507 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:17:06.507 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:17:06.507 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=15 2026-02-18T08:17:06.508 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=215, tag_tid=14, entry_tid=30979' 2026-02-18T08:17:06.508 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:17:06.508 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:17:06.510 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=14 2026-02-18T08:17:06.510 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=233, tag_tid=15, entry_tid=1' 2026-02-18T08:17:06.511 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:17:06.511 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:17:06.512 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:17:06.513 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=215, tag_tid=14, entry_tid=30979' 2026-02-18T08:17:06.513 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:17:06.513 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:17:06.514 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=30979 2026-02-18T08:17:06.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 15 = 14 -a 1 -le 30979 2026-02-18T08:17:06.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:17:06.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:17:06.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:17:06.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:17:06.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:17:06.717 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:17:06.717 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:17:06.717 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:17:06.717 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:17:06.717 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:17:06.717 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:17:06.717 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:17:06.717 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:06.717 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:17:06.717 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:17:06.717 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:17:06.717 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:17:06.718 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:17:06.718 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:17:06.720 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:17:06.720 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:17:06.720 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:17:06.721 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:17:06.721 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:06.721 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:17:06.721 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:17:06.721 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:17:06.721 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:17:06.721 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:17:06.721 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:17:06.721 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:17:06.721 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:17:06.721 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:17:06.722 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:06.722 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:17:06.722 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:17:06.722 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:17:06.722 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:17:06.722 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:17:06.722 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:17:06.722 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:17:06.722 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:17:06.722 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:17:06.722 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:17:06.722 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:06.722 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:17:06.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:17:06.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:17:06.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:17:06.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:17:06.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:17:06.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:17:06.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:17:06.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:17:06.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:17:06.731 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:17:06.779 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:17:06.779 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:17:06.779 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:17:06.782 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:17:06.782 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:17:06.782 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:17:06.782 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:06.782 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:17:06.782 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:17:06.782 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:17:06.783 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:17:06.783 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:17:06.783 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:17:06.783 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:17:06.783 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:17:06.783 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:17:06.783 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:06.783 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:17:06.783 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:17:06.783 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:17:06.783 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:17:06.784 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:17:06.784 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:17:06.784 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:17:06.784 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:17:06.784 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:17:06.784 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:17:06.784 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:06.784 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:17:06.784 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:17:06.784 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:17:06.784 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:17:06.784 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:17:06.784 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:17:06.785 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:17:06.793 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:17:06.793 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:17:06.793 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:17:06.793 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:17:06.841 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:17:06.842 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:17:06.842 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:17:06.844 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:17:06.844 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:17:06.845 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:17:06.845 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:06.845 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:17:06.845 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:17:06.845 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:17:06.845 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:17:06.845 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:17:06.845 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:17:06.845 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:17:06.845 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:17:06.845 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:17:06.845 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:06.846 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:17:06.846 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:17:06.846 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:17:06.846 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:17:06.846 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:17:06.846 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:17:06.846 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:17:06.846 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:17:06.846 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:17:06.846 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:17:06.846 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:06.846 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:17:06.847 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:17:06.847 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:17:06.847 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:17:06.847 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:17:06.847 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:17:06.847 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:17:06.855 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:17:06.856 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:17:06.856 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:17:06.856 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:17:06.904 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:17:06.904 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:17:06.904 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:17:06.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:17:06.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:17:06.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:17:06.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:06.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:17:06.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:17:06.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:17:06.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:17:06.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:17:06.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:17:06.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:17:06.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:17:06.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:17:06.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:06.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:17:06.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:17:06.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:17:06.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:17:06.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:17:06.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:17:06.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:17:06.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:17:06.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:17:06.909 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:17:06.909 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:06.909 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:17:06.909 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:17:06.909 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:17:06.909 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:17:06.909 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:17:06.909 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:17:06.909 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:17:06.917 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:17:06.917 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:17:06.917 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:17:06.918 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:17:06.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:17:06.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:17:06.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:17:06.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:17:06.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:17:06.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:17:06.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:17:06.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:17:06.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:17:06.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:17:06.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:17:06.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:17:06.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:17:06.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:06.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:17:06.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:07.005 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 53 2026-02-18T08:17:07.005 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 58 2026-02-18T08:17:07.006 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:17:07.006 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=233, tag_tid=15, entry_tid=1], [object_number=232, tag_tid=15, entry_tid=0], [object_number=234, tag_tid=14, entry_tid=52758], [object_number=235, tag_tid=14, entry_tid=52755]]], state=connected] 2026-02-18T08:17:07.006 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=212, tag_tid=14, entry_tid=31336], [object_number=215, tag_tid=14, entry_tid=31335], [object_number=214, tag_tid=14, entry_tid=31334], [object_number=213, tag_tid=14, entry_tid=31333]]], state=connected] 2026-02-18T08:17:07.008 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:07.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=233, tag_tid=15, entry_tid=1' 2026-02-18T08:17:07.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:17:07.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:17:07.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:17:07.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:17:07.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:17:07.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:17:07.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:17:07.012 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:17:07.012 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:17:07.012 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:17:07.012 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:17:07.012 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:17:07.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:07.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:17:07.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:07.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 53 2026-02-18T08:17:07.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 58 2026-02-18T08:17:07.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:17:07.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=233, tag_tid=15, entry_tid=1], [object_number=232, tag_tid=15, entry_tid=0], [object_number=234, tag_tid=14, entry_tid=52758], [object_number=235, tag_tid=14, entry_tid=52755]]], state=connected] 2026-02-18T08:17:07.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=212, tag_tid=14, entry_tid=31336], [object_number=215, tag_tid=14, entry_tid=31335], [object_number=214, tag_tid=14, entry_tid=31334], [object_number=213, tag_tid=14, entry_tid=31333]]], state=connected] 2026-02-18T08:17:07.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:07.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=212, tag_tid=14, entry_tid=31336' 2026-02-18T08:17:07.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=233, tag_tid=15, entry_tid=1' -a 'object_number=233, tag_tid=15, entry_tid=1' = 'object_number=212, tag_tid=14, entry_tid=31336' 2026-02-18T08:17:07.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=212, tag_tid=14, entry_tid=31336' '!=' 'object_number=215, tag_tid=14, entry_tid=30979' 2026-02-18T08:17:07.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:17:07.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=212, tag_tid=14, entry_tid=31336' = 'object_number=215, tag_tid=14, entry_tid=30979' 2026-02-18T08:17:07.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=212, tag_tid=14, entry_tid=31336' 2026-02-18T08:17:07.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=233, tag_tid=15, entry_tid=1' 2026-02-18T08:17:07.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:17:07.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:17:07.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=15 2026-02-18T08:17:07.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=212, tag_tid=14, entry_tid=31336' 2026-02-18T08:17:07.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:17:07.055 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:17:07.055 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=14 2026-02-18T08:17:07.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=233, tag_tid=15, entry_tid=1' 2026-02-18T08:17:07.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:17:07.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:17:07.058 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:17:07.059 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=212, tag_tid=14, entry_tid=31336' 2026-02-18T08:17:07.059 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:17:07.059 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:17:07.060 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=31336 2026-02-18T08:17:07.060 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 15 = 14 -a 1 -le 31336 2026-02-18T08:17:07.060 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:17:07.060 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:17:07.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:17:07.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:17:07.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:17:07.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:17:07.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:17:07.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:17:07.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:17:07.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:17:07.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:17:07.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:17:07.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:07.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:17:07.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:17:07.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:17:07.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:17:07.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:17:07.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:17:07.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:17:07.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:17:07.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:17:07.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:17:07.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:07.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:17:07.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:17:07.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:17:07.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:17:07.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:17:07.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:17:07.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:17:07.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:17:07.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:17:07.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:07.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:17:07.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:17:07.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:17:07.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:17:07.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:17:07.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:17:07.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:17:07.269 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:17:07.269 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:17:07.269 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:17:07.269 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:07.269 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:17:07.269 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:17:07.269 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:17:07.269 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:17:07.269 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:17:07.269 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:17:07.269 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:17:07.276 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:17:07.277 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:17:07.277 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:17:07.277 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:17:07.327 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:17:07.327 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:17:07.327 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:17:07.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:17:07.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:17:07.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:17:07.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:07.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:17:07.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:17:07.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:17:07.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:17:07.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:17:07.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:17:07.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:17:07.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:17:07.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:17:07.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:07.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:17:07.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:17:07.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:17:07.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:17:07.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:17:07.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:17:07.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:17:07.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:17:07.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:17:07.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:17:07.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:07.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:17:07.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:17:07.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:17:07.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:17:07.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:17:07.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:17:07.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:17:07.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:17:07.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:17:07.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:17:07.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:17:07.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:17:07.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:17:07.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:17:07.395 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:17:07.395 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:17:07.395 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:17:07.395 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:07.396 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:17:07.396 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:17:07.396 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:17:07.396 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:17:07.396 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:17:07.396 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:17:07.396 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:17:07.396 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:17:07.396 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:17:07.396 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:07.396 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:17:07.396 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:17:07.397 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:17:07.397 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:17:07.397 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:17:07.397 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:17:07.397 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:17:07.397 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:17:07.397 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:17:07.397 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:17:07.397 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:07.397 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:17:07.397 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:17:07.397 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:17:07.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:17:07.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:17:07.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:17:07.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:17:07.405 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:17:07.405 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:17:07.406 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:17:07.406 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:17:07.454 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:17:07.454 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:17:07.455 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:17:07.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:17:07.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:17:07.458 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:17:07.458 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:07.458 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:17:07.458 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:17:07.458 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:17:07.458 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:17:07.458 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:17:07.458 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:17:07.458 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:17:07.458 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:17:07.458 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:17:07.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:07.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:17:07.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:17:07.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:17:07.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:17:07.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:17:07.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:17:07.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:17:07.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:17:07.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:17:07.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:17:07.460 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:07.460 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:17:07.460 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:17:07.460 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:17:07.460 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:17:07.460 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:17:07.460 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:17:07.460 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:17:07.468 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:17:07.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:17:07.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:17:07.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:17:07.531 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:17:07.531 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:17:07.531 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:17:07.531 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:17:07.531 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:17:07.531 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:17:07.532 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:17:07.532 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:17:07.532 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:17:07.532 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:17:07.532 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:17:07.532 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:17:07.532 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:17:07.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:07.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:17:07.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:07.560 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 53 2026-02-18T08:17:07.560 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 58 2026-02-18T08:17:07.560 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:17:07.560 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=233, tag_tid=15, entry_tid=1], [object_number=232, tag_tid=15, entry_tid=0], [object_number=234, tag_tid=14, entry_tid=52758], [object_number=235, tag_tid=14, entry_tid=52755]]], state=connected] 2026-02-18T08:17:07.561 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=215, tag_tid=14, entry_tid=31703], [object_number=214, tag_tid=14, entry_tid=31702], [object_number=213, tag_tid=14, entry_tid=31701], [object_number=212, tag_tid=14, entry_tid=31700]]], state=connected] 2026-02-18T08:17:07.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:07.565 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=233, tag_tid=15, entry_tid=1' 2026-02-18T08:17:07.566 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:17:07.566 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:17:07.566 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:17:07.566 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:17:07.566 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:17:07.566 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:17:07.566 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:17:07.566 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:17:07.567 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:17:07.567 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:17:07.567 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:17:07.567 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:17:07.568 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:07.568 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:17:07.568 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:07.597 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 53 2026-02-18T08:17:07.597 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 58 2026-02-18T08:17:07.597 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:17:07.597 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=233, tag_tid=15, entry_tid=1], [object_number=232, tag_tid=15, entry_tid=0], [object_number=234, tag_tid=14, entry_tid=52758], [object_number=235, tag_tid=14, entry_tid=52755]]], state=connected] 2026-02-18T08:17:07.597 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=215, tag_tid=14, entry_tid=31703], [object_number=214, tag_tid=14, entry_tid=31702], [object_number=213, tag_tid=14, entry_tid=31701], [object_number=212, tag_tid=14, entry_tid=31700]]], state=connected] 2026-02-18T08:17:07.600 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:07.601 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=215, tag_tid=14, entry_tid=31703' 2026-02-18T08:17:07.601 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=233, tag_tid=15, entry_tid=1' -a 'object_number=233, tag_tid=15, entry_tid=1' = 'object_number=215, tag_tid=14, entry_tid=31703' 2026-02-18T08:17:07.601 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=215, tag_tid=14, entry_tid=31703' '!=' 'object_number=212, tag_tid=14, entry_tid=31336' 2026-02-18T08:17:07.601 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:17:07.601 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=215, tag_tid=14, entry_tid=31703' = 'object_number=212, tag_tid=14, entry_tid=31336' 2026-02-18T08:17:07.602 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=215, tag_tid=14, entry_tid=31703' 2026-02-18T08:17:07.602 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=233, tag_tid=15, entry_tid=1' 2026-02-18T08:17:07.602 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:17:07.602 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:17:07.603 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=15 2026-02-18T08:17:07.604 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=215, tag_tid=14, entry_tid=31703' 2026-02-18T08:17:07.604 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:17:07.604 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:17:07.606 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=14 2026-02-18T08:17:07.606 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=233, tag_tid=15, entry_tid=1' 2026-02-18T08:17:07.606 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:17:07.607 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:17:07.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:17:07.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=215, tag_tid=14, entry_tid=31703' 2026-02-18T08:17:07.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:17:07.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:17:07.610 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=31703 2026-02-18T08:17:07.610 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 15 = 14 -a 1 -le 31703 2026-02-18T08:17:07.610 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:17:07.611 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:17:07.611 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:17:07.812 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:17:07.812 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:17:07.813 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:17:07.813 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:17:07.813 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:17:07.813 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:17:07.813 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:17:07.813 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:17:07.813 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:17:07.813 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:07.813 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:17:07.814 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:17:07.814 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:17:07.814 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:17:07.814 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:17:07.814 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:17:07.814 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:17:07.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:17:07.817 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:17:07.817 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:17:07.817 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:07.817 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:17:07.817 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:17:07.817 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:17:07.817 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:17:07.817 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:17:07.817 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:17:07.817 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:17:07.818 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:17:07.818 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:17:07.818 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:07.818 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:17:07.818 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:17:07.818 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:17:07.818 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:17:07.818 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:17:07.818 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:17:07.818 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:17:07.818 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:17:07.818 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:17:07.818 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:17:07.819 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:07.819 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:17:07.819 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:17:07.819 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:17:07.819 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:17:07.819 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:17:07.819 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:17:07.819 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:17:07.826 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:17:07.827 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:17:07.827 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:17:07.827 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:17:07.875 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:17:07.875 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:17:07.876 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:17:07.878 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:17:07.878 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:17:07.878 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:17:07.878 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:07.878 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:17:07.879 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:17:07.879 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:17:07.879 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:17:07.879 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:17:07.879 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:17:07.879 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:17:07.879 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:17:07.879 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:17:07.879 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:07.879 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:17:07.879 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:17:07.879 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:17:07.880 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:17:07.880 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:17:07.880 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:17:07.880 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:17:07.880 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:17:07.880 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:17:07.880 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:17:07.880 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:07.880 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:17:07.880 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:17:07.880 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:17:07.880 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:17:07.881 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:17:07.881 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:17:07.881 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:17:07.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:17:07.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:17:07.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:17:07.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:17:07.946 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:17:07.947 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:17:07.947 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:17:07.950 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:17:07.950 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:17:07.950 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:17:07.950 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:07.950 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:17:07.950 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:17:07.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:17:07.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:17:07.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:17:07.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:17:07.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:17:07.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:17:07.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:17:07.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:07.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:17:07.952 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:17:07.952 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:17:07.952 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:17:07.952 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:17:07.952 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:17:07.952 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:17:07.952 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:17:07.952 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:17:07.952 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:17:07.952 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:07.952 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:17:07.952 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:17:07.953 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:17:07.953 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:17:07.953 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:17:07.953 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:17:07.953 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:17:07.961 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:17:07.962 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:17:07.962 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:17:07.962 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:17:08.018 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:17:08.018 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:17:08.018 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:17:08.021 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:17:08.021 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:17:08.021 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:17:08.021 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:08.021 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:17:08.021 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:17:08.021 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:17:08.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:17:08.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:17:08.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:17:08.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:17:08.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:17:08.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:17:08.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:08.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:17:08.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:17:08.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:17:08.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:17:08.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:17:08.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:17:08.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:17:08.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:17:08.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:17:08.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:17:08.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:08.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:17:08.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:17:08.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:17:08.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:17:08.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:17:08.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:17:08.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:17:08.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:17:08.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:17:08.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:17:08.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:17:08.100 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:17:08.101 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:17:08.101 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:17:08.102 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:17:08.102 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:17:08.102 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:17:08.102 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:17:08.102 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:17:08.102 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:17:08.102 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:17:08.102 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:17:08.102 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:17:08.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:17:08.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:08.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:17:08.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:08.130 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 53 2026-02-18T08:17:08.131 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 58 2026-02-18T08:17:08.131 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:17:08.131 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=233, tag_tid=15, entry_tid=1], [object_number=232, tag_tid=15, entry_tid=0], [object_number=234, tag_tid=14, entry_tid=52758], [object_number=235, tag_tid=14, entry_tid=52755]]], state=connected] 2026-02-18T08:17:08.131 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=212, tag_tid=14, entry_tid=32076], [object_number=215, tag_tid=14, entry_tid=32075], [object_number=214, tag_tid=14, entry_tid=32074], [object_number=213, tag_tid=14, entry_tid=32073]]], state=connected] 2026-02-18T08:17:08.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:08.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=233, tag_tid=15, entry_tid=1' 2026-02-18T08:17:08.136 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:17:08.136 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:17:08.137 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:17:08.137 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:17:08.137 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:17:08.137 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:17:08.137 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:17:08.137 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:17:08.137 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:17:08.137 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:17:08.137 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:17:08.137 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:17:08.138 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:08.138 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:08.138 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:17:08.165 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 53 2026-02-18T08:17:08.166 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 58 2026-02-18T08:17:08.166 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:17:08.166 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=233, tag_tid=15, entry_tid=1], [object_number=232, tag_tid=15, entry_tid=0], [object_number=234, tag_tid=14, entry_tid=52758], [object_number=235, tag_tid=14, entry_tid=52755]]], state=connected] 2026-02-18T08:17:08.166 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=212, tag_tid=14, entry_tid=32076], [object_number=215, tag_tid=14, entry_tid=32075], [object_number=214, tag_tid=14, entry_tid=32074], [object_number=213, tag_tid=14, entry_tid=32073]]], state=connected] 2026-02-18T08:17:08.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:08.170 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=212, tag_tid=14, entry_tid=32076' 2026-02-18T08:17:08.170 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=233, tag_tid=15, entry_tid=1' -a 'object_number=233, tag_tid=15, entry_tid=1' = 'object_number=212, tag_tid=14, entry_tid=32076' 2026-02-18T08:17:08.171 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=212, tag_tid=14, entry_tid=32076' '!=' 'object_number=215, tag_tid=14, entry_tid=31703' 2026-02-18T08:17:08.171 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:17:08.171 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=212, tag_tid=14, entry_tid=32076' = 'object_number=215, tag_tid=14, entry_tid=31703' 2026-02-18T08:17:08.171 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=212, tag_tid=14, entry_tid=32076' 2026-02-18T08:17:08.171 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=233, tag_tid=15, entry_tid=1' 2026-02-18T08:17:08.171 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:17:08.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:17:08.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=15 2026-02-18T08:17:08.173 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=212, tag_tid=14, entry_tid=32076' 2026-02-18T08:17:08.173 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:17:08.173 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:17:08.174 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=14 2026-02-18T08:17:08.175 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=233, tag_tid=15, entry_tid=1' 2026-02-18T08:17:08.175 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:17:08.175 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:17:08.177 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:17:08.177 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=212, tag_tid=14, entry_tid=32076' 2026-02-18T08:17:08.178 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:17:08.178 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:17:08.179 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=32076 2026-02-18T08:17:08.179 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 15 = 14 -a 1 -le 32076 2026-02-18T08:17:08.179 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:17:08.180 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:17:08.180 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:17:08.381 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:17:08.381 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:17:08.381 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:17:08.381 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:17:08.382 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:17:08.382 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:17:08.382 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:17:08.382 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:17:08.382 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:17:08.382 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:08.382 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:17:08.382 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:17:08.382 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:17:08.382 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:17:08.382 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:17:08.383 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:17:08.385 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:17:08.385 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:17:08.385 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:17:08.385 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:17:08.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:08.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:17:08.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:17:08.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:17:08.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:17:08.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:17:08.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:17:08.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:17:08.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:17:08.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:17:08.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:08.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:17:08.387 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:17:08.387 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:17:08.387 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:17:08.387 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:17:08.387 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:17:08.387 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:17:08.387 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:17:08.387 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:17:08.387 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:17:08.387 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:08.387 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:17:08.387 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:17:08.388 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:17:08.388 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:17:08.388 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:17:08.388 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:17:08.388 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:17:08.395 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:17:08.396 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:17:08.396 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:17:08.396 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:17:08.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:17:08.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:17:08.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:17:08.449 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:17:08.449 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:17:08.450 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:17:08.450 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:08.450 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:17:08.450 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:17:08.450 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:17:08.450 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:17:08.450 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:17:08.450 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:17:08.450 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:17:08.450 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:17:08.450 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:17:08.451 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:08.451 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:17:08.451 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:17:08.451 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:17:08.451 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:17:08.451 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:17:08.451 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:17:08.451 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:17:08.451 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:17:08.451 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:17:08.451 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:17:08.451 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:08.451 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:17:08.452 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:17:08.452 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:17:08.452 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:17:08.452 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:17:08.452 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:17:08.452 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:17:08.460 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:17:08.460 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:17:08.460 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:17:08.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:17:08.510 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:17:08.511 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:17:08.511 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:17:08.513 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:17:08.513 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:17:08.513 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:17:08.513 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:08.514 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:17:08.514 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:17:08.514 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:17:08.514 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:17:08.514 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:17:08.514 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:17:08.514 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:17:08.514 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:17:08.514 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:17:08.514 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:08.514 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:17:08.514 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:17:08.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:17:08.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:17:08.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:17:08.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:17:08.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:17:08.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:17:08.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:17:08.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:17:08.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:08.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:17:08.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:17:08.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:17:08.516 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:17:08.516 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:17:08.516 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:17:08.516 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:17:08.524 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:17:08.524 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:17:08.524 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:17:08.525 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:17:08.573 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:17:08.574 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:17:08.574 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:17:08.576 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:17:08.577 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:17:08.577 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:17:08.577 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:08.577 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:17:08.577 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:17:08.577 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:17:08.577 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:17:08.577 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:17:08.577 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:17:08.577 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:17:08.577 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:17:08.578 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:17:08.578 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:08.578 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:17:08.578 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:17:08.578 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:17:08.578 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:17:08.578 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:17:08.578 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:17:08.578 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:17:08.578 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:17:08.578 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:17:08.578 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:17:08.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:08.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:17:08.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:17:08.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:17:08.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:17:08.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:17:08.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:17:08.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:17:08.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:17:08.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:17:08.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:17:08.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:17:08.663 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:17:08.664 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:17:08.664 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:17:08.664 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:17:08.664 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:17:08.664 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:17:08.664 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:17:08.664 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:17:08.664 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:17:08.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:17:08.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:17:08.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:17:08.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:17:08.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:08.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:17:08.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:08.701 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 53 2026-02-18T08:17:08.701 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 58 2026-02-18T08:17:08.701 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:17:08.701 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=233, tag_tid=15, entry_tid=1], [object_number=232, tag_tid=15, entry_tid=0], [object_number=234, tag_tid=14, entry_tid=52758], [object_number=235, tag_tid=14, entry_tid=52755]]], state=connected] 2026-02-18T08:17:08.702 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=212, tag_tid=14, entry_tid=32468], [object_number=215, tag_tid=14, entry_tid=32467], [object_number=214, tag_tid=14, entry_tid=32466], [object_number=213, tag_tid=14, entry_tid=32465]]], state=connected] 2026-02-18T08:17:08.705 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:08.707 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=233, tag_tid=15, entry_tid=1' 2026-02-18T08:17:08.708 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:17:08.708 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:17:08.708 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:17:08.709 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:17:08.709 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:17:08.709 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:17:08.709 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:17:08.709 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:17:08.709 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:17:08.709 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:17:08.709 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:17:08.709 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:17:08.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:08.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:17:08.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:08.738 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 53 2026-02-18T08:17:08.738 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 58 2026-02-18T08:17:08.739 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:17:08.739 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=233, tag_tid=15, entry_tid=1], [object_number=232, tag_tid=15, entry_tid=0], [object_number=234, tag_tid=14, entry_tid=52758], [object_number=235, tag_tid=14, entry_tid=52755]]], state=connected] 2026-02-18T08:17:08.739 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=212, tag_tid=14, entry_tid=32468], [object_number=215, tag_tid=14, entry_tid=32467], [object_number=214, tag_tid=14, entry_tid=32466], [object_number=213, tag_tid=14, entry_tid=32465]]], state=connected] 2026-02-18T08:17:08.741 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:08.743 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=212, tag_tid=14, entry_tid=32468' 2026-02-18T08:17:08.743 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=233, tag_tid=15, entry_tid=1' -a 'object_number=233, tag_tid=15, entry_tid=1' = 'object_number=212, tag_tid=14, entry_tid=32468' 2026-02-18T08:17:08.743 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=212, tag_tid=14, entry_tid=32468' '!=' 'object_number=212, tag_tid=14, entry_tid=32076' 2026-02-18T08:17:08.743 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:17:08.743 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=212, tag_tid=14, entry_tid=32468' = 'object_number=212, tag_tid=14, entry_tid=32076' 2026-02-18T08:17:08.744 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=212, tag_tid=14, entry_tid=32468' 2026-02-18T08:17:08.744 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=233, tag_tid=15, entry_tid=1' 2026-02-18T08:17:08.744 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:17:08.744 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:17:08.745 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=15 2026-02-18T08:17:08.746 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=212, tag_tid=14, entry_tid=32468' 2026-02-18T08:17:08.746 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:17:08.746 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:17:08.747 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=14 2026-02-18T08:17:08.748 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=233, tag_tid=15, entry_tid=1' 2026-02-18T08:17:08.748 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:17:08.748 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:17:08.749 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:17:08.750 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=212, tag_tid=14, entry_tid=32468' 2026-02-18T08:17:08.750 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:17:08.751 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:17:08.752 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=32468 2026-02-18T08:17:08.752 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 15 = 14 -a 1 -le 32468 2026-02-18T08:17:08.752 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:17:08.752 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:17:08.752 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:17:08.954 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:17:08.954 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:17:08.954 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:17:08.954 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:17:08.954 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:17:08.954 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:17:08.954 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:17:08.954 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:17:08.954 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:17:08.955 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:08.955 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:17:08.955 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:17:08.955 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:17:08.955 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:17:08.955 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:17:08.958 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:17:08.958 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:17:08.958 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:17:08.958 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:17:08.958 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:17:08.959 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:08.959 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:17:08.959 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:17:08.959 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:17:08.959 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:17:08.959 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:17:08.959 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:17:08.959 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:17:08.959 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:17:08.959 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:17:08.959 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:08.959 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:17:08.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:17:08.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:17:08.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:17:08.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:17:08.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:17:08.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:17:08.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:17:08.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:17:08.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:17:08.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:08.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:17:08.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:17:08.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:17:08.961 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:17:08.961 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:17:08.961 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:17:08.961 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:17:08.969 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:17:08.969 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:17:08.969 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:17:08.969 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:17:09.018 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:17:09.018 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:17:09.018 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:17:09.021 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:17:09.021 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:17:09.021 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:17:09.021 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:09.021 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:17:09.021 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:17:09.021 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:17:09.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:17:09.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:17:09.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:17:09.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:17:09.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:17:09.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:17:09.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:09.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:17:09.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:17:09.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:17:09.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:17:09.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:17:09.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:17:09.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:17:09.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:17:09.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:17:09.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:17:09.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:09.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:17:09.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:17:09.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:17:09.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:17:09.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:17:09.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:17:09.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:17:09.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:17:09.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:17:09.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:17:09.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:17:09.080 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:17:09.080 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:17:09.080 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:17:09.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:17:09.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:17:09.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:17:09.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:09.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:17:09.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:17:09.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:17:09.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:17:09.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:17:09.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:17:09.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:17:09.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:17:09.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:17:09.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:09.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:17:09.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:17:09.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:17:09.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:17:09.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:17:09.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:17:09.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:17:09.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:17:09.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:17:09.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:17:09.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:09.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:17:09.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:17:09.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:17:09.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:17:09.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:17:09.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:17:09.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:17:09.093 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:17:09.094 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:17:09.094 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:17:09.094 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:17:09.156 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:17:09.156 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:17:09.156 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:17:09.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:17:09.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:17:09.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:17:09.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:09.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:17:09.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:17:09.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:17:09.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:17:09.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:17:09.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:17:09.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:17:09.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:17:09.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:17:09.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:09.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:17:09.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:17:09.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:17:09.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:17:09.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:17:09.161 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:17:09.161 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:17:09.161 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:17:09.161 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:17:09.161 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:17:09.161 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:09.161 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:17:09.161 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:17:09.161 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:17:09.161 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:17:09.161 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:17:09.161 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:17:09.162 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:17:09.169 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:17:09.169 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:17:09.169 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:17:09.169 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:17:09.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:17:09.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:17:09.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:17:09.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:17:09.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:17:09.234 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:17:09.234 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:17:09.234 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:17:09.234 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:17:09.234 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:17:09.234 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:17:09.234 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:17:09.234 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:17:09.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:09.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:17:09.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:09.269 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 53 2026-02-18T08:17:09.270 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 58 2026-02-18T08:17:09.270 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:17:09.270 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=233, tag_tid=15, entry_tid=1], [object_number=232, tag_tid=15, entry_tid=0], [object_number=234, tag_tid=14, entry_tid=52758], [object_number=235, tag_tid=14, entry_tid=52755]]], state=connected] 2026-02-18T08:17:09.270 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=213, tag_tid=14, entry_tid=32837], [object_number=212, tag_tid=14, entry_tid=32836], [object_number=215, tag_tid=14, entry_tid=32835], [object_number=214, tag_tid=14, entry_tid=32834]]], state=connected] 2026-02-18T08:17:09.273 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:09.275 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=233, tag_tid=15, entry_tid=1' 2026-02-18T08:17:09.276 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:17:09.276 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:17:09.276 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:17:09.276 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:17:09.276 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:17:09.276 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:17:09.276 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:17:09.276 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:17:09.277 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:17:09.277 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:17:09.277 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:17:09.277 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:17:09.278 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:09.278 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:17:09.278 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:09.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 53 2026-02-18T08:17:09.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 58 2026-02-18T08:17:09.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:17:09.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=233, tag_tid=15, entry_tid=1], [object_number=232, tag_tid=15, entry_tid=0], [object_number=234, tag_tid=14, entry_tid=52758], [object_number=235, tag_tid=14, entry_tid=52755]]], state=connected] 2026-02-18T08:17:09.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=213, tag_tid=14, entry_tid=32837], [object_number=212, tag_tid=14, entry_tid=32836], [object_number=215, tag_tid=14, entry_tid=32835], [object_number=214, tag_tid=14, entry_tid=32834]]], state=connected] 2026-02-18T08:17:09.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:09.310 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=213, tag_tid=14, entry_tid=32837' 2026-02-18T08:17:09.310 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=233, tag_tid=15, entry_tid=1' -a 'object_number=233, tag_tid=15, entry_tid=1' = 'object_number=213, tag_tid=14, entry_tid=32837' 2026-02-18T08:17:09.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=213, tag_tid=14, entry_tid=32837' '!=' 'object_number=212, tag_tid=14, entry_tid=32468' 2026-02-18T08:17:09.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:17:09.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=213, tag_tid=14, entry_tid=32837' = 'object_number=212, tag_tid=14, entry_tid=32468' 2026-02-18T08:17:09.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=213, tag_tid=14, entry_tid=32837' 2026-02-18T08:17:09.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=233, tag_tid=15, entry_tid=1' 2026-02-18T08:17:09.312 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:17:09.312 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:17:09.312 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=15 2026-02-18T08:17:09.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=213, tag_tid=14, entry_tid=32837' 2026-02-18T08:17:09.314 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:17:09.314 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:17:09.315 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=14 2026-02-18T08:17:09.316 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=233, tag_tid=15, entry_tid=1' 2026-02-18T08:17:09.316 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:17:09.316 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:17:09.317 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:17:09.318 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=213, tag_tid=14, entry_tid=32837' 2026-02-18T08:17:09.318 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:17:09.318 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:17:09.320 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=32837 2026-02-18T08:17:09.320 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 15 = 14 -a 1 -le 32837 2026-02-18T08:17:09.320 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:17:09.320 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:17:09.320 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:17:09.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:17:09.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:17:09.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:17:09.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:17:09.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:17:09.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:17:09.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:17:09.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:17:09.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:17:09.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:09.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:17:09.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:17:09.523 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:17:09.523 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:17:09.523 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:17:09.525 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:17:09.525 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:17:09.526 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:17:09.526 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:17:09.526 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:17:09.526 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:09.526 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:17:09.526 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:17:09.526 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:17:09.526 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:17:09.526 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:17:09.526 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:17:09.526 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:17:09.527 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:17:09.527 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:17:09.527 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:09.527 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:17:09.527 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:17:09.527 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:17:09.527 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:17:09.527 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:17:09.527 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:17:09.527 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:17:09.527 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:17:09.527 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:17:09.528 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:17:09.528 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:09.528 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:17:09.528 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:17:09.528 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:17:09.528 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:17:09.528 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:17:09.528 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:17:09.528 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:17:09.536 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:17:09.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:17:09.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:17:09.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:17:09.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:17:09.594 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:17:09.594 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:17:09.596 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:17:09.596 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:17:09.596 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:17:09.596 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:09.597 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:17:09.597 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:17:09.597 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:17:09.597 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:17:09.597 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:17:09.597 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:17:09.597 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:17:09.597 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:17:09.597 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:17:09.597 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:09.597 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:17:09.597 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:17:09.598 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:17:09.598 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:17:09.598 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:17:09.598 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:17:09.598 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:17:09.598 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:17:09.598 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:17:09.598 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:17:09.598 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:09.598 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:17:09.598 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:17:09.599 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:17:09.599 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:17:09.599 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:17:09.599 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:17:09.599 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:17:09.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:17:09.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:17:09.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:17:09.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:17:09.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:17:09.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:17:09.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:17:09.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:17:09.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:17:09.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:17:09.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:09.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:17:09.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:17:09.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:17:09.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:17:09.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:17:09.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:17:09.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:17:09.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:17:09.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:17:09.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:09.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:17:09.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:17:09.670 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:17:09.670 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:17:09.670 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:17:09.670 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:17:09.670 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:17:09.670 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:17:09.670 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:17:09.670 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:17:09.670 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:09.670 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:17:09.670 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:17:09.670 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:17:09.670 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:17:09.671 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:17:09.671 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:17:09.671 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:17:09.679 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:17:09.680 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:17:09.680 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:17:09.680 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:17:09.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:17:09.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:17:09.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:17:09.729 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:17:09.729 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:17:09.729 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:17:09.729 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:09.729 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:17:09.729 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:17:09.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:17:09.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:17:09.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:17:09.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:17:09.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:17:09.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:17:09.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:17:09.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:09.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:17:09.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:17:09.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:17:09.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:17:09.731 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:17:09.731 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:17:09.731 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:17:09.731 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:17:09.731 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:17:09.731 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:17:09.731 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:09.731 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:17:09.731 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:17:09.731 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:17:09.731 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:17:09.731 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:17:09.732 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:17:09.732 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:17:09.739 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:17:09.740 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:17:09.740 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:17:09.740 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:17:09.806 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:17:09.807 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:17:09.807 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:17:09.807 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:17:09.808 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:17:09.808 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:17:09.808 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:17:09.808 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:17:09.808 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:17:09.808 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:17:09.808 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:17:09.808 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:17:09.808 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:17:09.809 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:09.809 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:17:09.809 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:09.845 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 53 2026-02-18T08:17:09.845 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 58 2026-02-18T08:17:09.845 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:17:09.845 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=233, tag_tid=15, entry_tid=1], [object_number=232, tag_tid=15, entry_tid=0], [object_number=234, tag_tid=14, entry_tid=52758], [object_number=235, tag_tid=14, entry_tid=52755]]], state=connected] 2026-02-18T08:17:09.845 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=213, tag_tid=14, entry_tid=33197], [object_number=212, tag_tid=14, entry_tid=33196], [object_number=215, tag_tid=14, entry_tid=33195], [object_number=214, tag_tid=14, entry_tid=33194]]], state=connected] 2026-02-18T08:17:09.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:09.850 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=233, tag_tid=15, entry_tid=1' 2026-02-18T08:17:09.851 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:17:09.851 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:17:09.851 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:17:09.852 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:17:09.852 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:17:09.852 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:17:09.852 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:17:09.852 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:17:09.852 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:17:09.852 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:17:09.853 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:17:09.853 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:17:09.853 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:09.854 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:17:09.854 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:09.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 53 2026-02-18T08:17:09.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 58 2026-02-18T08:17:09.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:17:09.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=233, tag_tid=15, entry_tid=1], [object_number=232, tag_tid=15, entry_tid=0], [object_number=234, tag_tid=14, entry_tid=52758], [object_number=235, tag_tid=14, entry_tid=52755]]], state=connected] 2026-02-18T08:17:09.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=213, tag_tid=14, entry_tid=33197], [object_number=212, tag_tid=14, entry_tid=33196], [object_number=215, tag_tid=14, entry_tid=33195], [object_number=214, tag_tid=14, entry_tid=33194]]], state=connected] 2026-02-18T08:17:09.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:09.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=213, tag_tid=14, entry_tid=33197' 2026-02-18T08:17:09.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=233, tag_tid=15, entry_tid=1' -a 'object_number=233, tag_tid=15, entry_tid=1' = 'object_number=213, tag_tid=14, entry_tid=33197' 2026-02-18T08:17:09.894 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=213, tag_tid=14, entry_tid=33197' '!=' 'object_number=213, tag_tid=14, entry_tid=32837' 2026-02-18T08:17:09.894 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:17:09.894 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=213, tag_tid=14, entry_tid=33197' = 'object_number=213, tag_tid=14, entry_tid=32837' 2026-02-18T08:17:09.894 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=213, tag_tid=14, entry_tid=33197' 2026-02-18T08:17:09.894 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=233, tag_tid=15, entry_tid=1' 2026-02-18T08:17:09.895 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:17:09.895 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:17:09.895 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=15 2026-02-18T08:17:09.896 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=213, tag_tid=14, entry_tid=33197' 2026-02-18T08:17:09.896 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:17:09.897 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:17:09.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=14 2026-02-18T08:17:09.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=233, tag_tid=15, entry_tid=1' 2026-02-18T08:17:09.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:17:09.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:17:09.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:17:09.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=213, tag_tid=14, entry_tid=33197' 2026-02-18T08:17:09.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:17:09.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:17:09.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=33197 2026-02-18T08:17:09.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 15 = 14 -a 1 -le 33197 2026-02-18T08:17:09.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:17:09.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:17:09.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:17:10.105 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:17:10.105 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:17:10.105 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:17:10.105 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:17:10.105 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:17:10.105 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:17:10.105 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:17:10.105 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:17:10.105 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:17:10.106 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:10.106 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:17:10.106 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:17:10.106 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:17:10.106 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:17:10.106 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:17:10.106 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:17:10.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:17:10.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:17:10.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:17:10.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:17:10.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:10.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:17:10.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:17:10.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:17:10.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:17:10.110 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:17:10.110 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:17:10.110 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:17:10.110 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:17:10.110 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:17:10.110 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:10.110 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:17:10.110 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:17:10.110 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:17:10.110 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:17:10.110 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:17:10.110 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:17:10.111 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:17:10.111 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:17:10.111 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:17:10.111 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:17:10.111 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:10.111 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:17:10.111 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:17:10.111 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:17:10.111 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:17:10.111 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:17:10.111 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:17:10.111 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:17:10.118 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:17:10.118 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:17:10.118 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:17:10.118 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:17:10.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:17:10.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:17:10.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:17:10.171 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:17:10.171 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:17:10.171 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:17:10.171 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:10.171 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:17:10.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:17:10.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:17:10.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:17:10.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:17:10.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:17:10.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:17:10.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:17:10.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:17:10.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:10.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:17:10.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:17:10.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:17:10.173 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:17:10.173 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:17:10.173 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:17:10.173 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:17:10.173 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:17:10.173 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:17:10.173 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:17:10.173 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:10.173 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:17:10.173 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:17:10.173 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:17:10.173 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:17:10.174 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:17:10.174 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:17:10.174 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:17:10.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:17:10.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:17:10.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:17:10.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:17:10.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:17:10.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:17:10.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:17:10.238 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:17:10.238 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:17:10.238 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:17:10.238 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:10.238 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:17:10.238 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:17:10.238 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:17:10.238 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:17:10.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:17:10.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:17:10.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:17:10.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:17:10.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:17:10.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:10.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:17:10.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:17:10.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:17:10.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:17:10.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:17:10.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:17:10.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:17:10.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:17:10.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:17:10.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:17:10.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:10.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:17:10.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:17:10.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:17:10.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:17:10.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:17:10.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:17:10.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:17:10.248 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:17:10.249 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:17:10.249 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:17:10.249 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:17:10.298 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:17:10.298 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:17:10.298 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:17:10.301 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:17:10.301 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:17:10.301 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:17:10.301 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:10.301 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:17:10.301 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:17:10.301 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:17:10.301 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:17:10.301 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:17:10.301 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:17:10.302 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:17:10.302 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:17:10.302 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:17:10.302 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:10.302 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:17:10.302 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:17:10.302 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:17:10.302 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:17:10.302 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:17:10.302 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:17:10.302 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:17:10.302 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:17:10.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:17:10.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:17:10.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:10.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:17:10.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:17:10.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:17:10.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:17:10.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:17:10.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:17:10.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:17:10.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:17:10.312 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:17:10.312 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:17:10.312 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:17:10.374 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:17:10.376 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:17:10.376 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:17:10.376 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:17:10.376 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:17:10.376 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:17:10.376 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:17:10.377 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:17:10.377 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:17:10.377 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:17:10.377 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:17:10.377 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:17:10.377 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:17:10.377 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:10.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:17:10.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:10.406 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 53 2026-02-18T08:17:10.407 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 58 2026-02-18T08:17:10.407 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:17:10.407 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=233, tag_tid=15, entry_tid=1], [object_number=232, tag_tid=15, entry_tid=0], [object_number=234, tag_tid=14, entry_tid=52758], [object_number=235, tag_tid=14, entry_tid=52755]]], state=connected] 2026-02-18T08:17:10.407 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=213, tag_tid=14, entry_tid=33561], [object_number=212, tag_tid=14, entry_tid=33560], [object_number=215, tag_tid=14, entry_tid=33559], [object_number=214, tag_tid=14, entry_tid=33558]]], state=connected] 2026-02-18T08:17:10.409 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:10.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=233, tag_tid=15, entry_tid=1' 2026-02-18T08:17:10.412 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:17:10.412 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:17:10.412 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:17:10.412 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:17:10.412 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:17:10.412 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:17:10.412 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:17:10.412 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:17:10.412 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:17:10.412 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:17:10.413 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:17:10.413 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:17:10.414 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:10.414 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:17:10.414 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:10.440 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 53 2026-02-18T08:17:10.440 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 58 2026-02-18T08:17:10.440 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:17:10.440 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=233, tag_tid=15, entry_tid=1], [object_number=232, tag_tid=15, entry_tid=0], [object_number=234, tag_tid=14, entry_tid=52758], [object_number=235, tag_tid=14, entry_tid=52755]]], state=connected] 2026-02-18T08:17:10.440 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=213, tag_tid=14, entry_tid=33561], [object_number=212, tag_tid=14, entry_tid=33560], [object_number=215, tag_tid=14, entry_tid=33559], [object_number=214, tag_tid=14, entry_tid=33558]]], state=connected] 2026-02-18T08:17:10.443 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:10.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=213, tag_tid=14, entry_tid=33561' 2026-02-18T08:17:10.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=233, tag_tid=15, entry_tid=1' -a 'object_number=233, tag_tid=15, entry_tid=1' = 'object_number=213, tag_tid=14, entry_tid=33561' 2026-02-18T08:17:10.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=213, tag_tid=14, entry_tid=33561' '!=' 'object_number=213, tag_tid=14, entry_tid=33197' 2026-02-18T08:17:10.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:17:10.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=213, tag_tid=14, entry_tid=33561' = 'object_number=213, tag_tid=14, entry_tid=33197' 2026-02-18T08:17:10.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=213, tag_tid=14, entry_tid=33561' 2026-02-18T08:17:10.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=233, tag_tid=15, entry_tid=1' 2026-02-18T08:17:10.446 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:17:10.446 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:17:10.446 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=15 2026-02-18T08:17:10.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=213, tag_tid=14, entry_tid=33561' 2026-02-18T08:17:10.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:17:10.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:17:10.449 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=14 2026-02-18T08:17:10.449 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=233, tag_tid=15, entry_tid=1' 2026-02-18T08:17:10.450 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:17:10.450 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:17:10.451 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:17:10.452 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=213, tag_tid=14, entry_tid=33561' 2026-02-18T08:17:10.452 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:17:10.452 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:17:10.454 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=33561 2026-02-18T08:17:10.454 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 15 = 14 -a 1 -le 33561 2026-02-18T08:17:10.454 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:17:10.454 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:17:10.455 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:17:10.655 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:17:10.655 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:17:10.655 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:17:10.655 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:17:10.655 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:17:10.655 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:17:10.655 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:17:10.655 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:17:10.655 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:17:10.655 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:10.655 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:17:10.655 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:17:10.655 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:17:10.655 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:17:10.655 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:17:10.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:17:10.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:17:10.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:17:10.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:17:10.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:17:10.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:10.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:17:10.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:17:10.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:17:10.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:17:10.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:17:10.658 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:17:10.658 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:17:10.658 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:17:10.658 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:17:10.658 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:10.658 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:17:10.658 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:17:10.658 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:17:10.658 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:17:10.658 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:17:10.658 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:17:10.658 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:17:10.658 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:17:10.658 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:17:10.658 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:17:10.658 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:10.658 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:17:10.658 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:17:10.658 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:17:10.658 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:17:10.658 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:17:10.658 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:17:10.658 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:17:10.670 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:17:10.670 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:17:10.670 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:17:10.670 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:17:10.720 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:17:10.720 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:17:10.720 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:17:10.722 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:17:10.722 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:17:10.722 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:17:10.722 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:10.722 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:17:10.722 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:17:10.722 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:17:10.722 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:17:10.722 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:17:10.722 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:17:10.722 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:17:10.722 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:17:10.722 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:17:10.722 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:10.722 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:17:10.722 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:17:10.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:17:10.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:17:10.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:17:10.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:17:10.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:17:10.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:17:10.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:17:10.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:17:10.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:10.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:17:10.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:17:10.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:17:10.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:17:10.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:17:10.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:17:10.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:17:10.734 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:17:10.735 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:17:10.735 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:17:10.735 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:17:10.784 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:17:10.785 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:17:10.785 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:17:10.788 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:17:10.788 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:17:10.788 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:17:10.788 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:10.788 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:17:10.788 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:17:10.788 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:17:10.788 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:17:10.788 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:17:10.788 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:17:10.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:17:10.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:17:10.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:17:10.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:10.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:17:10.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:17:10.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:17:10.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:17:10.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:17:10.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:17:10.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:17:10.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:17:10.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:17:10.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:17:10.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:10.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:17:10.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:17:10.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:17:10.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:17:10.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:17:10.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:17:10.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:17:10.800 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:17:10.801 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:17:10.801 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:17:10.801 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:17:10.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:17:10.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:17:10.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:17:10.852 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:17:10.852 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:17:10.852 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:17:10.852 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:10.852 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:17:10.852 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:17:10.853 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:17:10.853 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:17:10.853 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:17:10.853 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:17:10.853 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:17:10.853 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:17:10.853 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:17:10.853 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:10.853 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:17:10.853 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:17:10.853 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:17:10.854 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:17:10.854 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:17:10.854 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:17:10.854 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:17:10.854 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:17:10.854 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:17:10.854 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:17:10.854 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:10.854 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:17:10.854 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:17:10.854 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:17:10.854 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:17:10.854 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:17:10.855 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:17:10.855 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:17:10.865 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:17:10.866 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:17:10.866 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:17:10.866 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:17:10.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:17:10.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:17:10.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:17:10.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:17:10.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:17:10.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:17:10.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:17:10.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:17:10.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:17:10.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:17:10.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:17:10.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:17:10.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:17:10.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:10.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:10.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:17:11.170 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 54 2026-02-18T08:17:11.170 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 58 2026-02-18T08:17:11.170 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:17:11.171 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=233, tag_tid=15, entry_tid=1], [object_number=232, tag_tid=15, entry_tid=0], [object_number=234, tag_tid=14, entry_tid=52758], [object_number=235, tag_tid=14, entry_tid=52755]]], state=connected] 2026-02-18T08:17:11.171 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=217, tag_tid=14, entry_tid=33837], [object_number=216, tag_tid=14, entry_tid=33836], [object_number=219, tag_tid=14, entry_tid=33835], [object_number=218, tag_tid=14, entry_tid=33834]]], state=connected] 2026-02-18T08:17:11.174 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:11.175 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=233, tag_tid=15, entry_tid=1' 2026-02-18T08:17:11.177 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:17:11.177 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:17:11.177 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:17:11.177 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:17:11.177 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:17:11.177 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:17:11.177 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:17:11.177 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:17:11.177 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:17:11.178 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:17:11.178 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:17:11.178 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:17:11.179 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:11.179 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:17:11.179 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:11.207 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 54 2026-02-18T08:17:11.207 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 58 2026-02-18T08:17:11.207 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:17:11.207 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=233, tag_tid=15, entry_tid=1], [object_number=232, tag_tid=15, entry_tid=0], [object_number=234, tag_tid=14, entry_tid=52758], [object_number=235, tag_tid=14, entry_tid=52755]]], state=connected] 2026-02-18T08:17:11.207 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=217, tag_tid=14, entry_tid=33837], [object_number=216, tag_tid=14, entry_tid=33836], [object_number=219, tag_tid=14, entry_tid=33835], [object_number=218, tag_tid=14, entry_tid=33834]]], state=connected] 2026-02-18T08:17:11.210 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:11.212 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=217, tag_tid=14, entry_tid=33837' 2026-02-18T08:17:11.212 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=233, tag_tid=15, entry_tid=1' -a 'object_number=233, tag_tid=15, entry_tid=1' = 'object_number=217, tag_tid=14, entry_tid=33837' 2026-02-18T08:17:11.213 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=217, tag_tid=14, entry_tid=33837' '!=' 'object_number=213, tag_tid=14, entry_tid=33561' 2026-02-18T08:17:11.213 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:17:11.213 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=217, tag_tid=14, entry_tid=33837' = 'object_number=213, tag_tid=14, entry_tid=33561' 2026-02-18T08:17:11.213 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=217, tag_tid=14, entry_tid=33837' 2026-02-18T08:17:11.213 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=233, tag_tid=15, entry_tid=1' 2026-02-18T08:17:11.213 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:17:11.213 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:17:11.214 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=15 2026-02-18T08:17:11.215 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=217, tag_tid=14, entry_tid=33837' 2026-02-18T08:17:11.215 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:17:11.215 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:17:11.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=14 2026-02-18T08:17:11.217 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=233, tag_tid=15, entry_tid=1' 2026-02-18T08:17:11.217 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:17:11.217 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:17:11.218 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:17:11.219 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=217, tag_tid=14, entry_tid=33837' 2026-02-18T08:17:11.219 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:17:11.219 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:17:11.221 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=33837 2026-02-18T08:17:11.221 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 15 = 14 -a 1 -le 33837 2026-02-18T08:17:11.221 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:17:11.221 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:17:11.222 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:17:11.423 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:17:11.423 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:17:11.423 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:17:11.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:17:11.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:17:11.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:17:11.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:17:11.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:17:11.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:17:11.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:11.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:17:11.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:17:11.425 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:17:11.425 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:17:11.425 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:17:11.425 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:17:11.428 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:17:11.428 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:17:11.428 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:17:11.428 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:17:11.428 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:11.428 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:17:11.428 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:17:11.429 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:17:11.429 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:17:11.429 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:17:11.429 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:17:11.429 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:17:11.429 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:17:11.429 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:17:11.429 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:11.429 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:17:11.429 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:17:11.429 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:17:11.430 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:17:11.430 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:17:11.430 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:17:11.430 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:17:11.430 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:17:11.430 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:17:11.430 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:17:11.430 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:11.430 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:17:11.430 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:17:11.431 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:17:11.431 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:17:11.431 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:17:11.431 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:17:11.431 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:17:11.437 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:17:11.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:17:11.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:17:11.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:17:11.487 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:17:11.487 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:17:11.487 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:17:11.490 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:17:11.490 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:17:11.490 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:17:11.490 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:11.490 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:17:11.491 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:17:11.491 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:17:11.491 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:17:11.491 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:17:11.491 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:17:11.491 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:17:11.491 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:17:11.491 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:17:11.491 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:11.491 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:17:11.491 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:17:11.491 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:17:11.492 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:17:11.492 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:17:11.492 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:17:11.492 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:17:11.492 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:17:11.492 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:17:11.492 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:17:11.492 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:11.492 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:17:11.492 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:17:11.492 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:17:11.492 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:17:11.493 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:17:11.493 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:17:11.493 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:17:11.501 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:17:11.502 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:17:11.502 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:17:11.502 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:17:11.552 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:17:11.553 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:17:11.553 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:17:11.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:17:11.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:17:11.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:17:11.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:11.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:17:11.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:17:11.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:17:11.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:17:11.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:17:11.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:17:11.557 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:17:11.557 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:17:11.557 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:17:11.557 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:11.557 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:17:11.557 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:17:11.557 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:17:11.557 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:17:11.557 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:17:11.557 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:17:11.557 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:17:11.557 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:17:11.558 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:17:11.558 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:17:11.558 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:11.558 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:17:11.558 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:17:11.558 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:17:11.558 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:17:11.558 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:17:11.558 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:17:11.558 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:17:11.568 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:17:11.568 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:17:11.568 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:17:11.568 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:17:11.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:17:11.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:17:11.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:17:11.629 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:17:11.629 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:17:11.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:17:11.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:11.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:17:11.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:17:11.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:17:11.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:17:11.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:17:11.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:17:11.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:17:11.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:17:11.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:17:11.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:11.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:17:11.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:17:11.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:17:11.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:17:11.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:17:11.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:17:11.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:17:11.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:17:11.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:17:11.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:17:11.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:11.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:17:11.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:17:11.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:17:11.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:17:11.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:17:11.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:17:11.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:17:11.640 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:17:11.640 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:17:11.641 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:17:11.641 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:17:11.701 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:17:11.702 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:17:11.702 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:17:11.703 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:17:11.703 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:17:11.703 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:17:11.703 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:17:11.703 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:17:11.703 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:17:11.703 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:17:11.703 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:17:11.703 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:17:11.703 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:17:11.704 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:11.704 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:17:11.705 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:11.733 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 54 2026-02-18T08:17:11.734 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 58 2026-02-18T08:17:11.734 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:17:11.734 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=233, tag_tid=15, entry_tid=1], [object_number=232, tag_tid=15, entry_tid=0], [object_number=234, tag_tid=14, entry_tid=52758], [object_number=235, tag_tid=14, entry_tid=52755]]], state=connected] 2026-02-18T08:17:11.734 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=216, tag_tid=14, entry_tid=34324], [object_number=219, tag_tid=14, entry_tid=34323], [object_number=218, tag_tid=14, entry_tid=34322], [object_number=217, tag_tid=14, entry_tid=34321]]], state=connected] 2026-02-18T08:17:11.736 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:11.737 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=233, tag_tid=15, entry_tid=1' 2026-02-18T08:17:11.739 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:17:11.739 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:17:11.739 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:17:11.739 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:17:11.739 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:17:11.739 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:17:11.739 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:17:11.740 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:17:11.740 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:17:11.740 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:17:11.740 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:17:11.740 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:17:11.741 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:11.741 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:17:11.741 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:11.769 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 54 2026-02-18T08:17:11.769 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 58 2026-02-18T08:17:11.769 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:17:11.769 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=233, tag_tid=15, entry_tid=1], [object_number=232, tag_tid=15, entry_tid=0], [object_number=234, tag_tid=14, entry_tid=52758], [object_number=235, tag_tid=14, entry_tid=52755]]], state=connected] 2026-02-18T08:17:11.769 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=216, tag_tid=14, entry_tid=34324], [object_number=219, tag_tid=14, entry_tid=34323], [object_number=218, tag_tid=14, entry_tid=34322], [object_number=217, tag_tid=14, entry_tid=34321]]], state=connected] 2026-02-18T08:17:11.771 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:11.773 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=216, tag_tid=14, entry_tid=34324' 2026-02-18T08:17:11.774 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=233, tag_tid=15, entry_tid=1' -a 'object_number=233, tag_tid=15, entry_tid=1' = 'object_number=216, tag_tid=14, entry_tid=34324' 2026-02-18T08:17:11.774 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=216, tag_tid=14, entry_tid=34324' '!=' 'object_number=217, tag_tid=14, entry_tid=33837' 2026-02-18T08:17:11.774 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:17:11.774 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=216, tag_tid=14, entry_tid=34324' = 'object_number=217, tag_tid=14, entry_tid=33837' 2026-02-18T08:17:11.774 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=216, tag_tid=14, entry_tid=34324' 2026-02-18T08:17:11.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=233, tag_tid=15, entry_tid=1' 2026-02-18T08:17:11.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:17:11.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:17:11.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=15 2026-02-18T08:17:11.776 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=216, tag_tid=14, entry_tid=34324' 2026-02-18T08:17:11.776 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:17:11.776 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:17:11.778 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=14 2026-02-18T08:17:11.778 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=233, tag_tid=15, entry_tid=1' 2026-02-18T08:17:11.779 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:17:11.779 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:17:11.780 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:17:11.780 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=216, tag_tid=14, entry_tid=34324' 2026-02-18T08:17:11.780 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:17:11.780 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:17:11.782 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=34324 2026-02-18T08:17:11.782 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 15 = 14 -a 1 -le 34324 2026-02-18T08:17:11.782 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:17:11.782 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:17:11.782 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:17:11.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:17:11.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:17:11.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:17:11.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:17:11.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:17:11.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:17:11.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:17:11.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:17:11.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:17:11.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:11.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:17:11.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:17:11.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:17:11.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:17:11.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:17:11.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:17:11.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:17:11.985 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:17:11.985 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:17:11.985 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:17:11.985 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:11.985 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:17:11.985 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:17:11.986 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:17:11.986 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:17:11.986 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:17:11.986 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:17:11.986 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:17:11.986 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:17:11.986 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:17:11.986 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:11.986 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:17:11.986 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:17:11.986 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:17:11.986 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:17:11.986 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:17:11.986 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:17:11.986 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:17:11.986 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:17:11.986 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:17:11.986 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:17:11.986 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:11.986 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:17:11.986 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:17:11.986 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:17:11.986 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:17:11.986 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:17:11.986 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:17:11.987 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:17:11.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:17:11.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:17:11.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:17:11.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:17:12.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:17:12.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:17:12.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:17:12.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:17:12.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:17:12.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:17:12.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:12.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:17:12.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:17:12.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:17:12.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:17:12.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:17:12.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:17:12.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:17:12.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:17:12.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:17:12.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:12.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:17:12.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:17:12.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:17:12.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:17:12.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:17:12.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:17:12.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:17:12.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:17:12.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:17:12.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:17:12.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:12.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:17:12.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:17:12.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:17:12.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:17:12.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:17:12.055 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:17:12.055 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:17:12.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:17:12.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:17:12.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:17:12.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:17:12.111 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:17:12.111 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:17:12.111 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:17:12.114 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:17:12.114 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:17:12.114 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:17:12.114 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:12.114 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:17:12.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:17:12.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:17:12.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:17:12.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:17:12.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:17:12.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:17:12.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:17:12.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:17:12.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:12.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:17:12.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:17:12.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:17:12.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:17:12.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:17:12.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:17:12.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:17:12.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:17:12.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:17:12.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:17:12.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:12.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:17:12.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:17:12.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:17:12.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:17:12.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:17:12.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:17:12.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:17:12.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:17:12.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:17:12.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:17:12.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:17:12.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:17:12.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:17:12.183 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:17:12.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:17:12.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:17:12.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:17:12.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:12.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:17:12.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:17:12.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:17:12.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:17:12.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:17:12.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:17:12.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:17:12.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:17:12.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:17:12.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:12.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:17:12.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:17:12.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:17:12.187 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:17:12.187 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:17:12.187 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:17:12.187 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:17:12.187 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:17:12.187 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:17:12.187 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:17:12.187 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:12.187 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:17:12.187 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:17:12.187 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:17:12.187 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:17:12.187 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:17:12.188 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:17:12.188 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:17:12.196 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:17:12.197 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:17:12.197 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:17:12.197 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:17:12.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:17:12.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:17:12.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:17:12.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:17:12.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:17:12.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:17:12.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:17:12.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:17:12.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:17:12.266 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:17:12.266 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:17:12.266 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:17:12.266 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:17:12.266 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:12.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:17:12.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:12.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 54 2026-02-18T08:17:12.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 58 2026-02-18T08:17:12.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:17:12.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=233, tag_tid=15, entry_tid=1], [object_number=232, tag_tid=15, entry_tid=0], [object_number=234, tag_tid=14, entry_tid=52758], [object_number=235, tag_tid=14, entry_tid=52755]]], state=connected] 2026-02-18T08:17:12.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=217, tag_tid=14, entry_tid=34721], [object_number=216, tag_tid=14, entry_tid=34720], [object_number=219, tag_tid=14, entry_tid=34719], [object_number=218, tag_tid=14, entry_tid=34718]]], state=connected] 2026-02-18T08:17:12.297 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:12.299 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=233, tag_tid=15, entry_tid=1' 2026-02-18T08:17:12.300 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:17:12.300 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:17:12.300 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:17:12.300 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:17:12.301 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:17:12.301 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:17:12.301 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:17:12.301 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:17:12.301 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:17:12.301 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:17:12.302 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:17:12.302 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:17:12.302 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:12.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:17:12.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:12.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 54 2026-02-18T08:17:12.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 58 2026-02-18T08:17:12.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:17:12.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=233, tag_tid=15, entry_tid=1], [object_number=232, tag_tid=15, entry_tid=0], [object_number=234, tag_tid=14, entry_tid=52758], [object_number=235, tag_tid=14, entry_tid=52755]]], state=connected] 2026-02-18T08:17:12.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=217, tag_tid=14, entry_tid=34721], [object_number=216, tag_tid=14, entry_tid=34720], [object_number=219, tag_tid=14, entry_tid=34719], [object_number=218, tag_tid=14, entry_tid=34718]]], state=connected] 2026-02-18T08:17:12.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:12.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=217, tag_tid=14, entry_tid=34721' 2026-02-18T08:17:12.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=233, tag_tid=15, entry_tid=1' -a 'object_number=233, tag_tid=15, entry_tid=1' = 'object_number=217, tag_tid=14, entry_tid=34721' 2026-02-18T08:17:12.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=217, tag_tid=14, entry_tid=34721' '!=' 'object_number=216, tag_tid=14, entry_tid=34324' 2026-02-18T08:17:12.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:17:12.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=217, tag_tid=14, entry_tid=34721' = 'object_number=216, tag_tid=14, entry_tid=34324' 2026-02-18T08:17:12.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=217, tag_tid=14, entry_tid=34721' 2026-02-18T08:17:12.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=233, tag_tid=15, entry_tid=1' 2026-02-18T08:17:12.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:17:12.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:17:12.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=15 2026-02-18T08:17:12.348 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=217, tag_tid=14, entry_tid=34721' 2026-02-18T08:17:12.348 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:17:12.348 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:17:12.350 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=14 2026-02-18T08:17:12.350 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=233, tag_tid=15, entry_tid=1' 2026-02-18T08:17:12.351 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:17:12.351 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:17:12.352 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:17:12.352 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=217, tag_tid=14, entry_tid=34721' 2026-02-18T08:17:12.352 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:17:12.353 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:17:12.354 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=34721 2026-02-18T08:17:12.354 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 15 = 14 -a 1 -le 34721 2026-02-18T08:17:12.354 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:17:12.354 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:17:12.354 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:17:12.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:17:12.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:17:12.557 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:17:12.557 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:17:12.557 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:17:12.557 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:17:12.557 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:17:12.557 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:17:12.557 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:17:12.557 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:12.558 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:17:12.558 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:17:12.558 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:17:12.558 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:17:12.558 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:17:12.558 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:17:12.558 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:17:12.561 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:17:12.561 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:17:12.561 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:17:12.561 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:12.561 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:17:12.561 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:17:12.561 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:17:12.562 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:17:12.562 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:17:12.562 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:17:12.562 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:17:12.562 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:17:12.562 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:17:12.562 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:12.562 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:17:12.562 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:17:12.562 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:17:12.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:17:12.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:17:12.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:17:12.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:17:12.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:17:12.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:17:12.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:17:12.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:12.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:17:12.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:17:12.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:17:12.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:17:12.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:17:12.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:17:12.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:17:12.571 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:17:12.571 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:17:12.571 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:17:12.571 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:17:12.629 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:17:12.629 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:17:12.629 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:17:12.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:17:12.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:17:12.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:17:12.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:12.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:17:12.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:17:12.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:17:12.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:17:12.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:17:12.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:17:12.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:17:12.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:17:12.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:17:12.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:12.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:17:12.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:17:12.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:17:12.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:17:12.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:17:12.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:17:12.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:17:12.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:17:12.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:17:12.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:17:12.635 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:12.635 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:17:12.635 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:17:12.635 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:17:12.635 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:17:12.635 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:17:12.635 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:17:12.635 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:17:12.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:17:12.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:17:12.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:17:12.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:17:12.705 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:17:12.705 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:17:12.705 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:17:12.708 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:17:12.708 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:17:12.708 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:17:12.708 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:12.708 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:17:12.708 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:17:12.708 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:17:12.708 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:17:12.708 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:17:12.709 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:17:12.709 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:17:12.709 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:17:12.709 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:17:12.709 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:12.709 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:17:12.709 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:17:12.709 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:17:12.709 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:17:12.709 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:17:12.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:17:12.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:17:12.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:17:12.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:17:12.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:17:12.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:12.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:17:12.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:17:12.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:17:12.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:17:12.711 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:17:12.711 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:17:12.711 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:17:12.718 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:17:12.718 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:17:12.718 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:17:12.718 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:17:12.774 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:17:12.774 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:17:12.774 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:17:12.777 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:17:12.777 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:17:12.777 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:17:12.777 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:12.777 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:17:12.777 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:17:12.777 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:17:12.777 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:17:12.778 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:17:12.778 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:17:12.778 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:17:12.778 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:17:12.778 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:17:12.778 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:12.778 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:17:12.778 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:17:12.778 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:17:12.778 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:17:12.778 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:17:12.779 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:17:12.779 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:17:12.779 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:17:12.779 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:17:12.779 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:17:12.779 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:12.779 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:17:12.779 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:17:12.779 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:17:12.779 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:17:12.779 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:17:12.780 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:17:12.780 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:17:12.788 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:17:12.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:17:12.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:17:12.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:17:12.848 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:17:12.848 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:17:12.848 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:17:12.848 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:17:12.848 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:17:12.848 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:17:12.848 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:17:12.848 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:17:12.848 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:17:12.848 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:17:12.848 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:17:12.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:17:12.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:17:12.850 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:12.851 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:17:12.851 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:12.879 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 54 2026-02-18T08:17:12.879 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 58 2026-02-18T08:17:12.879 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:17:12.879 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=233, tag_tid=15, entry_tid=1], [object_number=232, tag_tid=15, entry_tid=0], [object_number=234, tag_tid=14, entry_tid=52758], [object_number=235, tag_tid=14, entry_tid=52755]]], state=connected] 2026-02-18T08:17:12.879 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=216, tag_tid=14, entry_tid=35112], [object_number=219, tag_tid=14, entry_tid=35111], [object_number=218, tag_tid=14, entry_tid=35110], [object_number=217, tag_tid=14, entry_tid=35109]]], state=connected] 2026-02-18T08:17:12.883 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:12.884 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=233, tag_tid=15, entry_tid=1' 2026-02-18T08:17:12.885 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:17:12.885 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:17:12.885 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:17:12.885 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:17:12.885 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:17:12.885 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:17:12.885 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:17:12.885 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:17:12.885 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:17:12.885 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:17:12.886 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:17:12.886 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:17:12.887 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:12.887 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:17:12.887 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:12.917 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 54 2026-02-18T08:17:12.917 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 58 2026-02-18T08:17:12.918 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:17:12.918 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=233, tag_tid=15, entry_tid=1], [object_number=232, tag_tid=15, entry_tid=0], [object_number=234, tag_tid=14, entry_tid=52758], [object_number=235, tag_tid=14, entry_tid=52755]]], state=connected] 2026-02-18T08:17:12.918 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=216, tag_tid=14, entry_tid=35112], [object_number=219, tag_tid=14, entry_tid=35111], [object_number=218, tag_tid=14, entry_tid=35110], [object_number=217, tag_tid=14, entry_tid=35109]]], state=connected] 2026-02-18T08:17:12.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:12.922 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=216, tag_tid=14, entry_tid=35112' 2026-02-18T08:17:12.922 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=233, tag_tid=15, entry_tid=1' -a 'object_number=233, tag_tid=15, entry_tid=1' = 'object_number=216, tag_tid=14, entry_tid=35112' 2026-02-18T08:17:12.922 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=216, tag_tid=14, entry_tid=35112' '!=' 'object_number=217, tag_tid=14, entry_tid=34721' 2026-02-18T08:17:12.922 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:17:12.922 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=216, tag_tid=14, entry_tid=35112' = 'object_number=217, tag_tid=14, entry_tid=34721' 2026-02-18T08:17:12.922 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=216, tag_tid=14, entry_tid=35112' 2026-02-18T08:17:12.922 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=233, tag_tid=15, entry_tid=1' 2026-02-18T08:17:12.922 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:17:12.922 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:17:12.924 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=15 2026-02-18T08:17:12.924 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=216, tag_tid=14, entry_tid=35112' 2026-02-18T08:17:12.924 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:17:12.924 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:17:12.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=14 2026-02-18T08:17:12.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=233, tag_tid=15, entry_tid=1' 2026-02-18T08:17:12.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:17:12.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:17:12.928 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:17:12.929 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=216, tag_tid=14, entry_tid=35112' 2026-02-18T08:17:12.929 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:17:12.929 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:17:12.930 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=35112 2026-02-18T08:17:12.930 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 15 = 14 -a 1 -le 35112 2026-02-18T08:17:12.930 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:17:12.931 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:17:12.931 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:17:13.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:17:13.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:17:13.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:17:13.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:17:13.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:17:13.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:17:13.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:17:13.134 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:17:13.134 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:17:13.134 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:13.134 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:17:13.134 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:17:13.134 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:17:13.134 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:17:13.134 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:17:13.137 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:17:13.137 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:17:13.137 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:17:13.137 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:17:13.137 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:17:13.138 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:13.138 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:17:13.138 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:17:13.138 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:17:13.138 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:17:13.138 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:17:13.138 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:17:13.138 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:17:13.138 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:17:13.138 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:17:13.138 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:13.138 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:17:13.139 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:17:13.139 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:17:13.139 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:17:13.139 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:17:13.139 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:17:13.139 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:17:13.139 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:17:13.139 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:17:13.139 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:17:13.139 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:13.140 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:17:13.140 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:17:13.140 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:17:13.140 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:17:13.140 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:17:13.140 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:17:13.140 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:17:13.148 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:17:13.149 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:17:13.149 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:17:13.149 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:17:13.201 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:17:13.201 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:17:13.201 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:17:13.204 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:17:13.204 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:17:13.204 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:17:13.204 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:13.204 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:17:13.204 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:17:13.204 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:17:13.204 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:17:13.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:17:13.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:17:13.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:17:13.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:17:13.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:17:13.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:13.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:17:13.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:17:13.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:17:13.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:17:13.206 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:17:13.206 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:17:13.206 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:17:13.206 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:17:13.206 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:17:13.206 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:17:13.206 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:13.206 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:17:13.206 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:17:13.206 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:17:13.206 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:17:13.207 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:17:13.207 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:17:13.207 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:17:13.215 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:17:13.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:17:13.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:17:13.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:17:13.266 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:17:13.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:17:13.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:17:13.270 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:17:13.270 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:17:13.270 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:17:13.270 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:13.270 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:17:13.270 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:17:13.270 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:17:13.270 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:17:13.270 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:17:13.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:17:13.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:17:13.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:17:13.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:17:13.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:13.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:17:13.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:17:13.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:17:13.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:17:13.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:17:13.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:17:13.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:17:13.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:17:13.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:17:13.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:17:13.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:13.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:17:13.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:17:13.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:17:13.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:17:13.273 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:17:13.273 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:17:13.273 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:17:13.281 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:17:13.281 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:17:13.281 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:17:13.281 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:17:13.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:17:13.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:17:13.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:17:13.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:17:13.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:17:13.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:17:13.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:13.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:17:13.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:17:13.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:17:13.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:17:13.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:17:13.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:17:13.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:17:13.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:17:13.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:17:13.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:13.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:17:13.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:17:13.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:17:13.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:17:13.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:17:13.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:17:13.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:17:13.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:17:13.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:17:13.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:17:13.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:13.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:17:13.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:17:13.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:17:13.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:17:13.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:17:13.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:17:13.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:17:13.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:17:13.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:17:13.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:17:13.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:17:13.406 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:17:13.408 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:17:13.408 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:17:13.408 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:17:13.408 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:17:13.408 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:17:13.408 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:17:13.408 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:17:13.408 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:17:13.408 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:17:13.409 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:17:13.409 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:17:13.409 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:17:13.409 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:13.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:17:13.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:13.446 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 54 2026-02-18T08:17:13.446 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 58 2026-02-18T08:17:13.446 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:17:13.446 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=233, tag_tid=15, entry_tid=1], [object_number=232, tag_tid=15, entry_tid=0], [object_number=234, tag_tid=14, entry_tid=52758], [object_number=235, tag_tid=14, entry_tid=52755]]], state=connected] 2026-02-18T08:17:13.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=219, tag_tid=14, entry_tid=35455], [object_number=218, tag_tid=14, entry_tid=35454], [object_number=217, tag_tid=14, entry_tid=35453], [object_number=216, tag_tid=14, entry_tid=35452]]], state=connected] 2026-02-18T08:17:13.450 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:13.451 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=233, tag_tid=15, entry_tid=1' 2026-02-18T08:17:13.453 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:17:13.453 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:17:13.453 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:17:13.453 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:17:13.453 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:17:13.453 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:17:13.453 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:17:13.454 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:17:13.454 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:17:13.454 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:17:13.454 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:17:13.454 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:17:13.455 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:13.455 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:17:13.455 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:13.492 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 54 2026-02-18T08:17:13.492 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 58 2026-02-18T08:17:13.492 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:17:13.493 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=233, tag_tid=15, entry_tid=1], [object_number=232, tag_tid=15, entry_tid=0], [object_number=234, tag_tid=14, entry_tid=52758], [object_number=235, tag_tid=14, entry_tid=52755]]], state=connected] 2026-02-18T08:17:13.493 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=219, tag_tid=14, entry_tid=35455], [object_number=218, tag_tid=14, entry_tid=35454], [object_number=217, tag_tid=14, entry_tid=35453], [object_number=216, tag_tid=14, entry_tid=35452]]], state=connected] 2026-02-18T08:17:13.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:13.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=219, tag_tid=14, entry_tid=35455' 2026-02-18T08:17:13.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=233, tag_tid=15, entry_tid=1' -a 'object_number=233, tag_tid=15, entry_tid=1' = 'object_number=219, tag_tid=14, entry_tid=35455' 2026-02-18T08:17:13.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=219, tag_tid=14, entry_tid=35455' '!=' 'object_number=216, tag_tid=14, entry_tid=35112' 2026-02-18T08:17:13.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:17:13.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=219, tag_tid=14, entry_tid=35455' = 'object_number=216, tag_tid=14, entry_tid=35112' 2026-02-18T08:17:13.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=219, tag_tid=14, entry_tid=35455' 2026-02-18T08:17:13.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=233, tag_tid=15, entry_tid=1' 2026-02-18T08:17:13.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:17:13.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:17:13.500 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=15 2026-02-18T08:17:13.500 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=219, tag_tid=14, entry_tid=35455' 2026-02-18T08:17:13.501 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:17:13.501 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:17:13.502 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=14 2026-02-18T08:17:13.503 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=233, tag_tid=15, entry_tid=1' 2026-02-18T08:17:13.503 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:17:13.503 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:17:13.504 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:17:13.505 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=219, tag_tid=14, entry_tid=35455' 2026-02-18T08:17:13.505 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:17:13.505 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:17:13.507 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=35455 2026-02-18T08:17:13.507 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 15 = 14 -a 1 -le 35455 2026-02-18T08:17:13.507 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:17:13.507 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:17:13.508 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:17:13.709 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:17:13.709 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:17:13.709 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:17:13.709 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:17:13.709 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:17:13.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:17:13.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:17:13.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:17:13.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:17:13.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:13.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:17:13.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:17:13.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:17:13.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:17:13.711 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:17:13.711 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:17:13.713 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:17:13.714 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:17:13.714 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:17:13.714 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:17:13.714 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:13.714 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:17:13.714 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:17:13.714 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:17:13.714 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:17:13.714 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:17:13.714 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:17:13.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:17:13.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:17:13.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:17:13.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:13.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:17:13.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:17:13.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:17:13.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:17:13.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:17:13.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:17:13.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:17:13.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:17:13.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:17:13.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:17:13.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:13.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:17:13.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:17:13.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:17:13.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:17:13.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:17:13.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:17:13.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:17:13.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:17:13.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:17:13.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:17:13.724 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:17:13.773 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:17:13.773 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:17:13.773 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:17:13.776 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:17:13.777 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:17:13.777 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:17:13.777 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:13.777 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:17:13.777 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:17:13.777 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:17:13.777 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:17:13.777 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:17:13.777 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:17:13.777 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:17:13.777 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:17:13.778 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:17:13.778 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:13.778 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:17:13.778 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:17:13.778 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:17:13.778 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:17:13.778 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:17:13.778 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:17:13.778 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:17:13.778 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:17:13.778 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:17:13.778 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:17:13.779 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:13.779 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:17:13.779 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:17:13.779 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:17:13.779 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:17:13.779 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:17:13.779 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:17:13.779 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:17:13.788 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:17:13.788 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:17:13.788 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:17:13.788 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:17:13.847 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:17:13.847 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:17:13.847 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:17:13.851 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:17:13.851 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:17:13.851 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:17:13.851 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:13.851 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:17:13.851 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:17:13.851 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:17:13.851 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:17:13.851 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:17:13.852 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:17:13.852 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:17:13.852 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:17:13.852 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:17:13.852 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:13.852 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:17:13.852 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:17:13.852 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:17:13.852 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:17:13.852 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:17:13.853 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:17:13.853 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:17:13.853 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:17:13.853 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:17:13.853 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:17:13.853 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:13.853 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:17:13.853 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:17:13.853 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:17:13.853 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:17:13.854 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:17:13.854 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:17:13.854 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:17:13.860 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:17:13.861 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:17:13.861 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:17:13.861 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:17:13.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:17:13.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:17:13.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:17:13.922 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:17:13.922 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:17:13.922 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:17:13.923 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:13.923 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:17:13.923 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:17:13.923 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:17:13.923 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:17:13.923 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:17:13.923 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:17:13.923 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:17:13.923 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:17:13.923 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:17:13.924 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:13.924 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:17:13.924 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:17:13.924 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:17:13.924 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:17:13.924 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:17:13.924 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:17:13.924 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:17:13.924 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:17:13.924 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:17:13.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:17:13.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:13.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:17:13.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:17:13.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:17:13.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:17:13.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:17:13.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:17:13.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:17:13.932 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:17:13.933 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:17:13.933 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:17:13.933 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:17:13.994 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:17:13.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:17:13.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:17:13.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:17:13.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:17:13.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:17:13.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:17:13.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:17:13.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:17:13.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:17:13.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:17:13.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:17:13.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:17:13.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:13.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:17:13.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:14.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 54 2026-02-18T08:17:14.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 58 2026-02-18T08:17:14.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:17:14.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=233, tag_tid=15, entry_tid=1], [object_number=232, tag_tid=15, entry_tid=0], [object_number=234, tag_tid=14, entry_tid=52758], [object_number=235, tag_tid=14, entry_tid=52755]]], state=connected] 2026-02-18T08:17:14.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=216, tag_tid=14, entry_tid=35800], [object_number=219, tag_tid=14, entry_tid=35799], [object_number=218, tag_tid=14, entry_tid=35798], [object_number=217, tag_tid=14, entry_tid=35797]]], state=connected] 2026-02-18T08:17:14.027 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:14.028 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=233, tag_tid=15, entry_tid=1' 2026-02-18T08:17:14.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:17:14.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:17:14.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:17:14.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:17:14.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:17:14.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:17:14.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:17:14.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:17:14.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:17:14.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:17:14.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:17:14.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:17:14.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:14.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:17:14.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:14.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 54 2026-02-18T08:17:14.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 58 2026-02-18T08:17:14.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:17:14.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=233, tag_tid=15, entry_tid=1], [object_number=232, tag_tid=15, entry_tid=0], [object_number=234, tag_tid=14, entry_tid=52758], [object_number=235, tag_tid=14, entry_tid=52755]]], state=connected] 2026-02-18T08:17:14.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=216, tag_tid=14, entry_tid=35800], [object_number=219, tag_tid=14, entry_tid=35799], [object_number=218, tag_tid=14, entry_tid=35798], [object_number=217, tag_tid=14, entry_tid=35797]]], state=connected] 2026-02-18T08:17:14.068 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:14.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=216, tag_tid=14, entry_tid=35800' 2026-02-18T08:17:14.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=233, tag_tid=15, entry_tid=1' -a 'object_number=233, tag_tid=15, entry_tid=1' = 'object_number=216, tag_tid=14, entry_tid=35800' 2026-02-18T08:17:14.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=216, tag_tid=14, entry_tid=35800' '!=' 'object_number=219, tag_tid=14, entry_tid=35455' 2026-02-18T08:17:14.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:17:14.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=216, tag_tid=14, entry_tid=35800' = 'object_number=219, tag_tid=14, entry_tid=35455' 2026-02-18T08:17:14.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=216, tag_tid=14, entry_tid=35800' 2026-02-18T08:17:14.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=233, tag_tid=15, entry_tid=1' 2026-02-18T08:17:14.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:17:14.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:17:14.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=15 2026-02-18T08:17:14.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=216, tag_tid=14, entry_tid=35800' 2026-02-18T08:17:14.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:17:14.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:17:14.074 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=14 2026-02-18T08:17:14.075 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=233, tag_tid=15, entry_tid=1' 2026-02-18T08:17:14.075 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:17:14.075 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:17:14.076 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:17:14.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=216, tag_tid=14, entry_tid=35800' 2026-02-18T08:17:14.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:17:14.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:17:14.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=35800 2026-02-18T08:17:14.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 15 = 14 -a 1 -le 35800 2026-02-18T08:17:14.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:17:14.080 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:17:14.080 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:17:14.281 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:17:14.281 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:17:14.281 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:17:14.282 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:17:14.282 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:17:14.282 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:17:14.282 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:17:14.282 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:17:14.282 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:17:14.282 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:14.282 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:17:14.282 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:17:14.282 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:17:14.283 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:17:14.283 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:17:14.283 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:17:14.283 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:17:14.286 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:17:14.286 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:17:14.286 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:17:14.286 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:14.286 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:17:14.286 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:17:14.286 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:17:14.286 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:17:14.286 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:17:14.287 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:17:14.287 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:17:14.287 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:17:14.287 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:17:14.287 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:14.287 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:17:14.287 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:17:14.287 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:17:14.287 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:17:14.287 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:17:14.288 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:17:14.288 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:17:14.288 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:17:14.288 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:17:14.288 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:17:14.288 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:14.288 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:17:14.288 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:17:14.288 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:17:14.288 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:17:14.289 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:17:14.289 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:17:14.289 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:17:14.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:17:14.297 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:17:14.297 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:17:14.297 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:17:14.355 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:17:14.355 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:17:14.355 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:17:14.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:17:14.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:17:14.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:17:14.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:14.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:17:14.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:17:14.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:17:14.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:17:14.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:17:14.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:17:14.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:17:14.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:17:14.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:17:14.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:14.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:17:14.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:17:14.360 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:17:14.360 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:17:14.360 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:17:14.360 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:17:14.360 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:17:14.360 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:17:14.360 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:17:14.360 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:17:14.360 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:14.360 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:17:14.360 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:17:14.360 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:17:14.361 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:17:14.361 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:17:14.361 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:17:14.361 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:17:14.368 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:17:14.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:17:14.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:17:14.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:17:14.419 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:17:14.419 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:17:14.419 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:17:14.422 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:17:14.422 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:17:14.422 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:17:14.422 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:14.423 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:17:14.423 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:17:14.423 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:17:14.423 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:17:14.423 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:17:14.423 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:17:14.423 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:17:14.423 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:17:14.423 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:17:14.423 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:14.423 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:17:14.423 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:17:14.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:17:14.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:17:14.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:17:14.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:17:14.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:17:14.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:17:14.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:17:14.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:17:14.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:14.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:17:14.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:17:14.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:17:14.425 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:17:14.425 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:17:14.425 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:17:14.425 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:17:14.432 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:17:14.432 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:17:14.433 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:17:14.433 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:17:14.483 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:17:14.483 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:17:14.483 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:17:14.486 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:17:14.486 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:17:14.486 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:17:14.487 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:14.487 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:17:14.487 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:17:14.487 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:17:14.487 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:17:14.487 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:17:14.487 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:17:14.487 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:17:14.487 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:17:14.487 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:17:14.488 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:14.488 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:17:14.488 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:17:14.488 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:17:14.488 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:17:14.488 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:17:14.488 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:17:14.488 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:17:14.488 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:17:14.488 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:17:14.488 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:17:14.489 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:14.489 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:17:14.489 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:17:14.489 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:17:14.489 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:17:14.489 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:17:14.489 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:17:14.489 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:17:14.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:17:14.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:17:14.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:17:14.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:17:14.566 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:17:14.567 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:17:14.567 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:17:14.568 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:17:14.568 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:17:14.568 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:17:14.568 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:17:14.568 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:17:14.568 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:17:14.568 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:17:14.568 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:17:14.568 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:17:14.569 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:17:14.569 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:14.569 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:17:14.570 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:14.598 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 54 2026-02-18T08:17:14.598 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 58 2026-02-18T08:17:14.598 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:17:14.598 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=233, tag_tid=15, entry_tid=1], [object_number=232, tag_tid=15, entry_tid=0], [object_number=234, tag_tid=14, entry_tid=52758], [object_number=235, tag_tid=14, entry_tid=52755]]], state=connected] 2026-02-18T08:17:14.598 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=216, tag_tid=14, entry_tid=36168], [object_number=219, tag_tid=14, entry_tid=36167], [object_number=218, tag_tid=14, entry_tid=36166], [object_number=217, tag_tid=14, entry_tid=36165]]], state=connected] 2026-02-18T08:17:14.601 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:14.602 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=233, tag_tid=15, entry_tid=1' 2026-02-18T08:17:14.604 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:17:14.604 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:17:14.604 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:17:14.604 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:17:14.604 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:17:14.604 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:17:14.604 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:17:14.604 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:17:14.605 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:17:14.605 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:17:14.605 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:17:14.605 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:17:14.606 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:14.606 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:17:14.606 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:14.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 54 2026-02-18T08:17:14.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 58 2026-02-18T08:17:14.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:17:14.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=233, tag_tid=15, entry_tid=1], [object_number=232, tag_tid=15, entry_tid=0], [object_number=234, tag_tid=14, entry_tid=52758], [object_number=235, tag_tid=14, entry_tid=52755]]], state=connected] 2026-02-18T08:17:14.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=216, tag_tid=14, entry_tid=36168], [object_number=219, tag_tid=14, entry_tid=36167], [object_number=218, tag_tid=14, entry_tid=36166], [object_number=217, tag_tid=14, entry_tid=36165]]], state=connected] 2026-02-18T08:17:14.635 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:14.637 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=216, tag_tid=14, entry_tid=36168' 2026-02-18T08:17:14.637 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=233, tag_tid=15, entry_tid=1' -a 'object_number=233, tag_tid=15, entry_tid=1' = 'object_number=216, tag_tid=14, entry_tid=36168' 2026-02-18T08:17:14.637 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=216, tag_tid=14, entry_tid=36168' '!=' 'object_number=216, tag_tid=14, entry_tid=35800' 2026-02-18T08:17:14.637 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:17:14.638 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=216, tag_tid=14, entry_tid=36168' = 'object_number=216, tag_tid=14, entry_tid=35800' 2026-02-18T08:17:14.638 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=216, tag_tid=14, entry_tid=36168' 2026-02-18T08:17:14.638 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=233, tag_tid=15, entry_tid=1' 2026-02-18T08:17:14.638 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:17:14.638 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:17:14.639 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=15 2026-02-18T08:17:14.640 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=216, tag_tid=14, entry_tid=36168' 2026-02-18T08:17:14.640 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:17:14.640 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:17:14.642 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=14 2026-02-18T08:17:14.642 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=233, tag_tid=15, entry_tid=1' 2026-02-18T08:17:14.643 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:17:14.643 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:17:14.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:17:14.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=216, tag_tid=14, entry_tid=36168' 2026-02-18T08:17:14.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:17:14.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:17:14.646 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=36168 2026-02-18T08:17:14.647 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 15 = 14 -a 1 -le 36168 2026-02-18T08:17:14.647 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:17:14.647 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:17:14.647 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:17:14.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:17:14.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:17:14.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:17:14.850 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:17:14.850 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:17:14.850 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:17:14.850 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:17:14.850 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:17:14.850 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:17:14.850 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:14.850 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:17:14.851 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:17:14.851 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:17:14.851 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:17:14.851 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:17:14.851 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:17:14.851 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:17:14.854 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:17:14.854 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:17:14.854 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:17:14.854 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:14.854 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:17:14.855 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:17:14.855 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:17:14.855 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:17:14.855 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:17:14.855 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:17:14.855 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:17:14.855 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:17:14.855 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:17:14.855 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:14.855 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:17:14.856 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:17:14.856 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:17:14.856 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:17:14.856 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:17:14.856 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:17:14.856 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:17:14.856 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:17:14.856 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:17:14.856 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:17:14.856 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:14.856 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:17:14.857 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:17:14.857 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:17:14.857 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:17:14.857 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:17:14.857 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:17:14.857 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:17:14.864 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:17:14.864 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:17:14.865 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:17:14.865 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:17:14.915 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:17:14.915 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:17:14.915 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:17:14.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:17:14.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:17:14.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:17:14.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:14.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:17:14.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:17:14.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:17:14.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:17:14.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:17:14.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:17:14.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:17:14.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:17:14.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:17:14.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:14.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:17:14.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:17:14.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:17:14.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:17:14.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:17:14.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:17:14.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:17:14.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:17:14.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:17:14.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:17:14.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:14.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:17:14.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:17:14.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:17:14.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:17:14.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:17:14.922 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:17:14.922 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:17:14.930 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:17:14.930 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:17:14.930 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:17:14.931 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:17:14.989 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:17:14.989 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:17:14.990 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:17:14.993 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:17:14.993 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:17:14.993 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:17:14.993 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:14.993 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:17:14.993 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:17:14.993 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:17:14.994 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:17:14.994 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:17:14.994 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:17:14.994 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:17:14.994 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:17:14.994 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:17:14.994 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:14.994 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:17:14.994 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:17:14.994 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:17:14.994 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:17:14.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:17:14.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:17:14.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:17:14.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:17:14.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:17:14.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:17:14.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:14.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:17:14.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:17:14.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:17:14.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:17:14.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:17:14.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:17:14.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:17:15.004 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:17:15.004 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:17:15.004 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:17:15.004 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:17:15.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:17:15.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:17:15.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:17:15.058 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:17:15.058 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:17:15.058 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:17:15.058 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:15.058 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:17:15.058 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:17:15.058 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:17:15.058 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:17:15.058 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:17:15.059 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:17:15.059 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:17:15.059 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:17:15.059 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:17:15.059 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:15.059 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:17:15.059 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:17:15.059 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:17:15.059 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:17:15.059 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:17:15.060 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:17:15.060 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:17:15.060 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:17:15.060 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:17:15.060 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:17:15.060 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:15.060 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:17:15.060 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:17:15.060 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:17:15.060 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:17:15.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:17:15.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:17:15.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:17:15.068 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:17:15.069 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:17:15.069 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:17:15.069 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:17:15.130 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:17:15.131 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:17:15.131 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:17:15.131 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:17:15.131 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:17:15.132 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:17:15.132 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:17:15.132 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:17:15.132 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:17:15.132 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:17:15.132 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:17:15.132 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:17:15.132 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:17:15.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:15.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:17:15.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:15.161 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 54 2026-02-18T08:17:15.161 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 58 2026-02-18T08:17:15.161 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:17:15.161 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=233, tag_tid=15, entry_tid=1], [object_number=232, tag_tid=15, entry_tid=0], [object_number=234, tag_tid=14, entry_tid=52758], [object_number=235, tag_tid=14, entry_tid=52755]]], state=connected] 2026-02-18T08:17:15.161 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=217, tag_tid=14, entry_tid=36541], [object_number=216, tag_tid=14, entry_tid=36540], [object_number=219, tag_tid=14, entry_tid=36539], [object_number=218, tag_tid=14, entry_tid=36538]]], state=connected] 2026-02-18T08:17:15.164 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:15.165 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=233, tag_tid=15, entry_tid=1' 2026-02-18T08:17:15.166 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:17:15.166 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:17:15.166 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:17:15.166 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:17:15.166 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:17:15.166 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:17:15.166 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:17:15.166 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:17:15.167 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:17:15.167 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:17:15.167 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:17:15.167 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:17:15.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:15.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:17:15.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:15.202 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 54 2026-02-18T08:17:15.202 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 58 2026-02-18T08:17:15.202 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:17:15.202 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=233, tag_tid=15, entry_tid=1], [object_number=232, tag_tid=15, entry_tid=0], [object_number=234, tag_tid=14, entry_tid=52758], [object_number=235, tag_tid=14, entry_tid=52755]]], state=connected] 2026-02-18T08:17:15.202 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=217, tag_tid=14, entry_tid=36541], [object_number=216, tag_tid=14, entry_tid=36540], [object_number=219, tag_tid=14, entry_tid=36539], [object_number=218, tag_tid=14, entry_tid=36538]]], state=connected] 2026-02-18T08:17:15.206 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:15.207 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=217, tag_tid=14, entry_tid=36541' 2026-02-18T08:17:15.207 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=233, tag_tid=15, entry_tid=1' -a 'object_number=233, tag_tid=15, entry_tid=1' = 'object_number=217, tag_tid=14, entry_tid=36541' 2026-02-18T08:17:15.208 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=217, tag_tid=14, entry_tid=36541' '!=' 'object_number=216, tag_tid=14, entry_tid=36168' 2026-02-18T08:17:15.208 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:17:15.208 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=217, tag_tid=14, entry_tid=36541' = 'object_number=216, tag_tid=14, entry_tid=36168' 2026-02-18T08:17:15.208 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=217, tag_tid=14, entry_tid=36541' 2026-02-18T08:17:15.209 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=233, tag_tid=15, entry_tid=1' 2026-02-18T08:17:15.209 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:17:15.209 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:17:15.209 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=15 2026-02-18T08:17:15.210 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=217, tag_tid=14, entry_tid=36541' 2026-02-18T08:17:15.210 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:17:15.210 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:17:15.211 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=14 2026-02-18T08:17:15.212 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=233, tag_tid=15, entry_tid=1' 2026-02-18T08:17:15.213 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:17:15.213 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:17:15.214 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:17:15.215 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=217, tag_tid=14, entry_tid=36541' 2026-02-18T08:17:15.215 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:17:15.215 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:17:15.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=36541 2026-02-18T08:17:15.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 15 = 14 -a 1 -le 36541 2026-02-18T08:17:15.217 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:17:15.217 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:17:15.217 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:17:15.418 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:17:15.419 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:17:15.419 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:17:15.419 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:17:15.419 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:17:15.419 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:17:15.419 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:17:15.419 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:17:15.420 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:17:15.420 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:15.420 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:17:15.420 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:17:15.420 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:17:15.420 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:17:15.420 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:17:15.420 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:17:15.423 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:17:15.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:17:15.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:17:15.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:17:15.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:15.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:17:15.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:17:15.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:17:15.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:17:15.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:17:15.425 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:17:15.425 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:17:15.425 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:17:15.425 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:17:15.425 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:15.425 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:17:15.425 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:17:15.425 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:17:15.425 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:17:15.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:17:15.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:17:15.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:17:15.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:17:15.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:17:15.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:17:15.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:15.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:17:15.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:17:15.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:17:15.427 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:17:15.427 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:17:15.427 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:17:15.427 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:17:15.434 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:17:15.434 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:17:15.434 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:17:15.434 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:17:15.484 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:17:15.484 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:17:15.485 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:17:15.488 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:17:15.488 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:17:15.488 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:17:15.488 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:15.488 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:17:15.489 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:17:15.489 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:17:15.489 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:17:15.489 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:17:15.489 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:17:15.489 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:17:15.489 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:17:15.489 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:17:15.489 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:15.489 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:17:15.489 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:17:15.490 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:17:15.490 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:17:15.490 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:17:15.490 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:17:15.490 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:17:15.490 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:17:15.490 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:17:15.490 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:17:15.490 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:15.490 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:17:15.491 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:17:15.491 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:17:15.491 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:17:15.491 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:17:15.491 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:17:15.491 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:17:15.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:17:15.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:17:15.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:17:15.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:17:15.558 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:17:15.558 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:17:15.558 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:17:15.561 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:17:15.561 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:17:15.561 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:17:15.561 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:15.561 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:17:15.561 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:17:15.561 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:17:15.562 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:17:15.562 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:17:15.562 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:17:15.562 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:17:15.562 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:17:15.562 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:17:15.562 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:15.562 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:17:15.562 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:17:15.562 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:17:15.562 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:17:15.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:17:15.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:17:15.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:17:15.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:17:15.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:17:15.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:17:15.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:15.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:17:15.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:17:15.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:17:15.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:17:15.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:17:15.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:17:15.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:17:15.572 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:17:15.573 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:17:15.573 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:17:15.573 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:17:15.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:17:15.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:17:15.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:17:15.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:17:15.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:17:15.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:17:15.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:15.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:17:15.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:17:15.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:17:15.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:17:15.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:17:15.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:17:15.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:17:15.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:17:15.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:17:15.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:15.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:17:15.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:17:15.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:17:15.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:17:15.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:17:15.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:17:15.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:17:15.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:17:15.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:17:15.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:17:15.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:15.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:17:15.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:17:15.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:17:15.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:17:15.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:17:15.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:17:15.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:17:15.636 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:17:15.637 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:17:15.637 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:17:15.637 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:17:15.695 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:17:15.695 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:17:15.695 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:17:15.695 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:17:15.695 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:17:15.695 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:17:15.696 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:17:15.696 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:17:15.696 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:17:15.696 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:17:15.696 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:17:15.696 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:17:15.696 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:17:15.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:15.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:17:15.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:15.735 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 54 2026-02-18T08:17:15.735 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 58 2026-02-18T08:17:15.735 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:17:15.735 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=233, tag_tid=15, entry_tid=1], [object_number=232, tag_tid=15, entry_tid=0], [object_number=234, tag_tid=14, entry_tid=52758], [object_number=235, tag_tid=14, entry_tid=52755]]], state=connected] 2026-02-18T08:17:15.735 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=216, tag_tid=14, entry_tid=36888], [object_number=219, tag_tid=14, entry_tid=36887], [object_number=218, tag_tid=14, entry_tid=36886], [object_number=217, tag_tid=14, entry_tid=36885]]], state=connected] 2026-02-18T08:17:15.737 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:15.739 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=233, tag_tid=15, entry_tid=1' 2026-02-18T08:17:15.740 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:17:15.740 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:17:15.740 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:17:15.740 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:17:15.740 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:17:15.740 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:17:15.740 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:17:15.740 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:17:15.740 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:17:15.740 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:17:15.740 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:17:15.740 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:17:15.742 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:15.742 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:17:15.742 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:15.780 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 54 2026-02-18T08:17:15.780 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 58 2026-02-18T08:17:15.780 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:17:15.780 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=233, tag_tid=15, entry_tid=1], [object_number=232, tag_tid=15, entry_tid=0], [object_number=234, tag_tid=14, entry_tid=52758], [object_number=235, tag_tid=14, entry_tid=52755]]], state=connected] 2026-02-18T08:17:15.780 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=216, tag_tid=14, entry_tid=36888], [object_number=219, tag_tid=14, entry_tid=36887], [object_number=218, tag_tid=14, entry_tid=36886], [object_number=217, tag_tid=14, entry_tid=36885]]], state=connected] 2026-02-18T08:17:15.783 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:15.784 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=216, tag_tid=14, entry_tid=36888' 2026-02-18T08:17:15.785 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=233, tag_tid=15, entry_tid=1' -a 'object_number=233, tag_tid=15, entry_tid=1' = 'object_number=216, tag_tid=14, entry_tid=36888' 2026-02-18T08:17:15.785 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=216, tag_tid=14, entry_tid=36888' '!=' 'object_number=217, tag_tid=14, entry_tid=36541' 2026-02-18T08:17:15.785 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:17:15.785 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=216, tag_tid=14, entry_tid=36888' = 'object_number=217, tag_tid=14, entry_tid=36541' 2026-02-18T08:17:15.785 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=216, tag_tid=14, entry_tid=36888' 2026-02-18T08:17:15.786 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=233, tag_tid=15, entry_tid=1' 2026-02-18T08:17:15.786 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:17:15.786 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:17:15.787 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=15 2026-02-18T08:17:15.788 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=216, tag_tid=14, entry_tid=36888' 2026-02-18T08:17:15.788 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:17:15.788 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:17:15.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=14 2026-02-18T08:17:15.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=233, tag_tid=15, entry_tid=1' 2026-02-18T08:17:15.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:17:15.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:17:15.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:17:15.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=216, tag_tid=14, entry_tid=36888' 2026-02-18T08:17:15.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:17:15.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:17:15.794 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=36888 2026-02-18T08:17:15.794 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 15 = 14 -a 1 -le 36888 2026-02-18T08:17:15.794 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:17:15.794 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:17:15.794 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:17:15.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:17:15.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:17:15.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:17:15.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:17:15.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:17:15.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:17:15.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:17:15.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:17:15.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:17:15.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:15.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:17:15.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:17:15.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:17:15.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:17:15.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:17:15.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:17:16.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:17:16.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:17:16.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:17:16.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:17:16.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:16.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:17:16.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:17:16.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:17:16.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:17:16.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:17:16.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:17:16.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:17:16.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:17:16.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:17:16.003 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:16.003 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:17:16.003 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:17:16.003 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:17:16.003 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:17:16.003 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:17:16.003 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:17:16.003 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:17:16.003 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:17:16.003 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:17:16.004 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:17:16.004 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:16.004 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:17:16.004 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:17:16.004 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:17:16.004 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:17:16.004 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:17:16.004 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:17:16.004 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:17:16.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:17:16.010 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:17:16.010 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:17:16.010 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:17:16.060 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:17:16.060 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:17:16.060 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:17:16.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:17:16.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:17:16.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:17:16.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:16.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:17:16.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:17:16.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:17:16.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:17:16.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:17:16.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:17:16.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:17:16.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:17:16.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:17:16.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:16.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:17:16.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:17:16.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:17:16.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:17:16.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:17:16.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:17:16.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:17:16.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:17:16.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:17:16.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:17:16.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:16.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:17:16.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:17:16.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:17:16.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:17:16.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:17:16.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:17:16.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:17:16.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:17:16.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:17:16.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:17:16.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:17:16.122 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:17:16.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:17:16.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:17:16.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:17:16.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:17:16.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:17:16.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:16.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:17:16.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:17:16.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:17:16.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:17:16.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:17:16.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:17:16.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:17:16.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:17:16.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:17:16.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:16.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:17:16.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:17:16.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:17:16.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:17:16.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:17:16.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:17:16.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:17:16.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:17:16.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:17:16.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:17:16.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:16.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:17:16.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:17:16.129 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:17:16.129 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:17:16.129 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:17:16.129 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:17:16.129 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:17:16.136 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:17:16.136 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:17:16.136 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:17:16.137 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:17:16.194 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:17:16.194 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:17:16.194 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:17:16.197 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:17:16.198 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:17:16.198 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:17:16.198 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:16.198 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:17:16.198 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:17:16.198 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:17:16.198 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:17:16.198 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:17:16.198 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:17:16.199 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:17:16.199 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:17:16.199 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:17:16.199 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:16.199 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:17:16.199 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:17:16.199 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:17:16.199 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:17:16.199 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:17:16.199 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:17:16.199 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:17:16.199 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:17:16.200 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:17:16.200 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:17:16.200 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:16.200 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:17:16.200 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:17:16.200 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:17:16.200 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:17:16.200 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:17:16.200 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:17:16.200 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:17:16.207 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:17:16.208 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:17:16.208 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:17:16.208 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:17:16.269 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:17:16.270 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:17:16.270 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:17:16.270 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:17:16.270 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:17:16.270 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:17:16.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:17:16.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:17:16.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:17:16.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:17:16.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:17:16.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:17:16.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:17:16.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:16.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:17:16.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:16.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 54 2026-02-18T08:17:16.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 58 2026-02-18T08:17:16.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:17:16.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=233, tag_tid=15, entry_tid=1], [object_number=232, tag_tid=15, entry_tid=0], [object_number=234, tag_tid=14, entry_tid=52758], [object_number=235, tag_tid=14, entry_tid=52755]]], state=connected] 2026-02-18T08:17:16.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=217, tag_tid=14, entry_tid=37249], [object_number=216, tag_tid=14, entry_tid=37248], [object_number=219, tag_tid=14, entry_tid=37247], [object_number=218, tag_tid=14, entry_tid=37246]]], state=connected] 2026-02-18T08:17:16.312 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:16.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=233, tag_tid=15, entry_tid=1' 2026-02-18T08:17:16.314 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:17:16.314 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:17:16.314 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:17:16.314 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:17:16.314 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:17:16.314 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:17:16.315 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:17:16.315 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:17:16.315 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:17:16.315 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:17:16.315 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:17:16.315 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:17:16.316 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:16.316 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:17:16.316 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:16.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 54 2026-02-18T08:17:16.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 58 2026-02-18T08:17:16.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:17:16.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=233, tag_tid=15, entry_tid=1], [object_number=232, tag_tid=15, entry_tid=0], [object_number=234, tag_tid=14, entry_tid=52758], [object_number=235, tag_tid=14, entry_tid=52755]]], state=connected] 2026-02-18T08:17:16.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=217, tag_tid=14, entry_tid=37249], [object_number=216, tag_tid=14, entry_tid=37248], [object_number=219, tag_tid=14, entry_tid=37247], [object_number=218, tag_tid=14, entry_tid=37246]]], state=connected] 2026-02-18T08:17:16.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:16.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=217, tag_tid=14, entry_tid=37249' 2026-02-18T08:17:16.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=233, tag_tid=15, entry_tid=1' -a 'object_number=233, tag_tid=15, entry_tid=1' = 'object_number=217, tag_tid=14, entry_tid=37249' 2026-02-18T08:17:16.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=217, tag_tid=14, entry_tid=37249' '!=' 'object_number=216, tag_tid=14, entry_tid=36888' 2026-02-18T08:17:16.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:17:16.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=217, tag_tid=14, entry_tid=37249' = 'object_number=216, tag_tid=14, entry_tid=36888' 2026-02-18T08:17:16.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=217, tag_tid=14, entry_tid=37249' 2026-02-18T08:17:16.348 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=233, tag_tid=15, entry_tid=1' 2026-02-18T08:17:16.348 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:17:16.348 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:17:16.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=15 2026-02-18T08:17:16.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=217, tag_tid=14, entry_tid=37249' 2026-02-18T08:17:16.350 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:17:16.350 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:17:16.351 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=14 2026-02-18T08:17:16.351 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=233, tag_tid=15, entry_tid=1' 2026-02-18T08:17:16.352 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:17:16.352 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:17:16.353 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:17:16.354 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=217, tag_tid=14, entry_tid=37249' 2026-02-18T08:17:16.354 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:17:16.354 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:17:16.355 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=37249 2026-02-18T08:17:16.356 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 15 = 14 -a 1 -le 37249 2026-02-18T08:17:16.356 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:17:16.356 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:17:16.356 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:17:16.558 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:17:16.558 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:17:16.558 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:17:16.558 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:17:16.558 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:17:16.558 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:17:16.558 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:17:16.559 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:17:16.559 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:17:16.559 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:16.559 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:17:16.559 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:17:16.559 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:17:16.559 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:17:16.559 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:17:16.559 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:17:16.559 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:17:16.562 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:17:16.562 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:17:16.562 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:17:16.562 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:16.562 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:17:16.562 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:17:16.562 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:17:16.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:17:16.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:17:16.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:17:16.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:17:16.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:17:16.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:17:16.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:16.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:17:16.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:17:16.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:17:16.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:17:16.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:17:16.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:17:16.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:17:16.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:17:16.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:17:16.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:17:16.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:16.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:17:16.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:17:16.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:17:16.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:17:16.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:17:16.565 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:17:16.565 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:17:16.572 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:17:16.572 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:17:16.572 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:17:16.573 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:17:16.629 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:17:16.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:17:16.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:17:16.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:17:16.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:17:16.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:17:16.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:16.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:17:16.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:17:16.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:17:16.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:17:16.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:17:16.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:17:16.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:17:16.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:17:16.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:17:16.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:16.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:17:16.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:17:16.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:17:16.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:17:16.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:17:16.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:17:16.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:17:16.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:17:16.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:17:16.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:17:16.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:16.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:17:16.635 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:17:16.635 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:17:16.635 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:17:16.635 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:17:16.635 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:17:16.635 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:17:16.643 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:17:16.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:17:16.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:17:16.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:17:16.693 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:17:16.693 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:17:16.693 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:17:16.696 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:17:16.696 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:17:16.696 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:17:16.696 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:16.696 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:17:16.696 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:17:16.696 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:17:16.696 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:17:16.696 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:17:16.697 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:17:16.697 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:17:16.697 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:17:16.697 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:17:16.697 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:16.697 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:17:16.697 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:17:16.697 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:17:16.697 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:17:16.697 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:17:16.697 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:17:16.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:17:16.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:17:16.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:17:16.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:17:16.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:16.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:17:16.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:17:16.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:17:16.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:17:16.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:17:16.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:17:16.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:17:16.707 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:17:16.708 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:17:16.708 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:17:16.708 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:17:16.756 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:17:16.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:17:16.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:17:16.759 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:17:16.759 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:17:16.760 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:17:16.760 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:16.760 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:17:16.760 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:17:16.760 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:17:16.760 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:17:16.760 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:17:16.760 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:17:16.760 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:17:16.760 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:17:16.760 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:17:16.761 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:16.761 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:17:16.761 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:17:16.761 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:17:16.761 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:17:16.761 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:17:16.761 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:17:16.761 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:17:16.761 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:17:16.761 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:17:16.761 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:17:16.761 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:16.762 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:17:16.762 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:17:16.762 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:17:16.762 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:17:16.762 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:17:16.762 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:17:16.762 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:17:16.770 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:17:16.770 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:17:16.771 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:17:16.771 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:17:16.838 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:17:16.839 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:17:16.839 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:17:16.839 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:17:16.839 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:17:16.839 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:17:16.839 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:17:16.840 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:17:16.840 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:17:16.840 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:17:16.840 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:17:16.840 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:17:16.840 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:17:16.841 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:16.841 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:17:16.841 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:16.879 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 54 2026-02-18T08:17:16.879 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 58 2026-02-18T08:17:16.879 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:17:16.879 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=233, tag_tid=15, entry_tid=1], [object_number=232, tag_tid=15, entry_tid=0], [object_number=234, tag_tid=14, entry_tid=52758], [object_number=235, tag_tid=14, entry_tid=52755]]], state=connected] 2026-02-18T08:17:16.879 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=216, tag_tid=14, entry_tid=37632], [object_number=219, tag_tid=14, entry_tid=37631], [object_number=218, tag_tid=14, entry_tid=37630], [object_number=217, tag_tid=14, entry_tid=37629]]], state=connected] 2026-02-18T08:17:16.882 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:16.884 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=233, tag_tid=15, entry_tid=1' 2026-02-18T08:17:16.885 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:17:16.885 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:17:16.885 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:17:16.885 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:17:16.886 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:17:16.886 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:17:16.886 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:17:16.886 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:17:16.886 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:17:16.886 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:17:16.886 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:17:16.887 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:17:16.887 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:16.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:17:16.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:16.914 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 54 2026-02-18T08:17:16.914 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 58 2026-02-18T08:17:16.914 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:17:16.914 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=233, tag_tid=15, entry_tid=1], [object_number=232, tag_tid=15, entry_tid=0], [object_number=234, tag_tid=14, entry_tid=52758], [object_number=235, tag_tid=14, entry_tid=52755]]], state=connected] 2026-02-18T08:17:16.914 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=216, tag_tid=14, entry_tid=37632], [object_number=219, tag_tid=14, entry_tid=37631], [object_number=218, tag_tid=14, entry_tid=37630], [object_number=217, tag_tid=14, entry_tid=37629]]], state=connected] 2026-02-18T08:17:16.917 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:16.918 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=216, tag_tid=14, entry_tid=37632' 2026-02-18T08:17:16.918 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=233, tag_tid=15, entry_tid=1' -a 'object_number=233, tag_tid=15, entry_tid=1' = 'object_number=216, tag_tid=14, entry_tid=37632' 2026-02-18T08:17:16.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=216, tag_tid=14, entry_tid=37632' '!=' 'object_number=217, tag_tid=14, entry_tid=37249' 2026-02-18T08:17:16.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:17:16.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=216, tag_tid=14, entry_tid=37632' = 'object_number=217, tag_tid=14, entry_tid=37249' 2026-02-18T08:17:16.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=216, tag_tid=14, entry_tid=37632' 2026-02-18T08:17:16.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=233, tag_tid=15, entry_tid=1' 2026-02-18T08:17:16.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:17:16.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:17:16.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=15 2026-02-18T08:17:16.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=216, tag_tid=14, entry_tid=37632' 2026-02-18T08:17:16.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:17:16.922 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:17:16.923 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=14 2026-02-18T08:17:16.924 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=233, tag_tid=15, entry_tid=1' 2026-02-18T08:17:16.924 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:17:16.924 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:17:16.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:17:16.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=216, tag_tid=14, entry_tid=37632' 2026-02-18T08:17:16.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:17:16.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:17:16.928 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=37632 2026-02-18T08:17:16.928 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 15 = 14 -a 1 -le 37632 2026-02-18T08:17:16.928 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:17:16.928 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:17:16.928 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:17:17.130 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:17:17.130 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:17:17.130 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:17:17.131 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:17:17.131 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:17:17.131 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:17:17.131 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:17:17.131 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:17:17.131 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:17:17.131 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:17.131 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:17:17.131 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:17:17.131 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:17:17.131 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:17:17.131 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:17:17.132 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:17:17.132 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:17:17.134 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:17:17.134 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:17:17.134 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:17:17.134 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:17.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:17:17.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:17:17.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:17:17.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:17:17.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:17:17.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:17:17.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:17:17.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:17:17.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:17:17.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:17.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:17:17.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:17:17.136 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:17:17.136 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:17:17.136 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:17:17.136 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:17:17.136 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:17:17.136 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:17:17.136 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:17:17.136 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:17:17.136 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:17.136 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:17:17.136 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:17:17.136 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:17:17.137 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:17:17.137 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:17:17.137 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:17:17.137 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:17:17.145 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:17:17.145 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:17:17.145 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:17:17.145 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:17:17.202 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:17:17.202 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:17:17.202 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:17:17.204 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:17:17.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:17:17.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:17:17.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:17.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:17:17.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:17:17.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:17:17.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:17:17.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:17:17.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:17:17.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:17:17.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:17:17.206 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:17:17.206 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:17.206 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:17:17.206 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:17:17.206 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:17:17.206 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:17:17.206 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:17:17.206 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:17:17.206 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:17:17.206 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:17:17.206 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:17:17.206 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:17:17.207 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:17.207 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:17:17.207 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:17:17.207 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:17:17.207 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:17:17.207 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:17:17.207 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:17:17.207 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:17:17.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:17:17.217 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:17:17.217 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:17:17.217 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:17:17.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:17:17.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:17:17.266 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:17:17.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:17:17.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:17:17.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:17:17.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:17.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:17:17.269 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:17:17.269 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:17:17.269 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:17:17.269 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:17:17.269 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:17:17.269 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:17:17.269 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:17:17.269 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:17:17.269 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:17.269 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:17:17.269 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:17:17.270 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:17:17.270 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:17:17.270 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:17:17.270 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:17:17.270 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:17:17.270 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:17:17.270 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:17:17.270 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:17:17.270 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:17.270 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:17:17.270 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:17:17.270 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:17:17.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:17:17.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:17:17.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:17:17.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:17:17.280 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:17:17.280 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:17:17.281 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:17:17.281 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:17:17.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:17:17.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:17:17.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:17:17.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:17:17.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:17:17.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:17:17.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:17.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:17:17.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:17:17.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:17:17.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:17:17.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:17:17.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:17:17.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:17:17.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:17:17.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:17:17.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:17.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:17:17.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:17:17.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:17:17.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:17:17.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:17:17.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:17:17.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:17:17.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:17:17.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:17:17.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:17:17.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:17.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:17:17.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:17:17.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:17:17.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:17:17.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:17:17.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:17:17.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:17:17.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:17:17.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:17:17.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:17:17.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:17:17.423 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:17:17.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:17:17.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:17:17.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:17:17.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:17:17.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:17:17.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:17:17.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:17:17.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:17:17.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:17:17.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:17:17.425 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:17:17.425 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:17:17.425 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:17.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:17:17.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:17.462 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 55 2026-02-18T08:17:17.462 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 58 2026-02-18T08:17:17.462 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:17:17.462 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=233, tag_tid=15, entry_tid=1], [object_number=232, tag_tid=15, entry_tid=0], [object_number=234, tag_tid=14, entry_tid=52758], [object_number=235, tag_tid=14, entry_tid=52755]]], state=connected] 2026-02-18T08:17:17.462 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=223, tag_tid=14, entry_tid=37927], [object_number=222, tag_tid=14, entry_tid=37926], [object_number=221, tag_tid=14, entry_tid=37925], [object_number=220, tag_tid=14, entry_tid=37924]]], state=connected] 2026-02-18T08:17:17.465 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:17.467 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=233, tag_tid=15, entry_tid=1' 2026-02-18T08:17:17.468 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:17:17.468 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:17:17.468 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:17:17.468 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:17:17.468 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:17:17.468 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:17:17.468 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:17:17.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:17:17.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:17:17.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:17:17.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:17:17.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:17:17.470 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:17.470 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:17:17.470 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:17.712 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 55 2026-02-18T08:17:17.712 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 58 2026-02-18T08:17:17.712 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:17:17.713 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=233, tag_tid=15, entry_tid=1], [object_number=232, tag_tid=15, entry_tid=0], [object_number=234, tag_tid=14, entry_tid=52758], [object_number=235, tag_tid=14, entry_tid=52755]]], state=connected] 2026-02-18T08:17:17.713 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=223, tag_tid=14, entry_tid=37927], [object_number=222, tag_tid=14, entry_tid=37926], [object_number=221, tag_tid=14, entry_tid=37925], [object_number=220, tag_tid=14, entry_tid=37924]]], state=connected] 2026-02-18T08:17:17.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:17.718 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=223, tag_tid=14, entry_tid=37927' 2026-02-18T08:17:17.718 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=233, tag_tid=15, entry_tid=1' -a 'object_number=233, tag_tid=15, entry_tid=1' = 'object_number=223, tag_tid=14, entry_tid=37927' 2026-02-18T08:17:17.718 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=223, tag_tid=14, entry_tid=37927' '!=' 'object_number=216, tag_tid=14, entry_tid=37632' 2026-02-18T08:17:17.718 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:17:17.718 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=223, tag_tid=14, entry_tid=37927' = 'object_number=216, tag_tid=14, entry_tid=37632' 2026-02-18T08:17:17.718 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=223, tag_tid=14, entry_tid=37927' 2026-02-18T08:17:17.718 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=233, tag_tid=15, entry_tid=1' 2026-02-18T08:17:17.718 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:17:17.718 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:17:17.720 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=15 2026-02-18T08:17:17.720 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=223, tag_tid=14, entry_tid=37927' 2026-02-18T08:17:17.720 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:17:17.721 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:17:17.722 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=14 2026-02-18T08:17:17.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=233, tag_tid=15, entry_tid=1' 2026-02-18T08:17:17.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:17:17.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:17:17.724 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:17:17.725 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=223, tag_tid=14, entry_tid=37927' 2026-02-18T08:17:17.725 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:17:17.725 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:17:17.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=37927 2026-02-18T08:17:17.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 15 = 14 -a 1 -le 37927 2026-02-18T08:17:17.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:17:17.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:17:17.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:17:17.930 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:17:17.930 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:17:17.930 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:17:17.930 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:17:17.930 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:17:17.930 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:17:17.930 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:17:17.931 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:17:17.931 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:17:17.931 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:17.931 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:17:17.931 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:17:17.931 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:17:17.931 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:17:17.931 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:17:17.931 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:17:17.931 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:17:17.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:17:17.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:17:17.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:17:17.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:17.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:17:17.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:17:17.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:17:17.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:17:17.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:17:17.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:17:17.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:17:17.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:17:17.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:17:17.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:17.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:17:17.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:17:17.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:17:17.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:17:17.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:17:17.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:17:17.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:17:17.936 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:17:17.936 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:17:17.936 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:17:17.936 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:17.936 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:17:17.936 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:17:17.936 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:17:17.936 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:17:17.936 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:17:17.936 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:17:17.936 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:17:17.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:17:17.944 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:17:17.944 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:17:17.944 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:17:17.994 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:17:17.994 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:17:17.994 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:17:17.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:17:17.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:17:17.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:17:17.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:17.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:17:17.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:17:17.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:17:17.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:17:17.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:17:17.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:17:17.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:17:17.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:17:17.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:17:17.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:17.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:17:17.999 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:17:17.999 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:17:17.999 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:17:17.999 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:17:17.999 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:17:17.999 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:17:17.999 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:17:17.999 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:17:17.999 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:17:17.999 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:17.999 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:17:17.999 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:17:18.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:17:18.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:17:18.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:17:18.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:17:18.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:17:18.008 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:17:18.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:17:18.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:17:18.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:17:18.059 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:17:18.059 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:17:18.059 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:17:18.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:17:18.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:17:18.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:17:18.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:18.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:17:18.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:17:18.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:17:18.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:17:18.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:17:18.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:17:18.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:17:18.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:17:18.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:17:18.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:18.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:17:18.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:17:18.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:17:18.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:17:18.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:17:18.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:17:18.067 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:17:18.067 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:17:18.067 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:17:18.067 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:17:18.067 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:18.067 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:17:18.067 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:17:18.068 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:17:18.068 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:17:18.068 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:17:18.068 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:17:18.068 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:17:18.074 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:17:18.074 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:17:18.075 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:17:18.075 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:17:18.132 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:17:18.132 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:17:18.132 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:17:18.134 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:17:18.134 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:17:18.134 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:17:18.134 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:18.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:17:18.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:17:18.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:17:18.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:17:18.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:17:18.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:17:18.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:17:18.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:17:18.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:17:18.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:18.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:17:18.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:17:18.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:17:18.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:17:18.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:17:18.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:17:18.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:17:18.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:17:18.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:17:18.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:17:18.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:18.136 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:17:18.136 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:17:18.136 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:17:18.136 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:17:18.136 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:17:18.136 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:17:18.136 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:17:18.146 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:17:18.146 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:17:18.146 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:17:18.146 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:17:18.204 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:17:18.204 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:17:18.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:17:18.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:17:18.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:17:18.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:17:18.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:17:18.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:17:18.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:17:18.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:17:18.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:17:18.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:17:18.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:17:18.207 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:18.207 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:17:18.207 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:18.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 55 2026-02-18T08:17:18.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 58 2026-02-18T08:17:18.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:17:18.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=233, tag_tid=15, entry_tid=1], [object_number=232, tag_tid=15, entry_tid=0], [object_number=234, tag_tid=14, entry_tid=52758], [object_number=235, tag_tid=14, entry_tid=52755]]], state=connected] 2026-02-18T08:17:18.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=222, tag_tid=14, entry_tid=38474], [object_number=221, tag_tid=14, entry_tid=38473], [object_number=220, tag_tid=14, entry_tid=38472], [object_number=223, tag_tid=14, entry_tid=38471]]], state=connected] 2026-02-18T08:17:18.238 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:18.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=233, tag_tid=15, entry_tid=1' 2026-02-18T08:17:18.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:17:18.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:17:18.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:17:18.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:17:18.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:17:18.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:17:18.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:17:18.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:17:18.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:17:18.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:17:18.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:17:18.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:17:18.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:18.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:17:18.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:18.278 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 55 2026-02-18T08:17:18.278 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 58 2026-02-18T08:17:18.279 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:17:18.279 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=233, tag_tid=15, entry_tid=1], [object_number=232, tag_tid=15, entry_tid=0], [object_number=234, tag_tid=14, entry_tid=52758], [object_number=235, tag_tid=14, entry_tid=52755]]], state=connected] 2026-02-18T08:17:18.279 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=222, tag_tid=14, entry_tid=38474], [object_number=221, tag_tid=14, entry_tid=38473], [object_number=220, tag_tid=14, entry_tid=38472], [object_number=223, tag_tid=14, entry_tid=38471]]], state=connected] 2026-02-18T08:17:18.282 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:18.284 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=222, tag_tid=14, entry_tid=38474' 2026-02-18T08:17:18.284 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=233, tag_tid=15, entry_tid=1' -a 'object_number=233, tag_tid=15, entry_tid=1' = 'object_number=222, tag_tid=14, entry_tid=38474' 2026-02-18T08:17:18.284 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=222, tag_tid=14, entry_tid=38474' '!=' 'object_number=223, tag_tid=14, entry_tid=37927' 2026-02-18T08:17:18.284 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:17:18.284 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=222, tag_tid=14, entry_tid=38474' = 'object_number=223, tag_tid=14, entry_tid=37927' 2026-02-18T08:17:18.284 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=222, tag_tid=14, entry_tid=38474' 2026-02-18T08:17:18.284 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=233, tag_tid=15, entry_tid=1' 2026-02-18T08:17:18.284 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:17:18.284 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:17:18.286 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=15 2026-02-18T08:17:18.287 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=222, tag_tid=14, entry_tid=38474' 2026-02-18T08:17:18.287 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:17:18.287 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:17:18.288 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=14 2026-02-18T08:17:18.289 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=233, tag_tid=15, entry_tid=1' 2026-02-18T08:17:18.289 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:17:18.289 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:17:18.290 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:17:18.291 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=222, tag_tid=14, entry_tid=38474' 2026-02-18T08:17:18.291 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:17:18.291 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:17:18.292 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=38474 2026-02-18T08:17:18.293 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 15 = 14 -a 1 -le 38474 2026-02-18T08:17:18.293 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:17:18.293 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:17:18.293 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:17:18.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:17:18.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:17:18.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:17:18.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:17:18.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:17:18.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:17:18.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:17:18.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:17:18.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:17:18.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:18.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:17:18.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:17:18.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:17:18.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:17:18.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:17:18.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:17:18.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:17:18.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:17:18.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:17:18.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:17:18.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:18.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:17:18.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:17:18.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:17:18.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:17:18.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:17:18.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:17:18.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:17:18.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:17:18.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:17:18.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:18.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:17:18.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:17:18.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:17:18.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:17:18.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:17:18.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:17:18.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:17:18.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:17:18.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:17:18.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:17:18.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:18.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:17:18.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:17:18.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:17:18.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:17:18.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:17:18.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:17:18.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:17:18.508 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:17:18.509 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:17:18.509 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:17:18.509 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:17:18.559 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:17:18.560 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:17:18.560 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:17:18.562 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:17:18.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:17:18.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:17:18.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:18.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:17:18.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:17:18.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:17:18.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:17:18.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:17:18.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:17:18.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:17:18.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:17:18.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:17:18.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:18.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:17:18.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:17:18.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:17:18.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:17:18.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:17:18.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:17:18.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:17:18.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:17:18.565 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:17:18.565 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:17:18.565 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:18.565 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:17:18.565 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:17:18.565 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:17:18.565 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:17:18.565 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:17:18.565 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:17:18.565 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:17:18.573 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:17:18.574 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:17:18.574 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:17:18.574 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:17:18.622 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:17:18.623 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:17:18.623 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:17:18.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:17:18.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:17:18.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:17:18.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:18.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:17:18.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:17:18.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:17:18.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:17:18.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:17:18.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:17:18.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:17:18.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:17:18.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:17:18.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:18.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:17:18.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:17:18.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:17:18.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:17:18.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:17:18.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:17:18.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:17:18.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:17:18.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:17:18.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:17:18.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:18.628 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:17:18.628 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:17:18.628 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:17:18.628 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:17:18.628 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:17:18.628 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:17:18.628 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:17:18.636 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:17:18.636 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:17:18.636 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:17:18.636 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:17:18.686 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:17:18.686 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:17:18.686 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:17:18.687 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:17:18.687 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:17:18.687 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:17:18.687 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:18.687 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:17:18.687 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:17:18.687 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:17:18.687 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:17:18.687 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:17:18.687 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:17:18.687 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:17:18.687 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:17:18.688 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:17:18.688 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:18.688 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:17:18.688 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:17:18.688 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:17:18.688 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:17:18.688 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:17:18.688 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:17:18.688 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:17:18.688 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:17:18.688 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:17:18.688 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:17:18.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:18.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:17:18.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:17:18.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:17:18.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:17:18.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:17:18.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:17:18.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:17:18.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:17:18.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:17:18.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:17:18.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:17:18.759 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:17:18.759 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:17:18.759 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:17:18.759 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:17:18.759 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:17:18.759 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:17:18.759 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:17:18.759 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:17:18.759 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:17:18.759 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:17:18.759 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:17:18.760 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:17:18.760 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:17:18.761 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:18.762 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:17:18.762 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:18.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 55 2026-02-18T08:17:18.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 58 2026-02-18T08:17:18.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:17:18.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=233, tag_tid=15, entry_tid=1], [object_number=232, tag_tid=15, entry_tid=0], [object_number=234, tag_tid=14, entry_tid=52758], [object_number=235, tag_tid=14, entry_tid=52755]]], state=connected] 2026-02-18T08:17:18.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=221, tag_tid=14, entry_tid=38825], [object_number=220, tag_tid=14, entry_tid=38824], [object_number=223, tag_tid=14, entry_tid=38823], [object_number=222, tag_tid=14, entry_tid=38822]]], state=connected] 2026-02-18T08:17:18.793 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:18.795 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=233, tag_tid=15, entry_tid=1' 2026-02-18T08:17:18.796 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:17:18.796 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:17:18.796 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:17:18.796 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:17:18.796 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:17:18.796 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:17:18.796 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:17:18.796 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:17:18.796 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:17:18.796 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:17:18.796 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:17:18.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:17:18.798 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:18.798 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:17:18.798 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:18.826 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 55 2026-02-18T08:17:18.826 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 58 2026-02-18T08:17:18.826 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:17:18.826 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=233, tag_tid=15, entry_tid=1], [object_number=232, tag_tid=15, entry_tid=0], [object_number=234, tag_tid=14, entry_tid=52758], [object_number=235, tag_tid=14, entry_tid=52755]]], state=connected] 2026-02-18T08:17:18.826 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=221, tag_tid=14, entry_tid=38825], [object_number=220, tag_tid=14, entry_tid=38824], [object_number=223, tag_tid=14, entry_tid=38823], [object_number=222, tag_tid=14, entry_tid=38822]]], state=connected] 2026-02-18T08:17:18.829 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:18.830 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=221, tag_tid=14, entry_tid=38825' 2026-02-18T08:17:18.830 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=233, tag_tid=15, entry_tid=1' -a 'object_number=233, tag_tid=15, entry_tid=1' = 'object_number=221, tag_tid=14, entry_tid=38825' 2026-02-18T08:17:18.830 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=221, tag_tid=14, entry_tid=38825' '!=' 'object_number=222, tag_tid=14, entry_tid=38474' 2026-02-18T08:17:18.830 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:17:18.830 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=221, tag_tid=14, entry_tid=38825' = 'object_number=222, tag_tid=14, entry_tid=38474' 2026-02-18T08:17:18.830 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=221, tag_tid=14, entry_tid=38825' 2026-02-18T08:17:18.831 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=233, tag_tid=15, entry_tid=1' 2026-02-18T08:17:18.831 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:17:18.831 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:17:18.832 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=15 2026-02-18T08:17:18.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=221, tag_tid=14, entry_tid=38825' 2026-02-18T08:17:18.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:17:18.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:17:18.835 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=14 2026-02-18T08:17:18.835 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=233, tag_tid=15, entry_tid=1' 2026-02-18T08:17:18.836 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:17:18.836 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:17:18.837 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:17:18.838 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=221, tag_tid=14, entry_tid=38825' 2026-02-18T08:17:18.838 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:17:18.838 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:17:18.839 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=38825 2026-02-18T08:17:18.839 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 15 = 14 -a 1 -le 38825 2026-02-18T08:17:18.839 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:17:18.839 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:17:18.840 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:17:19.041 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:17:19.041 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:17:19.041 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:17:19.041 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:17:19.041 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:17:19.041 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:17:19.041 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:17:19.041 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:17:19.041 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:17:19.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:19.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:17:19.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:17:19.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:17:19.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:17:19.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:17:19.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:17:19.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:17:19.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:17:19.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:17:19.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:17:19.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:19.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:17:19.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:17:19.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:17:19.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:17:19.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:17:19.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:17:19.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:17:19.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:17:19.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:17:19.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:19.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:17:19.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:17:19.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:17:19.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:17:19.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:17:19.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:17:19.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:17:19.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:17:19.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:17:19.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:17:19.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:19.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:17:19.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:17:19.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:17:19.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:17:19.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:17:19.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:17:19.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:17:19.056 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:17:19.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:17:19.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:17:19.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:17:19.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:17:19.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:17:19.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:17:19.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:17:19.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:17:19.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:17:19.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:19.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:17:19.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:17:19.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:17:19.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:17:19.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:17:19.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:17:19.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:17:19.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:17:19.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:17:19.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:19.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:17:19.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:17:19.118 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:17:19.118 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:17:19.118 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:17:19.118 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:17:19.118 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:17:19.118 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:17:19.118 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:17:19.118 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:17:19.118 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:19.118 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:17:19.118 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:17:19.118 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:17:19.118 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:17:19.118 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:17:19.118 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:17:19.118 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:17:19.130 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:17:19.130 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:17:19.130 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:17:19.130 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:17:19.188 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:17:19.189 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:17:19.189 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:17:19.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:17:19.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:17:19.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:17:19.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:19.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:17:19.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:17:19.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:17:19.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:17:19.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:17:19.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:17:19.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:17:19.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:17:19.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:17:19.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:19.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:17:19.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:17:19.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:17:19.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:17:19.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:17:19.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:17:19.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:17:19.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:17:19.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:17:19.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:17:19.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:19.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:17:19.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:17:19.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:17:19.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:17:19.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:17:19.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:17:19.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:17:19.203 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:17:19.204 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:17:19.204 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:17:19.204 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:17:19.260 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:17:19.260 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:17:19.260 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:17:19.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:17:19.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:17:19.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:17:19.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:19.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:17:19.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:17:19.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:17:19.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:17:19.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:17:19.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:17:19.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:17:19.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:17:19.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:17:19.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:19.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:17:19.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:17:19.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:17:19.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:17:19.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:17:19.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:17:19.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:17:19.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:17:19.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:17:19.266 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:17:19.266 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:19.266 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:17:19.266 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:17:19.266 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:17:19.266 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:17:19.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:17:19.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:17:19.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:17:19.274 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:17:19.274 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:17:19.275 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:17:19.275 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:17:19.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:17:19.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:17:19.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:17:19.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:17:19.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:17:19.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:17:19.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:17:19.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:17:19.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:17:19.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:17:19.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:17:19.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:17:19.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:17:19.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:19.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:17:19.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:19.371 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 55 2026-02-18T08:17:19.371 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 58 2026-02-18T08:17:19.371 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:17:19.371 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=233, tag_tid=15, entry_tid=1], [object_number=232, tag_tid=15, entry_tid=0], [object_number=234, tag_tid=14, entry_tid=52758], [object_number=235, tag_tid=14, entry_tid=52755]]], state=connected] 2026-02-18T08:17:19.372 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=220, tag_tid=14, entry_tid=39216], [object_number=223, tag_tid=14, entry_tid=39215], [object_number=222, tag_tid=14, entry_tid=39214], [object_number=221, tag_tid=14, entry_tid=39213]]], state=connected] 2026-02-18T08:17:19.374 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:19.376 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=233, tag_tid=15, entry_tid=1' 2026-02-18T08:17:19.376 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:17:19.377 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:17:19.377 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:17:19.377 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:17:19.377 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:17:19.377 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:17:19.377 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:17:19.377 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:17:19.377 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:17:19.377 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:17:19.377 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:17:19.377 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:17:19.379 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:19.379 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:17:19.379 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:19.414 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 55 2026-02-18T08:17:19.414 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 58 2026-02-18T08:17:19.414 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:17:19.414 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=233, tag_tid=15, entry_tid=1], [object_number=232, tag_tid=15, entry_tid=0], [object_number=234, tag_tid=14, entry_tid=52758], [object_number=235, tag_tid=14, entry_tid=52755]]], state=connected] 2026-02-18T08:17:19.414 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=220, tag_tid=14, entry_tid=39216], [object_number=223, tag_tid=14, entry_tid=39215], [object_number=222, tag_tid=14, entry_tid=39214], [object_number=221, tag_tid=14, entry_tid=39213]]], state=connected] 2026-02-18T08:17:19.418 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:19.419 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=220, tag_tid=14, entry_tid=39216' 2026-02-18T08:17:19.419 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=233, tag_tid=15, entry_tid=1' -a 'object_number=233, tag_tid=15, entry_tid=1' = 'object_number=220, tag_tid=14, entry_tid=39216' 2026-02-18T08:17:19.419 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=220, tag_tid=14, entry_tid=39216' '!=' 'object_number=221, tag_tid=14, entry_tid=38825' 2026-02-18T08:17:19.419 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:17:19.419 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=220, tag_tid=14, entry_tid=39216' = 'object_number=221, tag_tid=14, entry_tid=38825' 2026-02-18T08:17:19.419 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=220, tag_tid=14, entry_tid=39216' 2026-02-18T08:17:19.420 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=233, tag_tid=15, entry_tid=1' 2026-02-18T08:17:19.420 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:17:19.420 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:17:19.421 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=15 2026-02-18T08:17:19.422 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=220, tag_tid=14, entry_tid=39216' 2026-02-18T08:17:19.422 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:17:19.422 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:17:19.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=14 2026-02-18T08:17:19.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=233, tag_tid=15, entry_tid=1' 2026-02-18T08:17:19.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:17:19.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:17:19.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:17:19.427 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=220, tag_tid=14, entry_tid=39216' 2026-02-18T08:17:19.427 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:17:19.427 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:17:19.428 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=39216 2026-02-18T08:17:19.428 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 15 = 14 -a 1 -le 39216 2026-02-18T08:17:19.428 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:17:19.428 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:17:19.428 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:17:19.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:17:19.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:17:19.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:17:19.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:17:19.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:17:19.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:17:19.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:17:19.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:17:19.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:17:19.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:19.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:17:19.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:17:19.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:17:19.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:17:19.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:17:19.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:17:19.635 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:17:19.635 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:17:19.635 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:17:19.635 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:17:19.635 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:19.635 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:17:19.635 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:17:19.635 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:17:19.636 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:17:19.636 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:17:19.636 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:17:19.636 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:17:19.636 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:17:19.636 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:17:19.636 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:19.636 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:17:19.636 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:17:19.636 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:17:19.636 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:17:19.636 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:17:19.637 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:17:19.637 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:17:19.637 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:17:19.637 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:17:19.637 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:17:19.637 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:19.637 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:17:19.637 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:17:19.637 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:17:19.637 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:17:19.637 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:17:19.637 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:17:19.638 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:17:19.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:17:19.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:17:19.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:17:19.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:17:19.693 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:17:19.694 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:17:19.694 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:17:19.696 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:17:19.697 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:17:19.697 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:17:19.697 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:19.697 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:17:19.697 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:17:19.697 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:17:19.697 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:17:19.697 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:17:19.697 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:17:19.697 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:17:19.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:17:19.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:17:19.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:19.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:17:19.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:17:19.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:17:19.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:17:19.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:17:19.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:17:19.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:17:19.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:17:19.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:17:19.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:17:19.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:19.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:17:19.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:17:19.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:17:19.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:17:19.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:17:19.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:17:19.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:17:19.708 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:17:19.708 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:17:19.708 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:17:19.708 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:17:19.760 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:17:19.760 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:17:19.760 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:17:19.763 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:17:19.763 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:17:19.763 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:17:19.763 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:19.763 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:17:19.763 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:17:19.763 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:17:19.763 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:17:19.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:17:19.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:17:19.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:17:19.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:17:19.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:17:19.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:19.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:17:19.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:17:19.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:17:19.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:17:19.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:17:19.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:17:19.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:17:19.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:17:19.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:17:19.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:17:19.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:19.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:17:19.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:17:19.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:17:19.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:17:19.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:17:19.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:17:19.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:17:19.773 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:17:19.774 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:17:19.774 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:17:19.774 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:17:19.823 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:17:19.824 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:17:19.824 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:17:19.826 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:17:19.826 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:17:19.827 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:17:19.827 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:19.827 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:17:19.827 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:17:19.827 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:17:19.827 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:17:19.827 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:17:19.827 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:17:19.827 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:17:19.827 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:17:19.827 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:17:19.827 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:19.828 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:17:19.828 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:17:19.828 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:17:19.828 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:17:19.828 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:17:19.828 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:17:19.828 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:17:19.828 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:17:19.828 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:17:19.828 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:17:19.828 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:19.828 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:17:19.829 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:17:19.829 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:17:19.829 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:17:19.829 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:17:19.829 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:17:19.829 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:17:19.836 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:17:19.837 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:17:19.837 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:17:19.837 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:17:19.896 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:17:19.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:17:19.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:17:19.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:17:19.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:17:19.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:17:19.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:17:19.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:17:19.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:17:19.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:17:19.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:17:19.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:17:19.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:17:19.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:19.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:17:19.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:19.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 55 2026-02-18T08:17:19.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 58 2026-02-18T08:17:19.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:17:19.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=233, tag_tid=15, entry_tid=1], [object_number=232, tag_tid=15, entry_tid=0], [object_number=234, tag_tid=14, entry_tid=52758], [object_number=235, tag_tid=14, entry_tid=52755]]], state=connected] 2026-02-18T08:17:19.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=223, tag_tid=14, entry_tid=39579], [object_number=222, tag_tid=14, entry_tid=39578], [object_number=221, tag_tid=14, entry_tid=39577], [object_number=220, tag_tid=14, entry_tid=39576]]], state=connected] 2026-02-18T08:17:19.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:19.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=233, tag_tid=15, entry_tid=1' 2026-02-18T08:17:19.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:17:19.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:17:19.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:17:19.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:17:19.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:17:19.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:17:19.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:17:19.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:17:19.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:17:19.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:17:19.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:17:19.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:17:19.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:19.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:17:19.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:19.968 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 55 2026-02-18T08:17:19.968 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 58 2026-02-18T08:17:19.968 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:17:19.968 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=233, tag_tid=15, entry_tid=1], [object_number=232, tag_tid=15, entry_tid=0], [object_number=234, tag_tid=14, entry_tid=52758], [object_number=235, tag_tid=14, entry_tid=52755]]], state=connected] 2026-02-18T08:17:19.968 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=223, tag_tid=14, entry_tid=39579], [object_number=222, tag_tid=14, entry_tid=39578], [object_number=221, tag_tid=14, entry_tid=39577], [object_number=220, tag_tid=14, entry_tid=39576]]], state=connected] 2026-02-18T08:17:19.971 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:19.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=223, tag_tid=14, entry_tid=39579' 2026-02-18T08:17:19.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=233, tag_tid=15, entry_tid=1' -a 'object_number=233, tag_tid=15, entry_tid=1' = 'object_number=223, tag_tid=14, entry_tid=39579' 2026-02-18T08:17:19.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=223, tag_tid=14, entry_tid=39579' '!=' 'object_number=220, tag_tid=14, entry_tid=39216' 2026-02-18T08:17:19.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:17:19.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=223, tag_tid=14, entry_tid=39579' = 'object_number=220, tag_tid=14, entry_tid=39216' 2026-02-18T08:17:19.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=223, tag_tid=14, entry_tid=39579' 2026-02-18T08:17:19.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=233, tag_tid=15, entry_tid=1' 2026-02-18T08:17:19.974 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:17:19.974 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:17:19.974 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=15 2026-02-18T08:17:19.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=223, tag_tid=14, entry_tid=39579' 2026-02-18T08:17:19.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:17:19.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:17:19.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=14 2026-02-18T08:17:19.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=233, tag_tid=15, entry_tid=1' 2026-02-18T08:17:19.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:17:19.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:17:19.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:17:19.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=223, tag_tid=14, entry_tid=39579' 2026-02-18T08:17:19.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:17:19.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:17:19.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=39579 2026-02-18T08:17:19.982 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 15 = 14 -a 1 -le 39579 2026-02-18T08:17:19.982 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:17:19.982 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:17:19.982 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:17:20.183 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:17:20.183 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:17:20.183 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:17:20.183 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:17:20.183 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:17:20.183 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:17:20.183 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:17:20.183 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:17:20.183 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:17:20.183 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:20.183 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:17:20.183 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:17:20.183 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:17:20.183 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:17:20.183 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:17:20.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:17:20.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:17:20.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:17:20.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:17:20.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:17:20.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:20.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:17:20.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:17:20.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:17:20.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:17:20.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:17:20.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:17:20.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:17:20.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:17:20.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:17:20.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:20.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:17:20.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:17:20.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:17:20.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:17:20.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:17:20.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:17:20.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:17:20.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:17:20.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:17:20.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:17:20.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:20.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:17:20.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:17:20.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:17:20.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:17:20.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:17:20.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:17:20.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:17:20.197 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:17:20.197 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:17:20.198 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:17:20.198 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:17:20.247 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:17:20.247 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:17:20.247 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:17:20.249 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:17:20.249 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:17:20.249 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:17:20.249 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:20.249 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:17:20.249 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:17:20.249 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:17:20.249 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:17:20.249 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:17:20.249 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:17:20.249 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:17:20.249 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:17:20.249 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:17:20.249 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:20.249 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:17:20.249 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:17:20.249 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:17:20.250 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:17:20.250 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:17:20.250 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:17:20.250 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:17:20.250 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:17:20.250 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:17:20.250 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:17:20.250 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:20.250 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:17:20.250 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:17:20.250 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:17:20.250 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:17:20.250 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:17:20.250 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:17:20.250 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:17:20.260 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:17:20.261 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:17:20.261 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:17:20.261 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:17:20.318 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:17:20.318 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:17:20.318 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:17:20.320 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:17:20.320 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:17:20.320 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:17:20.320 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:20.320 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:17:20.320 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:17:20.320 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:17:20.320 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:17:20.320 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:17:20.320 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:17:20.320 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:17:20.320 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:17:20.320 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:17:20.320 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:20.320 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:17:20.320 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:17:20.320 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:17:20.320 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:17:20.320 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:17:20.320 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:17:20.321 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:17:20.321 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:17:20.321 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:17:20.321 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:17:20.321 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:20.321 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:17:20.321 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:17:20.321 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:17:20.321 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:17:20.321 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:17:20.321 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:17:20.321 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:17:20.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:17:20.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:17:20.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:17:20.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:17:20.389 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:17:20.389 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:17:20.389 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:17:20.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:17:20.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:17:20.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:17:20.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:20.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:17:20.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:17:20.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:17:20.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:17:20.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:17:20.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:17:20.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:17:20.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:17:20.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:17:20.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:20.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:17:20.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:17:20.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:17:20.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:17:20.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:17:20.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:17:20.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:17:20.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:17:20.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:17:20.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:17:20.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:20.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:17:20.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:17:20.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:17:20.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:17:20.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:17:20.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:17:20.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:17:20.403 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:17:20.403 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:17:20.403 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:17:20.403 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:17:20.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:17:20.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:17:20.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:17:20.467 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:17:20.467 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:17:20.467 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:17:20.467 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:17:20.467 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:17:20.467 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:17:20.467 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:17:20.467 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:17:20.467 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:17:20.467 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:17:20.468 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:20.468 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:17:20.468 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:20.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 55 2026-02-18T08:17:20.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 58 2026-02-18T08:17:20.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:17:20.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=233, tag_tid=15, entry_tid=1], [object_number=232, tag_tid=15, entry_tid=0], [object_number=234, tag_tid=14, entry_tid=52758], [object_number=235, tag_tid=14, entry_tid=52755]]], state=connected] 2026-02-18T08:17:20.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=221, tag_tid=14, entry_tid=39809], [object_number=220, tag_tid=14, entry_tid=39808], [object_number=223, tag_tid=14, entry_tid=39807], [object_number=222, tag_tid=14, entry_tid=39806]]], state=connected] 2026-02-18T08:17:20.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:20.500 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=233, tag_tid=15, entry_tid=1' 2026-02-18T08:17:20.501 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:17:20.502 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:17:20.502 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:17:20.502 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:17:20.502 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:17:20.502 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:17:20.502 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:17:20.502 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:17:20.503 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:17:20.503 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:17:20.503 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:17:20.503 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:17:20.504 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:20.504 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:17:20.504 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:20.532 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 55 2026-02-18T08:17:20.532 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 58 2026-02-18T08:17:20.532 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:17:20.532 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=233, tag_tid=15, entry_tid=1], [object_number=232, tag_tid=15, entry_tid=0], [object_number=234, tag_tid=14, entry_tid=52758], [object_number=235, tag_tid=14, entry_tid=52755]]], state=connected] 2026-02-18T08:17:20.532 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=221, tag_tid=14, entry_tid=39809], [object_number=220, tag_tid=14, entry_tid=39808], [object_number=223, tag_tid=14, entry_tid=39807], [object_number=222, tag_tid=14, entry_tid=39806]]], state=connected] 2026-02-18T08:17:20.535 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:20.536 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=221, tag_tid=14, entry_tid=39809' 2026-02-18T08:17:20.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=233, tag_tid=15, entry_tid=1' -a 'object_number=233, tag_tid=15, entry_tid=1' = 'object_number=221, tag_tid=14, entry_tid=39809' 2026-02-18T08:17:20.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=221, tag_tid=14, entry_tid=39809' '!=' 'object_number=223, tag_tid=14, entry_tid=39579' 2026-02-18T08:17:20.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:17:20.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=221, tag_tid=14, entry_tid=39809' = 'object_number=223, tag_tid=14, entry_tid=39579' 2026-02-18T08:17:20.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=221, tag_tid=14, entry_tid=39809' 2026-02-18T08:17:20.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=233, tag_tid=15, entry_tid=1' 2026-02-18T08:17:20.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:17:20.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:17:20.539 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=15 2026-02-18T08:17:20.539 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=221, tag_tid=14, entry_tid=39809' 2026-02-18T08:17:20.540 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:17:20.540 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:17:20.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=14 2026-02-18T08:17:20.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=233, tag_tid=15, entry_tid=1' 2026-02-18T08:17:20.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:17:20.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:17:20.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:17:20.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=221, tag_tid=14, entry_tid=39809' 2026-02-18T08:17:20.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:17:20.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:17:20.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=39809 2026-02-18T08:17:20.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 15 = 14 -a 1 -le 39809 2026-02-18T08:17:20.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:17:20.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:17:20.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:17:20.748 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:17:20.748 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:17:20.748 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:17:20.748 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:17:20.748 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:17:20.748 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:17:20.748 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:17:20.749 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:17:20.749 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:17:20.749 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:20.749 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:17:20.749 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:17:20.749 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:17:20.749 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:17:20.749 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:17:20.752 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:17:20.752 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:17:20.752 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:17:20.752 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:17:20.752 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:17:20.752 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:20.752 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:17:20.752 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:17:20.753 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:17:20.753 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:17:20.753 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:17:20.753 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:17:20.753 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:17:20.753 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:17:20.753 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:17:20.753 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:20.753 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:17:20.753 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:17:20.753 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:17:20.753 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:17:20.754 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:17:20.754 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:17:20.754 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:17:20.754 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:17:20.754 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:17:20.754 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:17:20.754 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:20.754 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:17:20.754 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:17:20.754 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:17:20.754 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:17:20.754 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:17:20.755 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:17:20.755 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:17:20.761 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:17:20.762 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:17:20.762 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:17:20.762 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:17:20.813 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:17:20.814 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:17:20.814 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:17:20.817 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:17:20.817 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:17:20.817 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:17:20.817 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:20.817 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:17:20.817 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:17:20.817 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:17:20.817 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:17:20.817 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:17:20.817 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:17:20.818 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:17:20.818 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:17:20.818 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:17:20.818 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:20.818 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:17:20.818 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:17:20.818 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:17:20.818 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:17:20.818 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:17:20.818 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:17:20.818 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:17:20.818 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:17:20.819 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:17:20.819 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:17:20.819 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:20.819 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:17:20.819 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:17:20.819 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:17:20.819 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:17:20.819 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:17:20.819 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:17:20.819 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:17:20.828 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:17:20.828 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:17:20.828 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:17:20.829 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:17:20.878 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:17:20.878 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:17:20.878 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:17:20.881 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:17:20.881 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:17:20.881 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:17:20.881 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:20.881 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:17:20.881 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:17:20.882 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:17:20.882 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:17:20.882 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:17:20.882 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:17:20.882 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:17:20.882 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:17:20.882 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:17:20.882 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:20.882 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:17:20.882 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:17:20.882 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:17:20.882 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:17:20.883 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:17:20.883 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:17:20.883 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:17:20.883 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:17:20.883 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:17:20.883 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:17:20.883 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:20.883 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:17:20.883 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:17:20.883 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:17:20.883 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:17:20.883 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:17:20.884 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:17:20.884 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:17:20.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:17:20.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:17:20.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:17:20.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:17:20.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:17:20.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:17:20.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:17:20.946 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:17:20.946 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:17:20.946 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:17:20.946 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:20.946 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:17:20.946 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:17:20.946 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:17:20.947 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:17:20.947 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:17:20.947 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:17:20.947 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:17:20.947 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:17:20.947 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:17:20.947 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:20.947 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:17:20.947 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:17:20.947 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:17:20.947 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:17:20.947 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:17:20.948 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:17:20.948 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:17:20.948 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:17:20.948 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:17:20.948 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:17:20.948 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:20.948 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:17:20.948 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:17:20.948 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:17:20.948 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:17:20.948 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:17:20.949 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:17:20.949 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:17:20.956 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:17:20.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:17:20.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:17:20.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:17:21.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:17:21.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:17:21.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:17:21.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:17:21.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:17:21.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:17:21.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:17:21.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:17:21.018 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:17:21.018 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:17:21.018 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:17:21.018 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:17:21.018 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:17:21.019 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:21.019 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:17:21.019 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:21.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 55 2026-02-18T08:17:21.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 58 2026-02-18T08:17:21.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:17:21.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=233, tag_tid=15, entry_tid=1], [object_number=232, tag_tid=15, entry_tid=0], [object_number=234, tag_tid=14, entry_tid=52758], [object_number=235, tag_tid=14, entry_tid=52755]]], state=connected] 2026-02-18T08:17:21.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=221, tag_tid=14, entry_tid=40165], [object_number=220, tag_tid=14, entry_tid=40164], [object_number=223, tag_tid=14, entry_tid=40163], [object_number=222, tag_tid=14, entry_tid=40162]]], state=connected] 2026-02-18T08:17:21.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:21.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=233, tag_tid=15, entry_tid=1' 2026-02-18T08:17:21.055 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:17:21.055 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:17:21.055 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:17:21.055 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:17:21.056 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:17:21.056 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:17:21.056 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:17:21.056 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:17:21.056 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:17:21.056 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:17:21.056 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:17:21.056 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:17:21.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:21.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:21.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:17:21.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 55 2026-02-18T08:17:21.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 58 2026-02-18T08:17:21.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:17:21.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=233, tag_tid=15, entry_tid=1], [object_number=232, tag_tid=15, entry_tid=0], [object_number=234, tag_tid=14, entry_tid=52758], [object_number=235, tag_tid=14, entry_tid=52755]]], state=connected] 2026-02-18T08:17:21.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=221, tag_tid=14, entry_tid=40165], [object_number=220, tag_tid=14, entry_tid=40164], [object_number=223, tag_tid=14, entry_tid=40163], [object_number=222, tag_tid=14, entry_tid=40162]]], state=connected] 2026-02-18T08:17:21.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:21.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=221, tag_tid=14, entry_tid=40165' 2026-02-18T08:17:21.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=233, tag_tid=15, entry_tid=1' -a 'object_number=233, tag_tid=15, entry_tid=1' = 'object_number=221, tag_tid=14, entry_tid=40165' 2026-02-18T08:17:21.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=221, tag_tid=14, entry_tid=40165' '!=' 'object_number=221, tag_tid=14, entry_tid=39809' 2026-02-18T08:17:21.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:17:21.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=221, tag_tid=14, entry_tid=40165' = 'object_number=221, tag_tid=14, entry_tid=39809' 2026-02-18T08:17:21.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=221, tag_tid=14, entry_tid=40165' 2026-02-18T08:17:21.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=233, tag_tid=15, entry_tid=1' 2026-02-18T08:17:21.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:17:21.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:17:21.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=15 2026-02-18T08:17:21.092 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=221, tag_tid=14, entry_tid=40165' 2026-02-18T08:17:21.092 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:17:21.092 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:17:21.094 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=14 2026-02-18T08:17:21.095 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=233, tag_tid=15, entry_tid=1' 2026-02-18T08:17:21.095 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:17:21.095 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:17:21.096 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:17:21.097 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=221, tag_tid=14, entry_tid=40165' 2026-02-18T08:17:21.097 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:17:21.097 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:17:21.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=40165 2026-02-18T08:17:21.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 15 = 14 -a 1 -le 40165 2026-02-18T08:17:21.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:17:21.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:17:21.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:17:21.300 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:17:21.300 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:17:21.300 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:17:21.300 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:17:21.300 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:17:21.300 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:17:21.300 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:17:21.300 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:17:21.300 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:17:21.300 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:21.300 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:17:21.300 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:17:21.301 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:17:21.301 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:17:21.301 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:17:21.301 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:17:21.301 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:17:21.302 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:17:21.302 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:17:21.302 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:17:21.302 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:21.302 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:17:21.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:17:21.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:17:21.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:17:21.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:17:21.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:17:21.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:17:21.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:17:21.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:17:21.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:21.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:17:21.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:17:21.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:17:21.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:17:21.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:17:21.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:17:21.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:17:21.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:17:21.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:17:21.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:17:21.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:21.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:17:21.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:17:21.304 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:17:21.304 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:17:21.304 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:17:21.304 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:17:21.304 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:17:21.314 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:17:21.314 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:17:21.314 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:17:21.314 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:17:21.364 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:17:21.364 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:17:21.364 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:17:21.366 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:17:21.366 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:17:21.366 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:17:21.366 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:21.366 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:17:21.366 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:17:21.366 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:17:21.366 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:17:21.366 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:17:21.366 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:17:21.366 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:17:21.366 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:17:21.366 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:17:21.366 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:21.366 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:17:21.367 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:17:21.367 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:17:21.367 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:17:21.367 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:17:21.367 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:17:21.367 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:17:21.368 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:17:21.368 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:17:21.368 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:17:21.368 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:21.368 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:17:21.368 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:17:21.368 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:17:21.368 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:17:21.368 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:17:21.368 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:17:21.368 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:17:21.379 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:17:21.379 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:17:21.379 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:17:21.379 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:17:21.430 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:17:21.430 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:17:21.430 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:17:21.433 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:17:21.433 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:17:21.433 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:17:21.433 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:21.433 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:17:21.433 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:17:21.433 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:17:21.433 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:17:21.433 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:17:21.433 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:17:21.434 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:17:21.434 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:17:21.434 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:17:21.434 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:21.434 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:17:21.434 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:17:21.434 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:17:21.434 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:17:21.434 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:17:21.434 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:17:21.434 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:17:21.435 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:17:21.435 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:17:21.435 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:17:21.435 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:21.435 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:17:21.435 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:17:21.435 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:17:21.435 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:17:21.435 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:17:21.435 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:17:21.435 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:17:21.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:17:21.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:17:21.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:17:21.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:17:21.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:17:21.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:17:21.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:17:21.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:17:21.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:17:21.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:17:21.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:21.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:17:21.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:17:21.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:17:21.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:17:21.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:17:21.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:17:21.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:17:21.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:17:21.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:17:21.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:21.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:17:21.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:17:21.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:17:21.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:17:21.500 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:17:21.500 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:17:21.500 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:17:21.500 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:17:21.500 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:17:21.500 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:17:21.500 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:21.500 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:17:21.500 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:17:21.500 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:17:21.500 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:17:21.500 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:17:21.501 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:17:21.501 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:17:21.509 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:17:21.509 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:17:21.510 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:17:21.510 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:17:21.570 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:17:21.572 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:17:21.572 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:17:21.572 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:17:21.572 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:17:21.572 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:17:21.572 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:17:21.573 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:17:21.573 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:17:21.573 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:17:21.573 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:17:21.573 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:17:21.573 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:17:21.574 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:21.574 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:17:21.574 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:21.602 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 55 2026-02-18T08:17:21.602 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 58 2026-02-18T08:17:21.602 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:17:21.602 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=233, tag_tid=15, entry_tid=1], [object_number=232, tag_tid=15, entry_tid=0], [object_number=234, tag_tid=14, entry_tid=52758], [object_number=235, tag_tid=14, entry_tid=52755]]], state=connected] 2026-02-18T08:17:21.602 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=222, tag_tid=14, entry_tid=40518], [object_number=221, tag_tid=14, entry_tid=40517], [object_number=220, tag_tid=14, entry_tid=40516], [object_number=223, tag_tid=14, entry_tid=40515]]], state=connected] 2026-02-18T08:17:21.604 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:21.606 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=233, tag_tid=15, entry_tid=1' 2026-02-18T08:17:21.607 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:17:21.607 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:17:21.607 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:17:21.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:17:21.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:17:21.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:17:21.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:17:21.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:17:21.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:17:21.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:17:21.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:17:21.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:17:21.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:21.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:17:21.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:21.635 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 55 2026-02-18T08:17:21.636 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 58 2026-02-18T08:17:21.636 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:17:21.636 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=233, tag_tid=15, entry_tid=1], [object_number=232, tag_tid=15, entry_tid=0], [object_number=234, tag_tid=14, entry_tid=52758], [object_number=235, tag_tid=14, entry_tid=52755]]], state=connected] 2026-02-18T08:17:21.636 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=222, tag_tid=14, entry_tid=40518], [object_number=221, tag_tid=14, entry_tid=40517], [object_number=220, tag_tid=14, entry_tid=40516], [object_number=223, tag_tid=14, entry_tid=40515]]], state=connected] 2026-02-18T08:17:21.638 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:21.640 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=222, tag_tid=14, entry_tid=40518' 2026-02-18T08:17:21.640 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=233, tag_tid=15, entry_tid=1' -a 'object_number=233, tag_tid=15, entry_tid=1' = 'object_number=222, tag_tid=14, entry_tid=40518' 2026-02-18T08:17:21.640 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=222, tag_tid=14, entry_tid=40518' '!=' 'object_number=221, tag_tid=14, entry_tid=40165' 2026-02-18T08:17:21.640 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:17:21.640 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=222, tag_tid=14, entry_tid=40518' = 'object_number=221, tag_tid=14, entry_tid=40165' 2026-02-18T08:17:21.640 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=222, tag_tid=14, entry_tid=40518' 2026-02-18T08:17:21.641 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=233, tag_tid=15, entry_tid=1' 2026-02-18T08:17:21.641 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:17:21.641 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:17:21.642 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=15 2026-02-18T08:17:21.642 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=222, tag_tid=14, entry_tid=40518' 2026-02-18T08:17:21.643 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:17:21.643 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:17:21.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=14 2026-02-18T08:17:21.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=233, tag_tid=15, entry_tid=1' 2026-02-18T08:17:21.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:17:21.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:17:21.646 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:17:21.647 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=222, tag_tid=14, entry_tid=40518' 2026-02-18T08:17:21.647 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:17:21.647 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:17:21.649 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=40518 2026-02-18T08:17:21.649 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 15 = 14 -a 1 -le 40518 2026-02-18T08:17:21.649 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:17:21.649 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:17:21.649 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:17:21.851 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:17:21.851 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:17:21.851 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:17:21.851 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:17:21.851 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:17:21.851 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:17:21.851 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:17:21.851 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:17:21.852 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:17:21.852 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:21.852 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:17:21.852 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:17:21.852 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:17:21.852 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:17:21.852 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:17:21.855 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:17:21.855 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:17:21.855 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:17:21.855 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:17:21.855 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:17:21.855 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:21.855 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:17:21.855 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:17:21.855 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:17:21.855 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:17:21.856 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:17:21.856 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:17:21.856 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:17:21.856 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:17:21.856 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:17:21.856 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:21.856 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:17:21.856 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:17:21.856 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:17:21.856 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:17:21.856 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:17:21.856 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:17:21.856 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:17:21.857 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:17:21.857 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:17:21.857 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:17:21.857 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:21.857 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:17:21.857 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:17:21.857 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:17:21.857 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:17:21.857 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:17:21.857 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:17:21.857 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:17:21.865 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:17:21.866 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:17:21.866 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:17:21.866 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:17:21.923 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:17:21.923 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:17:21.923 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:17:21.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:17:21.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:17:21.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:17:21.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:21.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:17:21.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:17:21.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:17:21.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:17:21.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:17:21.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:17:21.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:17:21.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:17:21.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:17:21.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:21.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:17:21.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:17:21.928 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:17:21.928 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:17:21.928 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:17:21.928 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:17:21.928 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:17:21.928 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:17:21.928 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:17:21.928 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:17:21.928 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:21.928 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:17:21.928 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:17:21.928 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:17:21.929 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:17:21.929 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:17:21.929 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:17:21.929 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:17:21.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:17:21.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:17:21.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:17:21.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:17:21.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:17:21.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:17:21.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:17:21.999 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:17:21.999 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:17:21.999 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:17:21.999 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:21.999 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:17:21.999 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:17:21.999 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:17:21.999 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:17:22.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:17:22.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:17:22.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:17:22.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:17:22.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:17:22.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:22.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:17:22.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:17:22.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:17:22.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:17:22.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:17:22.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:17:22.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:17:22.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:17:22.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:17:22.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:17:22.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:22.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:17:22.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:17:22.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:17:22.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:17:22.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:17:22.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:17:22.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:17:22.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:17:22.010 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:17:22.010 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:17:22.010 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:17:22.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:17:22.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:17:22.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:17:22.068 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:17:22.068 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:17:22.068 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:17:22.069 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:22.069 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:17:22.069 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:17:22.069 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:17:22.069 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:17:22.069 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:17:22.069 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:17:22.069 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:17:22.069 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:17:22.069 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:17:22.069 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:22.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:17:22.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:17:22.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:17:22.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:17:22.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:17:22.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:17:22.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:17:22.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:17:22.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:17:22.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:17:22.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:22.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:17:22.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:17:22.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:17:22.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:17:22.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:17:22.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:17:22.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:17:22.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:17:22.080 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:17:22.080 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:17:22.080 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:17:22.146 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:17:22.147 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:17:22.148 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:17:22.148 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:17:22.148 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:17:22.148 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:17:22.148 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:17:22.148 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:17:22.148 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:17:22.148 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:17:22.148 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:17:22.148 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:17:22.148 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:17:22.149 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:22.149 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:17:22.150 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:22.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 55 2026-02-18T08:17:22.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 58 2026-02-18T08:17:22.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:17:22.187 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=233, tag_tid=15, entry_tid=1], [object_number=232, tag_tid=15, entry_tid=0], [object_number=234, tag_tid=14, entry_tid=52758], [object_number=235, tag_tid=14, entry_tid=52755]]], state=connected] 2026-02-18T08:17:22.187 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=222, tag_tid=14, entry_tid=40890], [object_number=221, tag_tid=14, entry_tid=40889], [object_number=220, tag_tid=14, entry_tid=40888], [object_number=223, tag_tid=14, entry_tid=40887]]], state=connected] 2026-02-18T08:17:22.190 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:22.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=233, tag_tid=15, entry_tid=1' 2026-02-18T08:17:22.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:17:22.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:17:22.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:17:22.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:17:22.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:17:22.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:17:22.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:17:22.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:17:22.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:17:22.194 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:17:22.194 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:17:22.194 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:17:22.195 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:22.195 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:17:22.195 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:22.222 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 55 2026-02-18T08:17:22.222 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 58 2026-02-18T08:17:22.222 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:17:22.222 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=233, tag_tid=15, entry_tid=1], [object_number=232, tag_tid=15, entry_tid=0], [object_number=234, tag_tid=14, entry_tid=52758], [object_number=235, tag_tid=14, entry_tid=52755]]], state=connected] 2026-02-18T08:17:22.223 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=222, tag_tid=14, entry_tid=40890], [object_number=221, tag_tid=14, entry_tid=40889], [object_number=220, tag_tid=14, entry_tid=40888], [object_number=223, tag_tid=14, entry_tid=40887]]], state=connected] 2026-02-18T08:17:22.225 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:22.227 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=222, tag_tid=14, entry_tid=40890' 2026-02-18T08:17:22.227 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=233, tag_tid=15, entry_tid=1' -a 'object_number=233, tag_tid=15, entry_tid=1' = 'object_number=222, tag_tid=14, entry_tid=40890' 2026-02-18T08:17:22.227 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=222, tag_tid=14, entry_tid=40890' '!=' 'object_number=222, tag_tid=14, entry_tid=40518' 2026-02-18T08:17:22.227 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:17:22.227 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=222, tag_tid=14, entry_tid=40890' = 'object_number=222, tag_tid=14, entry_tid=40518' 2026-02-18T08:17:22.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=222, tag_tid=14, entry_tid=40890' 2026-02-18T08:17:22.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=233, tag_tid=15, entry_tid=1' 2026-02-18T08:17:22.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:17:22.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:17:22.229 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=15 2026-02-18T08:17:22.230 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=222, tag_tid=14, entry_tid=40890' 2026-02-18T08:17:22.230 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:17:22.230 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:17:22.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=14 2026-02-18T08:17:22.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=233, tag_tid=15, entry_tid=1' 2026-02-18T08:17:22.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:17:22.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:17:22.234 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:17:22.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=222, tag_tid=14, entry_tid=40890' 2026-02-18T08:17:22.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:17:22.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:17:22.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=40890 2026-02-18T08:17:22.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 15 = 14 -a 1 -le 40890 2026-02-18T08:17:22.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:17:22.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:17:22.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:17:22.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:17:22.439 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:17:22.439 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:17:22.439 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:17:22.439 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:17:22.439 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:17:22.439 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:17:22.439 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:17:22.439 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:17:22.440 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:22.440 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:17:22.440 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:17:22.440 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:17:22.440 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:17:22.440 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:17:22.440 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:17:22.440 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:17:22.443 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:17:22.443 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:17:22.443 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:17:22.443 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:22.443 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:17:22.443 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:17:22.443 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:17:22.443 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:17:22.443 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:17:22.443 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:17:22.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:17:22.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:17:22.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:17:22.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:22.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:17:22.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:17:22.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:17:22.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:17:22.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:17:22.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:17:22.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:17:22.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:17:22.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:17:22.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:17:22.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:22.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:17:22.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:17:22.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:17:22.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:17:22.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:17:22.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:17:22.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:17:22.453 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:17:22.453 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:17:22.454 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:17:22.454 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:17:22.510 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:17:22.510 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:17:22.510 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:17:22.513 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:17:22.513 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:17:22.513 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:17:22.513 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:22.513 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:17:22.513 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:17:22.513 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:17:22.513 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:17:22.513 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:17:22.514 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:17:22.514 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:17:22.514 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:17:22.514 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:17:22.514 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:22.514 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:17:22.514 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:17:22.514 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:17:22.514 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:17:22.514 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:17:22.514 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:17:22.514 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:17:22.514 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:17:22.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:17:22.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:17:22.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:22.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:17:22.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:17:22.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:17:22.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:17:22.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:17:22.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:17:22.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:17:22.524 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:17:22.524 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:17:22.525 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:17:22.525 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:17:22.582 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:17:22.582 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:17:22.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:17:22.585 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:17:22.585 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:17:22.585 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:17:22.586 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:22.586 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:17:22.586 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:17:22.586 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:17:22.586 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:17:22.586 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:17:22.586 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:17:22.586 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:17:22.586 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:17:22.586 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:17:22.586 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:22.587 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:17:22.587 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:17:22.587 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:17:22.587 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:17:22.587 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:17:22.587 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:17:22.587 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:17:22.587 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:17:22.587 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:17:22.587 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:17:22.587 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:22.587 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:17:22.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:17:22.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:17:22.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:17:22.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:17:22.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:17:22.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:17:22.596 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:17:22.597 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:17:22.597 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:17:22.597 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:17:22.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:17:22.653 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:17:22.653 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:17:22.655 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:17:22.655 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:17:22.655 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:17:22.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:22.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:17:22.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:17:22.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:17:22.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:17:22.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:17:22.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:17:22.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:17:22.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:17:22.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:17:22.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:22.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:17:22.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:17:22.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:17:22.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:17:22.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:17:22.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:17:22.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:17:22.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:17:22.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:17:22.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:17:22.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:22.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:17:22.658 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:17:22.658 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:17:22.658 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:17:22.658 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:17:22.658 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:17:22.658 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:17:22.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:17:22.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:17:22.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:17:22.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:17:22.725 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:17:22.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:17:22.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:17:22.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:17:22.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:17:22.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:17:22.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:17:22.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:17:22.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:17:22.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:17:22.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:17:22.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:17:22.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:17:22.728 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:22.728 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:17:22.728 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:22.756 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 55 2026-02-18T08:17:22.756 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 58 2026-02-18T08:17:22.756 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:17:22.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=233, tag_tid=15, entry_tid=1], [object_number=232, tag_tid=15, entry_tid=0], [object_number=234, tag_tid=14, entry_tid=52758], [object_number=235, tag_tid=14, entry_tid=52755]]], state=connected] 2026-02-18T08:17:22.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=221, tag_tid=14, entry_tid=41241], [object_number=220, tag_tid=14, entry_tid=41240], [object_number=223, tag_tid=14, entry_tid=41239], [object_number=222, tag_tid=14, entry_tid=41238]]], state=connected] 2026-02-18T08:17:22.759 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:22.760 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=233, tag_tid=15, entry_tid=1' 2026-02-18T08:17:22.761 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:17:22.761 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:17:22.762 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:17:22.762 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:17:22.762 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:17:22.762 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:17:22.762 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:17:22.762 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:17:22.762 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:17:22.762 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:17:22.763 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:17:22.763 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:17:22.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:22.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:17:22.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:22.796 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 55 2026-02-18T08:17:22.796 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 58 2026-02-18T08:17:22.796 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:17:22.796 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=233, tag_tid=15, entry_tid=1], [object_number=232, tag_tid=15, entry_tid=0], [object_number=234, tag_tid=14, entry_tid=52758], [object_number=235, tag_tid=14, entry_tid=52755]]], state=connected] 2026-02-18T08:17:22.796 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=221, tag_tid=14, entry_tid=41241], [object_number=220, tag_tid=14, entry_tid=41240], [object_number=223, tag_tid=14, entry_tid=41239], [object_number=222, tag_tid=14, entry_tid=41238]]], state=connected] 2026-02-18T08:17:22.800 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:22.802 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=221, tag_tid=14, entry_tid=41241' 2026-02-18T08:17:22.802 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=233, tag_tid=15, entry_tid=1' -a 'object_number=233, tag_tid=15, entry_tid=1' = 'object_number=221, tag_tid=14, entry_tid=41241' 2026-02-18T08:17:22.802 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=221, tag_tid=14, entry_tid=41241' '!=' 'object_number=222, tag_tid=14, entry_tid=40890' 2026-02-18T08:17:22.802 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:17:22.802 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=221, tag_tid=14, entry_tid=41241' = 'object_number=222, tag_tid=14, entry_tid=40890' 2026-02-18T08:17:22.802 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=221, tag_tid=14, entry_tid=41241' 2026-02-18T08:17:22.803 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=233, tag_tid=15, entry_tid=1' 2026-02-18T08:17:22.803 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:17:22.803 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:17:22.804 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=15 2026-02-18T08:17:22.805 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=221, tag_tid=14, entry_tid=41241' 2026-02-18T08:17:22.805 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:17:22.805 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:17:22.806 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=14 2026-02-18T08:17:22.807 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=233, tag_tid=15, entry_tid=1' 2026-02-18T08:17:22.807 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:17:22.807 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:17:22.808 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:17:22.809 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=221, tag_tid=14, entry_tid=41241' 2026-02-18T08:17:22.809 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:17:22.809 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:17:22.811 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=41241 2026-02-18T08:17:22.811 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 15 = 14 -a 1 -le 41241 2026-02-18T08:17:22.811 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:17:22.812 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:17:22.812 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:17:23.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:17:23.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:17:23.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:17:23.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:17:23.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:17:23.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:17:23.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:17:23.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:17:23.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:17:23.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:23.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:17:23.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:17:23.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:17:23.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:17:23.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:17:23.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:17:23.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:17:23.018 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:17:23.018 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:17:23.018 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:17:23.018 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:23.018 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:17:23.018 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:17:23.018 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:17:23.018 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:17:23.018 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:17:23.018 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:17:23.018 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:17:23.018 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:17:23.019 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:17:23.019 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:23.019 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:17:23.019 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:17:23.019 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:17:23.019 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:17:23.019 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:17:23.019 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:17:23.019 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:17:23.019 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:17:23.019 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:17:23.019 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:17:23.020 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:23.020 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:17:23.020 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:17:23.020 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:17:23.020 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:17:23.020 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:17:23.020 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:17:23.020 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:17:23.028 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:17:23.028 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:17:23.029 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:17:23.029 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:17:23.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:17:23.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:17:23.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:17:23.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:17:23.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:17:23.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:17:23.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:23.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:17:23.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:17:23.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:17:23.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:17:23.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:17:23.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:17:23.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:17:23.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:17:23.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:17:23.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:23.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:17:23.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:17:23.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:17:23.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:17:23.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:17:23.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:17:23.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:17:23.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:17:23.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:17:23.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:17:23.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:23.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:17:23.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:17:23.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:17:23.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:17:23.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:17:23.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:17:23.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:17:23.094 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:17:23.094 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:17:23.094 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:17:23.094 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:17:23.143 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:17:23.144 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:17:23.144 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:17:23.146 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:17:23.147 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:17:23.147 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:17:23.147 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:23.147 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:17:23.147 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:17:23.147 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:17:23.147 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:17:23.147 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:17:23.147 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:17:23.147 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:17:23.147 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:17:23.148 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:17:23.148 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:23.148 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:17:23.148 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:17:23.148 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:17:23.148 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:17:23.148 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:17:23.148 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:17:23.148 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:17:23.148 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:17:23.148 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:17:23.148 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:17:23.149 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:23.149 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:17:23.149 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:17:23.149 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:17:23.149 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:17:23.149 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:17:23.149 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:17:23.149 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:17:23.157 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:17:23.157 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:17:23.157 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:17:23.158 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:17:23.207 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:17:23.207 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:17:23.207 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:17:23.210 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:17:23.210 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:17:23.210 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:17:23.210 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:23.210 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:17:23.210 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:17:23.210 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:17:23.210 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:17:23.210 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:17:23.210 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:17:23.211 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:17:23.211 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:17:23.211 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:17:23.211 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:23.211 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:17:23.211 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:17:23.211 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:17:23.211 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:17:23.211 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:17:23.211 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:17:23.211 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:17:23.211 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:17:23.212 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:17:23.212 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:17:23.212 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:23.212 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:17:23.212 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:17:23.212 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:17:23.212 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:17:23.212 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:17:23.212 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:17:23.212 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:17:23.220 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:17:23.221 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:17:23.221 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:17:23.221 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:17:23.281 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:17:23.282 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:17:23.282 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:17:23.282 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:17:23.282 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:17:23.282 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:17:23.282 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:17:23.282 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:17:23.282 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:17:23.283 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:17:23.283 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:17:23.283 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:17:23.283 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:17:23.284 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:23.284 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:17:23.284 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:23.317 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 55 2026-02-18T08:17:23.318 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 58 2026-02-18T08:17:23.318 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:17:23.318 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=233, tag_tid=15, entry_tid=1], [object_number=232, tag_tid=15, entry_tid=0], [object_number=234, tag_tid=14, entry_tid=52758], [object_number=235, tag_tid=14, entry_tid=52755]]], state=connected] 2026-02-18T08:17:23.318 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=221, tag_tid=14, entry_tid=41597], [object_number=220, tag_tid=14, entry_tid=41596], [object_number=223, tag_tid=14, entry_tid=41595], [object_number=222, tag_tid=14, entry_tid=41594]]], state=connected] 2026-02-18T08:17:23.321 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:23.323 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=233, tag_tid=15, entry_tid=1' 2026-02-18T08:17:23.324 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:17:23.324 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:17:23.324 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:17:23.324 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:17:23.325 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:17:23.325 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:17:23.325 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:17:23.325 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:17:23.325 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:17:23.325 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:17:23.325 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:17:23.326 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:17:23.326 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:23.327 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:17:23.327 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:23.360 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 55 2026-02-18T08:17:23.361 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 58 2026-02-18T08:17:23.361 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:17:23.361 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=233, tag_tid=15, entry_tid=1], [object_number=232, tag_tid=15, entry_tid=0], [object_number=234, tag_tid=14, entry_tid=52758], [object_number=235, tag_tid=14, entry_tid=52755]]], state=connected] 2026-02-18T08:17:23.361 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=221, tag_tid=14, entry_tid=41597], [object_number=220, tag_tid=14, entry_tid=41596], [object_number=223, tag_tid=14, entry_tid=41595], [object_number=222, tag_tid=14, entry_tid=41594]]], state=connected] 2026-02-18T08:17:23.363 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:23.365 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=221, tag_tid=14, entry_tid=41597' 2026-02-18T08:17:23.365 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=233, tag_tid=15, entry_tid=1' -a 'object_number=233, tag_tid=15, entry_tid=1' = 'object_number=221, tag_tid=14, entry_tid=41597' 2026-02-18T08:17:23.366 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=221, tag_tid=14, entry_tid=41597' '!=' 'object_number=221, tag_tid=14, entry_tid=41241' 2026-02-18T08:17:23.366 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:17:23.366 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=221, tag_tid=14, entry_tid=41597' = 'object_number=221, tag_tid=14, entry_tid=41241' 2026-02-18T08:17:23.366 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=221, tag_tid=14, entry_tid=41597' 2026-02-18T08:17:23.367 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=233, tag_tid=15, entry_tid=1' 2026-02-18T08:17:23.367 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:17:23.367 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:17:23.368 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=15 2026-02-18T08:17:23.368 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=221, tag_tid=14, entry_tid=41597' 2026-02-18T08:17:23.368 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:17:23.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:17:23.370 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=14 2026-02-18T08:17:23.371 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=233, tag_tid=15, entry_tid=1' 2026-02-18T08:17:23.371 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:17:23.371 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:17:23.372 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:17:23.373 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=221, tag_tid=14, entry_tid=41597' 2026-02-18T08:17:23.373 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:17:23.373 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:17:23.375 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=41597 2026-02-18T08:17:23.375 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 15 = 14 -a 1 -le 41597 2026-02-18T08:17:23.375 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:17:23.375 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:17:23.375 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:17:23.577 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:17:23.577 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:17:23.577 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:17:23.577 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:17:23.577 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:17:23.577 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:17:23.577 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:17:23.577 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:17:23.577 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:17:23.577 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:23.578 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:17:23.578 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:17:23.578 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:17:23.578 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:17:23.578 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:17:23.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:17:23.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:17:23.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:17:23.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:17:23.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:17:23.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:23.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:17:23.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:17:23.582 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:17:23.582 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:17:23.582 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:17:23.582 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:17:23.582 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:17:23.582 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:17:23.582 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:17:23.582 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:23.582 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:17:23.582 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:17:23.582 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:17:23.582 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:17:23.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:17:23.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:17:23.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:17:23.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:17:23.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:17:23.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:17:23.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:23.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:17:23.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:17:23.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:17:23.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:17:23.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:17:23.584 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:17:23.584 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:17:23.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:17:23.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:17:23.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:17:23.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:17:23.643 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:17:23.643 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:17:23.643 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:17:23.646 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:17:23.646 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:17:23.646 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:17:23.646 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:23.646 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:17:23.646 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:17:23.646 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:17:23.646 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:17:23.646 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:17:23.647 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:17:23.647 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:17:23.647 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:17:23.647 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:17:23.647 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:23.647 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:17:23.647 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:17:23.647 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:17:23.647 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:17:23.647 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:17:23.647 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:17:23.647 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:17:23.648 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:17:23.648 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:17:23.648 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:17:23.648 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:23.648 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:17:23.648 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:17:23.648 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:17:23.648 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:17:23.648 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:17:23.648 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:17:23.648 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:17:23.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:17:23.658 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:17:23.658 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:17:23.658 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:17:23.707 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:17:23.708 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:17:23.708 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:17:23.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:17:23.711 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:17:23.711 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:17:23.711 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:23.711 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:17:23.711 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:17:23.711 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:17:23.711 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:17:23.711 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:17:23.711 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:17:23.711 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:17:23.711 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:17:23.712 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:17:23.712 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:23.712 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:17:23.712 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:17:23.712 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:17:23.712 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:17:23.712 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:17:23.712 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:17:23.712 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:17:23.712 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:17:23.712 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:17:23.712 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:17:23.713 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:23.713 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:17:23.713 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:17:23.713 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:17:23.713 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:17:23.713 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:17:23.713 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:17:23.713 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:17:23.720 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:17:23.721 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:17:23.721 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:17:23.721 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:17:23.770 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:17:23.770 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:17:23.771 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:17:23.774 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:17:23.774 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:17:23.774 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:17:23.774 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:23.774 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:17:23.774 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:17:23.774 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:17:23.774 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:17:23.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:17:23.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:17:23.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:17:23.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:17:23.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:17:23.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:23.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:17:23.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:17:23.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:17:23.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:17:23.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:17:23.776 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:17:23.776 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:17:23.776 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:17:23.776 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:17:23.776 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:17:23.776 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:23.776 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:17:23.776 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:17:23.776 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:17:23.776 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:17:23.776 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:17:23.776 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:17:23.776 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:17:23.784 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:17:23.785 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:17:23.785 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:17:23.785 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:17:23.848 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:17:23.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:17:23.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:17:23.850 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:17:23.850 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:17:23.850 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:17:23.850 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:17:23.850 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:17:23.850 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:17:23.850 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:17:23.850 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:17:23.850 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:17:23.850 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:17:23.851 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:23.851 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:17:23.852 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:23.880 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 55 2026-02-18T08:17:23.880 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 58 2026-02-18T08:17:23.880 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:17:23.880 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=233, tag_tid=15, entry_tid=1], [object_number=232, tag_tid=15, entry_tid=0], [object_number=234, tag_tid=14, entry_tid=52758], [object_number=235, tag_tid=14, entry_tid=52755]]], state=connected] 2026-02-18T08:17:23.880 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=222, tag_tid=14, entry_tid=41954], [object_number=221, tag_tid=14, entry_tid=41953], [object_number=220, tag_tid=14, entry_tid=41952], [object_number=223, tag_tid=14, entry_tid=41951]]], state=connected] 2026-02-18T08:17:23.882 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:23.884 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=233, tag_tid=15, entry_tid=1' 2026-02-18T08:17:23.886 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:17:23.886 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:17:23.886 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:17:23.886 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:17:23.886 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:17:23.886 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:17:23.886 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:17:23.886 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:17:23.887 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:17:23.887 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:17:23.887 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:17:23.887 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:17:23.887 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:23.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:17:23.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:23.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 55 2026-02-18T08:17:23.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 58 2026-02-18T08:17:23.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:17:23.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=233, tag_tid=15, entry_tid=1], [object_number=232, tag_tid=15, entry_tid=0], [object_number=234, tag_tid=14, entry_tid=52758], [object_number=235, tag_tid=14, entry_tid=52755]]], state=connected] 2026-02-18T08:17:23.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=222, tag_tid=14, entry_tid=41954], [object_number=221, tag_tid=14, entry_tid=41953], [object_number=220, tag_tid=14, entry_tid=41952], [object_number=223, tag_tid=14, entry_tid=41951]]], state=connected] 2026-02-18T08:17:23.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:23.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=222, tag_tid=14, entry_tid=41954' 2026-02-18T08:17:23.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=233, tag_tid=15, entry_tid=1' -a 'object_number=233, tag_tid=15, entry_tid=1' = 'object_number=222, tag_tid=14, entry_tid=41954' 2026-02-18T08:17:23.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=222, tag_tid=14, entry_tid=41954' '!=' 'object_number=221, tag_tid=14, entry_tid=41597' 2026-02-18T08:17:23.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:17:23.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=222, tag_tid=14, entry_tid=41954' = 'object_number=221, tag_tid=14, entry_tid=41597' 2026-02-18T08:17:23.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=222, tag_tid=14, entry_tid=41954' 2026-02-18T08:17:23.928 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=233, tag_tid=15, entry_tid=1' 2026-02-18T08:17:23.928 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:17:23.928 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:17:23.929 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=15 2026-02-18T08:17:23.929 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=222, tag_tid=14, entry_tid=41954' 2026-02-18T08:17:23.930 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:17:23.930 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:17:23.931 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=14 2026-02-18T08:17:23.932 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=233, tag_tid=15, entry_tid=1' 2026-02-18T08:17:23.932 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:17:23.932 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:17:23.933 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:17:23.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=222, tag_tid=14, entry_tid=41954' 2026-02-18T08:17:23.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:17:23.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:17:23.936 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=41954 2026-02-18T08:17:23.936 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 15 = 14 -a 1 -le 41954 2026-02-18T08:17:23.936 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:17:23.936 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:17:23.936 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:17:24.137 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:17:24.138 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:17:24.138 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:17:24.138 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:17:24.138 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:17:24.138 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:17:24.138 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:17:24.138 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:17:24.138 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:17:24.138 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:24.138 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:17:24.138 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:17:24.139 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:17:24.139 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:17:24.139 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:17:24.141 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:17:24.142 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:17:24.142 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:17:24.142 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:17:24.142 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:17:24.142 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:24.142 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:17:24.142 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:17:24.142 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:17:24.142 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:17:24.142 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:17:24.142 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:17:24.143 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:17:24.143 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:17:24.143 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:17:24.143 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:24.143 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:17:24.143 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:17:24.143 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:17:24.143 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:17:24.143 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:17:24.143 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:17:24.143 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:17:24.143 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:17:24.144 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:17:24.144 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:17:24.144 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:24.144 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:17:24.144 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:17:24.144 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:17:24.144 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:17:24.144 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:17:24.144 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:17:24.144 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:17:24.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:17:24.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:17:24.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:17:24.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:17:24.211 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:17:24.212 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:17:24.212 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:17:24.214 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:17:24.215 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:17:24.215 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:17:24.215 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:24.215 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:17:24.215 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:17:24.215 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:17:24.215 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:17:24.215 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:17:24.215 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:17:24.215 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:17:24.215 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:17:24.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:17:24.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:24.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:17:24.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:17:24.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:17:24.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:17:24.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:17:24.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:17:24.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:17:24.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:17:24.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:17:24.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:17:24.217 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:24.217 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:17:24.217 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:17:24.217 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:17:24.217 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:17:24.217 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:17:24.217 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:17:24.217 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:17:24.224 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:17:24.225 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:17:24.225 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:17:24.225 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:17:24.274 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:17:24.274 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:17:24.274 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:17:24.277 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:17:24.277 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:17:24.277 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:17:24.277 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:24.278 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:17:24.278 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:17:24.278 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:17:24.278 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:17:24.278 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:17:24.278 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:17:24.278 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:17:24.278 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:17:24.278 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:17:24.278 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:24.278 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:17:24.278 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:17:24.279 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:17:24.279 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:17:24.279 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:17:24.279 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:17:24.279 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:17:24.279 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:17:24.279 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:17:24.279 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:17:24.279 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:24.279 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:17:24.279 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:17:24.280 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:17:24.280 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:17:24.280 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:17:24.280 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:17:24.280 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:17:24.287 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:17:24.288 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:17:24.288 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:17:24.288 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:17:24.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:17:24.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:17:24.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:17:24.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:17:24.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:17:24.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:17:24.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:24.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:17:24.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:17:24.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:17:24.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:17:24.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:17:24.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:17:24.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:17:24.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:17:24.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:17:24.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:24.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:17:24.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:17:24.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:17:24.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:17:24.344 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:17:24.344 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:17:24.344 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:17:24.344 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:17:24.344 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:17:24.344 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:17:24.344 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:24.344 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:17:24.344 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:17:24.344 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:17:24.344 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:17:24.344 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:17:24.344 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:17:24.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:17:24.352 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:17:24.352 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:17:24.353 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:17:24.353 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:17:24.418 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:17:24.420 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:17:24.420 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:17:24.420 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:17:24.420 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:17:24.420 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:17:24.420 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:17:24.420 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:17:24.420 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:17:24.420 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:17:24.420 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:17:24.420 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:17:24.421 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:17:24.421 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:24.422 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:17:24.422 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:24.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 56 2026-02-18T08:17:24.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 58 2026-02-18T08:17:24.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:17:24.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=233, tag_tid=15, entry_tid=1], [object_number=232, tag_tid=15, entry_tid=0], [object_number=234, tag_tid=14, entry_tid=52758], [object_number=235, tag_tid=14, entry_tid=52755]]], state=connected] 2026-02-18T08:17:24.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=226, tag_tid=14, entry_tid=42238], [object_number=225, tag_tid=14, entry_tid=42237], [object_number=224, tag_tid=14, entry_tid=42236], [object_number=227, tag_tid=14, entry_tid=42235]]], state=connected] 2026-02-18T08:17:24.462 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:24.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=233, tag_tid=15, entry_tid=1' 2026-02-18T08:17:24.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:17:24.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:17:24.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:17:24.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:17:24.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:17:24.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:17:24.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:17:24.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:17:24.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:17:24.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:17:24.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:17:24.467 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:17:24.467 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:24.468 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:17:24.468 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:24.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 56 2026-02-18T08:17:24.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 58 2026-02-18T08:17:24.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:17:24.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=233, tag_tid=15, entry_tid=1], [object_number=232, tag_tid=15, entry_tid=0], [object_number=234, tag_tid=14, entry_tid=52758], [object_number=235, tag_tid=14, entry_tid=52755]]], state=connected] 2026-02-18T08:17:24.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=226, tag_tid=14, entry_tid=42238], [object_number=225, tag_tid=14, entry_tid=42237], [object_number=224, tag_tid=14, entry_tid=42236], [object_number=227, tag_tid=14, entry_tid=42235]]], state=connected] 2026-02-18T08:17:24.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:24.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=226, tag_tid=14, entry_tid=42238' 2026-02-18T08:17:24.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=233, tag_tid=15, entry_tid=1' -a 'object_number=233, tag_tid=15, entry_tid=1' = 'object_number=226, tag_tid=14, entry_tid=42238' 2026-02-18T08:17:24.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=226, tag_tid=14, entry_tid=42238' '!=' 'object_number=222, tag_tid=14, entry_tid=41954' 2026-02-18T08:17:24.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:17:24.500 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=226, tag_tid=14, entry_tid=42238' = 'object_number=222, tag_tid=14, entry_tid=41954' 2026-02-18T08:17:24.500 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=226, tag_tid=14, entry_tid=42238' 2026-02-18T08:17:24.500 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=233, tag_tid=15, entry_tid=1' 2026-02-18T08:17:24.500 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:17:24.500 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:17:24.501 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=15 2026-02-18T08:17:24.502 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=226, tag_tid=14, entry_tid=42238' 2026-02-18T08:17:24.502 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:17:24.502 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:17:24.503 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=14 2026-02-18T08:17:24.504 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=233, tag_tid=15, entry_tid=1' 2026-02-18T08:17:24.504 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:17:24.505 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:17:24.506 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:17:24.507 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=226, tag_tid=14, entry_tid=42238' 2026-02-18T08:17:24.507 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:17:24.507 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:17:24.508 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=42238 2026-02-18T08:17:24.508 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 15 = 14 -a 1 -le 42238 2026-02-18T08:17:24.509 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:17:24.509 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:17:24.509 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:17:24.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:17:24.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:17:24.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:17:24.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:17:24.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:17:24.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:17:24.711 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:17:24.711 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:17:24.711 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:17:24.711 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:24.711 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:17:24.711 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:17:24.711 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:17:24.711 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:17:24.711 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:17:24.714 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:17:24.714 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:17:24.714 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:17:24.714 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:17:24.714 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:17:24.714 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:24.714 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:17:24.714 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:17:24.714 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:17:24.714 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:17:24.714 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:17:24.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:17:24.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:17:24.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:17:24.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:17:24.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:24.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:17:24.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:17:24.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:17:24.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:17:24.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:17:24.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:17:24.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:17:24.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:17:24.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:17:24.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:17:24.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:24.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:17:24.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:17:24.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:17:24.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:17:24.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:17:24.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:17:24.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:17:24.724 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:17:24.725 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:17:24.725 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:17:24.725 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:17:24.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:17:24.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:17:24.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:17:24.778 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:17:24.778 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:17:24.778 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:17:24.778 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:24.778 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:17:24.778 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:17:24.778 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:17:24.778 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:17:24.778 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:17:24.779 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:17:24.779 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:17:24.779 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:17:24.779 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:17:24.779 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:24.779 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:17:24.779 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:17:24.779 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:17:24.779 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:17:24.779 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:17:24.779 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:17:24.779 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:17:24.780 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:17:24.780 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:17:24.780 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:17:24.780 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:24.780 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:17:24.780 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:17:24.780 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:17:24.780 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:17:24.780 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:17:24.780 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:17:24.780 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:17:24.788 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:17:24.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:17:24.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:17:24.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:17:24.838 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:17:24.839 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:17:24.839 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:17:24.841 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:17:24.842 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:17:24.842 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:17:24.842 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:24.842 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:17:24.842 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:17:24.842 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:17:24.842 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:17:24.842 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:17:24.842 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:17:24.842 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:17:24.842 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:17:24.843 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:17:24.843 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:24.843 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:17:24.843 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:17:24.843 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:17:24.843 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:17:24.843 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:17:24.843 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:17:24.843 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:17:24.843 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:17:24.843 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:17:24.843 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:17:24.844 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:24.844 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:17:24.844 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:17:24.844 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:17:24.844 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:17:24.844 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:17:24.844 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:17:24.844 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:17:24.852 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:17:24.852 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:17:24.853 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:17:24.853 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:17:24.890 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:17:24.886+0000 7fb27e7f3640 -1 reset not still connected to 0x55e17071e820 2026-02-18T08:17:24.891 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:17:24.890+0000 7fb27e7f3640 -1 reset not still connected to 0x55e171bf05b0 2026-02-18T08:17:24.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:17:24.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:17:24.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:17:24.904 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:17:24.904 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:17:24.904 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:17:24.904 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:24.904 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:17:24.904 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:17:24.904 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:17:24.904 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:17:24.904 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:17:24.905 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:17:24.905 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:17:24.905 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:17:24.905 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:17:24.905 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:24.905 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:17:24.905 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:17:24.905 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:17:24.905 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:17:24.905 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:17:24.905 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:17:24.905 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:17:24.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:17:24.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:17:24.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:17:24.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:24.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:17:24.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:17:24.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:17:24.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:17:24.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:17:24.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:17:24.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:17:24.915 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:17:24.916 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:17:24.916 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:17:24.916 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:17:24.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:17:24.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:17:24.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:17:24.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:17:24.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:17:24.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:17:24.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:17:24.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:17:24.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:17:24.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:17:24.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:17:24.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:17:24.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:17:24.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:24.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:17:24.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:25.007 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 56 2026-02-18T08:17:25.007 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 58 2026-02-18T08:17:25.007 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:17:25.007 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=233, tag_tid=15, entry_tid=1], [object_number=232, tag_tid=15, entry_tid=0], [object_number=234, tag_tid=14, entry_tid=52758], [object_number=235, tag_tid=14, entry_tid=52755]]], state=connected] 2026-02-18T08:17:25.007 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=226, tag_tid=14, entry_tid=42594], [object_number=225, tag_tid=14, entry_tid=42593], [object_number=224, tag_tid=14, entry_tid=42592], [object_number=227, tag_tid=14, entry_tid=42591]]], state=connected] 2026-02-18T08:17:25.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:25.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=233, tag_tid=15, entry_tid=1' 2026-02-18T08:17:25.012 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:17:25.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:17:25.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:17:25.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:17:25.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:17:25.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:17:25.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:17:25.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:17:25.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:17:25.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:17:25.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:17:25.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:17:25.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:25.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:17:25.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:25.038 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.0 with skip (0.31649788401979917) > weight (0.25) 2026-02-18T08:17:25.038 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.1 with skip (0.3906418159731986) > weight (0.25) 2026-02-18T08:17:25.038 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.2 with skip (0.5313197283659599) > weight (0.25) 2026-02-18T08:17:25.038 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.3 with skip (0.9863472561022301) > weight (0.25) 2026-02-18T08:17:25.038 INFO:tasks.rbd_mirror_thrash:waiting for 62.173692242594825 secs before thrashing 2026-02-18T08:17:25.040 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 56 2026-02-18T08:17:25.040 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 58 2026-02-18T08:17:25.041 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:17:25.041 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=233, tag_tid=15, entry_tid=1], [object_number=232, tag_tid=15, entry_tid=0], [object_number=234, tag_tid=14, entry_tid=52758], [object_number=235, tag_tid=14, entry_tid=52755]]], state=connected] 2026-02-18T08:17:25.041 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=226, tag_tid=14, entry_tid=42594], [object_number=225, tag_tid=14, entry_tid=42593], [object_number=224, tag_tid=14, entry_tid=42592], [object_number=227, tag_tid=14, entry_tid=42591]]], state=connected] 2026-02-18T08:17:25.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:25.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=226, tag_tid=14, entry_tid=42594' 2026-02-18T08:17:25.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=233, tag_tid=15, entry_tid=1' -a 'object_number=233, tag_tid=15, entry_tid=1' = 'object_number=226, tag_tid=14, entry_tid=42594' 2026-02-18T08:17:25.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=226, tag_tid=14, entry_tid=42594' '!=' 'object_number=226, tag_tid=14, entry_tid=42238' 2026-02-18T08:17:25.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:17:25.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=226, tag_tid=14, entry_tid=42594' = 'object_number=226, tag_tid=14, entry_tid=42238' 2026-02-18T08:17:25.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=226, tag_tid=14, entry_tid=42594' 2026-02-18T08:17:25.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=233, tag_tid=15, entry_tid=1' 2026-02-18T08:17:25.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:17:25.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:17:25.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=15 2026-02-18T08:17:25.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=226, tag_tid=14, entry_tid=42594' 2026-02-18T08:17:25.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:17:25.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:17:25.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=14 2026-02-18T08:17:25.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=233, tag_tid=15, entry_tid=1' 2026-02-18T08:17:25.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:17:25.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:17:25.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:17:25.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=226, tag_tid=14, entry_tid=42594' 2026-02-18T08:17:25.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:17:25.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:17:25.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=42594 2026-02-18T08:17:25.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 15 = 14 -a 1 -le 42594 2026-02-18T08:17:25.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:17:25.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:17:25.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:17:25.256 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:17:25.256 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:17:25.257 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:17:25.257 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:17:25.257 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:17:25.257 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:17:25.257 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:17:25.257 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:17:25.257 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:17:25.257 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:25.257 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:17:25.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:17:25.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:17:25.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:17:25.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:17:25.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:17:25.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:17:25.261 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:17:25.261 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:17:25.261 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:17:25.261 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:25.261 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:17:25.261 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:17:25.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:17:25.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:17:25.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:17:25.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:17:25.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:17:25.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:17:25.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:17:25.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:25.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:17:25.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:17:25.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:17:25.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:17:25.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:17:25.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:17:25.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:17:25.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:17:25.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:17:25.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:17:25.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:25.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:17:25.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:17:25.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:17:25.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:17:25.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:17:25.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:17:25.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:17:25.270 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:17:25.270 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:17:25.270 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:17:25.270 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:17:25.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:17:25.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:17:25.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:17:25.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:17:25.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:17:25.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:17:25.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:25.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:17:25.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:17:25.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:17:25.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:17:25.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:17:25.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:17:25.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:17:25.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:17:25.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:17:25.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:25.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:17:25.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:17:25.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:17:25.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:17:25.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:17:25.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:17:25.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:17:25.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:17:25.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:17:25.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:17:25.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:25.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:17:25.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:17:25.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:17:25.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:17:25.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:17:25.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:17:25.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:17:25.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:17:25.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:17:25.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:17:25.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:17:25.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:17:25.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:17:25.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:17:25.405 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:17:25.405 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:17:25.405 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:17:25.405 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:25.405 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:17:25.405 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:17:25.405 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:17:25.405 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:17:25.405 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:17:25.406 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:17:25.406 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:17:25.406 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:17:25.406 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:17:25.406 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:25.406 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:17:25.406 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:17:25.406 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:17:25.406 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:17:25.406 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:17:25.406 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:17:25.406 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:17:25.407 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:17:25.407 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:17:25.407 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:17:25.407 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:25.407 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:17:25.407 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:17:25.407 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:17:25.407 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:17:25.407 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:17:25.407 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:17:25.407 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:17:25.416 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:17:25.416 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:17:25.416 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:17:25.416 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:17:25.465 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:17:25.465 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:17:25.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:17:25.468 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:17:25.468 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:17:25.468 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:17:25.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:25.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:17:25.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:17:25.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:17:25.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:17:25.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:17:25.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:17:25.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:17:25.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:17:25.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:17:25.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:25.470 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:17:25.470 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:17:25.470 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:17:25.470 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:17:25.470 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:17:25.470 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:17:25.470 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:17:25.470 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:17:25.470 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:17:25.470 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:17:25.470 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:25.470 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:17:25.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:17:25.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:17:25.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:17:25.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:17:25.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:17:25.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:17:25.480 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:17:25.480 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:17:25.480 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:17:25.480 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:17:25.545 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:17:25.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:17:25.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:17:25.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:17:25.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:17:25.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:17:25.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:17:25.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:17:25.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:17:25.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:17:25.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:17:25.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:17:25.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:17:25.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:25.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:17:25.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:25.576 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 56 2026-02-18T08:17:25.576 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 58 2026-02-18T08:17:25.576 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:17:25.576 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=233, tag_tid=15, entry_tid=1], [object_number=232, tag_tid=15, entry_tid=0], [object_number=234, tag_tid=14, entry_tid=52758], [object_number=235, tag_tid=14, entry_tid=52755]]], state=connected] 2026-02-18T08:17:25.576 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=227, tag_tid=14, entry_tid=42931], [object_number=226, tag_tid=14, entry_tid=42930], [object_number=225, tag_tid=14, entry_tid=42929], [object_number=224, tag_tid=14, entry_tid=42928]]], state=connected] 2026-02-18T08:17:25.578 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:25.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=233, tag_tid=15, entry_tid=1' 2026-02-18T08:17:25.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:17:25.582 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:17:25.582 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:17:25.582 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:17:25.582 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:17:25.582 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:17:25.582 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:17:25.582 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:17:25.582 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:17:25.582 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:17:25.582 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:17:25.582 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:17:25.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:25.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:17:25.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:25.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 56 2026-02-18T08:17:25.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 58 2026-02-18T08:17:25.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:17:25.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=233, tag_tid=15, entry_tid=1], [object_number=232, tag_tid=15, entry_tid=0], [object_number=234, tag_tid=14, entry_tid=52758], [object_number=235, tag_tid=14, entry_tid=52755]]], state=connected] 2026-02-18T08:17:25.618 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=227, tag_tid=14, entry_tid=42931], [object_number=226, tag_tid=14, entry_tid=42930], [object_number=225, tag_tid=14, entry_tid=42929], [object_number=224, tag_tid=14, entry_tid=42928]]], state=connected] 2026-02-18T08:17:25.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:25.623 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=227, tag_tid=14, entry_tid=42931' 2026-02-18T08:17:25.623 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=233, tag_tid=15, entry_tid=1' -a 'object_number=233, tag_tid=15, entry_tid=1' = 'object_number=227, tag_tid=14, entry_tid=42931' 2026-02-18T08:17:25.623 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=227, tag_tid=14, entry_tid=42931' '!=' 'object_number=226, tag_tid=14, entry_tid=42594' 2026-02-18T08:17:25.623 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:17:25.623 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=227, tag_tid=14, entry_tid=42931' = 'object_number=226, tag_tid=14, entry_tid=42594' 2026-02-18T08:17:25.623 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=227, tag_tid=14, entry_tid=42931' 2026-02-18T08:17:25.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=233, tag_tid=15, entry_tid=1' 2026-02-18T08:17:25.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:17:25.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:17:25.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=15 2026-02-18T08:17:25.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=227, tag_tid=14, entry_tid=42931' 2026-02-18T08:17:25.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:17:25.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:17:25.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=14 2026-02-18T08:17:25.628 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=233, tag_tid=15, entry_tid=1' 2026-02-18T08:17:25.628 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:17:25.628 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:17:25.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:17:25.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=227, tag_tid=14, entry_tid=42931' 2026-02-18T08:17:25.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:17:25.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:17:25.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=42931 2026-02-18T08:17:25.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 15 = 14 -a 1 -le 42931 2026-02-18T08:17:25.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:17:25.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:17:25.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:17:25.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:17:25.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:17:25.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:17:25.835 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:17:25.835 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:17:25.835 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:17:25.835 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:17:25.835 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:17:25.835 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:17:25.835 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:25.835 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:17:25.835 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:17:25.835 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:17:25.835 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:17:25.836 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:17:25.838 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:17:25.839 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:17:25.839 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:17:25.839 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:17:25.839 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:17:25.839 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:25.839 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:17:25.839 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:17:25.839 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:17:25.839 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:17:25.839 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:17:25.839 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:17:25.840 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:17:25.840 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:17:25.840 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:17:25.840 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:25.840 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:17:25.840 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:17:25.840 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:17:25.840 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:17:25.840 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:17:25.840 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:17:25.840 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:17:25.840 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:17:25.841 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:17:25.841 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:17:25.841 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:25.841 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:17:25.841 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:17:25.841 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:17:25.841 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:17:25.841 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:17:25.841 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:17:25.841 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:17:25.850 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:17:25.850 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:17:25.850 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:17:25.850 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:17:25.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:17:25.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:17:25.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:17:25.910 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:17:25.910 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:17:25.911 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:17:25.911 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:25.911 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:17:25.911 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:17:25.911 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:17:25.911 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:17:25.911 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:17:25.911 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:17:25.911 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:17:25.911 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:17:25.911 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:17:25.912 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:25.912 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:17:25.912 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:17:25.912 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:17:25.912 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:17:25.912 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:17:25.912 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:17:25.912 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:17:25.912 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:17:25.912 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:17:25.912 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:17:25.912 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:25.913 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:17:25.913 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:17:25.913 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:17:25.913 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:17:25.913 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:17:25.913 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:17:25.913 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:17:25.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:17:25.922 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:17:25.922 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:17:25.922 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:17:25.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:17:25.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:17:25.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:17:25.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:17:25.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:17:25.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:17:25.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:25.982 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:17:25.982 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:17:25.982 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:17:25.982 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:17:25.982 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:17:25.982 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:17:25.982 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:17:25.982 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:17:25.982 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:17:25.982 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:25.982 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:17:25.982 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:17:25.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:17:25.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:17:25.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:17:25.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:17:25.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:17:25.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:17:25.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:17:25.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:17:25.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:25.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:17:25.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:17:25.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:17:25.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:17:25.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:17:25.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:17:25.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:17:25.993 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:17:25.994 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:17:25.994 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:17:25.994 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:17:26.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:17:26.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:17:26.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:17:26.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:17:26.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:17:26.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:17:26.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:26.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:17:26.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:17:26.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:17:26.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:17:26.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:17:26.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:17:26.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:17:26.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:17:26.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:17:26.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:26.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:17:26.055 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:17:26.055 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:17:26.055 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:17:26.055 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:17:26.055 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:17:26.055 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:17:26.055 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:17:26.055 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:17:26.055 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:17:26.055 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:26.055 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:17:26.056 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:17:26.056 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:17:26.056 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:17:26.056 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:17:26.056 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:17:26.056 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:17:26.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:17:26.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:17:26.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:17:26.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:17:26.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:17:26.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:17:26.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:17:26.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:17:26.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:17:26.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:17:26.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:17:26.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:17:26.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:17:26.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:17:26.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:17:26.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:17:26.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:17:26.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:26.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:17:26.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:26.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 56 2026-02-18T08:17:26.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 58 2026-02-18T08:17:26.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:17:26.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=233, tag_tid=15, entry_tid=1], [object_number=232, tag_tid=15, entry_tid=0], [object_number=234, tag_tid=14, entry_tid=52758], [object_number=235, tag_tid=14, entry_tid=52755]]], state=connected] 2026-02-18T08:17:26.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=224, tag_tid=14, entry_tid=43348], [object_number=227, tag_tid=14, entry_tid=43347], [object_number=226, tag_tid=14, entry_tid=43346], [object_number=225, tag_tid=14, entry_tid=43345]]], state=connected] 2026-02-18T08:17:26.157 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:26.158 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=233, tag_tid=15, entry_tid=1' 2026-02-18T08:17:26.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:17:26.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:17:26.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:17:26.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:17:26.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:17:26.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:17:26.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:17:26.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:17:26.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:17:26.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:17:26.161 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:17:26.161 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:17:26.161 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:26.162 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:17:26.162 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:26.198 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 56 2026-02-18T08:17:26.198 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 58 2026-02-18T08:17:26.198 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:17:26.198 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=233, tag_tid=15, entry_tid=1], [object_number=232, tag_tid=15, entry_tid=0], [object_number=234, tag_tid=14, entry_tid=52758], [object_number=235, tag_tid=14, entry_tid=52755]]], state=connected] 2026-02-18T08:17:26.198 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=224, tag_tid=14, entry_tid=43348], [object_number=227, tag_tid=14, entry_tid=43347], [object_number=226, tag_tid=14, entry_tid=43346], [object_number=225, tag_tid=14, entry_tid=43345]]], state=connected] 2026-02-18T08:17:26.202 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:26.204 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=224, tag_tid=14, entry_tid=43348' 2026-02-18T08:17:26.204 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=233, tag_tid=15, entry_tid=1' -a 'object_number=233, tag_tid=15, entry_tid=1' = 'object_number=224, tag_tid=14, entry_tid=43348' 2026-02-18T08:17:26.204 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=224, tag_tid=14, entry_tid=43348' '!=' 'object_number=227, tag_tid=14, entry_tid=42931' 2026-02-18T08:17:26.204 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:17:26.204 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=224, tag_tid=14, entry_tid=43348' = 'object_number=227, tag_tid=14, entry_tid=42931' 2026-02-18T08:17:26.204 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=224, tag_tid=14, entry_tid=43348' 2026-02-18T08:17:26.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=233, tag_tid=15, entry_tid=1' 2026-02-18T08:17:26.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:17:26.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:17:26.206 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=15 2026-02-18T08:17:26.207 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=224, tag_tid=14, entry_tid=43348' 2026-02-18T08:17:26.207 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:17:26.207 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:17:26.208 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=14 2026-02-18T08:17:26.209 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=233, tag_tid=15, entry_tid=1' 2026-02-18T08:17:26.209 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:17:26.209 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:17:26.210 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:17:26.211 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=224, tag_tid=14, entry_tid=43348' 2026-02-18T08:17:26.211 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:17:26.212 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:17:26.213 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=43348 2026-02-18T08:17:26.213 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 15 = 14 -a 1 -le 43348 2026-02-18T08:17:26.213 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:17:26.213 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:17:26.213 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:17:26.415 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:17:26.415 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:17:26.415 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:17:26.415 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:17:26.415 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:17:26.415 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:17:26.416 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:17:26.416 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:17:26.416 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:17:26.416 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:26.416 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:17:26.416 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:17:26.416 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:17:26.416 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:17:26.416 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:17:26.419 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:17:26.419 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:17:26.419 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:17:26.420 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:17:26.420 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:17:26.420 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:26.420 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:17:26.420 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:17:26.420 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:17:26.420 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:17:26.420 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:17:26.420 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:17:26.420 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:17:26.420 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:17:26.420 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:17:26.421 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:26.421 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:17:26.421 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:17:26.421 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:17:26.421 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:17:26.421 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:17:26.421 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:17:26.421 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:17:26.421 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:17:26.421 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:17:26.421 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:17:26.421 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:26.422 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:17:26.422 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:17:26.422 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:17:26.422 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:17:26.422 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:17:26.422 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:17:26.422 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:17:26.430 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:17:26.430 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:17:26.430 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:17:26.430 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:17:26.480 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:17:26.480 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:17:26.480 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:17:26.483 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:17:26.484 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:17:26.484 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:17:26.484 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:26.484 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:17:26.484 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:17:26.484 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:17:26.484 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:17:26.484 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:17:26.484 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:17:26.484 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:17:26.484 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:17:26.485 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:17:26.485 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:26.485 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:17:26.485 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:17:26.485 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:17:26.485 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:17:26.485 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:17:26.485 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:17:26.485 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:17:26.485 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:17:26.485 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:17:26.485 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:17:26.486 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:26.486 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:17:26.486 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:17:26.486 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:17:26.486 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:17:26.486 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:17:26.486 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:17:26.486 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:17:26.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:17:26.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:17:26.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:17:26.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:17:26.545 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:17:26.545 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:17:26.545 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:17:26.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:17:26.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:17:26.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:17:26.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:26.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:17:26.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:17:26.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:17:26.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:17:26.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:17:26.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:17:26.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:17:26.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:17:26.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:17:26.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:26.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:17:26.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:17:26.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:17:26.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:17:26.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:17:26.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:17:26.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:17:26.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:17:26.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:17:26.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:17:26.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:26.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:17:26.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:17:26.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:17:26.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:17:26.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:17:26.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:17:26.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:17:26.558 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:17:26.559 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:17:26.559 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:17:26.559 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:17:26.611 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:17:26.611 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:17:26.611 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:17:26.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:17:26.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:17:26.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:17:26.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:26.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:17:26.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:17:26.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:17:26.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:17:26.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:17:26.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:17:26.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:17:26.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:17:26.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:17:26.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:26.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:17:26.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:17:26.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:17:26.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:17:26.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:17:26.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:17:26.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:17:26.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:17:26.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:17:26.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:17:26.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:26.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:17:26.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:17:26.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:17:26.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:17:26.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:17:26.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:17:26.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:17:26.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:17:26.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:17:26.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:17:26.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:17:26.686 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:17:26.687 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:17:26.687 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:17:26.687 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:17:26.687 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:17:26.687 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:17:26.688 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:17:26.688 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:17:26.688 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:17:26.688 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:17:26.688 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:17:26.688 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:17:26.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:17:26.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:26.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:17:26.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:26.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 56 2026-02-18T08:17:26.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 58 2026-02-18T08:17:26.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:17:26.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=233, tag_tid=15, entry_tid=1], [object_number=232, tag_tid=15, entry_tid=0], [object_number=234, tag_tid=14, entry_tid=52758], [object_number=235, tag_tid=14, entry_tid=52755]]], state=connected] 2026-02-18T08:17:26.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=226, tag_tid=14, entry_tid=43702], [object_number=225, tag_tid=14, entry_tid=43701], [object_number=224, tag_tid=14, entry_tid=43700], [object_number=227, tag_tid=14, entry_tid=43699]]], state=connected] 2026-02-18T08:17:26.719 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:26.721 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=233, tag_tid=15, entry_tid=1' 2026-02-18T08:17:26.721 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:17:26.721 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:17:26.722 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:17:26.722 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:17:26.722 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:17:26.722 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:17:26.722 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:17:26.722 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:17:26.722 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:17:26.722 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:17:26.722 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:17:26.722 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:17:26.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:26.724 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:17:26.724 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:26.760 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 56 2026-02-18T08:17:26.761 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 58 2026-02-18T08:17:26.761 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:17:26.761 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=233, tag_tid=15, entry_tid=1], [object_number=232, tag_tid=15, entry_tid=0], [object_number=234, tag_tid=14, entry_tid=52758], [object_number=235, tag_tid=14, entry_tid=52755]]], state=connected] 2026-02-18T08:17:26.761 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=226, tag_tid=14, entry_tid=43702], [object_number=225, tag_tid=14, entry_tid=43701], [object_number=224, tag_tid=14, entry_tid=43700], [object_number=227, tag_tid=14, entry_tid=43699]]], state=connected] 2026-02-18T08:17:26.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:26.767 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=226, tag_tid=14, entry_tid=43702' 2026-02-18T08:17:26.767 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=233, tag_tid=15, entry_tid=1' -a 'object_number=233, tag_tid=15, entry_tid=1' = 'object_number=226, tag_tid=14, entry_tid=43702' 2026-02-18T08:17:26.767 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=226, tag_tid=14, entry_tid=43702' '!=' 'object_number=224, tag_tid=14, entry_tid=43348' 2026-02-18T08:17:26.767 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:17:26.767 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=226, tag_tid=14, entry_tid=43702' = 'object_number=224, tag_tid=14, entry_tid=43348' 2026-02-18T08:17:26.767 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=226, tag_tid=14, entry_tid=43702' 2026-02-18T08:17:26.768 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=233, tag_tid=15, entry_tid=1' 2026-02-18T08:17:26.768 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:17:26.768 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:17:26.769 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=15 2026-02-18T08:17:26.769 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=226, tag_tid=14, entry_tid=43702' 2026-02-18T08:17:26.769 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:17:26.770 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:17:26.771 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=14 2026-02-18T08:17:26.772 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=233, tag_tid=15, entry_tid=1' 2026-02-18T08:17:26.772 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:17:26.772 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:17:26.773 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:17:26.774 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=226, tag_tid=14, entry_tid=43702' 2026-02-18T08:17:26.774 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:17:26.774 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:17:26.776 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=43702 2026-02-18T08:17:26.776 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 15 = 14 -a 1 -le 43702 2026-02-18T08:17:26.776 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:17:26.776 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:17:26.776 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:17:26.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:17:26.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:17:26.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:17:26.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:17:26.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:17:26.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:17:26.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:17:26.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:17:26.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:17:26.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:26.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:17:26.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:17:26.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:17:26.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:17:26.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:17:26.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:17:26.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:17:26.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:17:26.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:17:26.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:17:26.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:26.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:17:26.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:17:26.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:17:26.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:17:26.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:17:26.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:17:26.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:17:26.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:17:26.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:17:26.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:26.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:17:26.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:17:26.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:17:26.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:17:26.985 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:17:26.985 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:17:26.985 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:17:26.985 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:17:26.985 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:17:26.985 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:17:26.985 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:26.985 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:17:26.985 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:17:26.985 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:17:26.985 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:17:26.986 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:17:26.986 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:17:26.986 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:17:26.993 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:17:26.993 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:17:26.993 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:17:26.994 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:17:27.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:17:27.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:17:27.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:17:27.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:17:27.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:17:27.055 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:17:27.055 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:27.055 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:17:27.055 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:17:27.055 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:17:27.055 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:17:27.055 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:17:27.055 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:17:27.055 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:17:27.055 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:17:27.056 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:17:27.056 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:27.056 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:17:27.056 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:17:27.056 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:17:27.056 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:17:27.056 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:17:27.056 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:17:27.056 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:17:27.056 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:17:27.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:17:27.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:17:27.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:27.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:17:27.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:17:27.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:17:27.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:17:27.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:17:27.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:17:27.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:17:27.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:17:27.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:17:27.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:17:27.067 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:17:27.122 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:17:27.122 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:17:27.122 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:17:27.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:17:27.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:17:27.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:17:27.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:27.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:17:27.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:17:27.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:17:27.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:17:27.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:17:27.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:17:27.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:17:27.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:17:27.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:17:27.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:27.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:17:27.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:17:27.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:17:27.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:17:27.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:17:27.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:17:27.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:17:27.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:17:27.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:17:27.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:17:27.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:27.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:17:27.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:17:27.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:17:27.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:17:27.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:17:27.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:17:27.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:17:27.136 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:17:27.136 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:17:27.136 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:17:27.136 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:17:27.194 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:17:27.194 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:17:27.194 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:17:27.197 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:17:27.197 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:17:27.197 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:17:27.198 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:27.198 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:17:27.198 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:17:27.198 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:17:27.198 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:17:27.198 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:17:27.198 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:17:27.198 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:17:27.198 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:17:27.198 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:17:27.199 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:27.199 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:17:27.199 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:17:27.199 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:17:27.199 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:17:27.199 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:17:27.199 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:17:27.199 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:17:27.199 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:17:27.199 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:17:27.200 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:17:27.200 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:27.200 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:17:27.200 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:17:27.200 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:17:27.200 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:17:27.200 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:17:27.200 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:17:27.200 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:17:27.207 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:17:27.208 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:17:27.208 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:17:27.208 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:17:27.270 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:17:27.270 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:17:27.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:17:27.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:17:27.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:17:27.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:17:27.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:17:27.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:17:27.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:17:27.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:17:27.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:17:27.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:17:27.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:17:27.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:27.273 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:17:27.273 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:27.299 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 56 2026-02-18T08:17:27.299 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 58 2026-02-18T08:17:27.300 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:17:27.300 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=233, tag_tid=15, entry_tid=1], [object_number=232, tag_tid=15, entry_tid=0], [object_number=234, tag_tid=14, entry_tid=52758], [object_number=235, tag_tid=14, entry_tid=52755]]], state=connected] 2026-02-18T08:17:27.300 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=227, tag_tid=14, entry_tid=44119], [object_number=226, tag_tid=14, entry_tid=44118], [object_number=225, tag_tid=14, entry_tid=44117], [object_number=224, tag_tid=14, entry_tid=44116]]], state=connected] 2026-02-18T08:17:27.302 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:27.304 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=233, tag_tid=15, entry_tid=1' 2026-02-18T08:17:27.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:17:27.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:17:27.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:17:27.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:17:27.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:17:27.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:17:27.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:17:27.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:17:27.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:17:27.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:17:27.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:17:27.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:17:27.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:27.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:17:27.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:27.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 56 2026-02-18T08:17:27.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 58 2026-02-18T08:17:27.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:17:27.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=233, tag_tid=15, entry_tid=1], [object_number=232, tag_tid=15, entry_tid=0], [object_number=234, tag_tid=14, entry_tid=52758], [object_number=235, tag_tid=14, entry_tid=52755]]], state=connected] 2026-02-18T08:17:27.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=227, tag_tid=14, entry_tid=44119], [object_number=226, tag_tid=14, entry_tid=44118], [object_number=225, tag_tid=14, entry_tid=44117], [object_number=224, tag_tid=14, entry_tid=44116]]], state=connected] 2026-02-18T08:17:27.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:27.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=227, tag_tid=14, entry_tid=44119' 2026-02-18T08:17:27.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=233, tag_tid=15, entry_tid=1' -a 'object_number=233, tag_tid=15, entry_tid=1' = 'object_number=227, tag_tid=14, entry_tid=44119' 2026-02-18T08:17:27.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=227, tag_tid=14, entry_tid=44119' '!=' 'object_number=226, tag_tid=14, entry_tid=43702' 2026-02-18T08:17:27.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:17:27.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=227, tag_tid=14, entry_tid=44119' = 'object_number=226, tag_tid=14, entry_tid=43702' 2026-02-18T08:17:27.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=227, tag_tid=14, entry_tid=44119' 2026-02-18T08:17:27.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=233, tag_tid=15, entry_tid=1' 2026-02-18T08:17:27.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:17:27.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:17:27.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=15 2026-02-18T08:17:27.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=227, tag_tid=14, entry_tid=44119' 2026-02-18T08:17:27.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:17:27.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:17:27.344 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=14 2026-02-18T08:17:27.344 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=233, tag_tid=15, entry_tid=1' 2026-02-18T08:17:27.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:17:27.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:17:27.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:17:27.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=227, tag_tid=14, entry_tid=44119' 2026-02-18T08:17:27.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:17:27.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:17:27.348 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=44119 2026-02-18T08:17:27.348 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 15 = 14 -a 1 -le 44119 2026-02-18T08:17:27.348 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:17:27.348 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:17:27.348 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:17:27.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:17:27.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:17:27.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:17:27.551 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:17:27.551 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:17:27.551 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:17:27.551 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:17:27.551 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:17:27.551 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:17:27.551 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:27.551 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:17:27.551 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:17:27.551 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:17:27.552 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:17:27.552 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:17:27.552 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:17:27.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:17:27.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:17:27.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:17:27.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:17:27.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:27.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:17:27.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:17:27.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:17:27.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:17:27.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:17:27.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:17:27.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:17:27.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:17:27.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:17:27.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:27.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:17:27.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:17:27.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:17:27.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:17:27.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:17:27.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:17:27.557 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:17:27.557 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:17:27.557 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:17:27.557 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:17:27.557 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:27.557 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:17:27.557 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:17:27.557 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:17:27.557 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:17:27.557 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:17:27.557 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:17:27.557 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:17:27.565 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:17:27.565 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:17:27.565 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:17:27.565 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:17:27.622 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:17:27.622 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:17:27.623 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:17:27.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:17:27.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:17:27.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:17:27.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:27.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:17:27.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:17:27.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:17:27.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:17:27.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:17:27.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:17:27.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:17:27.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:17:27.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:17:27.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:27.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:17:27.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:17:27.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:17:27.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:17:27.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:17:27.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:17:27.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:17:27.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:17:27.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:17:27.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:17:27.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:27.628 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:17:27.628 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:17:27.628 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:17:27.628 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:17:27.628 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:17:27.628 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:17:27.628 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:17:27.637 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:17:27.637 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:17:27.637 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:17:27.638 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:17:27.693 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:17:27.693 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:17:27.693 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:17:27.696 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:17:27.696 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:17:27.696 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:17:27.696 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:27.696 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:17:27.696 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:17:27.696 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:17:27.697 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:17:27.697 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:17:27.697 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:17:27.697 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:17:27.697 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:17:27.697 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:17:27.697 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:27.697 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:17:27.697 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:17:27.697 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:17:27.697 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:17:27.697 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:17:27.697 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:17:27.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:17:27.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:17:27.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:17:27.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:17:27.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:27.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:17:27.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:17:27.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:17:27.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:17:27.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:17:27.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:17:27.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:17:27.708 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:17:27.709 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:17:27.709 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:17:27.709 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:17:27.756 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:17:27.756 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:17:27.756 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:17:27.759 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:17:27.759 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:17:27.759 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:17:27.759 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:27.760 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:17:27.760 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:17:27.760 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:17:27.760 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:17:27.760 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:17:27.760 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:17:27.760 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:17:27.760 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:17:27.760 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:17:27.760 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:27.760 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:17:27.760 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:17:27.761 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:17:27.761 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:17:27.761 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:17:27.761 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:17:27.761 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:17:27.761 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:17:27.761 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:17:27.761 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:17:27.761 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:27.761 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:17:27.761 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:17:27.761 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:17:27.761 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:17:27.762 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:17:27.762 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:17:27.762 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:17:27.769 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:17:27.770 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:17:27.770 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:17:27.770 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:17:27.836 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:17:27.837 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:17:27.837 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:17:27.837 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:17:27.837 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:17:27.837 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:17:27.838 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:17:27.838 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:17:27.838 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:17:27.838 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:17:27.838 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:17:27.838 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:17:27.838 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:17:27.839 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:27.839 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:17:27.839 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:27.867 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 56 2026-02-18T08:17:27.867 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 58 2026-02-18T08:17:27.867 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:17:27.867 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=233, tag_tid=15, entry_tid=1], [object_number=232, tag_tid=15, entry_tid=0], [object_number=234, tag_tid=14, entry_tid=52758], [object_number=235, tag_tid=14, entry_tid=52755]]], state=connected] 2026-02-18T08:17:27.867 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=226, tag_tid=14, entry_tid=44550], [object_number=225, tag_tid=14, entry_tid=44549], [object_number=224, tag_tid=14, entry_tid=44548], [object_number=227, tag_tid=14, entry_tid=44547]]], state=connected] 2026-02-18T08:17:27.869 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:27.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=233, tag_tid=15, entry_tid=1' 2026-02-18T08:17:27.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:17:27.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:17:27.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:17:27.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:17:27.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:17:27.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:17:27.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:17:27.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:17:27.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:17:27.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:17:27.874 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:17:27.874 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:17:27.874 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:27.875 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:17:27.875 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:27.911 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 56 2026-02-18T08:17:27.911 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 58 2026-02-18T08:17:27.911 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:17:27.912 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=233, tag_tid=15, entry_tid=1], [object_number=232, tag_tid=15, entry_tid=0], [object_number=234, tag_tid=14, entry_tid=52758], [object_number=235, tag_tid=14, entry_tid=52755]]], state=connected] 2026-02-18T08:17:27.912 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=226, tag_tid=14, entry_tid=44550], [object_number=225, tag_tid=14, entry_tid=44549], [object_number=224, tag_tid=14, entry_tid=44548], [object_number=227, tag_tid=14, entry_tid=44547]]], state=connected] 2026-02-18T08:17:27.914 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:27.916 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=226, tag_tid=14, entry_tid=44550' 2026-02-18T08:17:27.916 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=233, tag_tid=15, entry_tid=1' -a 'object_number=233, tag_tid=15, entry_tid=1' = 'object_number=226, tag_tid=14, entry_tid=44550' 2026-02-18T08:17:27.916 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=226, tag_tid=14, entry_tid=44550' '!=' 'object_number=227, tag_tid=14, entry_tid=44119' 2026-02-18T08:17:27.916 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:17:27.916 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=226, tag_tid=14, entry_tid=44550' = 'object_number=227, tag_tid=14, entry_tid=44119' 2026-02-18T08:17:27.916 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=226, tag_tid=14, entry_tid=44550' 2026-02-18T08:17:27.917 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=233, tag_tid=15, entry_tid=1' 2026-02-18T08:17:27.917 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:17:27.917 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:17:27.918 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=15 2026-02-18T08:17:27.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=226, tag_tid=14, entry_tid=44550' 2026-02-18T08:17:27.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:17:27.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:17:27.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=14 2026-02-18T08:17:27.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=233, tag_tid=15, entry_tid=1' 2026-02-18T08:17:27.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:17:27.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:17:27.922 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:17:27.923 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=226, tag_tid=14, entry_tid=44550' 2026-02-18T08:17:27.923 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:17:27.923 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:17:27.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=44550 2026-02-18T08:17:27.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 15 = 14 -a 1 -le 44550 2026-02-18T08:17:27.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:17:27.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:17:27.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:17:28.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:17:28.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:17:28.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:17:28.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:17:28.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:17:28.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:17:28.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:17:28.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:17:28.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:17:28.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:28.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:17:28.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:17:28.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:17:28.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:17:28.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:17:28.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:17:28.131 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:17:28.131 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:17:28.131 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:17:28.131 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:17:28.131 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:28.131 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:17:28.132 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:17:28.132 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:17:28.132 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:17:28.132 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:17:28.132 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:17:28.132 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:17:28.132 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:17:28.132 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:17:28.132 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:28.132 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:17:28.132 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:17:28.132 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:17:28.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:17:28.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:17:28.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:17:28.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:17:28.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:17:28.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:17:28.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:17:28.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:28.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:17:28.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:17:28.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:17:28.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:17:28.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:17:28.134 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:17:28.134 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:17:28.141 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:17:28.141 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:17:28.141 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:17:28.141 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:17:28.198 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:17:28.199 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:17:28.199 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:17:28.202 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:17:28.202 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:17:28.202 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:17:28.202 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:28.202 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:17:28.202 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:17:28.202 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:17:28.202 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:17:28.203 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:17:28.203 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:17:28.203 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:17:28.203 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:17:28.203 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:17:28.203 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:28.203 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:17:28.203 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:17:28.203 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:17:28.203 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:17:28.203 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:17:28.203 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:17:28.204 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:17:28.204 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:17:28.204 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:17:28.204 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:17:28.204 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:28.204 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:17:28.204 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:17:28.204 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:17:28.204 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:17:28.204 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:17:28.204 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:17:28.204 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:17:28.212 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:17:28.213 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:17:28.213 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:17:28.213 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:17:28.275 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:17:28.275 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:17:28.275 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:17:28.278 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:17:28.278 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:17:28.278 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:17:28.278 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:28.279 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:17:28.279 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:17:28.279 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:17:28.279 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:17:28.279 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:17:28.279 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:17:28.279 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:17:28.279 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:17:28.279 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:17:28.279 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:28.279 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:17:28.280 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:17:28.280 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:17:28.280 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:17:28.280 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:17:28.280 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:17:28.280 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:17:28.280 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:17:28.280 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:17:28.280 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:17:28.280 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:28.280 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:17:28.280 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:17:28.280 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:17:28.281 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:17:28.281 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:17:28.281 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:17:28.281 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:17:28.288 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:17:28.289 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:17:28.289 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:17:28.289 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:17:28.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:17:28.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:17:28.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:17:28.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:17:28.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:17:28.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:17:28.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:28.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:17:28.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:17:28.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:17:28.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:17:28.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:17:28.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:17:28.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:17:28.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:17:28.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:17:28.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:28.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:17:28.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:17:28.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:17:28.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:17:28.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:17:28.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:17:28.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:17:28.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:17:28.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:17:28.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:17:28.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:28.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:17:28.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:17:28.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:17:28.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:17:28.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:17:28.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:17:28.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:17:28.351 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:17:28.352 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:17:28.352 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:17:28.352 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:17:28.412 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:17:28.413 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:17:28.413 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:17:28.413 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:17:28.413 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:17:28.414 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:17:28.414 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:17:28.414 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:17:28.414 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:17:28.414 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:17:28.414 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:17:28.414 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:17:28.414 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:17:28.415 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:28.415 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:17:28.415 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:28.646 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 56 2026-02-18T08:17:28.646 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 58 2026-02-18T08:17:28.646 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:17:28.646 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=233, tag_tid=15, entry_tid=1], [object_number=232, tag_tid=15, entry_tid=0], [object_number=234, tag_tid=14, entry_tid=52758], [object_number=235, tag_tid=14, entry_tid=52755]]], state=connected] 2026-02-18T08:17:28.646 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=227, tag_tid=14, entry_tid=44907], [object_number=226, tag_tid=14, entry_tid=44906], [object_number=225, tag_tid=14, entry_tid=44905], [object_number=224, tag_tid=14, entry_tid=44904]]], state=connected] 2026-02-18T08:17:28.649 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:28.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=233, tag_tid=15, entry_tid=1' 2026-02-18T08:17:28.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:17:28.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:17:28.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:17:28.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:17:28.653 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:17:28.653 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:17:28.653 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:17:28.653 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:17:28.653 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:17:28.653 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:17:28.654 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:17:28.654 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:17:28.654 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:28.654 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:17:28.655 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:28.681 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 56 2026-02-18T08:17:28.681 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 58 2026-02-18T08:17:28.681 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:17:28.681 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=233, tag_tid=15, entry_tid=1], [object_number=232, tag_tid=15, entry_tid=0], [object_number=234, tag_tid=14, entry_tid=52758], [object_number=235, tag_tid=14, entry_tid=52755]]], state=connected] 2026-02-18T08:17:28.681 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=227, tag_tid=14, entry_tid=44907], [object_number=226, tag_tid=14, entry_tid=44906], [object_number=225, tag_tid=14, entry_tid=44905], [object_number=224, tag_tid=14, entry_tid=44904]]], state=connected] 2026-02-18T08:17:28.683 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:28.685 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=227, tag_tid=14, entry_tid=44907' 2026-02-18T08:17:28.685 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=233, tag_tid=15, entry_tid=1' -a 'object_number=233, tag_tid=15, entry_tid=1' = 'object_number=227, tag_tid=14, entry_tid=44907' 2026-02-18T08:17:28.685 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=227, tag_tid=14, entry_tid=44907' '!=' 'object_number=226, tag_tid=14, entry_tid=44550' 2026-02-18T08:17:28.685 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:17:28.685 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=227, tag_tid=14, entry_tid=44907' = 'object_number=226, tag_tid=14, entry_tid=44550' 2026-02-18T08:17:28.685 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=227, tag_tid=14, entry_tid=44907' 2026-02-18T08:17:28.686 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=233, tag_tid=15, entry_tid=1' 2026-02-18T08:17:28.686 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:17:28.686 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:17:28.687 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=15 2026-02-18T08:17:28.688 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=227, tag_tid=14, entry_tid=44907' 2026-02-18T08:17:28.688 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:17:28.688 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:17:28.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=14 2026-02-18T08:17:28.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=233, tag_tid=15, entry_tid=1' 2026-02-18T08:17:28.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:17:28.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:17:28.692 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:17:28.692 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=227, tag_tid=14, entry_tid=44907' 2026-02-18T08:17:28.693 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:17:28.693 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:17:28.694 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=44907 2026-02-18T08:17:28.694 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 15 = 14 -a 1 -le 44907 2026-02-18T08:17:28.694 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:17:28.694 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:17:28.694 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:17:28.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:17:28.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:17:28.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:17:28.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:17:28.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:17:28.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:17:28.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:17:28.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:17:28.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:17:28.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:28.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:17:28.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:17:28.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:17:28.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:17:28.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:17:28.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:17:28.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:17:28.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:17:28.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:17:28.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:17:28.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:28.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:17:28.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:17:28.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:17:28.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:17:28.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:17:28.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:17:28.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:17:28.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:17:28.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:17:28.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:28.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:17:28.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:17:28.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:17:28.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:17:28.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:17:28.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:17:28.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:17:28.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:17:28.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:17:28.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:17:28.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:28.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:17:28.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:17:28.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:17:28.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:17:28.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:17:28.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:17:28.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:17:28.910 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:17:28.910 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:17:28.910 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:17:28.910 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:17:28.959 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:17:28.959 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:17:28.959 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:17:28.962 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:17:28.962 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:17:28.962 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:17:28.962 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:28.962 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:17:28.962 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:17:28.962 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:17:28.963 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:17:28.963 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:17:28.963 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:17:28.963 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:17:28.963 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:17:28.963 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:17:28.963 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:28.963 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:17:28.963 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:17:28.963 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:17:28.963 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:17:28.963 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:17:28.964 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:17:28.964 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:17:28.964 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:17:28.964 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:17:28.964 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:17:28.964 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:28.964 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:17:28.964 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:17:28.964 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:17:28.964 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:17:28.964 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:17:28.964 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:17:28.965 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:17:28.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:17:28.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:17:28.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:17:28.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:17:29.021 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:17:29.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:17:29.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:17:29.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:17:29.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:17:29.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:17:29.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:29.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:17:29.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:17:29.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:17:29.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:17:29.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:17:29.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:17:29.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:17:29.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:17:29.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:17:29.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:29.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:17:29.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:17:29.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:17:29.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:17:29.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:17:29.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:17:29.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:17:29.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:17:29.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:17:29.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:17:29.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:29.027 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:17:29.027 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:17:29.027 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:17:29.027 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:17:29.027 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:17:29.027 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:17:29.027 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:17:29.035 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:17:29.036 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:17:29.036 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:17:29.036 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:17:29.086 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:17:29.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:17:29.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:17:29.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:17:29.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:17:29.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:17:29.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:29.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:17:29.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:17:29.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:17:29.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:17:29.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:17:29.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:17:29.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:17:29.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:17:29.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:17:29.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:29.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:17:29.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:17:29.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:17:29.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:17:29.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:17:29.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:17:29.092 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:17:29.092 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:17:29.092 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:17:29.092 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:17:29.092 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:29.092 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:17:29.092 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:17:29.092 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:17:29.092 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:17:29.092 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:17:29.092 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:17:29.092 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:17:29.100 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:17:29.101 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:17:29.101 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:17:29.101 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:17:29.164 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:17:29.165 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:17:29.165 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:17:29.165 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:17:29.166 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:17:29.166 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:17:29.166 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:17:29.166 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:17:29.166 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:17:29.166 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:17:29.166 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:17:29.166 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:17:29.166 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:17:29.167 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:29.167 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:17:29.167 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:29.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 56 2026-02-18T08:17:29.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 58 2026-02-18T08:17:29.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:17:29.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=233, tag_tid=15, entry_tid=1], [object_number=232, tag_tid=15, entry_tid=0], [object_number=234, tag_tid=14, entry_tid=52758], [object_number=235, tag_tid=14, entry_tid=52755]]], state=connected] 2026-02-18T08:17:29.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=227, tag_tid=14, entry_tid=45363], [object_number=226, tag_tid=14, entry_tid=45362], [object_number=225, tag_tid=14, entry_tid=45361], [object_number=224, tag_tid=14, entry_tid=45360]]], state=connected] 2026-02-18T08:17:29.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:29.403 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=233, tag_tid=15, entry_tid=1' 2026-02-18T08:17:29.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:17:29.405 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:17:29.405 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:17:29.405 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:17:29.405 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:17:29.405 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:17:29.405 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:17:29.405 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:17:29.405 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:17:29.405 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:17:29.405 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:17:29.406 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:17:29.406 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:29.407 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:17:29.407 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:29.434 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 56 2026-02-18T08:17:29.434 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 58 2026-02-18T08:17:29.434 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:17:29.434 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=233, tag_tid=15, entry_tid=1], [object_number=232, tag_tid=15, entry_tid=0], [object_number=234, tag_tid=14, entry_tid=52758], [object_number=235, tag_tid=14, entry_tid=52755]]], state=connected] 2026-02-18T08:17:29.434 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=227, tag_tid=14, entry_tid=45363], [object_number=226, tag_tid=14, entry_tid=45362], [object_number=225, tag_tid=14, entry_tid=45361], [object_number=224, tag_tid=14, entry_tid=45360]]], state=connected] 2026-02-18T08:17:29.436 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:29.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=227, tag_tid=14, entry_tid=45363' 2026-02-18T08:17:29.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=233, tag_tid=15, entry_tid=1' -a 'object_number=233, tag_tid=15, entry_tid=1' = 'object_number=227, tag_tid=14, entry_tid=45363' 2026-02-18T08:17:29.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=227, tag_tid=14, entry_tid=45363' '!=' 'object_number=227, tag_tid=14, entry_tid=44907' 2026-02-18T08:17:29.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:17:29.439 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=227, tag_tid=14, entry_tid=45363' = 'object_number=227, tag_tid=14, entry_tid=44907' 2026-02-18T08:17:29.439 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=227, tag_tid=14, entry_tid=45363' 2026-02-18T08:17:29.439 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=233, tag_tid=15, entry_tid=1' 2026-02-18T08:17:29.439 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:17:29.440 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:17:29.440 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=15 2026-02-18T08:17:29.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=227, tag_tid=14, entry_tid=45363' 2026-02-18T08:17:29.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:17:29.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:17:29.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=14 2026-02-18T08:17:29.443 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=233, tag_tid=15, entry_tid=1' 2026-02-18T08:17:29.443 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:17:29.443 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:17:29.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:17:29.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=227, tag_tid=14, entry_tid=45363' 2026-02-18T08:17:29.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:17:29.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:17:29.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=45363 2026-02-18T08:17:29.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 15 = 14 -a 1 -le 45363 2026-02-18T08:17:29.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:17:29.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:17:29.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:17:29.649 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:17:29.649 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:17:29.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:17:29.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:17:29.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:17:29.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:17:29.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:17:29.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:17:29.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:17:29.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:29.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:17:29.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:17:29.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:17:29.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:17:29.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:17:29.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:17:29.653 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:17:29.654 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:17:29.654 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:17:29.654 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:17:29.654 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:29.654 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:17:29.654 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:17:29.654 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:17:29.654 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:17:29.654 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:17:29.654 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:17:29.655 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:17:29.655 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:17:29.655 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:17:29.655 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:29.655 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:17:29.655 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:17:29.655 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:17:29.655 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:17:29.655 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:17:29.655 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:17:29.655 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:17:29.655 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:17:29.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:17:29.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:17:29.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:29.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:17:29.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:17:29.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:17:29.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:17:29.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:17:29.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:17:29.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:17:29.664 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:17:29.664 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:17:29.664 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:17:29.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:17:29.722 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:17:29.722 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:17:29.722 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:17:29.725 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:17:29.725 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:17:29.725 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:17:29.725 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:29.725 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:17:29.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:17:29.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:17:29.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:17:29.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:17:29.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:17:29.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:17:29.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:17:29.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:17:29.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:29.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:17:29.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:17:29.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:17:29.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:17:29.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:17:29.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:17:29.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:17:29.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:17:29.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:17:29.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:17:29.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:29.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:17:29.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:17:29.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:17:29.728 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:17:29.728 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:17:29.728 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:17:29.728 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:17:29.736 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:17:29.737 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:17:29.737 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:17:29.737 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:17:29.785 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:17:29.785 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:17:29.786 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:17:29.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:17:29.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:17:29.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:17:29.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:29.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:17:29.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:17:29.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:17:29.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:17:29.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:17:29.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:17:29.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:17:29.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:17:29.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:17:29.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:29.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:17:29.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:17:29.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:17:29.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:17:29.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:17:29.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:17:29.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:17:29.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:17:29.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:17:29.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:17:29.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:29.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:17:29.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:17:29.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:17:29.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:17:29.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:17:29.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:17:29.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:17:29.800 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:17:29.801 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:17:29.801 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:17:29.801 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:17:29.851 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:17:29.851 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:17:29.851 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:17:29.854 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:17:29.854 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:17:29.854 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:17:29.854 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:29.854 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:17:29.854 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:17:29.854 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:17:29.854 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:17:29.854 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:17:29.855 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:17:29.855 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:17:29.855 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:17:29.855 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:17:29.855 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:29.855 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:17:29.855 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:17:29.855 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:17:29.855 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:17:29.855 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:17:29.855 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:17:29.856 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:17:29.856 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:17:29.856 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:17:29.856 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:17:29.856 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:29.856 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:17:29.856 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:17:29.856 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:17:29.856 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:17:29.856 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:17:29.856 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:17:29.856 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:17:29.865 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:17:29.866 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:17:29.866 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:17:29.866 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:17:29.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:17:29.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:17:29.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:17:29.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:17:29.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:17:29.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:17:29.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:17:29.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:17:29.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:17:29.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:17:29.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:17:29.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:17:29.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:17:29.928 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:29.928 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:17:29.928 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:29.955 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 56 2026-02-18T08:17:29.956 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 58 2026-02-18T08:17:29.956 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:17:29.956 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=233, tag_tid=15, entry_tid=1], [object_number=232, tag_tid=15, entry_tid=0], [object_number=234, tag_tid=14, entry_tid=52758], [object_number=235, tag_tid=14, entry_tid=52755]]], state=connected] 2026-02-18T08:17:29.956 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=225, tag_tid=14, entry_tid=45861], [object_number=224, tag_tid=14, entry_tid=45860], [object_number=227, tag_tid=14, entry_tid=45859], [object_number=226, tag_tid=14, entry_tid=45858]]], state=connected] 2026-02-18T08:17:29.958 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:29.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=233, tag_tid=15, entry_tid=1' 2026-02-18T08:17:29.961 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:17:29.961 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:17:29.961 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:17:29.961 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:17:29.962 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:17:29.962 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:17:29.962 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:17:29.962 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:17:29.962 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:17:29.962 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:17:29.962 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:17:29.962 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:17:29.963 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:29.963 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:17:29.963 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:29.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 56 2026-02-18T08:17:29.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 58 2026-02-18T08:17:29.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:17:29.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=233, tag_tid=15, entry_tid=1], [object_number=232, tag_tid=15, entry_tid=0], [object_number=234, tag_tid=14, entry_tid=52758], [object_number=235, tag_tid=14, entry_tid=52755]]], state=connected] 2026-02-18T08:17:29.999 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=225, tag_tid=14, entry_tid=45861], [object_number=224, tag_tid=14, entry_tid=45860], [object_number=227, tag_tid=14, entry_tid=45859], [object_number=226, tag_tid=14, entry_tid=45858]]], state=connected] 2026-02-18T08:17:30.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:30.003 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=225, tag_tid=14, entry_tid=45861' 2026-02-18T08:17:30.004 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=233, tag_tid=15, entry_tid=1' -a 'object_number=233, tag_tid=15, entry_tid=1' = 'object_number=225, tag_tid=14, entry_tid=45861' 2026-02-18T08:17:30.004 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=225, tag_tid=14, entry_tid=45861' '!=' 'object_number=227, tag_tid=14, entry_tid=45363' 2026-02-18T08:17:30.004 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:17:30.004 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=225, tag_tid=14, entry_tid=45861' = 'object_number=227, tag_tid=14, entry_tid=45363' 2026-02-18T08:17:30.004 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=225, tag_tid=14, entry_tid=45861' 2026-02-18T08:17:30.005 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=233, tag_tid=15, entry_tid=1' 2026-02-18T08:17:30.005 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:17:30.005 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:17:30.006 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=15 2026-02-18T08:17:30.006 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=225, tag_tid=14, entry_tid=45861' 2026-02-18T08:17:30.007 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:17:30.007 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:17:30.008 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=14 2026-02-18T08:17:30.008 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=233, tag_tid=15, entry_tid=1' 2026-02-18T08:17:30.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:17:30.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:17:30.010 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:17:30.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=225, tag_tid=14, entry_tid=45861' 2026-02-18T08:17:30.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:17:30.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:17:30.012 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=45861 2026-02-18T08:17:30.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 15 = 14 -a 1 -le 45861 2026-02-18T08:17:30.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:17:30.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:17:30.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:17:30.214 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:17:30.214 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:17:30.215 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:17:30.215 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:17:30.215 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:17:30.215 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:17:30.215 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:17:30.215 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:17:30.215 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:17:30.215 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:30.215 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:17:30.215 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:17:30.215 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:17:30.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:17:30.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:17:30.218 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:17:30.219 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:17:30.219 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:17:30.219 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:17:30.219 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:17:30.219 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:30.219 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:17:30.219 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:17:30.219 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:17:30.219 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:17:30.220 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:17:30.220 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:17:30.220 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:17:30.220 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:17:30.220 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:17:30.220 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:30.220 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:17:30.220 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:17:30.220 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:17:30.220 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:17:30.220 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:17:30.220 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:17:30.221 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:17:30.221 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:17:30.221 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:17:30.221 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:17:30.221 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:30.221 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:17:30.221 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:17:30.221 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:17:30.221 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:17:30.221 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:17:30.221 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:17:30.221 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:17:30.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:17:30.229 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:17:30.229 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:17:30.229 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:17:30.280 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:17:30.281 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:17:30.281 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:17:30.283 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:17:30.284 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:17:30.284 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:17:30.284 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:30.284 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:17:30.284 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:17:30.284 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:17:30.284 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:17:30.284 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:17:30.284 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:17:30.284 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:17:30.285 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:17:30.285 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:17:30.285 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:30.285 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:17:30.285 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:17:30.285 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:17:30.285 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:17:30.285 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:17:30.285 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:17:30.285 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:17:30.285 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:17:30.286 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:17:30.286 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:17:30.286 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:30.286 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:17:30.286 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:17:30.286 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:17:30.286 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:17:30.286 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:17:30.286 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:17:30.286 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:17:30.293 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:17:30.294 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:17:30.294 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:17:30.294 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:17:30.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:17:30.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:17:30.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:17:30.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:17:30.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:17:30.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:17:30.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:30.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:17:30.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:17:30.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:17:30.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:17:30.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:17:30.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:17:30.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:17:30.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:17:30.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:17:30.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:30.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:17:30.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:17:30.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:17:30.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:17:30.348 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:17:30.348 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:17:30.348 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:17:30.348 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:17:30.348 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:17:30.348 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:17:30.348 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:30.348 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:17:30.348 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:17:30.348 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:17:30.348 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:17:30.348 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:17:30.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:17:30.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:17:30.356 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:17:30.356 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:17:30.356 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:17:30.357 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:17:30.414 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:17:30.414 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:17:30.414 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:17:30.416 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:17:30.416 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:17:30.417 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:17:30.417 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:30.417 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:17:30.417 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:17:30.417 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:17:30.417 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:17:30.417 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:17:30.417 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:17:30.417 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:17:30.417 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:17:30.417 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:17:30.417 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:30.418 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:17:30.418 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:17:30.418 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:17:30.418 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:17:30.418 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:17:30.418 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:17:30.418 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:17:30.418 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:17:30.418 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:17:30.418 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:17:30.418 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:30.418 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:17:30.419 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:17:30.419 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:17:30.419 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:17:30.419 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:17:30.419 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:17:30.419 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:17:30.428 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:17:30.428 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:17:30.428 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:17:30.428 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:17:30.487 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:17:30.488 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:17:30.488 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:17:30.488 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:17:30.488 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:17:30.489 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:17:30.489 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:17:30.489 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:17:30.489 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:17:30.489 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:17:30.489 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:17:30.489 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:17:30.489 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:17:30.490 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:30.490 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:17:30.490 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:30.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 56 2026-02-18T08:17:30.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 58 2026-02-18T08:17:30.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:17:30.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=233, tag_tid=15, entry_tid=1], [object_number=232, tag_tid=15, entry_tid=0], [object_number=234, tag_tid=14, entry_tid=52758], [object_number=235, tag_tid=14, entry_tid=52755]]], state=connected] 2026-02-18T08:17:30.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=226, tag_tid=14, entry_tid=46222], [object_number=225, tag_tid=14, entry_tid=46221], [object_number=224, tag_tid=14, entry_tid=46220], [object_number=227, tag_tid=14, entry_tid=46219]]], state=connected] 2026-02-18T08:17:30.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:30.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=233, tag_tid=15, entry_tid=1' 2026-02-18T08:17:30.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:17:30.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:17:30.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:17:30.523 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:17:30.523 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:17:30.523 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:17:30.523 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:17:30.523 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:17:30.523 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:17:30.523 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:17:30.524 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:17:30.524 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:17:30.524 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:30.524 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:17:30.525 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:30.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 56 2026-02-18T08:17:30.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 58 2026-02-18T08:17:30.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:17:30.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=233, tag_tid=15, entry_tid=1], [object_number=232, tag_tid=15, entry_tid=0], [object_number=234, tag_tid=14, entry_tid=52758], [object_number=235, tag_tid=14, entry_tid=52755]]], state=connected] 2026-02-18T08:17:30.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=226, tag_tid=14, entry_tid=46222], [object_number=225, tag_tid=14, entry_tid=46221], [object_number=224, tag_tid=14, entry_tid=46220], [object_number=227, tag_tid=14, entry_tid=46219]]], state=connected] 2026-02-18T08:17:30.557 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:30.559 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=226, tag_tid=14, entry_tid=46222' 2026-02-18T08:17:30.559 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=233, tag_tid=15, entry_tid=1' -a 'object_number=233, tag_tid=15, entry_tid=1' = 'object_number=226, tag_tid=14, entry_tid=46222' 2026-02-18T08:17:30.560 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=226, tag_tid=14, entry_tid=46222' '!=' 'object_number=225, tag_tid=14, entry_tid=45861' 2026-02-18T08:17:30.560 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:17:30.560 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=226, tag_tid=14, entry_tid=46222' = 'object_number=225, tag_tid=14, entry_tid=45861' 2026-02-18T08:17:30.560 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=226, tag_tid=14, entry_tid=46222' 2026-02-18T08:17:30.561 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=233, tag_tid=15, entry_tid=1' 2026-02-18T08:17:30.561 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:17:30.561 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:17:30.562 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=15 2026-02-18T08:17:30.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=226, tag_tid=14, entry_tid=46222' 2026-02-18T08:17:30.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:17:30.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:17:30.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=14 2026-02-18T08:17:30.565 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=233, tag_tid=15, entry_tid=1' 2026-02-18T08:17:30.565 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:17:30.565 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:17:30.566 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:17:30.567 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=226, tag_tid=14, entry_tid=46222' 2026-02-18T08:17:30.567 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:17:30.567 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:17:30.569 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=46222 2026-02-18T08:17:30.569 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 15 = 14 -a 1 -le 46222 2026-02-18T08:17:30.569 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:17:30.569 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:17:30.569 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:17:30.770 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:17:30.771 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:17:30.771 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:17:30.771 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:17:30.771 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:17:30.771 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:17:30.771 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:17:30.771 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:17:30.771 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:17:30.771 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:30.771 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:17:30.772 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:17:30.772 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:17:30.772 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:17:30.772 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:17:30.774 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:17:30.774 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:17:30.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:17:30.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:17:30.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:17:30.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:30.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:17:30.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:17:30.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:17:30.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:17:30.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:17:30.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:17:30.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:17:30.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:17:30.776 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:17:30.776 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:30.776 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:17:30.776 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:17:30.776 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:17:30.776 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:17:30.776 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:17:30.776 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:17:30.776 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:17:30.776 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:17:30.776 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:17:30.776 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:17:30.776 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:30.776 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:17:30.777 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:17:30.777 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:17:30.777 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:17:30.777 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:17:30.777 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:17:30.777 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:17:30.786 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:17:30.786 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:17:30.786 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:17:30.786 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:17:30.846 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:17:30.846 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:17:30.846 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:17:30.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:17:30.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:17:30.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:17:30.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:30.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:17:30.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:17:30.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:17:30.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:17:30.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:17:30.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:17:30.850 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:17:30.850 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:17:30.850 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:17:30.850 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:30.850 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:17:30.850 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:17:30.850 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:17:30.850 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:17:30.850 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:17:30.850 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:17:30.850 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:17:30.850 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:17:30.851 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:17:30.851 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:17:30.851 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:30.851 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:17:30.851 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:17:30.851 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:17:30.851 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:17:30.851 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:17:30.851 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:17:30.851 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:17:30.860 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:17:30.860 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:17:30.861 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:17:30.861 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:17:30.909 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:17:30.909 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:17:30.909 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:17:30.912 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:17:30.912 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:17:30.912 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:17:30.912 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:30.912 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:17:30.912 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:17:30.912 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:17:30.913 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:17:30.913 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:17:30.913 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:17:30.913 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:17:30.913 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:17:30.913 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:17:30.913 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:30.913 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:17:30.913 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:17:30.913 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:17:30.913 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:17:30.913 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:17:30.913 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:17:30.914 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:17:30.914 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:17:30.914 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:17:30.914 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:17:30.914 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:30.914 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:17:30.914 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:17:30.914 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:17:30.914 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:17:30.914 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:17:30.914 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:17:30.914 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:17:30.923 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:17:30.924 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:17:30.924 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:17:30.924 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:17:30.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:17:30.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:17:30.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:17:30.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:17:30.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:17:30.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:17:30.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:30.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:17:30.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:17:30.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:17:30.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:17:30.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:17:30.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:17:30.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:17:30.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:17:30.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:17:30.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:30.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:17:30.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:17:30.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:17:30.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:17:30.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:17:30.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:17:30.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:17:30.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:17:30.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:17:30.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:17:30.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:30.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:17:30.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:17:30.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:17:30.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:17:30.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:17:30.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:17:30.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:17:30.988 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:17:30.988 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:17:30.989 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:17:30.989 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:17:31.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:17:31.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:17:31.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:17:31.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:17:31.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:17:31.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:17:31.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:17:31.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:17:31.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:17:31.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:17:31.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:17:31.055 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:17:31.055 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:17:31.055 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:31.056 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:17:31.056 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:31.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 57 2026-02-18T08:17:31.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 58 2026-02-18T08:17:31.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:17:31.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=233, tag_tid=15, entry_tid=1], [object_number=232, tag_tid=15, entry_tid=0], [object_number=234, tag_tid=14, entry_tid=52758], [object_number=235, tag_tid=14, entry_tid=52755]]], state=connected] 2026-02-18T08:17:31.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=228, tag_tid=14, entry_tid=46484], [object_number=231, tag_tid=14, entry_tid=46483], [object_number=230, tag_tid=14, entry_tid=46482], [object_number=229, tag_tid=14, entry_tid=46481]]], state=connected] 2026-02-18T08:17:31.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:31.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=233, tag_tid=15, entry_tid=1' 2026-02-18T08:17:31.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:17:31.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:17:31.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:17:31.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:17:31.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:17:31.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:17:31.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:17:31.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:17:31.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:17:31.092 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:17:31.092 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:17:31.092 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:17:31.093 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:31.093 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:17:31.093 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:31.129 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 57 2026-02-18T08:17:31.129 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 58 2026-02-18T08:17:31.129 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:17:31.129 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=233, tag_tid=15, entry_tid=1], [object_number=232, tag_tid=15, entry_tid=0], [object_number=234, tag_tid=14, entry_tid=52758], [object_number=235, tag_tid=14, entry_tid=52755]]], state=connected] 2026-02-18T08:17:31.129 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=228, tag_tid=14, entry_tid=46484], [object_number=231, tag_tid=14, entry_tid=46483], [object_number=230, tag_tid=14, entry_tid=46482], [object_number=229, tag_tid=14, entry_tid=46481]]], state=connected] 2026-02-18T08:17:31.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:31.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=228, tag_tid=14, entry_tid=46484' 2026-02-18T08:17:31.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=233, tag_tid=15, entry_tid=1' -a 'object_number=233, tag_tid=15, entry_tid=1' = 'object_number=228, tag_tid=14, entry_tid=46484' 2026-02-18T08:17:31.136 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=228, tag_tid=14, entry_tid=46484' '!=' 'object_number=226, tag_tid=14, entry_tid=46222' 2026-02-18T08:17:31.136 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:17:31.136 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=228, tag_tid=14, entry_tid=46484' = 'object_number=226, tag_tid=14, entry_tid=46222' 2026-02-18T08:17:31.136 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=228, tag_tid=14, entry_tid=46484' 2026-02-18T08:17:31.136 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=233, tag_tid=15, entry_tid=1' 2026-02-18T08:17:31.136 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:17:31.136 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:17:31.137 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=15 2026-02-18T08:17:31.138 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=228, tag_tid=14, entry_tid=46484' 2026-02-18T08:17:31.138 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:17:31.138 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:17:31.139 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=14 2026-02-18T08:17:31.140 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=233, tag_tid=15, entry_tid=1' 2026-02-18T08:17:31.140 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:17:31.140 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:17:31.141 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:17:31.143 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=228, tag_tid=14, entry_tid=46484' 2026-02-18T08:17:31.143 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:17:31.143 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:17:31.144 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=46484 2026-02-18T08:17:31.145 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 15 = 14 -a 1 -le 46484 2026-02-18T08:17:31.145 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:17:31.145 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:17:31.145 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:17:31.344 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:17:31.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:17:31.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:17:31.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:17:31.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:17:31.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:17:31.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:17:31.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:17:31.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:17:31.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:31.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:17:31.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:17:31.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:17:31.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:17:31.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:17:31.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:17:31.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:17:31.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:17:31.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:17:31.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:17:31.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:31.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:17:31.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:17:31.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:17:31.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:17:31.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:17:31.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:17:31.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:17:31.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:17:31.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:17:31.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:31.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:17:31.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:17:31.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:17:31.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:17:31.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:17:31.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:17:31.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:17:31.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:17:31.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:17:31.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:17:31.348 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:31.348 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:17:31.348 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:17:31.348 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:17:31.348 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:17:31.348 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:17:31.348 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:17:31.348 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:17:31.360 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:17:31.360 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:17:31.360 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:17:31.360 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:17:31.418 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:17:31.418 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:17:31.418 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:17:31.420 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:17:31.420 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:17:31.420 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:17:31.420 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:31.420 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:17:31.420 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:17:31.421 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:17:31.421 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:17:31.421 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:17:31.421 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:17:31.421 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:17:31.421 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:17:31.421 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:17:31.421 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:31.421 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:17:31.421 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:17:31.421 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:17:31.421 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:17:31.421 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:17:31.421 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:17:31.421 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:17:31.421 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:17:31.421 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:17:31.421 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:17:31.421 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:31.421 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:17:31.421 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:17:31.421 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:17:31.421 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:17:31.421 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:17:31.421 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:17:31.421 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:17:31.432 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:17:31.433 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:17:31.433 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:17:31.433 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:17:31.484 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:17:31.484 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:17:31.484 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:17:31.486 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:17:31.486 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:17:31.486 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:17:31.486 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:31.486 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:17:31.487 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:17:31.487 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:17:31.487 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:17:31.487 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:17:31.487 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:17:31.487 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:17:31.487 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:17:31.487 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:17:31.487 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:31.487 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:17:31.487 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:17:31.487 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:17:31.487 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:17:31.487 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:17:31.487 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:17:31.487 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:17:31.487 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:17:31.487 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:17:31.487 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:17:31.487 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:31.487 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:17:31.487 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:17:31.487 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:17:31.487 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:17:31.487 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:17:31.488 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:17:31.488 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:17:31.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:17:31.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:17:31.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:17:31.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:17:31.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:17:31.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:17:31.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:17:31.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:17:31.551 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:17:31.551 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:17:31.551 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:31.551 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:17:31.551 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:17:31.551 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:17:31.551 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:17:31.551 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:17:31.552 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:17:31.552 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:17:31.552 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:17:31.552 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:17:31.552 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:31.552 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:17:31.552 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:17:31.552 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:17:31.552 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:17:31.552 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:17:31.552 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:17:31.553 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:17:31.553 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:17:31.553 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:17:31.553 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:17:31.553 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:31.553 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:17:31.553 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:17:31.553 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:17:31.553 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:17:31.553 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:17:31.553 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:17:31.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:17:31.560 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:17:31.561 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:17:31.561 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:17:31.561 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:17:31.622 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:17:31.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:17:31.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:17:31.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:17:31.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:17:31.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:17:31.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:17:31.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:17:31.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:17:31.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:17:31.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:17:31.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:17:31.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:17:31.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:31.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:17:31.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:31.655 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 57 2026-02-18T08:17:31.655 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 58 2026-02-18T08:17:31.655 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:17:31.655 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=233, tag_tid=15, entry_tid=1], [object_number=232, tag_tid=15, entry_tid=0], [object_number=234, tag_tid=14, entry_tid=52758], [object_number=235, tag_tid=14, entry_tid=52755]]], state=connected] 2026-02-18T08:17:31.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=229, tag_tid=14, entry_tid=46861], [object_number=228, tag_tid=14, entry_tid=46860], [object_number=231, tag_tid=14, entry_tid=46859], [object_number=230, tag_tid=14, entry_tid=46858]]], state=connected] 2026-02-18T08:17:31.658 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:31.659 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=233, tag_tid=15, entry_tid=1' 2026-02-18T08:17:31.660 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:17:31.660 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:17:31.660 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:17:31.660 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:17:31.661 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:17:31.661 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:17:31.661 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:17:31.661 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:17:31.661 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:17:31.661 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:17:31.662 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:17:31.662 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:17:31.662 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:31.663 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:17:31.663 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:31.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 57 2026-02-18T08:17:31.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 58 2026-02-18T08:17:31.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:17:31.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=233, tag_tid=15, entry_tid=1], [object_number=232, tag_tid=15, entry_tid=0], [object_number=234, tag_tid=14, entry_tid=52758], [object_number=235, tag_tid=14, entry_tid=52755]]], state=connected] 2026-02-18T08:17:31.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=229, tag_tid=14, entry_tid=46861], [object_number=228, tag_tid=14, entry_tid=46860], [object_number=231, tag_tid=14, entry_tid=46859], [object_number=230, tag_tid=14, entry_tid=46858]]], state=connected] 2026-02-18T08:17:31.702 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:31.704 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=229, tag_tid=14, entry_tid=46861' 2026-02-18T08:17:31.704 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=233, tag_tid=15, entry_tid=1' -a 'object_number=233, tag_tid=15, entry_tid=1' = 'object_number=229, tag_tid=14, entry_tid=46861' 2026-02-18T08:17:31.704 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=229, tag_tid=14, entry_tid=46861' '!=' 'object_number=228, tag_tid=14, entry_tid=46484' 2026-02-18T08:17:31.704 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:17:31.704 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=229, tag_tid=14, entry_tid=46861' = 'object_number=228, tag_tid=14, entry_tid=46484' 2026-02-18T08:17:31.704 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=229, tag_tid=14, entry_tid=46861' 2026-02-18T08:17:31.704 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=233, tag_tid=15, entry_tid=1' 2026-02-18T08:17:31.704 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:17:31.705 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:17:31.706 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=15 2026-02-18T08:17:31.707 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=229, tag_tid=14, entry_tid=46861' 2026-02-18T08:17:31.707 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:17:31.707 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:17:31.708 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=14 2026-02-18T08:17:31.709 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=233, tag_tid=15, entry_tid=1' 2026-02-18T08:17:31.709 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:17:31.709 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:17:31.711 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:17:31.711 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=229, tag_tid=14, entry_tid=46861' 2026-02-18T08:17:31.711 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:17:31.711 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:17:31.713 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=46861 2026-02-18T08:17:31.713 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 15 = 14 -a 1 -le 46861 2026-02-18T08:17:31.713 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:17:31.713 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:17:31.713 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:17:31.915 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:17:31.916 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:17:31.916 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:17:31.916 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:17:31.916 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:17:31.916 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:17:31.916 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:17:31.916 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:17:31.916 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:17:31.916 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:31.916 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:17:31.917 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:17:31.917 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:17:31.917 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:17:31.917 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:17:31.917 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:17:31.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:17:31.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:17:31.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:17:31.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:17:31.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:31.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:17:31.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:17:31.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:17:31.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:17:31.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:17:31.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:17:31.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:17:31.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:17:31.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:17:31.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:31.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:17:31.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:17:31.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:17:31.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:17:31.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:17:31.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:17:31.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:17:31.922 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:17:31.922 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:17:31.922 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:17:31.922 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:31.922 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:17:31.922 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:17:31.922 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:17:31.922 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:17:31.922 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:17:31.922 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:17:31.922 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:17:31.929 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:17:31.930 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:17:31.930 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:17:31.930 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:17:31.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:17:31.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:17:31.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:17:31.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:17:31.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:17:31.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:17:31.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:31.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:17:31.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:17:31.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:17:31.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:17:31.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:17:31.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:17:31.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:17:31.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:17:31.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:17:31.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:31.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:17:31.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:17:31.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:17:31.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:17:31.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:17:31.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:17:31.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:17:31.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:17:31.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:17:31.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:17:31.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:31.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:17:31.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:17:31.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:17:31.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:17:31.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:17:31.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:17:31.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:17:31.992 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:17:31.992 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:17:31.992 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:17:31.992 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:17:32.055 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:17:32.055 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:17:32.056 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:17:32.058 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:17:32.058 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:17:32.058 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:17:32.059 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:32.059 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:17:32.059 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:17:32.059 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:17:32.059 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:17:32.059 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:17:32.059 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:17:32.059 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:17:32.059 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:17:32.059 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:17:32.059 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:32.059 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:17:32.060 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:17:32.060 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:17:32.060 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:17:32.060 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:17:32.060 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:17:32.060 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:17:32.060 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:17:32.060 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:17:32.060 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:17:32.060 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:32.060 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:17:32.060 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:17:32.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:17:32.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:17:32.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:17:32.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:17:32.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:17:32.068 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:17:32.069 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:17:32.069 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:17:32.069 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:17:32.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:17:32.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:17:32.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:17:32.131 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:17:32.131 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:17:32.131 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:17:32.131 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:32.131 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:17:32.131 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:17:32.131 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:17:32.131 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:17:32.131 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:17:32.132 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:17:32.132 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:17:32.132 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:17:32.132 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:17:32.132 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:32.132 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:17:32.132 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:17:32.132 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:17:32.132 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:17:32.132 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:17:32.132 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:17:32.132 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:17:32.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:17:32.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:17:32.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:17:32.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:32.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:17:32.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:17:32.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:17:32.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:17:32.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:17:32.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:17:32.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:17:32.142 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:17:32.142 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:17:32.142 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:17:32.142 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:17:32.212 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:17:32.213 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:17:32.213 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:17:32.213 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:17:32.214 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:17:32.214 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:17:32.214 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:17:32.214 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:17:32.214 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:17:32.214 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:17:32.214 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:17:32.214 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:17:32.214 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:17:32.215 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:32.215 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:17:32.215 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:32.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 57 2026-02-18T08:17:32.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 58 2026-02-18T08:17:32.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:17:32.244 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=233, tag_tid=15, entry_tid=1], [object_number=232, tag_tid=15, entry_tid=0], [object_number=234, tag_tid=14, entry_tid=52758], [object_number=235, tag_tid=14, entry_tid=52755]]], state=connected] 2026-02-18T08:17:32.244 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=230, tag_tid=14, entry_tid=47234], [object_number=229, tag_tid=14, entry_tid=47233], [object_number=228, tag_tid=14, entry_tid=47232], [object_number=231, tag_tid=14, entry_tid=47231]]], state=connected] 2026-02-18T08:17:32.246 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:32.247 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=233, tag_tid=15, entry_tid=1' 2026-02-18T08:17:32.249 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:17:32.249 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:17:32.249 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:17:32.249 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:17:32.249 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:17:32.249 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:17:32.250 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:17:32.250 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:17:32.250 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:17:32.250 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:17:32.250 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:17:32.250 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:17:32.250 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:32.251 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:17:32.251 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:32.277 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 57 2026-02-18T08:17:32.277 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 58 2026-02-18T08:17:32.277 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:17:32.277 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=233, tag_tid=15, entry_tid=1], [object_number=232, tag_tid=15, entry_tid=0], [object_number=234, tag_tid=14, entry_tid=52758], [object_number=235, tag_tid=14, entry_tid=52755]]], state=connected] 2026-02-18T08:17:32.277 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=230, tag_tid=14, entry_tid=47234], [object_number=229, tag_tid=14, entry_tid=47233], [object_number=228, tag_tid=14, entry_tid=47232], [object_number=231, tag_tid=14, entry_tid=47231]]], state=connected] 2026-02-18T08:17:32.279 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:32.281 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=230, tag_tid=14, entry_tid=47234' 2026-02-18T08:17:32.281 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=233, tag_tid=15, entry_tid=1' -a 'object_number=233, tag_tid=15, entry_tid=1' = 'object_number=230, tag_tid=14, entry_tid=47234' 2026-02-18T08:17:32.281 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=230, tag_tid=14, entry_tid=47234' '!=' 'object_number=229, tag_tid=14, entry_tid=46861' 2026-02-18T08:17:32.281 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:17:32.281 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=230, tag_tid=14, entry_tid=47234' = 'object_number=229, tag_tid=14, entry_tid=46861' 2026-02-18T08:17:32.281 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=230, tag_tid=14, entry_tid=47234' 2026-02-18T08:17:32.282 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=233, tag_tid=15, entry_tid=1' 2026-02-18T08:17:32.282 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:17:32.282 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:17:32.283 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=15 2026-02-18T08:17:32.284 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=230, tag_tid=14, entry_tid=47234' 2026-02-18T08:17:32.284 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:17:32.284 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:17:32.285 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=14 2026-02-18T08:17:32.286 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=233, tag_tid=15, entry_tid=1' 2026-02-18T08:17:32.286 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:17:32.287 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:17:32.288 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:17:32.288 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=230, tag_tid=14, entry_tid=47234' 2026-02-18T08:17:32.289 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:17:32.289 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:17:32.290 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=47234 2026-02-18T08:17:32.290 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 15 = 14 -a 1 -le 47234 2026-02-18T08:17:32.290 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:17:32.290 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:17:32.290 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:17:32.492 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:17:32.492 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:17:32.492 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:17:32.492 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:17:32.492 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:17:32.492 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:17:32.492 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:17:32.492 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:17:32.492 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:17:32.492 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:32.493 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:17:32.493 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:17:32.493 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:17:32.493 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:17:32.493 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:17:32.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:17:32.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:17:32.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:17:32.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:17:32.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:17:32.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:32.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:17:32.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:17:32.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:17:32.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:17:32.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:17:32.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:17:32.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:17:32.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:17:32.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:17:32.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:32.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:17:32.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:17:32.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:17:32.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:17:32.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:17:32.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:17:32.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:17:32.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:17:32.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:17:32.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:17:32.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:32.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:17:32.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:17:32.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:17:32.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:17:32.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:17:32.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:17:32.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:17:32.506 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:17:32.506 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:17:32.507 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:17:32.507 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:17:32.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:17:32.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:17:32.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:17:32.558 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:17:32.558 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:17:32.558 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:17:32.558 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:32.558 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:17:32.558 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:17:32.558 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:17:32.558 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:17:32.558 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:17:32.558 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:17:32.558 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:17:32.559 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:17:32.559 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:17:32.559 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:32.559 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:17:32.559 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:17:32.559 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:17:32.559 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:17:32.559 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:17:32.559 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:17:32.559 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:17:32.559 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:17:32.559 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:17:32.560 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:17:32.560 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:32.560 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:17:32.560 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:17:32.560 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:17:32.560 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:17:32.560 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:17:32.560 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:17:32.560 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:17:32.569 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:17:32.569 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:17:32.570 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:17:32.570 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:17:32.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:17:32.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:17:32.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:17:32.628 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:17:32.628 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:17:32.628 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:17:32.629 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:32.629 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:17:32.629 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:17:32.629 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:17:32.629 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:17:32.629 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:17:32.629 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:17:32.629 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:17:32.629 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:17:32.629 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:17:32.629 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:32.629 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:17:32.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:17:32.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:17:32.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:17:32.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:17:32.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:17:32.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:17:32.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:17:32.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:17:32.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:17:32.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:32.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:17:32.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:17:32.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:17:32.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:17:32.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:17:32.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:17:32.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:17:32.639 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:17:32.640 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:17:32.640 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:17:32.640 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:17:32.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:17:32.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:17:32.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:17:32.693 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:17:32.693 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:17:32.693 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:17:32.694 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:32.694 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:17:32.694 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:17:32.694 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:17:32.694 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:17:32.694 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:17:32.694 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:17:32.694 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:17:32.694 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:17:32.694 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:17:32.694 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:32.694 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:17:32.695 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:17:32.695 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:17:32.695 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:17:32.695 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:17:32.695 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:17:32.695 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:17:32.695 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:17:32.695 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:17:32.695 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:17:32.695 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:32.695 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:17:32.695 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:17:32.696 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:17:32.696 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:17:32.696 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:17:32.696 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:17:32.696 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:17:32.704 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:17:32.705 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:17:32.705 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:17:32.705 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:17:32.766 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:17:32.767 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:17:32.767 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:17:32.767 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:17:32.767 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:17:32.768 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:17:32.768 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:17:32.768 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:17:32.768 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:17:32.768 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:17:32.768 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:17:32.769 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:17:32.769 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:17:32.769 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:32.770 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:17:32.770 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:32.799 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 57 2026-02-18T08:17:32.799 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 58 2026-02-18T08:17:32.799 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:17:32.799 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=233, tag_tid=15, entry_tid=1], [object_number=232, tag_tid=15, entry_tid=0], [object_number=234, tag_tid=14, entry_tid=52758], [object_number=235, tag_tid=14, entry_tid=52755]]], state=connected] 2026-02-18T08:17:32.799 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=230, tag_tid=14, entry_tid=47594], [object_number=229, tag_tid=14, entry_tid=47593], [object_number=228, tag_tid=14, entry_tid=47592], [object_number=231, tag_tid=14, entry_tid=47591]]], state=connected] 2026-02-18T08:17:32.802 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:32.804 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=233, tag_tid=15, entry_tid=1' 2026-02-18T08:17:32.805 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:17:32.805 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:17:32.806 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:17:32.806 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:17:32.806 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:17:32.806 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:17:32.806 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:17:32.806 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:17:32.806 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:17:32.806 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:17:32.806 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:17:32.806 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:17:32.807 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:32.807 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:17:32.807 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:32.842 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 57 2026-02-18T08:17:32.842 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 58 2026-02-18T08:17:32.842 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:17:32.842 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=233, tag_tid=15, entry_tid=1], [object_number=232, tag_tid=15, entry_tid=0], [object_number=234, tag_tid=14, entry_tid=52758], [object_number=235, tag_tid=14, entry_tid=52755]]], state=connected] 2026-02-18T08:17:32.842 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=230, tag_tid=14, entry_tid=47594], [object_number=229, tag_tid=14, entry_tid=47593], [object_number=228, tag_tid=14, entry_tid=47592], [object_number=231, tag_tid=14, entry_tid=47591]]], state=connected] 2026-02-18T08:17:32.846 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:32.848 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=230, tag_tid=14, entry_tid=47594' 2026-02-18T08:17:32.848 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=233, tag_tid=15, entry_tid=1' -a 'object_number=233, tag_tid=15, entry_tid=1' = 'object_number=230, tag_tid=14, entry_tid=47594' 2026-02-18T08:17:32.848 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=230, tag_tid=14, entry_tid=47594' '!=' 'object_number=230, tag_tid=14, entry_tid=47234' 2026-02-18T08:17:32.848 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:17:32.848 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=230, tag_tid=14, entry_tid=47594' = 'object_number=230, tag_tid=14, entry_tid=47234' 2026-02-18T08:17:32.848 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=230, tag_tid=14, entry_tid=47594' 2026-02-18T08:17:32.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=233, tag_tid=15, entry_tid=1' 2026-02-18T08:17:32.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:17:32.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:17:32.850 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=15 2026-02-18T08:17:32.851 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=230, tag_tid=14, entry_tid=47594' 2026-02-18T08:17:32.851 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:17:32.851 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:17:32.852 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=14 2026-02-18T08:17:32.853 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=233, tag_tid=15, entry_tid=1' 2026-02-18T08:17:32.853 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:17:32.853 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:17:32.854 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:17:32.855 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=230, tag_tid=14, entry_tid=47594' 2026-02-18T08:17:32.855 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:17:32.855 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:17:32.857 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=47594 2026-02-18T08:17:32.857 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 15 = 14 -a 1 -le 47594 2026-02-18T08:17:32.857 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:17:32.857 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:17:32.858 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:17:33.059 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:17:33.059 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:17:33.059 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:17:33.060 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:17:33.060 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:17:33.060 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:17:33.060 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:17:33.060 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:17:33.060 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:17:33.060 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:33.060 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:17:33.060 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:17:33.060 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:17:33.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:17:33.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:17:33.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:17:33.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:17:33.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:17:33.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:17:33.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:17:33.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:33.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:17:33.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:17:33.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:17:33.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:17:33.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:17:33.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:17:33.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:17:33.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:17:33.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:17:33.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:33.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:17:33.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:17:33.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:17:33.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:17:33.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:17:33.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:17:33.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:17:33.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:17:33.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:17:33.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:17:33.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:33.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:17:33.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:17:33.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:17:33.067 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:17:33.067 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:17:33.067 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:17:33.067 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:17:33.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:17:33.074 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:17:33.074 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:17:33.074 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:17:33.131 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:17:33.131 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:17:33.132 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:17:33.134 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:17:33.134 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:17:33.134 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:17:33.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:33.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:17:33.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:17:33.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:17:33.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:17:33.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:17:33.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:17:33.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:17:33.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:17:33.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:17:33.136 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:33.136 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:17:33.136 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:17:33.136 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:17:33.136 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:17:33.136 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:17:33.136 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:17:33.136 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:17:33.136 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:17:33.136 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:17:33.137 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:17:33.137 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:33.137 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:17:33.137 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:17:33.137 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:17:33.137 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:17:33.137 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:17:33.137 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:17:33.137 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:17:33.145 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:17:33.145 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:17:33.145 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:17:33.145 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:17:33.201 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:17:33.201 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:17:33.201 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:17:33.202 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:17:33.202 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:17:33.202 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:17:33.202 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:33.202 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:17:33.202 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:17:33.202 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:17:33.202 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:17:33.202 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:17:33.202 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:17:33.202 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:17:33.203 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:17:33.203 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:17:33.203 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:33.203 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:17:33.203 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:17:33.203 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:17:33.203 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:17:33.203 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:17:33.203 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:17:33.203 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:17:33.203 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:17:33.203 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:17:33.203 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:17:33.203 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:33.203 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:17:33.203 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:17:33.203 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:17:33.203 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:17:33.203 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:17:33.203 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:17:33.203 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:17:33.215 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:17:33.215 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:17:33.215 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:17:33.215 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:17:33.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:17:33.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:17:33.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:17:33.266 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:17:33.266 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:17:33.266 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:17:33.266 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:33.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:17:33.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:17:33.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:17:33.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:17:33.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:17:33.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:17:33.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:17:33.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:17:33.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:17:33.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:33.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:17:33.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:17:33.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:17:33.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:17:33.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:17:33.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:17:33.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:17:33.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:17:33.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:17:33.269 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:17:33.269 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:33.269 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:17:33.269 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:17:33.269 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:17:33.269 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:17:33.269 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:17:33.269 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:17:33.269 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:17:33.277 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:17:33.277 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:17:33.277 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:17:33.277 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:17:33.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:17:33.344 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:17:33.344 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:17:33.344 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:17:33.344 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:17:33.344 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:17:33.344 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:17:33.344 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:17:33.344 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:17:33.344 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:17:33.344 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:17:33.344 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:17:33.344 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:17:33.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:33.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:17:33.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:33.383 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 57 2026-02-18T08:17:33.383 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 58 2026-02-18T08:17:33.383 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:17:33.383 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=233, tag_tid=15, entry_tid=1], [object_number=232, tag_tid=15, entry_tid=0], [object_number=234, tag_tid=14, entry_tid=52758], [object_number=235, tag_tid=14, entry_tid=52755]]], state=connected] 2026-02-18T08:17:33.383 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=230, tag_tid=14, entry_tid=47982], [object_number=229, tag_tid=14, entry_tid=47981], [object_number=228, tag_tid=14, entry_tid=47980], [object_number=231, tag_tid=14, entry_tid=47979]]], state=connected] 2026-02-18T08:17:33.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:33.388 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=233, tag_tid=15, entry_tid=1' 2026-02-18T08:17:33.390 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:17:33.390 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:17:33.390 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:17:33.390 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:17:33.390 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:17:33.390 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:17:33.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:17:33.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:17:33.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:17:33.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:17:33.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:17:33.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:17:33.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:33.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:17:33.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:33.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 57 2026-02-18T08:17:33.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 58 2026-02-18T08:17:33.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:17:33.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=233, tag_tid=15, entry_tid=1], [object_number=232, tag_tid=15, entry_tid=0], [object_number=234, tag_tid=14, entry_tid=52758], [object_number=235, tag_tid=14, entry_tid=52755]]], state=connected] 2026-02-18T08:17:33.427 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=230, tag_tid=14, entry_tid=47982], [object_number=229, tag_tid=14, entry_tid=47981], [object_number=228, tag_tid=14, entry_tid=47980], [object_number=231, tag_tid=14, entry_tid=47979]]], state=connected] 2026-02-18T08:17:33.429 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:33.431 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=230, tag_tid=14, entry_tid=47982' 2026-02-18T08:17:33.431 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=233, tag_tid=15, entry_tid=1' -a 'object_number=233, tag_tid=15, entry_tid=1' = 'object_number=230, tag_tid=14, entry_tid=47982' 2026-02-18T08:17:33.431 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=230, tag_tid=14, entry_tid=47982' '!=' 'object_number=230, tag_tid=14, entry_tid=47594' 2026-02-18T08:17:33.431 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:17:33.432 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=230, tag_tid=14, entry_tid=47982' = 'object_number=230, tag_tid=14, entry_tid=47594' 2026-02-18T08:17:33.432 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=230, tag_tid=14, entry_tid=47982' 2026-02-18T08:17:33.432 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=233, tag_tid=15, entry_tid=1' 2026-02-18T08:17:33.432 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:17:33.433 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:17:33.433 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=15 2026-02-18T08:17:33.434 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=230, tag_tid=14, entry_tid=47982' 2026-02-18T08:17:33.434 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:17:33.434 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:17:33.435 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=14 2026-02-18T08:17:33.436 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=233, tag_tid=15, entry_tid=1' 2026-02-18T08:17:33.436 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:17:33.437 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:17:33.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:17:33.439 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=230, tag_tid=14, entry_tid=47982' 2026-02-18T08:17:33.439 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:17:33.439 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:17:33.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=47982 2026-02-18T08:17:33.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 15 = 14 -a 1 -le 47982 2026-02-18T08:17:33.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:17:33.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:17:33.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:17:33.642 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:17:33.643 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:17:33.643 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:17:33.643 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:17:33.643 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:17:33.643 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:17:33.643 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:17:33.643 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:17:33.643 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:17:33.643 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:33.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:17:33.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:17:33.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:17:33.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:17:33.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:17:33.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:17:33.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:17:33.646 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:17:33.647 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:17:33.647 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:17:33.647 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:33.647 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:17:33.647 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:17:33.647 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:17:33.647 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:17:33.647 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:17:33.647 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:17:33.647 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:17:33.648 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:17:33.648 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:17:33.648 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:33.648 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:17:33.648 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:17:33.648 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:17:33.648 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:17:33.648 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:17:33.648 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:17:33.648 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:17:33.648 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:17:33.648 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:17:33.649 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:17:33.649 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:33.649 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:17:33.649 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:17:33.649 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:17:33.649 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:17:33.649 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:17:33.649 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:17:33.649 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:17:33.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:17:33.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:17:33.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:17:33.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:17:33.708 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:17:33.708 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:17:33.708 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:17:33.711 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:17:33.711 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:17:33.711 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:17:33.711 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:33.711 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:17:33.711 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:17:33.711 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:17:33.711 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:17:33.712 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:17:33.712 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:17:33.712 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:17:33.712 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:17:33.712 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:17:33.712 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:33.712 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:17:33.712 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:17:33.712 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:17:33.712 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:17:33.712 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:17:33.712 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:17:33.713 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:17:33.713 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:17:33.713 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:17:33.713 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:17:33.713 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:33.713 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:17:33.713 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:17:33.713 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:17:33.713 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:17:33.713 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:17:33.713 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:17:33.713 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:17:33.721 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:17:33.721 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:17:33.721 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:17:33.721 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:17:33.771 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:17:33.771 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:17:33.771 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:17:33.774 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:17:33.774 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:17:33.774 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:17:33.774 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:33.774 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:17:33.774 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:17:33.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:17:33.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:17:33.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:17:33.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:17:33.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:17:33.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:17:33.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:17:33.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:33.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:17:33.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:17:33.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:17:33.776 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:17:33.776 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:17:33.776 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:17:33.776 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:17:33.776 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:17:33.776 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:17:33.776 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:17:33.776 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:33.776 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:17:33.776 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:17:33.776 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:17:33.776 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:17:33.777 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:17:33.777 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:17:33.777 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:17:33.784 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:17:33.785 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:17:33.785 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:17:33.785 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:17:33.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:17:33.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:17:33.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:17:33.837 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:17:33.837 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:17:33.837 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:17:33.838 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:33.838 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:17:33.838 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:17:33.838 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:17:33.838 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:17:33.838 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:17:33.838 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:17:33.838 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:17:33.838 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:17:33.838 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:17:33.838 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:33.838 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:17:33.839 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:17:33.839 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:17:33.839 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:17:33.839 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:17:33.839 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:17:33.839 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:17:33.839 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:17:33.839 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:17:33.839 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:17:33.839 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:33.839 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:17:33.839 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:17:33.840 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:17:33.840 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:17:33.840 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:17:33.840 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:17:33.840 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:17:33.848 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:17:33.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:17:33.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:17:33.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:17:33.913 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:17:33.915 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:17:33.915 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:17:33.915 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:17:33.915 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:17:33.915 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:17:33.915 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:17:33.915 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:17:33.915 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:17:33.916 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:17:33.916 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:17:33.916 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:17:33.916 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:17:33.917 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:33.917 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:17:33.917 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:33.948 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 57 2026-02-18T08:17:33.948 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 58 2026-02-18T08:17:33.948 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:17:33.948 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=233, tag_tid=15, entry_tid=1], [object_number=232, tag_tid=15, entry_tid=0], [object_number=234, tag_tid=14, entry_tid=52758], [object_number=235, tag_tid=14, entry_tid=52755]]], state=connected] 2026-02-18T08:17:33.948 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=231, tag_tid=14, entry_tid=48351], [object_number=230, tag_tid=14, entry_tid=48350], [object_number=229, tag_tid=14, entry_tid=48349], [object_number=228, tag_tid=14, entry_tid=48348]]], state=connected] 2026-02-18T08:17:33.950 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:33.952 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=233, tag_tid=15, entry_tid=1' 2026-02-18T08:17:33.953 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:17:33.953 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:17:33.953 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:17:33.953 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:17:33.953 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:17:33.953 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:17:33.953 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:17:33.953 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:17:33.953 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:17:33.954 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:17:33.954 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:17:33.954 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:17:33.955 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:33.955 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:17:33.955 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:33.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 57 2026-02-18T08:17:33.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 58 2026-02-18T08:17:33.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:17:33.982 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=233, tag_tid=15, entry_tid=1], [object_number=232, tag_tid=15, entry_tid=0], [object_number=234, tag_tid=14, entry_tid=52758], [object_number=235, tag_tid=14, entry_tid=52755]]], state=connected] 2026-02-18T08:17:33.982 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=231, tag_tid=14, entry_tid=48351], [object_number=230, tag_tid=14, entry_tid=48350], [object_number=229, tag_tid=14, entry_tid=48349], [object_number=228, tag_tid=14, entry_tid=48348]]], state=connected] 2026-02-18T08:17:33.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:33.986 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=231, tag_tid=14, entry_tid=48351' 2026-02-18T08:17:33.986 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=233, tag_tid=15, entry_tid=1' -a 'object_number=233, tag_tid=15, entry_tid=1' = 'object_number=231, tag_tid=14, entry_tid=48351' 2026-02-18T08:17:33.986 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=231, tag_tid=14, entry_tid=48351' '!=' 'object_number=230, tag_tid=14, entry_tid=47982' 2026-02-18T08:17:33.986 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:17:33.986 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=231, tag_tid=14, entry_tid=48351' = 'object_number=230, tag_tid=14, entry_tid=47982' 2026-02-18T08:17:33.986 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=231, tag_tid=14, entry_tid=48351' 2026-02-18T08:17:33.987 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=233, tag_tid=15, entry_tid=1' 2026-02-18T08:17:33.987 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:17:33.987 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:17:33.988 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=15 2026-02-18T08:17:33.988 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=231, tag_tid=14, entry_tid=48351' 2026-02-18T08:17:33.989 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:17:33.989 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:17:33.990 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=14 2026-02-18T08:17:33.990 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=233, tag_tid=15, entry_tid=1' 2026-02-18T08:17:33.990 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:17:33.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:17:33.992 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:17:33.993 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=231, tag_tid=14, entry_tid=48351' 2026-02-18T08:17:33.993 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:17:33.993 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:17:33.994 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=48351 2026-02-18T08:17:33.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 15 = 14 -a 1 -le 48351 2026-02-18T08:17:33.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:17:33.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:17:33.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:17:34.196 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:17:34.197 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:17:34.197 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:17:34.197 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:17:34.197 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:17:34.197 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:17:34.197 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:17:34.197 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:17:34.197 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:17:34.197 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:34.198 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:17:34.198 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:17:34.198 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:17:34.198 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:17:34.198 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:17:34.198 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:17:34.201 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:17:34.201 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:17:34.201 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:17:34.201 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:17:34.201 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:34.201 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:17:34.201 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:17:34.201 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:17:34.201 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:17:34.201 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:17:34.201 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:17:34.202 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:17:34.202 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:17:34.202 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:17:34.202 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:34.202 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:17:34.202 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:17:34.202 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:17:34.202 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:17:34.202 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:17:34.202 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:17:34.202 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:17:34.202 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:17:34.203 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:17:34.203 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:17:34.203 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:34.203 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:17:34.203 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:17:34.203 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:17:34.203 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:17:34.203 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:17:34.203 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:17:34.203 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:17:34.210 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:17:34.210 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:17:34.211 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:17:34.211 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:17:34.269 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:17:34.269 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:17:34.269 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:17:34.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:17:34.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:17:34.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:17:34.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:34.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:17:34.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:17:34.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:17:34.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:17:34.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:17:34.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:17:34.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:17:34.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:17:34.273 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:17:34.273 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:34.273 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:17:34.273 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:17:34.273 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:17:34.273 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:17:34.273 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:17:34.273 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:17:34.273 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:17:34.273 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:17:34.273 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:17:34.274 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:17:34.274 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:34.274 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:17:34.274 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:17:34.274 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:17:34.274 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:17:34.274 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:17:34.274 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:17:34.274 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:17:34.282 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:17:34.282 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:17:34.282 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:17:34.283 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:17:34.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:17:34.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:17:34.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:17:34.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:17:34.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:17:34.344 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:17:34.344 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:34.344 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:17:34.344 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:17:34.344 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:17:34.344 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:17:34.344 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:17:34.344 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:17:34.344 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:17:34.344 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:17:34.344 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:17:34.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:34.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:17:34.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:17:34.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:17:34.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:17:34.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:17:34.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:17:34.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:17:34.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:17:34.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:17:34.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:17:34.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:34.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:17:34.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:17:34.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:17:34.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:17:34.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:17:34.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:17:34.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:17:34.354 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:17:34.354 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:17:34.354 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:17:34.354 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:17:34.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:17:34.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:17:34.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:17:34.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:17:34.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:17:34.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:17:34.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:34.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:17:34.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:17:34.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:17:34.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:17:34.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:17:34.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:17:34.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:17:34.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:17:34.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:17:34.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:34.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:17:34.405 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:17:34.405 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:17:34.405 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:17:34.405 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:17:34.405 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:17:34.405 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:17:34.405 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:17:34.405 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:17:34.405 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:17:34.405 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:34.405 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:17:34.405 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:17:34.405 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:17:34.405 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:17:34.405 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:17:34.405 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:17:34.405 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:17:34.416 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:17:34.416 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:17:34.416 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:17:34.416 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:17:34.483 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:17:34.484 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:17:34.485 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:17:34.485 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:17:34.485 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:17:34.485 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:17:34.485 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:17:34.485 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:17:34.485 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:17:34.485 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:17:34.485 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:17:34.485 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:17:34.485 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:17:34.486 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:34.486 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:17:34.487 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:34.514 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 57 2026-02-18T08:17:34.514 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 58 2026-02-18T08:17:34.514 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:17:34.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=233, tag_tid=15, entry_tid=1], [object_number=232, tag_tid=15, entry_tid=0], [object_number=234, tag_tid=14, entry_tid=52758], [object_number=235, tag_tid=14, entry_tid=52755]]], state=connected] 2026-02-18T08:17:34.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=228, tag_tid=14, entry_tid=48712], [object_number=231, tag_tid=14, entry_tid=48711], [object_number=230, tag_tid=14, entry_tid=48710], [object_number=229, tag_tid=14, entry_tid=48709]]], state=connected] 2026-02-18T08:17:34.516 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:34.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=233, tag_tid=15, entry_tid=1' 2026-02-18T08:17:34.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:17:34.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:17:34.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:17:34.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:17:34.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:17:34.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:17:34.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:17:34.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:17:34.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:17:34.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:17:34.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:17:34.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:17:34.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:34.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:17:34.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:34.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 57 2026-02-18T08:17:34.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 58 2026-02-18T08:17:34.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:17:34.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=233, tag_tid=15, entry_tid=1], [object_number=232, tag_tid=15, entry_tid=0], [object_number=234, tag_tid=14, entry_tid=52758], [object_number=235, tag_tid=14, entry_tid=52755]]], state=connected] 2026-02-18T08:17:34.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=228, tag_tid=14, entry_tid=48712], [object_number=231, tag_tid=14, entry_tid=48711], [object_number=230, tag_tid=14, entry_tid=48710], [object_number=229, tag_tid=14, entry_tid=48709]]], state=connected] 2026-02-18T08:17:34.559 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:34.561 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=228, tag_tid=14, entry_tid=48712' 2026-02-18T08:17:34.561 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=233, tag_tid=15, entry_tid=1' -a 'object_number=233, tag_tid=15, entry_tid=1' = 'object_number=228, tag_tid=14, entry_tid=48712' 2026-02-18T08:17:34.562 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=228, tag_tid=14, entry_tid=48712' '!=' 'object_number=231, tag_tid=14, entry_tid=48351' 2026-02-18T08:17:34.562 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:17:34.562 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=228, tag_tid=14, entry_tid=48712' = 'object_number=231, tag_tid=14, entry_tid=48351' 2026-02-18T08:17:34.562 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=228, tag_tid=14, entry_tid=48712' 2026-02-18T08:17:34.562 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=233, tag_tid=15, entry_tid=1' 2026-02-18T08:17:34.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:17:34.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:17:34.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=15 2026-02-18T08:17:34.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=228, tag_tid=14, entry_tid=48712' 2026-02-18T08:17:34.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:17:34.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:17:34.566 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=14 2026-02-18T08:17:34.567 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=233, tag_tid=15, entry_tid=1' 2026-02-18T08:17:34.567 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:17:34.567 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:17:34.568 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:17:34.569 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=228, tag_tid=14, entry_tid=48712' 2026-02-18T08:17:34.569 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:17:34.569 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:17:34.570 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=48712 2026-02-18T08:17:34.571 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 15 = 14 -a 1 -le 48712 2026-02-18T08:17:34.571 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:17:34.571 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:17:34.571 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:17:34.772 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:17:34.772 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:17:34.773 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:17:34.773 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:17:34.773 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:17:34.773 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:17:34.773 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:17:34.773 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:17:34.773 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:17:34.773 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:34.773 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:17:34.773 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:17:34.773 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:17:34.774 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:17:34.774 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:17:34.774 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:17:34.776 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:17:34.776 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:17:34.776 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:17:34.777 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:17:34.777 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:34.777 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:17:34.777 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:17:34.777 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:17:34.777 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:17:34.777 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:17:34.777 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:17:34.777 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:17:34.777 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:17:34.777 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:17:34.778 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:34.778 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:17:34.778 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:17:34.778 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:17:34.778 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:17:34.778 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:17:34.778 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:17:34.778 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:17:34.778 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:17:34.778 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:17:34.778 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:17:34.778 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:34.779 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:17:34.779 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:17:34.779 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:17:34.779 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:17:34.779 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:17:34.779 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:17:34.779 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:17:34.786 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:17:34.786 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:17:34.786 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:17:34.786 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:17:34.846 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:17:34.846 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:17:34.846 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:17:34.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:17:34.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:17:34.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:17:34.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:34.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:17:34.850 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:17:34.850 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:17:34.850 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:17:34.850 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:17:34.850 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:17:34.850 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:17:34.850 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:17:34.850 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:17:34.850 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:34.850 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:17:34.850 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:17:34.851 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:17:34.851 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:17:34.851 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:17:34.851 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:17:34.851 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:17:34.851 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:17:34.851 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:17:34.851 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:17:34.851 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:34.851 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:17:34.851 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:17:34.851 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:17:34.852 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:17:34.852 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:17:34.852 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:17:34.852 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:17:34.860 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:17:34.860 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:17:34.861 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:17:34.861 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:17:34.909 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:17:34.910 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:17:34.910 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:17:34.912 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:17:34.913 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:17:34.913 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:17:34.913 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:34.913 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:17:34.913 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:17:34.913 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:17:34.913 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:17:34.913 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:17:34.913 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:17:34.913 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:17:34.914 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:17:34.914 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:17:34.914 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:34.914 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:17:34.914 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:17:34.914 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:17:34.914 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:17:34.914 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:17:34.914 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:17:34.914 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:17:34.914 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:17:34.914 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:17:34.915 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:17:34.915 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:34.915 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:17:34.915 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:17:34.915 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:17:34.915 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:17:34.915 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:17:34.915 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:17:34.915 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:17:34.923 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:17:34.924 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:17:34.924 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:17:34.924 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:17:34.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:17:34.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:17:34.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:17:34.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:17:34.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:17:34.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:17:34.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:34.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:17:34.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:17:34.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:17:34.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:17:34.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:17:34.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:17:34.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:17:34.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:17:34.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:17:34.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:34.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:17:34.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:17:34.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:17:34.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:17:34.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:17:34.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:17:34.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:17:34.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:17:34.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:17:34.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:17:34.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:34.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:17:34.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:17:34.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:17:34.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:17:34.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:17:34.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:17:34.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:17:34.985 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:17:34.986 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:17:34.986 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:17:34.986 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:17:35.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:17:35.055 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:17:35.055 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:17:35.055 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:17:35.055 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:17:35.055 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:17:35.056 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:17:35.056 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:17:35.056 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:17:35.056 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:17:35.056 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:17:35.056 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:17:35.056 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:17:35.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:35.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:17:35.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:35.086 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 57 2026-02-18T08:17:35.086 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 58 2026-02-18T08:17:35.086 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:17:35.086 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=233, tag_tid=15, entry_tid=1], [object_number=232, tag_tid=15, entry_tid=0], [object_number=234, tag_tid=14, entry_tid=52758], [object_number=235, tag_tid=14, entry_tid=52755]]], state=connected] 2026-02-18T08:17:35.086 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=230, tag_tid=14, entry_tid=49050], [object_number=229, tag_tid=14, entry_tid=49049], [object_number=228, tag_tid=14, entry_tid=49048], [object_number=231, tag_tid=14, entry_tid=49047]]], state=connected] 2026-02-18T08:17:35.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:35.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=233, tag_tid=15, entry_tid=1' 2026-02-18T08:17:35.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:17:35.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:17:35.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:17:35.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:17:35.092 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:17:35.092 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:17:35.092 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:17:35.092 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:17:35.092 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:17:35.092 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:17:35.093 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:17:35.093 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:17:35.093 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:35.093 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:17:35.093 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:35.121 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 57 2026-02-18T08:17:35.121 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 58 2026-02-18T08:17:35.121 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:17:35.121 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=233, tag_tid=15, entry_tid=1], [object_number=232, tag_tid=15, entry_tid=0], [object_number=234, tag_tid=14, entry_tid=52758], [object_number=235, tag_tid=14, entry_tid=52755]]], state=connected] 2026-02-18T08:17:35.121 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=230, tag_tid=14, entry_tid=49050], [object_number=229, tag_tid=14, entry_tid=49049], [object_number=228, tag_tid=14, entry_tid=49048], [object_number=231, tag_tid=14, entry_tid=49047]]], state=connected] 2026-02-18T08:17:35.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:35.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=230, tag_tid=14, entry_tid=49050' 2026-02-18T08:17:35.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=233, tag_tid=15, entry_tid=1' -a 'object_number=233, tag_tid=15, entry_tid=1' = 'object_number=230, tag_tid=14, entry_tid=49050' 2026-02-18T08:17:35.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=230, tag_tid=14, entry_tid=49050' '!=' 'object_number=228, tag_tid=14, entry_tid=48712' 2026-02-18T08:17:35.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:17:35.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=230, tag_tid=14, entry_tid=49050' = 'object_number=228, tag_tid=14, entry_tid=48712' 2026-02-18T08:17:35.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=230, tag_tid=14, entry_tid=49050' 2026-02-18T08:17:35.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=233, tag_tid=15, entry_tid=1' 2026-02-18T08:17:35.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:17:35.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:17:35.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=15 2026-02-18T08:17:35.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=230, tag_tid=14, entry_tid=49050' 2026-02-18T08:17:35.129 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:17:35.129 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:17:35.130 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=14 2026-02-18T08:17:35.131 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=233, tag_tid=15, entry_tid=1' 2026-02-18T08:17:35.131 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:17:35.131 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:17:35.132 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:17:35.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=230, tag_tid=14, entry_tid=49050' 2026-02-18T08:17:35.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:17:35.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:17:35.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=49050 2026-02-18T08:17:35.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 15 = 14 -a 1 -le 49050 2026-02-18T08:17:35.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:17:35.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:17:35.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:17:35.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:17:35.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:17:35.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:17:35.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:17:35.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:17:35.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:17:35.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:17:35.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:17:35.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:17:35.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:35.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:17:35.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:17:35.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:17:35.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:17:35.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:17:35.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:17:35.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:17:35.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:17:35.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:17:35.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:17:35.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:35.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:17:35.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:17:35.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:17:35.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:17:35.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:17:35.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:17:35.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:17:35.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:17:35.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:17:35.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:35.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:17:35.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:17:35.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:17:35.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:17:35.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:17:35.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:17:35.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:17:35.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:17:35.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:17:35.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:17:35.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:35.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:17:35.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:17:35.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:17:35.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:17:35.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:17:35.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:17:35.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:17:35.351 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:17:35.352 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:17:35.352 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:17:35.352 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:17:35.409 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:17:35.409 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:17:35.409 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:17:35.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:17:35.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:17:35.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:17:35.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:35.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:17:35.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:17:35.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:17:35.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:17:35.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:17:35.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:17:35.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:17:35.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:17:35.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:17:35.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:35.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:17:35.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:17:35.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:17:35.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:17:35.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:17:35.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:17:35.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:17:35.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:17:35.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:17:35.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:17:35.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:35.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:17:35.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:17:35.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:17:35.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:17:35.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:17:35.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:17:35.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:17:35.423 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:17:35.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:17:35.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:17:35.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:17:35.481 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:17:35.481 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:17:35.481 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:17:35.484 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:17:35.484 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:17:35.484 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:17:35.484 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:35.484 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:17:35.484 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:17:35.484 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:17:35.484 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:17:35.484 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:17:35.485 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:17:35.485 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:17:35.485 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:17:35.485 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:17:35.485 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:35.485 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:17:35.485 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:17:35.485 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:17:35.485 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:17:35.485 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:17:35.485 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:17:35.485 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:17:35.486 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:17:35.486 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:17:35.486 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:17:35.486 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:35.486 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:17:35.486 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:17:35.486 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:17:35.486 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:17:35.486 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:17:35.486 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:17:35.486 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:17:35.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:17:35.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:17:35.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:17:35.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:17:35.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:17:35.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:17:35.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:17:35.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:17:35.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:17:35.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:17:35.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:35.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:17:35.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:17:35.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:17:35.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:17:35.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:17:35.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:17:35.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:17:35.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:17:35.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:17:35.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:35.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:17:35.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:17:35.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:17:35.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:17:35.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:17:35.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:17:35.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:17:35.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:17:35.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:17:35.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:17:35.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:35.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:17:35.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:17:35.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:17:35.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:17:35.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:17:35.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:17:35.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:17:35.557 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:17:35.558 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:17:35.558 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:17:35.558 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:17:35.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:17:35.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:17:35.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:17:35.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:17:35.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:17:35.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:17:35.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:17:35.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:17:35.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:17:35.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:17:35.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:17:35.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:17:35.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:17:35.628 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:35.628 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:17:35.628 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:35.658 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 57 2026-02-18T08:17:35.658 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 58 2026-02-18T08:17:35.658 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:17:35.658 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=233, tag_tid=15, entry_tid=1], [object_number=232, tag_tid=15, entry_tid=0], [object_number=234, tag_tid=14, entry_tid=52758], [object_number=235, tag_tid=14, entry_tid=52755]]], state=connected] 2026-02-18T08:17:35.658 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=231, tag_tid=14, entry_tid=49379], [object_number=230, tag_tid=14, entry_tid=49378], [object_number=229, tag_tid=14, entry_tid=49377], [object_number=228, tag_tid=14, entry_tid=49376]]], state=connected] 2026-02-18T08:17:35.661 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:35.662 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=233, tag_tid=15, entry_tid=1' 2026-02-18T08:17:35.664 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:17:35.664 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:17:35.664 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:17:35.664 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:17:35.664 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:17:35.664 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:17:35.664 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:17:35.664 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:17:35.664 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:17:35.664 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:17:35.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:17:35.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:17:35.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:35.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:17:35.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:35.702 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 57 2026-02-18T08:17:35.702 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 58 2026-02-18T08:17:35.702 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:17:35.702 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=233, tag_tid=15, entry_tid=1], [object_number=232, tag_tid=15, entry_tid=0], [object_number=234, tag_tid=14, entry_tid=52758], [object_number=235, tag_tid=14, entry_tid=52755]]], state=connected] 2026-02-18T08:17:35.702 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=231, tag_tid=14, entry_tid=49379], [object_number=230, tag_tid=14, entry_tid=49378], [object_number=229, tag_tid=14, entry_tid=49377], [object_number=228, tag_tid=14, entry_tid=49376]]], state=connected] 2026-02-18T08:17:35.706 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:35.708 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=231, tag_tid=14, entry_tid=49379' 2026-02-18T08:17:35.708 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=233, tag_tid=15, entry_tid=1' -a 'object_number=233, tag_tid=15, entry_tid=1' = 'object_number=231, tag_tid=14, entry_tid=49379' 2026-02-18T08:17:35.708 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=231, tag_tid=14, entry_tid=49379' '!=' 'object_number=230, tag_tid=14, entry_tid=49050' 2026-02-18T08:17:35.708 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:17:35.708 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=231, tag_tid=14, entry_tid=49379' = 'object_number=230, tag_tid=14, entry_tid=49050' 2026-02-18T08:17:35.708 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=231, tag_tid=14, entry_tid=49379' 2026-02-18T08:17:35.709 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=233, tag_tid=15, entry_tid=1' 2026-02-18T08:17:35.709 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:17:35.709 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:17:35.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=15 2026-02-18T08:17:35.711 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=231, tag_tid=14, entry_tid=49379' 2026-02-18T08:17:35.711 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:17:35.711 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:17:35.712 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=14 2026-02-18T08:17:35.713 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=233, tag_tid=15, entry_tid=1' 2026-02-18T08:17:35.713 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:17:35.713 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:17:35.714 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:17:35.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=231, tag_tid=14, entry_tid=49379' 2026-02-18T08:17:35.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:17:35.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:17:35.717 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=49379 2026-02-18T08:17:35.717 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 15 = 14 -a 1 -le 49379 2026-02-18T08:17:35.717 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:17:35.717 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:17:35.717 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:17:35.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:17:35.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:17:35.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:17:35.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:17:35.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:17:35.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:17:35.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:17:35.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:17:35.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:17:35.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:35.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:17:35.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:17:35.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:17:35.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:17:35.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:17:35.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:17:35.923 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:17:35.923 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:17:35.923 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:17:35.923 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:17:35.923 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:35.923 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:17:35.923 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:17:35.923 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:17:35.923 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:17:35.924 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:17:35.924 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:17:35.924 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:17:35.924 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:17:35.924 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:17:35.924 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:35.924 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:17:35.924 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:17:35.924 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:17:35.924 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:17:35.924 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:17:35.924 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:17:35.924 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:17:35.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:17:35.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:17:35.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:17:35.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:35.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:17:35.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:17:35.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:17:35.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:17:35.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:17:35.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:17:35.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:17:35.933 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:17:35.933 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:17:35.933 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:17:35.933 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:17:35.982 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:17:35.982 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:17:35.982 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:17:35.985 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:17:35.985 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:17:35.985 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:17:35.985 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:35.986 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:17:35.986 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:17:35.986 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:17:35.986 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:17:35.986 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:17:35.986 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:17:35.986 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:17:35.986 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:17:35.986 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:17:35.986 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:35.986 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:17:35.986 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:17:35.987 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:17:35.987 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:17:35.987 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:17:35.987 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:17:35.987 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:17:35.987 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:17:35.987 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:17:35.987 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:17:35.987 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:35.987 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:17:35.987 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:17:35.987 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:17:35.988 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:17:35.988 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:17:35.988 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:17:35.988 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:17:35.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:17:35.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:17:35.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:17:35.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:17:36.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:17:36.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:17:36.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:17:36.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:17:36.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:17:36.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:17:36.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:36.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:17:36.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:17:36.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:17:36.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:17:36.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:17:36.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:17:36.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:17:36.058 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:17:36.058 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:17:36.058 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:36.058 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:17:36.058 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:17:36.058 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:17:36.058 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:17:36.058 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:17:36.058 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:17:36.058 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:17:36.058 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:17:36.058 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:17:36.059 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:17:36.059 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:36.059 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:17:36.059 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:17:36.059 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:17:36.059 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:17:36.059 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:17:36.059 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:17:36.059 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:17:36.068 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:17:36.068 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:17:36.068 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:17:36.068 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:17:36.119 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:17:36.120 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:17:36.120 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:17:36.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:17:36.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:17:36.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:17:36.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:36.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:17:36.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:17:36.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:17:36.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:17:36.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:17:36.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:17:36.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:17:36.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:17:36.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:17:36.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:36.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:17:36.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:17:36.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:17:36.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:17:36.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:17:36.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:17:36.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:17:36.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:17:36.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:17:36.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:17:36.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:36.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:17:36.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:17:36.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:17:36.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:17:36.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:17:36.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:17:36.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:17:36.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:17:36.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:17:36.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:17:36.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:17:36.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:17:36.194 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:17:36.195 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:17:36.195 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:17:36.195 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:17:36.195 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:17:36.195 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:17:36.195 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:17:36.195 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:17:36.195 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:17:36.195 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:17:36.195 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:17:36.195 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:17:36.196 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:36.196 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:17:36.196 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:36.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 57 2026-02-18T08:17:36.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 58 2026-02-18T08:17:36.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:17:36.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=233, tag_tid=15, entry_tid=1], [object_number=232, tag_tid=15, entry_tid=0], [object_number=234, tag_tid=14, entry_tid=52758], [object_number=235, tag_tid=14, entry_tid=52755]]], state=connected] 2026-02-18T08:17:36.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=228, tag_tid=14, entry_tid=49708], [object_number=231, tag_tid=14, entry_tid=49707], [object_number=230, tag_tid=14, entry_tid=49706], [object_number=229, tag_tid=14, entry_tid=49705]]], state=connected] 2026-02-18T08:17:36.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:36.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=233, tag_tid=15, entry_tid=1' 2026-02-18T08:17:36.238 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:17:36.238 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:17:36.238 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:17:36.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:17:36.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:17:36.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:17:36.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:17:36.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:17:36.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:17:36.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:17:36.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:17:36.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:17:36.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:36.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:17:36.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:36.276 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 57 2026-02-18T08:17:36.276 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 58 2026-02-18T08:17:36.276 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:17:36.276 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=233, tag_tid=15, entry_tid=1], [object_number=232, tag_tid=15, entry_tid=0], [object_number=234, tag_tid=14, entry_tid=52758], [object_number=235, tag_tid=14, entry_tid=52755]]], state=connected] 2026-02-18T08:17:36.276 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=228, tag_tid=14, entry_tid=49708], [object_number=231, tag_tid=14, entry_tid=49707], [object_number=230, tag_tid=14, entry_tid=49706], [object_number=229, tag_tid=14, entry_tid=49705]]], state=connected] 2026-02-18T08:17:36.279 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:36.281 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=228, tag_tid=14, entry_tid=49708' 2026-02-18T08:17:36.281 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=233, tag_tid=15, entry_tid=1' -a 'object_number=233, tag_tid=15, entry_tid=1' = 'object_number=228, tag_tid=14, entry_tid=49708' 2026-02-18T08:17:36.281 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=228, tag_tid=14, entry_tid=49708' '!=' 'object_number=231, tag_tid=14, entry_tid=49379' 2026-02-18T08:17:36.281 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:17:36.281 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=228, tag_tid=14, entry_tid=49708' = 'object_number=231, tag_tid=14, entry_tid=49379' 2026-02-18T08:17:36.282 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=228, tag_tid=14, entry_tid=49708' 2026-02-18T08:17:36.282 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=233, tag_tid=15, entry_tid=1' 2026-02-18T08:17:36.282 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:17:36.282 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:17:36.283 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=15 2026-02-18T08:17:36.284 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=228, tag_tid=14, entry_tid=49708' 2026-02-18T08:17:36.284 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:17:36.284 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:17:36.286 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=14 2026-02-18T08:17:36.286 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=233, tag_tid=15, entry_tid=1' 2026-02-18T08:17:36.287 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:17:36.287 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:17:36.288 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:17:36.289 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=228, tag_tid=14, entry_tid=49708' 2026-02-18T08:17:36.289 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:17:36.289 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:17:36.290 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=49708 2026-02-18T08:17:36.290 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 15 = 14 -a 1 -le 49708 2026-02-18T08:17:36.291 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:17:36.291 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:17:36.291 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:17:36.492 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:17:36.493 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:17:36.493 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:17:36.493 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:17:36.493 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:17:36.493 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:17:36.493 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:17:36.493 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:17:36.493 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:17:36.493 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:36.493 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:17:36.493 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:17:36.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:17:36.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:17:36.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:17:36.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:17:36.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:17:36.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:17:36.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:17:36.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:17:36.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:36.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:17:36.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:17:36.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:17:36.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:17:36.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:17:36.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:17:36.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:17:36.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:17:36.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:17:36.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:36.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:17:36.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:17:36.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:17:36.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:17:36.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:17:36.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:17:36.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:17:36.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:17:36.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:17:36.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:17:36.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:36.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:17:36.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:17:36.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:17:36.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:17:36.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:17:36.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:17:36.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:17:36.506 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:17:36.507 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:17:36.507 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:17:36.507 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:17:36.557 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:17:36.557 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:17:36.558 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:17:36.560 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:17:36.560 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:17:36.560 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:17:36.560 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:36.561 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:17:36.561 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:17:36.561 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:17:36.561 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:17:36.561 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:17:36.561 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:17:36.561 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:17:36.561 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:17:36.561 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:17:36.561 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:36.561 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:17:36.562 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:17:36.562 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:17:36.562 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:17:36.562 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:17:36.562 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:17:36.562 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:17:36.562 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:17:36.562 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:17:36.562 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:17:36.562 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:36.562 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:17:36.562 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:17:36.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:17:36.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:17:36.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:17:36.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:17:36.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:17:36.572 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:17:36.572 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:17:36.573 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:17:36.573 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:17:36.629 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:17:36.629 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:17:36.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:17:36.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:17:36.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:17:36.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:17:36.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:36.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:17:36.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:17:36.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:17:36.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:17:36.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:17:36.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:17:36.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:17:36.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:17:36.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:17:36.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:36.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:17:36.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:17:36.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:17:36.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:17:36.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:17:36.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:17:36.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:17:36.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:17:36.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:17:36.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:17:36.635 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:36.635 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:17:36.635 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:17:36.635 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:17:36.635 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:17:36.635 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:17:36.635 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:17:36.635 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:17:36.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:17:36.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:17:36.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:17:36.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:17:36.701 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:17:36.701 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:17:36.701 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:17:36.704 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:17:36.704 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:17:36.704 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:17:36.704 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:36.704 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:17:36.704 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:17:36.704 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:17:36.705 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:17:36.705 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:17:36.705 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:17:36.705 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:17:36.705 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:17:36.705 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:17:36.705 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:36.705 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:17:36.705 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:17:36.705 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:17:36.705 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:17:36.705 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:17:36.706 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:17:36.706 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:17:36.706 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:17:36.706 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:17:36.706 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:17:36.706 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:36.706 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:17:36.706 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:17:36.706 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:17:36.706 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:17:36.706 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:17:36.707 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:17:36.707 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:17:36.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:17:36.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:17:36.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:17:36.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:17:36.778 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:17:36.779 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:17:36.780 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:17:36.780 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:17:36.780 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:17:36.780 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:17:36.780 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:17:36.780 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:17:36.780 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:17:36.780 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:17:36.780 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:17:36.780 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:17:36.780 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:17:36.781 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:36.781 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:17:36.782 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:36.810 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 57 2026-02-18T08:17:36.810 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 58 2026-02-18T08:17:36.810 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:17:36.810 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=233, tag_tid=15, entry_tid=1], [object_number=232, tag_tid=15, entry_tid=0], [object_number=234, tag_tid=14, entry_tid=52758], [object_number=235, tag_tid=14, entry_tid=52755]]], state=connected] 2026-02-18T08:17:36.811 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=231, tag_tid=14, entry_tid=50047], [object_number=230, tag_tid=14, entry_tid=50046], [object_number=229, tag_tid=14, entry_tid=50045], [object_number=228, tag_tid=14, entry_tid=50044]]], state=connected] 2026-02-18T08:17:36.813 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:36.815 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=233, tag_tid=15, entry_tid=1' 2026-02-18T08:17:36.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:17:36.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:17:36.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:17:36.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:17:36.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:17:36.817 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:17:36.817 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:17:36.817 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:17:36.817 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:17:36.817 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:17:36.817 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:17:36.817 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:17:36.818 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:36.818 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:17:36.818 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:36.856 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 57 2026-02-18T08:17:36.856 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 58 2026-02-18T08:17:36.856 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:17:36.856 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=233, tag_tid=15, entry_tid=1], [object_number=232, tag_tid=15, entry_tid=0], [object_number=234, tag_tid=14, entry_tid=52758], [object_number=235, tag_tid=14, entry_tid=52755]]], state=connected] 2026-02-18T08:17:36.856 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=231, tag_tid=14, entry_tid=50047], [object_number=230, tag_tid=14, entry_tid=50046], [object_number=229, tag_tid=14, entry_tid=50045], [object_number=228, tag_tid=14, entry_tid=50044]]], state=connected] 2026-02-18T08:17:36.860 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:36.861 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=231, tag_tid=14, entry_tid=50047' 2026-02-18T08:17:36.861 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=233, tag_tid=15, entry_tid=1' -a 'object_number=233, tag_tid=15, entry_tid=1' = 'object_number=231, tag_tid=14, entry_tid=50047' 2026-02-18T08:17:36.862 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=231, tag_tid=14, entry_tid=50047' '!=' 'object_number=228, tag_tid=14, entry_tid=49708' 2026-02-18T08:17:36.862 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:17:36.862 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=231, tag_tid=14, entry_tid=50047' = 'object_number=228, tag_tid=14, entry_tid=49708' 2026-02-18T08:17:36.862 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=231, tag_tid=14, entry_tid=50047' 2026-02-18T08:17:36.862 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=233, tag_tid=15, entry_tid=1' 2026-02-18T08:17:36.863 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:17:36.863 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:17:36.864 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=15 2026-02-18T08:17:36.864 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=231, tag_tid=14, entry_tid=50047' 2026-02-18T08:17:36.865 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:17:36.865 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:17:36.866 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=14 2026-02-18T08:17:36.866 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=233, tag_tid=15, entry_tid=1' 2026-02-18T08:17:36.867 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:17:36.867 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:17:36.868 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:17:36.869 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=231, tag_tid=14, entry_tid=50047' 2026-02-18T08:17:36.869 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:17:36.869 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:17:36.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=50047 2026-02-18T08:17:36.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 15 = 14 -a 1 -le 50047 2026-02-18T08:17:36.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:17:36.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:17:36.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:17:37.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:17:37.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:17:37.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:17:37.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:17:37.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:17:37.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:17:37.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:17:37.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:17:37.074 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:17:37.074 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:37.074 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:17:37.074 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:17:37.074 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:17:37.074 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:17:37.074 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:17:37.074 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:17:37.074 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:17:37.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:17:37.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:17:37.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:17:37.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:37.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:17:37.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:17:37.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:17:37.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:17:37.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:17:37.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:17:37.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:17:37.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:17:37.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:17:37.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:37.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:17:37.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:17:37.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:17:37.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:17:37.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:17:37.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:17:37.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:17:37.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:17:37.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:17:37.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:17:37.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:37.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:17:37.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:17:37.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:17:37.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:17:37.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:17:37.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:17:37.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:17:37.086 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:17:37.086 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:17:37.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:17:37.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:17:37.149 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:17:37.149 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:17:37.149 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:17:37.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:17:37.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:17:37.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:17:37.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:37.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:17:37.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:17:37.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:17:37.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:17:37.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:17:37.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:17:37.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:17:37.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:17:37.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:17:37.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:37.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:17:37.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:17:37.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:17:37.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:17:37.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:17:37.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:17:37.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:17:37.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:17:37.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:17:37.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:17:37.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:37.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:17:37.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:17:37.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:17:37.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:17:37.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:17:37.155 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:17:37.155 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:17:37.164 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:17:37.164 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:17:37.164 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:17:37.164 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:17:37.217 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:17:37.217 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:17:37.218 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:17:37.221 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:17:37.221 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:17:37.221 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:17:37.221 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:37.221 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:17:37.221 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:17:37.221 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:17:37.221 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:17:37.221 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:17:37.222 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:17:37.222 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:17:37.222 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:17:37.222 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:17:37.222 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:37.222 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:17:37.222 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:17:37.222 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:17:37.222 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:17:37.222 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:17:37.222 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:17:37.222 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:17:37.223 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:17:37.223 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:17:37.223 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:17:37.223 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:37.223 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:17:37.223 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:17:37.223 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:17:37.223 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:17:37.223 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:17:37.223 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:17:37.223 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:17:37.231 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:17:37.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:17:37.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:17:37.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:17:37.282 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:17:37.283 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:17:37.283 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:17:37.285 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:17:37.285 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:17:37.286 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:17:37.286 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:37.286 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:17:37.286 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:17:37.286 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:17:37.286 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:17:37.286 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:17:37.286 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:17:37.286 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:17:37.286 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:17:37.286 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:17:37.287 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:37.287 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:17:37.287 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:17:37.287 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:17:37.287 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:17:37.287 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:17:37.287 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:17:37.287 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:17:37.287 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:17:37.287 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:17:37.287 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:17:37.287 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:37.287 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:17:37.288 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:17:37.288 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:17:37.288 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:17:37.288 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:17:37.288 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:17:37.288 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:17:37.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:17:37.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:17:37.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:17:37.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:17:37.363 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:17:37.364 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:17:37.364 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:17:37.364 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:17:37.364 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:17:37.364 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:17:37.365 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:17:37.365 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:17:37.365 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:17:37.365 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:17:37.365 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:17:37.365 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:17:37.366 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:17:37.366 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:37.367 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:17:37.367 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:37.395 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 57 2026-02-18T08:17:37.395 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 58 2026-02-18T08:17:37.395 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:17:37.395 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=233, tag_tid=15, entry_tid=1], [object_number=232, tag_tid=15, entry_tid=0], [object_number=234, tag_tid=14, entry_tid=52758], [object_number=235, tag_tid=14, entry_tid=52755]]], state=connected] 2026-02-18T08:17:37.395 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=232, tag_tid=14, entry_tid=50360], [object_number=235, tag_tid=14, entry_tid=50359], [object_number=234, tag_tid=14, entry_tid=50358], [object_number=229, tag_tid=14, entry_tid=50357]]], state=connected] 2026-02-18T08:17:37.397 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:37.399 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=233, tag_tid=15, entry_tid=1' 2026-02-18T08:17:37.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:17:37.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:17:37.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:17:37.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:17:37.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:17:37.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:17:37.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:17:37.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:17:37.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:17:37.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:17:37.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:17:37.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:17:37.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:37.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:17:37.403 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:37.430 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 57 2026-02-18T08:17:37.430 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 58 2026-02-18T08:17:37.430 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:17:37.431 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=233, tag_tid=15, entry_tid=1], [object_number=232, tag_tid=15, entry_tid=0], [object_number=234, tag_tid=14, entry_tid=52758], [object_number=235, tag_tid=14, entry_tid=52755]]], state=connected] 2026-02-18T08:17:37.431 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=232, tag_tid=14, entry_tid=50360], [object_number=235, tag_tid=14, entry_tid=50359], [object_number=234, tag_tid=14, entry_tid=50358], [object_number=229, tag_tid=14, entry_tid=50357]]], state=connected] 2026-02-18T08:17:37.433 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:37.434 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=232, tag_tid=14, entry_tid=50360' 2026-02-18T08:17:37.435 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=233, tag_tid=15, entry_tid=1' -a 'object_number=233, tag_tid=15, entry_tid=1' = 'object_number=232, tag_tid=14, entry_tid=50360' 2026-02-18T08:17:37.435 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=232, tag_tid=14, entry_tid=50360' '!=' 'object_number=231, tag_tid=14, entry_tid=50047' 2026-02-18T08:17:37.435 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:17:37.435 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=232, tag_tid=14, entry_tid=50360' = 'object_number=231, tag_tid=14, entry_tid=50047' 2026-02-18T08:17:37.435 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=232, tag_tid=14, entry_tid=50360' 2026-02-18T08:17:37.436 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=233, tag_tid=15, entry_tid=1' 2026-02-18T08:17:37.436 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:17:37.436 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:17:37.437 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=15 2026-02-18T08:17:37.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=232, tag_tid=14, entry_tid=50360' 2026-02-18T08:17:37.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:17:37.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:17:37.439 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=14 2026-02-18T08:17:37.440 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=233, tag_tid=15, entry_tid=1' 2026-02-18T08:17:37.440 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:17:37.440 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:17:37.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:17:37.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=232, tag_tid=14, entry_tid=50360' 2026-02-18T08:17:37.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:17:37.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:17:37.443 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=50360 2026-02-18T08:17:37.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 15 = 14 -a 1 -le 50360 2026-02-18T08:17:37.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:17:37.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:17:37.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:17:37.646 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:17:37.646 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:17:37.646 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:17:37.646 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:17:37.646 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:17:37.646 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:17:37.646 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:17:37.646 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:17:37.646 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:17:37.647 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:37.647 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:17:37.647 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:17:37.647 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:17:37.647 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:17:37.647 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:17:37.647 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:17:37.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:17:37.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:17:37.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:17:37.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:17:37.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:37.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:17:37.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:17:37.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:17:37.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:17:37.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:17:37.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:17:37.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:17:37.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:17:37.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:17:37.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:37.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:17:37.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:17:37.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:17:37.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:17:37.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:17:37.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:17:37.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:17:37.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:17:37.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:17:37.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:17:37.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:37.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:17:37.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:17:37.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:17:37.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:17:37.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:17:37.653 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:17:37.653 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:17:37.661 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:17:37.661 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:17:37.662 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:17:37.662 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:17:37.718 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:17:37.718 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:17:37.718 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:17:37.721 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:17:37.721 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:17:37.721 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:17:37.721 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:37.721 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:17:37.721 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:17:37.722 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:17:37.722 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:17:37.722 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:17:37.722 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:17:37.722 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:17:37.722 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:17:37.722 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:17:37.722 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:37.722 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:17:37.722 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:17:37.722 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:17:37.722 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:17:37.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:17:37.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:17:37.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:17:37.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:17:37.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:17:37.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:17:37.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:37.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:17:37.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:17:37.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:17:37.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:17:37.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:17:37.724 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:17:37.724 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:17:37.731 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:17:37.732 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:17:37.732 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:17:37.732 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:17:37.788 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:17:37.788 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:17:37.788 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:17:37.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:17:37.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:17:37.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:17:37.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:37.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:17:37.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:17:37.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:17:37.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:17:37.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:17:37.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:17:37.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:17:37.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:17:37.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:17:37.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:37.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:17:37.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:17:37.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:17:37.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:17:37.793 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:17:37.793 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:17:37.793 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:17:37.793 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:17:37.793 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:17:37.793 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:17:37.793 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:37.793 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:17:37.793 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:17:37.793 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:17:37.793 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:17:37.793 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:17:37.794 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:17:37.794 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:17:37.801 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:17:37.801 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:17:37.801 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:17:37.801 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:17:37.858 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:17:37.858 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:17:37.859 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:17:37.861 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:17:37.861 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:17:37.861 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:17:37.861 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:37.862 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:17:37.862 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:17:37.862 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:17:37.862 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:17:37.862 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:17:37.862 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:17:37.862 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:17:37.862 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:17:37.862 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:17:37.862 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:37.862 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:17:37.863 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:17:37.863 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:17:37.863 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:17:37.863 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:17:37.863 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:17:37.863 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:17:37.863 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:17:37.863 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:17:37.863 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:17:37.863 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:37.863 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:17:37.863 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:17:37.863 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:17:37.864 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:17:37.864 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:17:37.864 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:17:37.864 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:17:37.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:17:37.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:17:37.874 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:17:37.874 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:17:37.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:17:37.944 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:17:37.944 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:17:37.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:17:37.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:17:37.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:17:37.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:17:37.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:17:37.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:17:37.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:17:37.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:17:37.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:17:37.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:17:37.946 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:37.947 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:37.947 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:17:37.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 58 2026-02-18T08:17:37.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 58 2026-02-18T08:17:37.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:17:37.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=233, tag_tid=15, entry_tid=1], [object_number=232, tag_tid=15, entry_tid=0], [object_number=234, tag_tid=14, entry_tid=52758], [object_number=235, tag_tid=14, entry_tid=52755]]], state=connected] 2026-02-18T08:17:37.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=234, tag_tid=14, entry_tid=50706], [object_number=233, tag_tid=14, entry_tid=50705], [object_number=232, tag_tid=14, entry_tid=50704], [object_number=235, tag_tid=14, entry_tid=50703]]], state=connected] 2026-02-18T08:17:37.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:37.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=233, tag_tid=15, entry_tid=1' 2026-02-18T08:17:37.982 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:17:37.982 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:17:37.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:17:37.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:17:37.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:17:37.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:17:37.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:17:37.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:17:37.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:17:37.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:17:37.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:17:37.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:17:37.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:37.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:17:37.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:38.019 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 58 2026-02-18T08:17:38.020 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 58 2026-02-18T08:17:38.020 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:17:38.020 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=233, tag_tid=15, entry_tid=1], [object_number=232, tag_tid=15, entry_tid=0], [object_number=234, tag_tid=14, entry_tid=52758], [object_number=235, tag_tid=14, entry_tid=52755]]], state=connected] 2026-02-18T08:17:38.020 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=234, tag_tid=14, entry_tid=50706], [object_number=233, tag_tid=14, entry_tid=50705], [object_number=232, tag_tid=14, entry_tid=50704], [object_number=235, tag_tid=14, entry_tid=50703]]], state=connected] 2026-02-18T08:17:38.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:38.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=234, tag_tid=14, entry_tid=50706' 2026-02-18T08:17:38.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=233, tag_tid=15, entry_tid=1' -a 'object_number=233, tag_tid=15, entry_tid=1' = 'object_number=234, tag_tid=14, entry_tid=50706' 2026-02-18T08:17:38.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=234, tag_tid=14, entry_tid=50706' '!=' 'object_number=232, tag_tid=14, entry_tid=50360' 2026-02-18T08:17:38.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:17:38.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=234, tag_tid=14, entry_tid=50706' = 'object_number=232, tag_tid=14, entry_tid=50360' 2026-02-18T08:17:38.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=234, tag_tid=14, entry_tid=50706' 2026-02-18T08:17:38.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=233, tag_tid=15, entry_tid=1' 2026-02-18T08:17:38.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:17:38.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:17:38.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=15 2026-02-18T08:17:38.027 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=234, tag_tid=14, entry_tid=50706' 2026-02-18T08:17:38.027 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:17:38.027 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:17:38.028 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=14 2026-02-18T08:17:38.029 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=233, tag_tid=15, entry_tid=1' 2026-02-18T08:17:38.029 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:17:38.029 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:17:38.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:17:38.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=234, tag_tid=14, entry_tid=50706' 2026-02-18T08:17:38.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:17:38.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:17:38.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=50706 2026-02-18T08:17:38.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 15 = 14 -a 1 -le 50706 2026-02-18T08:17:38.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:17:38.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:17:38.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:17:38.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:17:38.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:17:38.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:17:38.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:17:38.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:17:38.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:17:38.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:17:38.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:17:38.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:17:38.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:38.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:17:38.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:17:38.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:17:38.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:17:38.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:17:38.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:17:38.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:17:38.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:17:38.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:17:38.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:17:38.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:38.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:17:38.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:17:38.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:17:38.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:17:38.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:17:38.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:17:38.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:17:38.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:17:38.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:17:38.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:38.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:17:38.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:17:38.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:17:38.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:17:38.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:17:38.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:17:38.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:17:38.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:17:38.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:17:38.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:17:38.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:38.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:17:38.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:17:38.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:17:38.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:17:38.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:17:38.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:17:38.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:17:38.249 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:17:38.249 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:17:38.249 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:17:38.250 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:17:38.299 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:17:38.299 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:17:38.299 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:17:38.302 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:17:38.302 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:17:38.302 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:17:38.302 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:38.302 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:17:38.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:17:38.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:17:38.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:17:38.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:17:38.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:17:38.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:17:38.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:17:38.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:17:38.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:38.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:17:38.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:17:38.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:17:38.304 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:17:38.304 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:17:38.304 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:17:38.304 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:17:38.304 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:17:38.304 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:17:38.304 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:17:38.304 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:38.304 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:17:38.304 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:17:38.304 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:17:38.304 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:17:38.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:17:38.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:17:38.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:17:38.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:17:38.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:17:38.314 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:17:38.314 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:17:38.370 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:17:38.370 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:17:38.370 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:17:38.372 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:17:38.373 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:17:38.373 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:17:38.373 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:38.373 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:17:38.373 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:17:38.373 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:17:38.373 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:17:38.373 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:17:38.373 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:17:38.373 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:17:38.373 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:17:38.373 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:17:38.374 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:38.374 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:17:38.374 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:17:38.374 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:17:38.374 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:17:38.374 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:17:38.374 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:17:38.374 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:17:38.374 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:17:38.374 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:17:38.374 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:17:38.374 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:38.375 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:17:38.375 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:17:38.375 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:17:38.375 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:17:38.375 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:17:38.375 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:17:38.375 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:17:38.385 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:17:38.385 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:17:38.385 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:17:38.385 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:17:38.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:17:38.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:17:38.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:17:38.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:17:38.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:17:38.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:17:38.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:38.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:17:38.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:17:38.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:17:38.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:17:38.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:17:38.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:17:38.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:17:38.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:17:38.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:17:38.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:38.446 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:17:38.446 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:17:38.446 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:17:38.446 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:17:38.446 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:17:38.446 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:17:38.446 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:17:38.446 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:17:38.446 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:17:38.446 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:17:38.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:38.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:17:38.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:17:38.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:17:38.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:17:38.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:17:38.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:17:38.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:17:38.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:17:38.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:17:38.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:17:38.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:17:38.513 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:17:38.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:17:38.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:17:38.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:17:38.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:17:38.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:17:38.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:17:38.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:17:38.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:17:38.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:17:38.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:17:38.516 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:17:38.516 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:17:38.516 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:38.516 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:17:38.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:38.545 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 58 2026-02-18T08:17:38.545 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 58 2026-02-18T08:17:38.545 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:17:38.545 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=233, tag_tid=15, entry_tid=1], [object_number=232, tag_tid=15, entry_tid=0], [object_number=234, tag_tid=14, entry_tid=52758], [object_number=235, tag_tid=14, entry_tid=52755]]], state=connected] 2026-02-18T08:17:38.545 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=233, tag_tid=14, entry_tid=51061], [object_number=232, tag_tid=14, entry_tid=51060], [object_number=235, tag_tid=14, entry_tid=51059], [object_number=234, tag_tid=14, entry_tid=51058]]], state=connected] 2026-02-18T08:17:38.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:38.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=233, tag_tid=15, entry_tid=1' 2026-02-18T08:17:38.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:17:38.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:17:38.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:17:38.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:17:38.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:17:38.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:17:38.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:17:38.551 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:17:38.551 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:17:38.551 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:17:38.551 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:17:38.551 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:17:38.552 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:38.552 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:17:38.552 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:38.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 58 2026-02-18T08:17:38.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 58 2026-02-18T08:17:38.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:17:38.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=233, tag_tid=15, entry_tid=1], [object_number=232, tag_tid=15, entry_tid=0], [object_number=234, tag_tid=14, entry_tid=52758], [object_number=235, tag_tid=14, entry_tid=52755]]], state=connected] 2026-02-18T08:17:38.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=233, tag_tid=14, entry_tid=51061], [object_number=232, tag_tid=14, entry_tid=51060], [object_number=235, tag_tid=14, entry_tid=51059], [object_number=234, tag_tid=14, entry_tid=51058]]], state=connected] 2026-02-18T08:17:38.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:38.585 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=233, tag_tid=14, entry_tid=51061' 2026-02-18T08:17:38.585 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=233, tag_tid=15, entry_tid=1' -a 'object_number=233, tag_tid=15, entry_tid=1' = 'object_number=233, tag_tid=14, entry_tid=51061' 2026-02-18T08:17:38.585 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=233, tag_tid=14, entry_tid=51061' '!=' 'object_number=234, tag_tid=14, entry_tid=50706' 2026-02-18T08:17:38.585 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:17:38.585 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=233, tag_tid=14, entry_tid=51061' = 'object_number=234, tag_tid=14, entry_tid=50706' 2026-02-18T08:17:38.585 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=233, tag_tid=14, entry_tid=51061' 2026-02-18T08:17:38.586 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=233, tag_tid=15, entry_tid=1' 2026-02-18T08:17:38.586 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:17:38.586 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:17:38.587 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=15 2026-02-18T08:17:38.587 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=233, tag_tid=14, entry_tid=51061' 2026-02-18T08:17:38.587 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:17:38.587 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:17:38.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=14 2026-02-18T08:17:38.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=233, tag_tid=15, entry_tid=1' 2026-02-18T08:17:38.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:17:38.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:17:38.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:17:38.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=233, tag_tid=14, entry_tid=51061' 2026-02-18T08:17:38.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:17:38.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:17:38.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=51061 2026-02-18T08:17:38.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 15 = 14 -a 1 -le 51061 2026-02-18T08:17:38.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:17:38.594 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:17:38.594 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:17:38.794 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:17:38.794 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:17:38.794 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:17:38.794 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:17:38.794 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:17:38.794 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:17:38.794 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:17:38.794 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:17:38.794 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:17:38.794 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:38.794 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:17:38.794 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:17:38.794 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:17:38.794 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:17:38.795 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:17:38.795 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:17:38.795 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:17:38.796 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:17:38.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:17:38.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:17:38.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:38.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:17:38.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:17:38.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:17:38.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:17:38.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:17:38.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:17:38.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:17:38.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:17:38.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:17:38.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:38.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:17:38.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:17:38.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:17:38.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:17:38.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:17:38.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:17:38.798 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:17:38.798 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:17:38.798 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:17:38.798 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:17:38.798 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:38.798 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:17:38.798 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:17:38.798 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:17:38.798 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:17:38.798 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:17:38.798 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:17:38.798 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:17:38.809 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:17:38.809 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:17:38.809 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:17:38.809 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:17:38.858 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:17:38.859 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:17:38.859 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:17:38.860 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:17:38.860 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:17:38.860 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:17:38.860 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:38.860 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:17:38.860 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:17:38.860 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:17:38.860 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:17:38.860 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:17:38.860 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:17:38.860 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:17:38.860 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:17:38.860 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:17:38.860 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:38.860 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:17:38.860 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:17:38.861 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:17:38.861 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:17:38.861 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:17:38.861 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:17:38.861 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:17:38.861 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:17:38.861 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:17:38.861 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:17:38.861 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:38.861 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:17:38.861 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:17:38.861 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:17:38.861 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:17:38.861 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:17:38.861 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:17:38.861 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:17:38.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:17:38.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:17:38.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:17:38.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:17:38.923 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:17:38.923 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:17:38.923 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:17:38.924 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:17:38.924 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:17:38.924 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:17:38.924 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:38.924 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:17:38.924 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:17:38.924 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:17:38.924 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:17:38.924 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:17:38.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:17:38.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:17:38.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:17:38.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:17:38.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:38.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:17:38.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:17:38.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:17:38.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:17:38.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:17:38.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:17:38.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:17:38.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:17:38.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:17:38.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:17:38.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:38.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:17:38.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:17:38.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:17:38.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:17:38.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:17:38.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:17:38.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:17:38.936 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:17:38.936 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:17:38.936 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:17:38.936 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:17:38.994 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:17:38.994 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:17:38.994 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:17:38.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:17:38.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:17:38.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:17:38.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:38.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:17:38.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:17:38.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:17:38.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:17:38.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:17:38.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:17:38.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:17:38.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:17:38.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:17:38.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:38.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:17:38.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:17:38.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:17:38.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:17:38.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:17:38.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:17:38.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:17:38.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:17:38.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:17:38.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:17:38.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:38.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:17:38.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:17:38.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:17:38.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:17:38.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:17:38.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:17:38.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:17:39.008 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:17:39.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:17:39.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:17:39.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:17:39.069 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:17:39.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:17:39.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:17:39.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:17:39.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:17:39.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:17:39.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:17:39.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:17:39.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:17:39.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:17:39.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:17:39.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:17:39.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:17:39.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:39.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:17:39.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:39.101 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 58 2026-02-18T08:17:39.101 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 58 2026-02-18T08:17:39.101 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:17:39.101 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=233, tag_tid=15, entry_tid=1], [object_number=232, tag_tid=15, entry_tid=0], [object_number=234, tag_tid=14, entry_tid=52758], [object_number=235, tag_tid=14, entry_tid=52755]]], state=connected] 2026-02-18T08:17:39.101 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=232, tag_tid=14, entry_tid=51416], [object_number=235, tag_tid=14, entry_tid=51415], [object_number=234, tag_tid=14, entry_tid=51414], [object_number=233, tag_tid=14, entry_tid=51413]]], state=connected] 2026-02-18T08:17:39.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:39.106 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=233, tag_tid=15, entry_tid=1' 2026-02-18T08:17:39.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:17:39.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:17:39.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:17:39.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:17:39.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:17:39.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:17:39.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:17:39.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:17:39.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:17:39.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:17:39.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:17:39.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:17:39.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:39.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:17:39.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:39.137 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 58 2026-02-18T08:17:39.137 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 58 2026-02-18T08:17:39.137 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:17:39.138 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=233, tag_tid=15, entry_tid=1], [object_number=232, tag_tid=15, entry_tid=0], [object_number=234, tag_tid=14, entry_tid=52758], [object_number=235, tag_tid=14, entry_tid=52755]]], state=connected] 2026-02-18T08:17:39.138 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=232, tag_tid=14, entry_tid=51416], [object_number=235, tag_tid=14, entry_tid=51415], [object_number=234, tag_tid=14, entry_tid=51414], [object_number=233, tag_tid=14, entry_tid=51413]]], state=connected] 2026-02-18T08:17:39.140 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:39.142 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=232, tag_tid=14, entry_tid=51416' 2026-02-18T08:17:39.142 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=233, tag_tid=15, entry_tid=1' -a 'object_number=233, tag_tid=15, entry_tid=1' = 'object_number=232, tag_tid=14, entry_tid=51416' 2026-02-18T08:17:39.142 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=232, tag_tid=14, entry_tid=51416' '!=' 'object_number=233, tag_tid=14, entry_tid=51061' 2026-02-18T08:17:39.142 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:17:39.142 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=232, tag_tid=14, entry_tid=51416' = 'object_number=233, tag_tid=14, entry_tid=51061' 2026-02-18T08:17:39.142 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=232, tag_tid=14, entry_tid=51416' 2026-02-18T08:17:39.143 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=233, tag_tid=15, entry_tid=1' 2026-02-18T08:17:39.143 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:17:39.143 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:17:39.144 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=15 2026-02-18T08:17:39.144 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=232, tag_tid=14, entry_tid=51416' 2026-02-18T08:17:39.144 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:17:39.145 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:17:39.146 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=14 2026-02-18T08:17:39.147 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=233, tag_tid=15, entry_tid=1' 2026-02-18T08:17:39.147 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:17:39.147 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:17:39.148 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:17:39.149 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=232, tag_tid=14, entry_tid=51416' 2026-02-18T08:17:39.149 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:17:39.149 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:17:39.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=51416 2026-02-18T08:17:39.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 15 = 14 -a 1 -le 51416 2026-02-18T08:17:39.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:17:39.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:17:39.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:17:39.353 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:17:39.353 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:17:39.353 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:17:39.353 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:17:39.354 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:17:39.354 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:17:39.354 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:17:39.354 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:17:39.354 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:17:39.354 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:39.354 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:17:39.354 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:17:39.354 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:17:39.354 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:17:39.355 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:17:39.355 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:17:39.355 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:17:39.357 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:17:39.357 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:17:39.357 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:17:39.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:39.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:17:39.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:17:39.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:17:39.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:17:39.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:17:39.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:17:39.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:17:39.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:17:39.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:17:39.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:39.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:17:39.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:17:39.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:17:39.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:17:39.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:17:39.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:17:39.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:17:39.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:17:39.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:17:39.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:17:39.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:39.360 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:17:39.360 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:17:39.360 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:17:39.360 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:17:39.360 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:17:39.360 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:17:39.360 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:17:39.367 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:17:39.367 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:17:39.367 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:17:39.367 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:17:39.415 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:17:39.415 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:17:39.415 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:17:39.418 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:17:39.418 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:17:39.418 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:17:39.418 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:39.418 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:17:39.418 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:17:39.418 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:17:39.418 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:17:39.418 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:17:39.419 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:17:39.419 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:17:39.419 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:17:39.419 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:17:39.419 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:39.419 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:17:39.419 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:17:39.419 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:17:39.419 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:17:39.419 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:17:39.419 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:17:39.419 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:17:39.420 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:17:39.420 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:17:39.420 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:17:39.420 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:39.420 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:17:39.420 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:17:39.420 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:17:39.420 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:17:39.420 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:17:39.420 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:17:39.420 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:17:39.429 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:17:39.429 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:17:39.429 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:17:39.430 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:17:39.477 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:17:39.478 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:17:39.478 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:17:39.480 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:17:39.480 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:17:39.481 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:17:39.481 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:39.481 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:17:39.481 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:17:39.481 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:17:39.481 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:17:39.481 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:17:39.481 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:17:39.481 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:17:39.481 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:17:39.481 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:17:39.482 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:39.482 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:17:39.482 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:17:39.482 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:17:39.482 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:17:39.482 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:17:39.482 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:17:39.482 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:17:39.482 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:17:39.482 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:17:39.482 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:17:39.483 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:39.483 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:17:39.483 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:17:39.483 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:17:39.483 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:17:39.483 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:17:39.483 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:17:39.483 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:17:39.492 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:17:39.492 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:17:39.493 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:17:39.493 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:17:39.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:17:39.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:17:39.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:17:39.551 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:17:39.552 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:17:39.552 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:17:39.552 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:39.552 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:17:39.552 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:17:39.552 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:17:39.553 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:17:39.553 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:17:39.553 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:17:39.553 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:17:39.553 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:17:39.553 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:17:39.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:39.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:17:39.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:17:39.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:17:39.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:17:39.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:17:39.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:17:39.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:17:39.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:17:39.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:17:39.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:17:39.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:39.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:17:39.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:17:39.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:17:39.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:17:39.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:17:39.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:17:39.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:17:39.561 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:17:39.562 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:17:39.562 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:17:39.562 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:17:39.622 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:17:39.622 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:17:39.622 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:17:39.622 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:17:39.623 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:17:39.623 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:17:39.623 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:17:39.623 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:17:39.623 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:17:39.623 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:17:39.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:17:39.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:17:39.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:17:39.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:39.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:17:39.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:39.653 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 58 2026-02-18T08:17:39.653 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 58 2026-02-18T08:17:39.653 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:17:39.653 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=233, tag_tid=15, entry_tid=1], [object_number=232, tag_tid=15, entry_tid=0], [object_number=234, tag_tid=14, entry_tid=52758], [object_number=235, tag_tid=14, entry_tid=52755]]], state=connected] 2026-02-18T08:17:39.653 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=232, tag_tid=14, entry_tid=51788], [object_number=235, tag_tid=14, entry_tid=51787], [object_number=234, tag_tid=14, entry_tid=51786], [object_number=233, tag_tid=14, entry_tid=51785]]], state=connected] 2026-02-18T08:17:39.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:39.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=233, tag_tid=15, entry_tid=1' 2026-02-18T08:17:39.658 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:17:39.658 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:17:39.658 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:17:39.658 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:17:39.659 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:17:39.659 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:17:39.659 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:17:39.659 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:17:39.659 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:17:39.659 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:17:39.659 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:17:39.659 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:17:39.660 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:39.660 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:17:39.660 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:39.695 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 58 2026-02-18T08:17:39.695 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 58 2026-02-18T08:17:39.695 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:17:39.695 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=233, tag_tid=15, entry_tid=1], [object_number=232, tag_tid=15, entry_tid=0], [object_number=234, tag_tid=14, entry_tid=52758], [object_number=235, tag_tid=14, entry_tid=52755]]], state=connected] 2026-02-18T08:17:39.695 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=232, tag_tid=14, entry_tid=51788], [object_number=235, tag_tid=14, entry_tid=51787], [object_number=234, tag_tid=14, entry_tid=51786], [object_number=233, tag_tid=14, entry_tid=51785]]], state=connected] 2026-02-18T08:17:39.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:39.700 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=232, tag_tid=14, entry_tid=51788' 2026-02-18T08:17:39.700 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=233, tag_tid=15, entry_tid=1' -a 'object_number=233, tag_tid=15, entry_tid=1' = 'object_number=232, tag_tid=14, entry_tid=51788' 2026-02-18T08:17:39.700 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=232, tag_tid=14, entry_tid=51788' '!=' 'object_number=232, tag_tid=14, entry_tid=51416' 2026-02-18T08:17:39.700 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:17:39.700 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=232, tag_tid=14, entry_tid=51788' = 'object_number=232, tag_tid=14, entry_tid=51416' 2026-02-18T08:17:39.700 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=232, tag_tid=14, entry_tid=51788' 2026-02-18T08:17:39.701 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=233, tag_tid=15, entry_tid=1' 2026-02-18T08:17:39.701 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:17:39.701 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:17:39.703 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=15 2026-02-18T08:17:39.703 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=232, tag_tid=14, entry_tid=51788' 2026-02-18T08:17:39.703 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:17:39.704 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:17:39.705 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=14 2026-02-18T08:17:39.706 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=233, tag_tid=15, entry_tid=1' 2026-02-18T08:17:39.706 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:17:39.706 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:17:39.707 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:17:39.708 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=232, tag_tid=14, entry_tid=51788' 2026-02-18T08:17:39.708 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:17:39.708 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:17:39.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=51788 2026-02-18T08:17:39.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 15 = 14 -a 1 -le 51788 2026-02-18T08:17:39.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:17:39.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:17:39.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:17:39.912 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:17:39.912 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:17:39.913 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:17:39.913 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:17:39.913 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:17:39.913 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:17:39.913 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:17:39.913 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:17:39.913 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:17:39.913 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:39.913 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:17:39.913 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:17:39.913 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:17:39.914 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:17:39.914 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:17:39.914 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:17:39.916 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:17:39.916 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:17:39.917 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:17:39.917 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:17:39.917 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:39.917 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:17:39.917 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:17:39.917 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:17:39.917 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:17:39.917 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:17:39.917 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:17:39.917 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:17:39.917 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:17:39.918 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:17:39.918 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:39.918 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:17:39.918 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:17:39.918 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:17:39.918 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:17:39.918 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:17:39.918 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:17:39.918 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:17:39.918 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:17:39.918 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:17:39.918 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:17:39.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:39.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:17:39.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:17:39.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:17:39.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:17:39.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:17:39.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:17:39.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:17:39.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:17:39.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:17:39.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:17:39.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:17:39.985 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:17:39.985 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:17:39.985 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:17:39.986 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:17:39.986 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:17:39.986 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:17:39.986 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:39.986 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:17:39.987 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:17:39.987 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:17:39.987 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:17:39.987 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:17:39.987 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:17:39.987 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:17:39.987 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:17:39.987 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:17:39.987 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:39.987 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:17:39.987 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:17:39.987 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:17:39.987 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:17:39.987 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:17:39.987 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:17:39.987 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:17:39.987 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:17:39.987 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:17:39.987 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:17:39.987 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:39.987 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:17:39.987 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:17:39.987 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:17:39.987 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:17:39.987 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:17:39.988 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:17:39.988 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:17:40.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:17:40.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:17:40.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:17:40.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:17:40.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:17:40.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:17:40.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:17:40.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:17:40.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:17:40.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:17:40.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:40.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:17:40.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:17:40.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:17:40.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:17:40.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:17:40.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:17:40.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:17:40.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:17:40.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:17:40.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:40.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:17:40.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:17:40.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:17:40.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:17:40.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:17:40.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:17:40.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:17:40.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:17:40.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:17:40.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:17:40.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:40.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:17:40.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:17:40.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:17:40.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:17:40.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:17:40.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:17:40.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:17:40.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:17:40.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:17:40.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:17:40.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:17:40.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:17:40.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:17:40.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:17:40.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:17:40.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:17:40.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:17:40.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:40.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:17:40.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:17:40.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:17:40.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:17:40.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:17:40.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:17:40.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:17:40.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:17:40.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:17:40.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:40.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:17:40.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:17:40.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:17:40.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:17:40.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:17:40.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:17:40.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:17:40.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:17:40.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:17:40.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:17:40.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:40.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:17:40.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:17:40.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:17:40.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:17:40.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:17:40.118 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:17:40.118 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:17:40.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:17:40.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:17:40.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:17:40.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:17:40.190 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:17:40.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:17:40.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:17:40.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:17:40.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:17:40.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:17:40.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:17:40.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:17:40.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:17:40.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:17:40.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:17:40.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:17:40.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:17:40.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:40.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:17:40.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:40.221 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 58 2026-02-18T08:17:40.221 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 58 2026-02-18T08:17:40.221 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:17:40.221 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=233, tag_tid=15, entry_tid=1], [object_number=232, tag_tid=15, entry_tid=0], [object_number=234, tag_tid=14, entry_tid=52758], [object_number=235, tag_tid=14, entry_tid=52755]]], state=connected] 2026-02-18T08:17:40.221 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=233, tag_tid=14, entry_tid=52157], [object_number=232, tag_tid=14, entry_tid=52156], [object_number=235, tag_tid=14, entry_tid=52155], [object_number=234, tag_tid=14, entry_tid=52154]]], state=connected] 2026-02-18T08:17:40.224 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:40.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=233, tag_tid=15, entry_tid=1' 2026-02-18T08:17:40.227 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:17:40.227 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:17:40.227 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:17:40.227 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:17:40.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:17:40.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:17:40.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:17:40.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:17:40.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:17:40.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:17:40.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:17:40.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:17:40.229 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:40.229 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:17:40.229 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:40.259 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 58 2026-02-18T08:17:40.259 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 58 2026-02-18T08:17:40.259 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:17:40.259 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=233, tag_tid=15, entry_tid=1], [object_number=232, tag_tid=15, entry_tid=0], [object_number=234, tag_tid=14, entry_tid=52758], [object_number=235, tag_tid=14, entry_tid=52755]]], state=connected] 2026-02-18T08:17:40.259 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=233, tag_tid=14, entry_tid=52157], [object_number=232, tag_tid=14, entry_tid=52156], [object_number=235, tag_tid=14, entry_tid=52155], [object_number=234, tag_tid=14, entry_tid=52154]]], state=connected] 2026-02-18T08:17:40.261 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:40.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=233, tag_tid=14, entry_tid=52157' 2026-02-18T08:17:40.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=233, tag_tid=15, entry_tid=1' -a 'object_number=233, tag_tid=15, entry_tid=1' = 'object_number=233, tag_tid=14, entry_tid=52157' 2026-02-18T08:17:40.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=233, tag_tid=14, entry_tid=52157' '!=' 'object_number=232, tag_tid=14, entry_tid=51788' 2026-02-18T08:17:40.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:17:40.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=233, tag_tid=14, entry_tid=52157' = 'object_number=232, tag_tid=14, entry_tid=51788' 2026-02-18T08:17:40.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=233, tag_tid=14, entry_tid=52157' 2026-02-18T08:17:40.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=233, tag_tid=15, entry_tid=1' 2026-02-18T08:17:40.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:17:40.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:17:40.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=15 2026-02-18T08:17:40.266 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=233, tag_tid=14, entry_tid=52157' 2026-02-18T08:17:40.266 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:17:40.266 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:17:40.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=14 2026-02-18T08:17:40.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=233, tag_tid=15, entry_tid=1' 2026-02-18T08:17:40.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:17:40.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:17:40.269 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:17:40.270 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=233, tag_tid=14, entry_tid=52157' 2026-02-18T08:17:40.270 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:17:40.270 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:17:40.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=52157 2026-02-18T08:17:40.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 15 = 14 -a 1 -le 52157 2026-02-18T08:17:40.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:17:40.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:17:40.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:17:40.473 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:17:40.473 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:17:40.473 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:17:40.473 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:17:40.473 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:17:40.474 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:17:40.474 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:17:40.474 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:17:40.474 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:17:40.474 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:40.474 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:17:40.474 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:17:40.474 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:17:40.474 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:17:40.474 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:17:40.474 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:17:40.474 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:17:40.476 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:17:40.476 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:17:40.476 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:17:40.476 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:40.476 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:17:40.476 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:17:40.476 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:17:40.476 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:17:40.476 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:17:40.476 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:17:40.476 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:17:40.476 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:17:40.476 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:17:40.476 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:40.476 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:17:40.476 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:17:40.476 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:17:40.476 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:17:40.477 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:17:40.477 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:17:40.477 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:17:40.477 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:17:40.477 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:17:40.477 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:17:40.477 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:40.477 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:17:40.477 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:17:40.477 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:17:40.477 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:17:40.477 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:17:40.477 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:17:40.477 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:17:40.489 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:17:40.489 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:17:40.489 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:17:40.489 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:17:40.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:17:40.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:17:40.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:17:40.551 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:17:40.551 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:17:40.551 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:17:40.551 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:40.552 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:17:40.552 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:17:40.552 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:17:40.552 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:17:40.552 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:17:40.552 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:17:40.552 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:17:40.553 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:17:40.553 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:17:40.553 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:40.553 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:17:40.553 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:17:40.553 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:17:40.553 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:17:40.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:17:40.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:17:40.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:17:40.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:17:40.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:17:40.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:17:40.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:40.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:17:40.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:17:40.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:17:40.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:17:40.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:17:40.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:17:40.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:17:40.561 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:17:40.561 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:17:40.561 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:17:40.561 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:17:40.607 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:17:40.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:17:40.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:17:40.610 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:17:40.610 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:17:40.611 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:17:40.611 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:40.611 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:17:40.611 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:17:40.611 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:17:40.611 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:17:40.611 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:17:40.611 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:17:40.611 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:17:40.611 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:17:40.611 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:17:40.612 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:40.612 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:17:40.612 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:17:40.612 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:17:40.612 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:17:40.612 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:17:40.612 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:17:40.612 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:17:40.612 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:17:40.612 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:17:40.612 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:17:40.612 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:40.613 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:17:40.613 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:17:40.613 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:17:40.613 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:17:40.613 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:17:40.613 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:17:40.613 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:17:40.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:17:40.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:17:40.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:17:40.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:17:40.677 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:17:40.677 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:17:40.677 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:17:40.679 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:17:40.680 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:17:40.680 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:17:40.680 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:40.680 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:17:40.680 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:17:40.680 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:17:40.680 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:17:40.680 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:17:40.680 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:17:40.680 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:17:40.680 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:17:40.681 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:17:40.681 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:40.681 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:17:40.681 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:17:40.681 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:17:40.681 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:17:40.681 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:17:40.681 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:17:40.681 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:17:40.681 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:17:40.681 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:17:40.681 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:17:40.681 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:40.682 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:17:40.682 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:17:40.682 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:17:40.682 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:17:40.682 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:17:40.682 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:17:40.682 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:17:40.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:17:40.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:17:40.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:17:40.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:17:40.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:17:40.758 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:17:40.758 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:17:40.758 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:17:40.758 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:17:40.759 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:17:40.759 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:17:40.759 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:17:40.759 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:17:40.759 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:17:40.759 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:17:40.759 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:17:40.759 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:17:40.760 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:40.760 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:17:40.760 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:40.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 58 2026-02-18T08:17:40.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 58 2026-02-18T08:17:40.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:17:40.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=233, tag_tid=15, entry_tid=1], [object_number=232, tag_tid=15, entry_tid=0], [object_number=234, tag_tid=14, entry_tid=52758], [object_number=235, tag_tid=14, entry_tid=52755]]], state=connected] 2026-02-18T08:17:40.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=233, tag_tid=14, entry_tid=52493], [object_number=232, tag_tid=14, entry_tid=52492], [object_number=235, tag_tid=14, entry_tid=52491], [object_number=234, tag_tid=14, entry_tid=52490]]], state=connected] 2026-02-18T08:17:40.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:40.793 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=233, tag_tid=15, entry_tid=1' 2026-02-18T08:17:40.795 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:17:40.795 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:17:40.795 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:17:40.795 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:17:40.795 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:17:40.795 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:17:40.795 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:17:40.796 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:17:40.796 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:17:40.796 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:17:40.796 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:17:40.796 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:17:40.796 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:40.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:17:40.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:41.029 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 58 2026-02-18T08:17:41.029 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 58 2026-02-18T08:17:41.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:17:41.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=233, tag_tid=15, entry_tid=1], [object_number=232, tag_tid=15, entry_tid=0], [object_number=234, tag_tid=14, entry_tid=52758], [object_number=235, tag_tid=14, entry_tid=52755]]], state=connected] 2026-02-18T08:17:41.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=233, tag_tid=14, entry_tid=52493], [object_number=232, tag_tid=14, entry_tid=52492], [object_number=235, tag_tid=14, entry_tid=52491], [object_number=234, tag_tid=14, entry_tid=52490]]], state=connected] 2026-02-18T08:17:41.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:41.035 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=233, tag_tid=14, entry_tid=52493' 2026-02-18T08:17:41.035 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=233, tag_tid=15, entry_tid=1' -a 'object_number=233, tag_tid=15, entry_tid=1' = 'object_number=233, tag_tid=14, entry_tid=52493' 2026-02-18T08:17:41.035 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=233, tag_tid=14, entry_tid=52493' '!=' 'object_number=233, tag_tid=14, entry_tid=52157' 2026-02-18T08:17:41.035 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:17:41.035 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=233, tag_tid=14, entry_tid=52493' = 'object_number=233, tag_tid=14, entry_tid=52157' 2026-02-18T08:17:41.035 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=233, tag_tid=14, entry_tid=52493' 2026-02-18T08:17:41.036 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=233, tag_tid=15, entry_tid=1' 2026-02-18T08:17:41.036 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:17:41.036 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:17:41.036 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=15 2026-02-18T08:17:41.037 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=233, tag_tid=14, entry_tid=52493' 2026-02-18T08:17:41.037 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:17:41.037 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:17:41.038 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=14 2026-02-18T08:17:41.039 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=233, tag_tid=15, entry_tid=1' 2026-02-18T08:17:41.039 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:17:41.040 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:17:41.041 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:17:41.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=233, tag_tid=14, entry_tid=52493' 2026-02-18T08:17:41.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:17:41.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:17:41.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=52493 2026-02-18T08:17:41.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 15 = 14 -a 1 -le 52493 2026-02-18T08:17:41.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:17:41.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:17:41.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:17:41.246 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:17:41.246 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:17:41.246 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:17:41.246 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:17:41.246 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:17:41.247 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:17:41.247 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:17:41.247 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:17:41.247 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:17:41.247 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:41.247 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:17:41.247 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:17:41.247 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:17:41.247 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:17:41.247 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:17:41.247 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:17:41.248 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:17:41.250 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:17:41.250 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:17:41.250 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:17:41.250 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:41.251 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:17:41.251 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:17:41.251 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:17:41.251 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:17:41.251 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:17:41.251 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:17:41.251 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:17:41.251 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:17:41.251 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:17:41.251 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:41.251 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:17:41.251 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:17:41.252 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:17:41.252 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:17:41.252 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:17:41.252 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:17:41.252 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:17:41.252 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:17:41.252 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:17:41.252 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:17:41.252 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:41.252 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:17:41.252 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:17:41.252 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:17:41.253 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:17:41.253 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:17:41.253 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:17:41.253 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:17:41.260 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:17:41.261 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:17:41.261 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:17:41.261 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:17:41.318 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:17:41.318 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:17:41.318 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:17:41.320 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:17:41.321 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:17:41.321 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:17:41.321 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:41.321 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:17:41.321 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:17:41.321 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:17:41.321 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:17:41.321 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:17:41.321 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:17:41.321 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:17:41.322 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:17:41.322 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:17:41.322 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:41.322 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:17:41.322 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:17:41.322 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:17:41.322 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:17:41.322 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:17:41.322 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:17:41.322 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:17:41.322 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:17:41.322 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:17:41.323 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:17:41.323 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:41.323 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:17:41.323 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:17:41.323 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:17:41.323 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:17:41.323 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:17:41.323 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:17:41.323 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:17:41.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:17:41.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:17:41.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:17:41.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:17:41.388 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:17:41.389 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:17:41.389 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:17:41.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:17:41.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:17:41.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:17:41.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:41.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:17:41.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:17:41.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:17:41.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:17:41.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:17:41.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:17:41.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:17:41.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:17:41.393 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:17:41.393 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:41.393 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:17:41.393 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:17:41.393 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:17:41.393 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:17:41.393 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:17:41.393 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:17:41.393 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:17:41.393 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:17:41.393 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:17:41.393 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:17:41.394 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:41.394 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:17:41.394 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:17:41.394 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:17:41.394 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:17:41.394 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:17:41.394 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:17:41.394 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:17:41.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:17:41.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:17:41.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:17:41.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:17:41.452 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:17:41.452 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:17:41.452 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:17:41.455 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:17:41.455 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:17:41.455 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:17:41.455 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:41.455 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:17:41.455 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:17:41.455 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:17:41.455 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:17:41.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:17:41.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:17:41.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:17:41.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:17:41.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:17:41.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:41.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:17:41.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:17:41.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:17:41.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:17:41.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:17:41.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:17:41.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:17:41.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:17:41.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:17:41.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:17:41.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:41.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:17:41.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:17:41.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:17:41.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:17:41.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:17:41.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:17:41.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:17:41.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:17:41.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:17:41.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:17:41.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:17:41.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:17:41.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:17:41.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:17:41.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:17:41.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:17:41.539 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:17:41.539 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:17:41.539 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:17:41.539 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:17:41.539 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:17:41.539 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:17:41.539 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:17:41.539 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:17:41.540 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:41.540 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:17:41.540 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:41.771 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 58 2026-02-18T08:17:41.772 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 58 2026-02-18T08:17:41.772 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:17:41.772 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=233, tag_tid=15, entry_tid=1], [object_number=232, tag_tid=15, entry_tid=0], [object_number=234, tag_tid=14, entry_tid=52758], [object_number=235, tag_tid=14, entry_tid=52755]]], state=connected] 2026-02-18T08:17:41.772 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=234, tag_tid=14, entry_tid=52758], [object_number=233, tag_tid=14, entry_tid=52757], [object_number=232, tag_tid=14, entry_tid=52756], [object_number=235, tag_tid=14, entry_tid=52755]]], state=connected] 2026-02-18T08:17:41.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:41.777 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=233, tag_tid=15, entry_tid=1' 2026-02-18T08:17:41.778 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:17:41.778 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:17:41.779 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:17:41.779 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:17:41.779 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:17:41.779 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:17:41.779 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:17:41.779 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:17:41.779 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:17:41.779 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:17:41.779 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:17:41.779 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:17:41.780 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:41.780 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:17:41.780 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:41.808 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 58 2026-02-18T08:17:41.808 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 58 2026-02-18T08:17:41.808 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:17:41.808 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=233, tag_tid=15, entry_tid=1], [object_number=232, tag_tid=15, entry_tid=0], [object_number=234, tag_tid=14, entry_tid=52758], [object_number=235, tag_tid=14, entry_tid=52755]]], state=connected] 2026-02-18T08:17:41.808 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=234, tag_tid=14, entry_tid=52758], [object_number=233, tag_tid=14, entry_tid=52757], [object_number=232, tag_tid=14, entry_tid=52756], [object_number=235, tag_tid=14, entry_tid=52755]]], state=connected] 2026-02-18T08:17:41.810 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:41.812 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=234, tag_tid=14, entry_tid=52758' 2026-02-18T08:17:41.812 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=233, tag_tid=15, entry_tid=1' -a 'object_number=233, tag_tid=15, entry_tid=1' = 'object_number=234, tag_tid=14, entry_tid=52758' 2026-02-18T08:17:41.812 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=234, tag_tid=14, entry_tid=52758' '!=' 'object_number=233, tag_tid=14, entry_tid=52493' 2026-02-18T08:17:41.812 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:17:41.812 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=234, tag_tid=14, entry_tid=52758' = 'object_number=233, tag_tid=14, entry_tid=52493' 2026-02-18T08:17:41.812 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=234, tag_tid=14, entry_tid=52758' 2026-02-18T08:17:41.813 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=233, tag_tid=15, entry_tid=1' 2026-02-18T08:17:41.813 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:17:41.813 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:17:41.814 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=15 2026-02-18T08:17:41.815 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=234, tag_tid=14, entry_tid=52758' 2026-02-18T08:17:41.815 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:17:41.815 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:17:41.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=14 2026-02-18T08:17:41.817 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=233, tag_tid=15, entry_tid=1' 2026-02-18T08:17:41.817 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:17:41.817 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:17:41.819 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:17:41.820 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=234, tag_tid=14, entry_tid=52758' 2026-02-18T08:17:41.820 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:17:41.820 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:17:41.821 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=52758 2026-02-18T08:17:41.821 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 15 = 14 -a 1 -le 52758 2026-02-18T08:17:41.821 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:17:41.822 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:17:41.822 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:17:42.014 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.0 with skip (0.9168920421423261) > weight (0.25) 2026-02-18T08:17:42.014 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.1 with skip (0.2984459675746133) > weight (0.25) 2026-02-18T08:17:42.014 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.2 with skip (0.41174251691075214) > weight (0.25) 2026-02-18T08:17:42.014 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.3 with skip (0.49581991648216417) > weight (0.25) 2026-02-18T08:17:42.014 INFO:tasks.rbd_mirror_thrash:waiting for 74.61291400007316 secs before thrashing 2026-02-18T08:17:42.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:17:42.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:17:42.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:17:42.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:17:42.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:17:42.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:17:42.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:17:42.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:17:42.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:17:42.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:42.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:17:42.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:17:42.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:17:42.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:17:42.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:17:42.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:17:42.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:17:42.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:17:42.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:17:42.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:17:42.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:42.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:17:42.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:17:42.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:17:42.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:17:42.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:17:42.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:17:42.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:17:42.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:17:42.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:17:42.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:42.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:17:42.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:17:42.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:17:42.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:17:42.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:17:42.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:17:42.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:17:42.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:17:42.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:17:42.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:17:42.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:42.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:17:42.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:17:42.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:17:42.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:17:42.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:17:42.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:17:42.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:17:42.038 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:17:42.038 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:17:42.038 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:17:42.038 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:17:42.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:17:42.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:17:42.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:17:42.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:17:42.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:17:42.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:17:42.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:42.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:17:42.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:17:42.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:17:42.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:17:42.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:17:42.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:17:42.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:17:42.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:17:42.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:17:42.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:42.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:17:42.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:17:42.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:17:42.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:17:42.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:17:42.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:17:42.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:17:42.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:17:42.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:17:42.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:17:42.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:42.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:17:42.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:17:42.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:17:42.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:17:42.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:17:42.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:17:42.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:17:42.101 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:17:42.101 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:17:42.101 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:17:42.101 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:17:42.158 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:17:42.158 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:17:42.158 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:17:42.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:17:42.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:17:42.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:17:42.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:42.161 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:17:42.161 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:17:42.161 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:17:42.161 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:17:42.161 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:17:42.161 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:17:42.161 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:17:42.161 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:17:42.161 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:17:42.161 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:42.161 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:17:42.161 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:17:42.161 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:17:42.161 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:17:42.161 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:17:42.161 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:17:42.161 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:17:42.161 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:17:42.162 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:17:42.162 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:17:42.162 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:42.162 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:17:42.162 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:17:42.162 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:17:42.162 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:17:42.162 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:17:42.162 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:17:42.162 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:17:42.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:17:42.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:17:42.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:17:42.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:17:42.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:17:42.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:17:42.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:17:42.230 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:17:42.231 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:17:42.231 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:17:42.231 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:42.231 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:17:42.231 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:17:42.231 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:17:42.231 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:17:42.231 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:17:42.231 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:17:42.231 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:17:42.231 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:17:42.231 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:17:42.231 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:42.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:17:42.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:17:42.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:17:42.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:17:42.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:17:42.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:17:42.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:17:42.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:17:42.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:17:42.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:17:42.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:42.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:17:42.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:17:42.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:17:42.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:17:42.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:17:42.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:17:42.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:17:42.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:17:42.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:17:42.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:17:42.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:17:42.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:17:42.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:17:42.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:17:42.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:17:42.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:17:42.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:17:42.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:17:42.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:17:42.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:17:42.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:17:42.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:17:42.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:17:42.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:17:42.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:42.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:17:42.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:42.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 58 2026-02-18T08:17:42.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 58 2026-02-18T08:17:42.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:17:42.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=233, tag_tid=15, entry_tid=1], [object_number=232, tag_tid=15, entry_tid=0], [object_number=234, tag_tid=14, entry_tid=52758], [object_number=235, tag_tid=14, entry_tid=52755]]], state=connected] 2026-02-18T08:17:42.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=234, tag_tid=14, entry_tid=52758], [object_number=233, tag_tid=14, entry_tid=52757], [object_number=232, tag_tid=14, entry_tid=52756], [object_number=235, tag_tid=14, entry_tid=52755]]], state=connected] 2026-02-18T08:17:42.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:42.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=233, tag_tid=15, entry_tid=1' 2026-02-18T08:17:42.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:17:42.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:17:42.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:17:42.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:17:42.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:17:42.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:17:42.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:17:42.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:17:42.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:17:42.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:17:42.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:17:42.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:17:42.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:42.344 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:17:42.344 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:42.372 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 58 2026-02-18T08:17:42.372 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 58 2026-02-18T08:17:42.372 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:17:42.372 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=233, tag_tid=15, entry_tid=1], [object_number=232, tag_tid=15, entry_tid=0], [object_number=234, tag_tid=14, entry_tid=52758], [object_number=235, tag_tid=14, entry_tid=52755]]], state=connected] 2026-02-18T08:17:42.372 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=234, tag_tid=14, entry_tid=52758], [object_number=233, tag_tid=14, entry_tid=52757], [object_number=232, tag_tid=14, entry_tid=52756], [object_number=235, tag_tid=14, entry_tid=52755]]], state=connected] 2026-02-18T08:17:42.374 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:42.376 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=234, tag_tid=14, entry_tid=52758' 2026-02-18T08:17:42.377 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=233, tag_tid=15, entry_tid=1' -a 'object_number=233, tag_tid=15, entry_tid=1' = 'object_number=234, tag_tid=14, entry_tid=52758' 2026-02-18T08:17:42.377 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=234, tag_tid=14, entry_tid=52758' '!=' 'object_number=234, tag_tid=14, entry_tid=52758' 2026-02-18T08:17:42.377 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:17:42.377 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.4 2026-02-18T08:17:42.778 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:17:42.778 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:17:42.778 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:17:42.778 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:17:42.778 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:17:42.778 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:17:42.778 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:17:42.779 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:17:42.779 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:17:42.779 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:42.779 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:17:42.779 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:17:42.779 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:17:42.779 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:17:42.779 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:17:42.781 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:17:42.781 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:17:42.781 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:17:42.781 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:17:42.781 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:17:42.781 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:42.781 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:17:42.781 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:17:42.781 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:17:42.781 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:17:42.781 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:17:42.781 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:17:42.781 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:17:42.781 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:17:42.781 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:17:42.781 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:42.781 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:17:42.781 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:17:42.782 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:17:42.782 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:17:42.782 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:17:42.782 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:17:42.782 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:17:42.782 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:17:42.782 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:17:42.782 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:17:42.782 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:42.782 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:17:42.782 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:17:42.782 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:17:42.782 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:17:42.782 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:17:42.782 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:17:42.782 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:17:42.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:17:42.793 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:17:42.793 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:17:42.793 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:17:42.843 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:17:42.843 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:17:42.843 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:17:42.845 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:17:42.845 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:17:42.845 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:17:42.845 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:42.845 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:17:42.845 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:17:42.845 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:17:42.845 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:17:42.845 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:17:42.846 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:17:42.846 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:17:42.846 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:17:42.846 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:17:42.846 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:42.846 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:17:42.846 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:17:42.846 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:17:42.846 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:17:42.846 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:17:42.846 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:17:42.846 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:17:42.846 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:17:42.846 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:17:42.846 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:17:42.846 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:42.846 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:17:42.846 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:17:42.846 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:17:42.846 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:17:42.846 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:17:42.847 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:17:42.847 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:17:42.856 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:17:42.857 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:17:42.857 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:17:42.857 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:17:42.914 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:17:42.915 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:17:42.915 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:17:42.917 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:17:42.917 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:17:42.917 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:17:42.918 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:42.918 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:17:42.918 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:17:42.918 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:17:42.918 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:17:42.918 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:17:42.918 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:17:42.918 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:17:42.918 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:17:42.918 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:17:42.918 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:42.918 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:17:42.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:17:42.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:17:42.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:17:42.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:17:42.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:17:42.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:17:42.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:17:42.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:17:42.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:17:42.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:42.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:17:42.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:17:42.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:17:42.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:17:42.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:17:42.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:17:42.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:17:42.928 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:17:42.929 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:17:42.929 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:17:42.929 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:17:42.986 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:17:42.986 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:17:42.986 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:17:42.988 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:17:42.989 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:17:42.989 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:17:42.989 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:42.989 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:17:42.989 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:17:42.989 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:17:42.989 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:17:42.989 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:17:42.989 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:17:42.989 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:17:42.989 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:17:42.989 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:17:42.989 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:42.989 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:17:42.989 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:17:42.989 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:17:42.989 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:17:42.989 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:17:42.989 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:17:42.989 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:17:42.990 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:17:42.990 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:17:42.990 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:17:42.990 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:42.990 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:17:42.990 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:17:42.990 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:17:42.990 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:17:42.990 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:17:42.990 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:17:42.990 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:17:43.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:17:43.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:17:43.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:17:43.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:17:43.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:17:43.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:17:43.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:17:43.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:17:43.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:17:43.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:17:43.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:17:43.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:17:43.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:17:43.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:17:43.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:17:43.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:17:43.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:17:43.067 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:43.068 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:17:43.068 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:43.097 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 58 2026-02-18T08:17:43.097 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 58 2026-02-18T08:17:43.097 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:17:43.097 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=233, tag_tid=15, entry_tid=1], [object_number=232, tag_tid=15, entry_tid=0], [object_number=234, tag_tid=14, entry_tid=52758], [object_number=235, tag_tid=14, entry_tid=52755]]], state=connected] 2026-02-18T08:17:43.097 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=234, tag_tid=14, entry_tid=52758], [object_number=233, tag_tid=14, entry_tid=52757], [object_number=232, tag_tid=14, entry_tid=52756], [object_number=235, tag_tid=14, entry_tid=52755]]], state=connected] 2026-02-18T08:17:43.100 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:43.102 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=233, tag_tid=15, entry_tid=1' 2026-02-18T08:17:43.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:17:43.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:17:43.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:17:43.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:17:43.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:17:43.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:17:43.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:17:43.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:17:43.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:17:43.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:17:43.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:17:43.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:17:43.105 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:43.105 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:17:43.105 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:43.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 58 2026-02-18T08:17:43.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 58 2026-02-18T08:17:43.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:17:43.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=233, tag_tid=15, entry_tid=1], [object_number=232, tag_tid=15, entry_tid=0], [object_number=234, tag_tid=14, entry_tid=52758], [object_number=235, tag_tid=14, entry_tid=52755]]], state=connected] 2026-02-18T08:17:43.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=234, tag_tid=14, entry_tid=52758], [object_number=233, tag_tid=14, entry_tid=52757], [object_number=232, tag_tid=14, entry_tid=52756], [object_number=235, tag_tid=14, entry_tid=52755]]], state=connected] 2026-02-18T08:17:43.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:43.137 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=234, tag_tid=14, entry_tid=52758' 2026-02-18T08:17:43.137 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=233, tag_tid=15, entry_tid=1' -a 'object_number=233, tag_tid=15, entry_tid=1' = 'object_number=234, tag_tid=14, entry_tid=52758' 2026-02-18T08:17:43.137 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=234, tag_tid=14, entry_tid=52758' '!=' 'object_number=234, tag_tid=14, entry_tid=52758' 2026-02-18T08:17:43.138 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:17:43.138 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.8 2026-02-18T08:17:43.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:17:43.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:17:43.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:17:43.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:17:43.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:17:43.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:17:43.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:17:43.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:17:43.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:17:43.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:43.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:17:43.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:17:43.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:17:43.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:17:43.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:17:43.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:17:43.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:17:43.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:17:43.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:17:43.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:17:43.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:43.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:17:43.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:17:43.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:17:43.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:17:43.946 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:17:43.946 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:17:43.946 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:17:43.946 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:17:43.946 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:17:43.946 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:43.946 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:17:43.946 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:17:43.946 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:17:43.946 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:17:43.946 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:17:43.947 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:17:43.947 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:17:43.947 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:17:43.947 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:17:43.947 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:17:43.947 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:43.947 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:17:43.947 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:17:43.947 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:17:43.947 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:17:43.947 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:17:43.948 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:17:43.948 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:17:43.956 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:17:43.956 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:17:43.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:17:43.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:17:44.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:17:44.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:17:44.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:17:44.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:17:44.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:17:44.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:17:44.018 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:44.018 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:17:44.018 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:17:44.018 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:17:44.018 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:17:44.018 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:17:44.018 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:17:44.018 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:17:44.018 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:17:44.018 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:17:44.019 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:44.019 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:17:44.019 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:17:44.019 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:17:44.019 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:17:44.019 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:17:44.019 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:17:44.019 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:17:44.019 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:17:44.019 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:17:44.020 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:17:44.020 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:44.020 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:17:44.020 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:17:44.020 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:17:44.020 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:17:44.020 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:17:44.020 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:17:44.020 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:17:44.028 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:17:44.029 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:17:44.029 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:17:44.029 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:17:44.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:17:44.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:17:44.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:17:44.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:17:44.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:17:44.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:17:44.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:44.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:17:44.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:17:44.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:17:44.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:17:44.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:17:44.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:17:44.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:17:44.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:17:44.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:17:44.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:44.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:17:44.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:17:44.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:17:44.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:17:44.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:17:44.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:17:44.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:17:44.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:17:44.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:17:44.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:17:44.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:44.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:17:44.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:17:44.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:17:44.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:17:44.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:17:44.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:17:44.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:17:44.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:17:44.100 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:17:44.100 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:17:44.100 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:17:44.147 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:17:44.147 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:17:44.147 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:17:44.150 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:17:44.150 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:17:44.150 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:17:44.150 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:44.150 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:17:44.150 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:17:44.150 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:17:44.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:17:44.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:17:44.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:17:44.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:17:44.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:17:44.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:17:44.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:44.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:17:44.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:17:44.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:17:44.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:17:44.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:17:44.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:17:44.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:17:44.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:17:44.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:17:44.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:17:44.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:44.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:17:44.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:17:44.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:17:44.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:17:44.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:17:44.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:17:44.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:17:44.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:17:44.161 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:17:44.161 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:17:44.161 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:17:44.223 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:17:44.224 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:17:44.224 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:17:44.224 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:17:44.224 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:17:44.224 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:17:44.224 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:17:44.224 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:17:44.224 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:17:44.224 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:17:44.224 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:17:44.225 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:17:44.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:17:44.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:44.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:17:44.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:44.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 58 2026-02-18T08:17:44.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 58 2026-02-18T08:17:44.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:17:44.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=233, tag_tid=15, entry_tid=1], [object_number=232, tag_tid=15, entry_tid=0], [object_number=234, tag_tid=14, entry_tid=52758], [object_number=235, tag_tid=14, entry_tid=52755]]], state=connected] 2026-02-18T08:17:44.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=234, tag_tid=14, entry_tid=52758], [object_number=233, tag_tid=14, entry_tid=52757], [object_number=232, tag_tid=14, entry_tid=52756], [object_number=235, tag_tid=14, entry_tid=52755]]], state=connected] 2026-02-18T08:17:44.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:44.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=233, tag_tid=15, entry_tid=1' 2026-02-18T08:17:44.270 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:17:44.270 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:17:44.270 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:17:44.270 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:17:44.270 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:17:44.270 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:17:44.270 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:17:44.270 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:17:44.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:17:44.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:17:44.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:17:44.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:17:44.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:44.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:17:44.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:44.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 58 2026-02-18T08:17:44.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 58 2026-02-18T08:17:44.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:17:44.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=233, tag_tid=15, entry_tid=1], [object_number=232, tag_tid=15, entry_tid=0], [object_number=234, tag_tid=14, entry_tid=52758], [object_number=235, tag_tid=14, entry_tid=52755]]], state=connected] 2026-02-18T08:17:44.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=234, tag_tid=14, entry_tid=52758], [object_number=233, tag_tid=14, entry_tid=52757], [object_number=232, tag_tid=14, entry_tid=52756], [object_number=235, tag_tid=14, entry_tid=52755]]], state=connected] 2026-02-18T08:17:44.310 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:44.312 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=234, tag_tid=14, entry_tid=52758' 2026-02-18T08:17:44.312 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=233, tag_tid=15, entry_tid=1' -a 'object_number=233, tag_tid=15, entry_tid=1' = 'object_number=234, tag_tid=14, entry_tid=52758' 2026-02-18T08:17:44.312 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=234, tag_tid=14, entry_tid=52758' '!=' 'object_number=234, tag_tid=14, entry_tid=52758' 2026-02-18T08:17:44.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:17:44.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1.6 2026-02-18T08:17:45.914 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:17:45.915 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:17:45.915 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:17:45.915 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:17:45.915 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:17:45.915 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:17:45.915 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:17:45.915 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:17:45.915 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:17:45.915 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:45.916 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:17:45.916 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:17:45.916 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:17:45.916 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:17:45.916 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:17:45.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:17:45.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:17:45.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:17:45.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:17:45.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:17:45.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:45.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:17:45.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:17:45.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:17:45.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:17:45.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:17:45.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:17:45.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:17:45.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:17:45.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:17:45.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:45.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:17:45.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:17:45.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:17:45.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:17:45.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:17:45.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:17:45.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:17:45.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:17:45.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:17:45.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:17:45.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:45.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:17:45.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:17:45.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:17:45.922 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:17:45.922 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:17:45.922 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:17:45.922 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:17:45.930 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:17:45.930 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:17:45.930 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:17:45.931 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:17:45.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:17:45.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:17:45.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:17:45.982 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:17:45.982 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:17:45.982 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:17:45.982 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:45.982 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:17:45.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:17:45.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:17:45.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:17:45.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:17:45.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:17:45.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:17:45.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:17:45.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:17:45.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:45.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:17:45.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:17:45.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:17:45.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:17:45.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:17:45.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:17:45.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:17:45.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:17:45.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:17:45.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:17:45.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:45.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:17:45.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:17:45.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:17:45.985 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:17:45.985 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:17:45.985 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:17:45.985 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:17:45.994 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:17:45.994 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:17:45.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:17:45.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:17:46.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:17:46.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:17:46.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:17:46.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:17:46.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:17:46.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:17:46.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:46.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:17:46.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:17:46.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:17:46.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:17:46.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:17:46.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:17:46.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:17:46.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:17:46.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:17:46.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:46.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:17:46.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:17:46.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:17:46.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:17:46.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:17:46.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:17:46.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:17:46.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:17:46.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:17:46.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:17:46.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:46.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:17:46.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:17:46.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:17:46.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:17:46.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:17:46.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:17:46.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:17:46.056 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:17:46.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:17:46.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:17:46.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:17:46.113 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:17:46.113 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:17:46.113 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:17:46.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:17:46.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:17:46.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:17:46.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:46.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:17:46.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:17:46.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:17:46.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:17:46.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:17:46.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:17:46.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:17:46.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:17:46.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:17:46.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:46.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:17:46.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:17:46.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:17:46.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:17:46.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:17:46.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:17:46.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:17:46.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:17:46.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:17:46.118 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:17:46.118 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:46.118 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:17:46.118 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:17:46.118 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:17:46.118 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:17:46.118 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:17:46.118 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:17:46.118 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:17:46.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:17:46.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:17:46.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:17:46.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:17:46.189 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:17:46.190 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:17:46.190 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:17:46.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:17:46.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:17:46.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:17:46.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:17:46.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:17:46.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:17:46.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:17:46.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:17:46.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:17:46.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:17:46.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:46.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:17:46.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:46.221 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 58 2026-02-18T08:17:46.221 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 58 2026-02-18T08:17:46.221 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:17:46.221 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=233, tag_tid=15, entry_tid=1], [object_number=232, tag_tid=15, entry_tid=0], [object_number=234, tag_tid=14, entry_tid=52758], [object_number=235, tag_tid=14, entry_tid=52755]]], state=connected] 2026-02-18T08:17:46.221 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=234, tag_tid=14, entry_tid=52758], [object_number=233, tag_tid=14, entry_tid=52757], [object_number=232, tag_tid=14, entry_tid=52756], [object_number=235, tag_tid=14, entry_tid=52755]]], state=connected] 2026-02-18T08:17:46.224 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:46.225 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=233, tag_tid=15, entry_tid=1' 2026-02-18T08:17:46.227 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:17:46.227 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:17:46.227 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:17:46.227 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:17:46.227 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:17:46.227 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:17:46.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:17:46.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:17:46.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:17:46.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:17:46.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:17:46.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:17:46.229 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:46.229 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:17:46.229 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:46.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 58 2026-02-18T08:17:46.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 58 2026-02-18T08:17:46.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:17:46.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=233, tag_tid=15, entry_tid=1], [object_number=232, tag_tid=15, entry_tid=0], [object_number=234, tag_tid=14, entry_tid=52758], [object_number=235, tag_tid=14, entry_tid=52755]]], state=connected] 2026-02-18T08:17:46.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=234, tag_tid=14, entry_tid=52758], [object_number=233, tag_tid=14, entry_tid=52757], [object_number=232, tag_tid=14, entry_tid=52756], [object_number=235, tag_tid=14, entry_tid=52755]]], state=connected] 2026-02-18T08:17:46.261 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:46.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=234, tag_tid=14, entry_tid=52758' 2026-02-18T08:17:46.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=233, tag_tid=15, entry_tid=1' -a 'object_number=233, tag_tid=15, entry_tid=1' = 'object_number=234, tag_tid=14, entry_tid=52758' 2026-02-18T08:17:46.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=234, tag_tid=14, entry_tid=52758' '!=' 'object_number=234, tag_tid=14, entry_tid=52758' 2026-02-18T08:17:46.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:17:46.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 2 2026-02-18T08:17:48.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:17:48.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:17:48.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:17:48.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:17:48.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:17:48.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:17:48.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:17:48.266 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:17:48.266 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:17:48.266 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:48.266 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:17:48.266 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:17:48.266 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:17:48.266 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:17:48.266 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:17:48.269 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:17:48.269 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:17:48.269 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:17:48.270 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:17:48.270 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:17:48.270 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:48.270 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:17:48.270 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:17:48.270 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:17:48.270 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:17:48.270 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:17:48.270 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:17:48.270 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:17:48.270 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:17:48.270 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:17:48.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:48.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:17:48.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:17:48.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:17:48.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:17:48.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:17:48.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:17:48.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:17:48.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:17:48.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:17:48.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:17:48.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:48.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:17:48.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:17:48.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:17:48.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:17:48.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:17:48.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:17:48.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:17:48.282 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:17:48.282 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:17:48.282 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:17:48.282 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:17:48.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:17:48.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:17:48.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:17:48.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:17:48.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:17:48.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:17:48.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:48.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:17:48.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:17:48.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:17:48.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:17:48.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:17:48.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:17:48.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:17:48.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:17:48.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:17:48.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:48.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:17:48.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:17:48.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:17:48.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:17:48.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:17:48.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:17:48.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:17:48.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:17:48.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:17:48.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:17:48.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:48.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:17:48.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:17:48.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:17:48.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:17:48.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:17:48.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:17:48.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:17:48.352 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:17:48.353 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:17:48.353 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:17:48.353 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:17:48.409 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:17:48.409 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:17:48.409 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:17:48.412 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:17:48.412 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:17:48.412 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:17:48.412 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:48.412 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:17:48.413 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:17:48.413 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:17:48.413 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:17:48.413 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:17:48.413 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:17:48.413 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:17:48.413 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:17:48.413 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:17:48.413 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:48.413 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:17:48.413 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:17:48.414 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:17:48.414 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:17:48.414 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:17:48.414 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:17:48.414 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:17:48.414 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:17:48.414 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:17:48.414 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:17:48.414 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:48.414 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:17:48.414 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:17:48.414 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:17:48.415 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:17:48.415 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:17:48.415 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:17:48.415 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:17:48.423 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:17:48.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:17:48.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:17:48.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:17:48.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:17:48.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:17:48.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:17:48.474 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:17:48.474 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:17:48.474 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:17:48.474 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:17:48.474 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:17:48.474 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:17:48.474 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:17:48.474 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:17:48.475 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:17:48.475 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:17:48.475 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:17:48.475 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:17:48.475 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:17:48.475 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:48.475 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:17:48.475 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:17:48.475 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:17:48.475 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:17:48.475 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:17:48.475 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:17:48.476 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:17:48.476 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:17:48.476 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:17:48.476 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:17:48.476 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:17:48.476 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:17:48.476 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:17:48.476 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:17:48.476 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:17:48.476 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:17:48.476 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:17:48.476 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:17:48.484 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:17:48.485 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:17:48.485 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:17:48.485 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:17:48.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:17:48.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:17:48.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:17:48.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:17:48.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:17:48.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:17:48.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:17:48.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:17:48.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:17:48.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:17:48.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:17:48.545 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:17:48.545 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:17:48.545 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:48.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:17:48.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:48.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 58 2026-02-18T08:17:48.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 58 2026-02-18T08:17:48.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:17:48.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=233, tag_tid=15, entry_tid=1], [object_number=232, tag_tid=15, entry_tid=0], [object_number=234, tag_tid=14, entry_tid=52758], [object_number=235, tag_tid=14, entry_tid=52755]]], state=connected] 2026-02-18T08:17:48.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=233, tag_tid=15, entry_tid=1], [object_number=232, tag_tid=15, entry_tid=0], [object_number=234, tag_tid=14, entry_tid=52758], [object_number=235, tag_tid=14, entry_tid=52755]]], state=connected] 2026-02-18T08:17:48.587 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:48.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=233, tag_tid=15, entry_tid=1' 2026-02-18T08:17:48.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:17:48.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:17:48.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:17:48.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:17:48.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:17:48.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:17:48.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:17:48.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:17:48.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:17:48.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:17:48.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:17:48.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:17:48.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:48.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:17:48.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:48.620 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 58 2026-02-18T08:17:48.620 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 58 2026-02-18T08:17:48.620 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:17:48.620 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=233, tag_tid=15, entry_tid=1], [object_number=232, tag_tid=15, entry_tid=0], [object_number=234, tag_tid=14, entry_tid=52758], [object_number=235, tag_tid=14, entry_tid=52755]]], state=connected] 2026-02-18T08:17:48.620 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=233, tag_tid=15, entry_tid=1], [object_number=232, tag_tid=15, entry_tid=0], [object_number=234, tag_tid=14, entry_tid=52758], [object_number=235, tag_tid=14, entry_tid=52755]]], state=connected] 2026-02-18T08:17:48.622 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:17:48.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=233, tag_tid=15, entry_tid=1' 2026-02-18T08:17:48.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=233, tag_tid=15, entry_tid=1' -a 'object_number=233, tag_tid=15, entry_tid=1' = 'object_number=233, tag_tid=15, entry_tid=1' 2026-02-18T08:17:48.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:17:48.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_snap_present cluster1 mirror test snap5 2026-02-18T08:17:48.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:17:48.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:17:48.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:17:48.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap5 2026-02-18T08:17:48.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-18T08:17:48.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-18T08:17:48.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1 2026-02-18T08:17:49.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster1 info mirror/test@snap5 2026-02-18T08:17:49.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:rbd image 'test': 2026-02-18T08:17:49.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: size 512 MiB in 128 objects 2026-02-18T08:17:49.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: order 22 (4 MiB objects) 2026-02-18T08:17:49.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: snapshot_count: 9 2026-02-18T08:17:49.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: id: 10a74aa8ebc0 2026-02-18T08:17:49.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: block_name_prefix: rbd_data.10a74aa8ebc0 2026-02-18T08:17:49.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: format: 2 2026-02-18T08:17:49.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: features: layering, exclusive-lock, journaling 2026-02-18T08:17:49.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: op_features: 2026-02-18T08:17:49.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: flags: 2026-02-18T08:17:49.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: create_timestamp: Wed Feb 18 08:10:48 2026 2026-02-18T08:17:49.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: access_timestamp: Wed Feb 18 08:16:08 2026 2026-02-18T08:17:49.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: modify_timestamp: Wed Feb 18 08:17:16 2026 2026-02-18T08:17:49.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: protected: False 2026-02-18T08:17:49.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: journal: 10a74aa8ebc0 2026-02-18T08:17:49.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring state: enabled 2026-02-18T08:17:49.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring mode: journal 2026-02-18T08:17:49.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring global id: 0b548823-d751-43df-a076-4efac2da4d3d 2026-02-18T08:17:49.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring primary: false 2026-02-18T08:17:49.659 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:17:49.659 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n snap4-clean ']' 2026-02-18T08:17:49.660 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ compare_image_snaps mirror test snap4-clean 2026-02-18T08:17:49.660 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:17:49.660 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:17:49.660 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap4-clean 2026-02-18T08:17:49.660 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local ret=0 2026-02-18T08:17:49.660 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local rmt_export=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.export 2026-02-18T08:17:49.660 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loc_export=/tmp/tmp.vybh1m9yJF/cluster1-mirror-test.export 2026-02-18T08:17:49.660 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.export /tmp/tmp.vybh1m9yJF/cluster1-mirror-test.export 2026-02-18T08:17:49.660 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 -p mirror export test@snap4-clean /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.export 2026-02-18T08:17:50.145 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-18T08:17:50.156 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster1 -p mirror export test@snap4-clean /tmp/tmp.vybh1m9yJF/cluster1-mirror-test.export 2026-02-18T08:17:50.742 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-18T08:17:50.756 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmp /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.export /tmp/tmp.vybh1m9yJF/cluster1-mirror-test.export 2026-02-18T08:17:50.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.export /tmp/tmp.vybh1m9yJF/cluster1-mirror-test.export 2026-02-18T08:17:51.086 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:17:51.086 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ compare_image_snaps mirror test snap5 2026-02-18T08:17:51.086 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:17:51.086 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:17:51.086 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap5 2026-02-18T08:17:51.086 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local ret=0 2026-02-18T08:17:51.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local rmt_export=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.export 2026-02-18T08:17:51.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loc_export=/tmp/tmp.vybh1m9yJF/cluster1-mirror-test.export 2026-02-18T08:17:51.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.export /tmp/tmp.vybh1m9yJF/cluster1-mirror-test.export 2026-02-18T08:17:51.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 -p mirror export test@snap5 /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.export 2026-02-18T08:17:51.561 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-18T08:17:51.570 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster1 -p mirror export test@snap5 /tmp/tmp.vybh1m9yJF/cluster1-mirror-test.export 2026-02-18T08:17:52.190 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-18T08:17:52.206 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmp /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.export /tmp/tmp.vybh1m9yJF/cluster1-mirror-test.export 2026-02-18T08:17:52.393 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.export /tmp/tmp.vybh1m9yJF/cluster1-mirror-test.export 2026-02-18T08:17:52.534 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:17:52.534 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ clean_snap_name=snap5-clean 2026-02-18T08:17:52.534 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_snap cluster2 mirror test snap5-clean 2026-02-18T08:17:52.534 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:17:52.534 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:17:52.534 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:17:52.534 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap5-clean 2026-02-18T08:17:52.534 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 -p mirror snap create test@snap5-clean --debug-rbd=20 --debug-journaler=20 2026-02-18T08:17:52.912 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 10` 2026-02-18T08:17:52.913 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ stress_write_image cluster2 mirror test 2026-02-18T08:17:52.913 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:17:52.913 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:17:52.913 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:17:52.913 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-18T08:17:52.914 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local duration=7 2026-02-18T08:17:52.915 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set +e 2026-02-18T08:17:52.915 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ timeout 7s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror test --debug-rbd=20 --debug-journaler=20 2026-02-18T08:17:53.147 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:100 IOs, 10236928 bytes 2026-02-18T08:17:54.397 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:200 IOs, 86723584 bytes 2026-02-18T08:17:54.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:300 IOs, 140259328 bytes 2026-02-18T08:17:55.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:400 IOs, 190121984 bytes 2026-02-18T08:17:55.885 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:500 IOs, 224422912 bytes 2026-02-18T08:17:56.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:600 IOs, 300667904 bytes 2026-02-18T08:17:58.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:700 IOs, 396037120 bytes 2026-02-18T08:17:59.430 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:800 IOs, 458811392 bytes 2026-02-18T08:17:59.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ error_code=124 2026-02-18T08:17:59.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set -e 2026-02-18T08:17:59.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 124 -eq 124 ']' 2026-02-18T08:17:59.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:17:59.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+replaying 2026-02-18T08:17:59.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:17:59.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:17:59.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:17:59.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local state_pattern=up+replaying 2026-02-18T08:17:59.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local description_pattern= 2026-02-18T08:17:59.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local service_pattern= 2026-02-18T08:17:59.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:17:59.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1 2026-02-18T08:18:00.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test_status_in_pool_dir cluster1 mirror test up+replaying '' '' 2026-02-18T08:18:00.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:18:00.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:18:00.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:18:00.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local state_pattern=up+replaying 2026-02-18T08:18:00.928 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local description_pattern= 2026-02-18T08:18:00.928 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local service_pattern= 2026-02-18T08:18:00.928 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local status 2026-02-18T08:18:00.928 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ CEPH_ARGS= 2026-02-18T08:18:00.928 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-18T08:18:00.967 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ status='test: 2026-02-18T08:18:00.967 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: global_id: 0b548823-d751-43df-a076-4efac2da4d3d 2026-02-18T08:18:00.968 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: state: up+replaying 2026-02-18T08:18:00.968 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: description: replaying, {"bytes_per_second":8424956.93,"entries_behind_primary":1,"entries_per_second":522.53,"non_primary_position":{"entry_tid":52758,"object_number":234,"tag_tid":14},"primary_position":{"entry_tid":1,"object_number":233,"tag_tid":15},"seconds_until_synced":0} 2026-02-18T08:18:00.968 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: service: mirror.3 on trial163 2026-02-18T08:18:00.968 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: last_update: 2026-02-18 08:17:46 2026-02-18T08:18:00.968 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: peer_sites: 2026-02-18T08:18:00.968 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: name: cluster2 2026-02-18T08:18:00.968 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: state: up+stopped 2026-02-18T08:18:00.968 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: description: local image is primary 2026-02-18T08:18:00.968 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: last_update: 2026-02-18 08:17:46' 2026-02-18T08:18:00.968 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ grep '^ state: .*up+replaying' 2026-02-18T08:18:00.969 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: state: up+replaying 2026-02-18T08:18:00.969 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ grep '^ description: .*' 2026-02-18T08:18:00.970 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: description: replaying, {"bytes_per_second":8424956.93,"entries_behind_primary":1,"entries_per_second":522.53,"non_primary_position":{"entry_tid":52758,"object_number":234,"tag_tid":14},"primary_position":{"entry_tid":1,"object_number":233,"tag_tid":15},"seconds_until_synced":0} 2026-02-18T08:18:00.971 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n '' ']' 2026-02-18T08:18:00.971 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ echo up+replaying 2026-02-18T08:18:00.971 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ grep '^up+' 2026-02-18T08:18:00.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:up+replaying 2026-02-18T08:18:00.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ grep 'service: *mirror..* on ' 2026-02-18T08:18:00.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: service: mirror.3 on trial163 2026-02-18T08:18:00.974 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local last_update 2026-02-18T08:18:00.974 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-18T08:18:00.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_update='2026-02-18 08:17:46' 2026-02-18T08:18:00.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+replaying '2026-02-18 08:17:46' 2026-02-18T08:18:00.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:18:00.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:18:00.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:18:00.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local state_pattern=up+replaying 2026-02-18T08:18:00.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'prev_last_update=2026-02-18 08:17:46' 2026-02-18T08:18:00.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local status 2026-02-18T08:18:00.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ CEPH_ARGS= 2026-02-18T08:18:00.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-18T08:18:01.020 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ status='OKOKOK142194268mirror.3trial16320.2.0-670-gadc19233falseOK43304333mirror.2trial16320.2.0-670-gadc19233falseOK43534356mirror.1trial16320.2.0-670-gadc19233trueOK43834386mirror.0trial16320.2.0-670-gadc19233falseOKtest0b548823-d751-43df-a076-4efac2da4d3dup+replayingreplaying, {"bytes_per_second":8424956.93,"entries_behind_primary":1,"entries_per_second":522.53,"non_primary_position":{"entry_tid":52758,"object_number":234,"tag_tid":14},"primary_position":{"entry_tid":1,"object_number":233,"tag_tid":15},"seconds_until_synced":0}42194268mirror.3trial1632026-02-18 08:17:46cluster2c5e4f85e-ec31-430b-a71d-56e000d0de0bup+stoppedlocal image is primary2026-02-18 08:17:46' 2026-02-18T08:18:01.020 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local last_update state 2026-02-18T08:18:01.020 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-18T08:18:01.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_update='2026-02-18 08:17:46' 2026-02-18T08:18:01.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-18T08:18:01.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ state=up+replaying 2026-02-18T08:18:01.027 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ echo up+replaying 2026-02-18T08:18:01.027 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ grep up+replaying 2026-02-18T08:18:01.028 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:up+replaying 2026-02-18T08:18:01.029 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:18:01.029 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:18:01.029 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ snap_name=snap6 2026-02-18T08:18:01.029 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_snap cluster2 mirror test snap6 2026-02-18T08:18:01.029 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:18:01.029 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:18:01.029 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:18:01.029 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap6 2026-02-18T08:18:01.029 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 -p mirror snap create test@snap6 --debug-rbd=20 --debug-journaler=20 2026-02-18T08:18:27.212 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.0 with skip (0.49011670390188056) > weight (0.25) 2026-02-18T08:18:27.212 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.1 with skip (0.6711697897875236) > weight (0.25) 2026-02-18T08:18:27.213 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.2 with skip (0.7960088894185788) > weight (0.25) 2026-02-18T08:18:27.213 INFO:tasks.rbd_mirror_thrash:kill cluster2.client.mirror.3 2026-02-18T08:18:27.213 INFO:tasks.rbd_mirror.cluster2.client.mirror.3:Sent signal 15 2026-02-18T08:18:27.213 INFO:tasks.rbd_mirror_thrash:waiting for 5.486871915448045 secs before reviving daemons 2026-02-18T08:18:27.213 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.trial163.stderr:2026-02-18T08:18:27.205+0000 7f3bb7b3d640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.3 (PID: 15053) UID: 1000 2026-02-18T08:18:31.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_replay_started cluster1 mirror test 2026-02-18T08:18:31.936 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:18:31.936 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:18:31.936 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:18:31.936 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_replay_state cluster1 mirror test started 2026-02-18T08:18:31.936 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:18:31.936 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:18:31.936 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:18:31.936 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local state=started 2026-02-18T08:18:31.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-18T08:18:31.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:18:31.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1 2026-02-18T08:18:32.700 INFO:tasks.rbd_mirror_thrash:waiting for cluster2.client.mirror.3 2026-02-18T08:18:32.701 INFO:teuthology.orchestra.run:waiting for 600 2026-02-18T08:18:32.701 INFO:tasks.rbd_mirror_thrash:reviving cluster2.client.mirror.3 2026-02-18T08:18:32.701 INFO:tasks.rbd_mirror.cluster2.client.mirror.3:Restarting daemon 2026-02-18T08:18:32.701 DEBUG:teuthology.orchestra.run.trial163:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.3 2026-02-18T08:18:32.704 INFO:tasks.rbd_mirror.cluster2.client.mirror.3:Started 2026-02-18T08:18:32.704 INFO:tasks.rbd_mirror_thrash:waiting for 76.14797430200016 secs before thrashing 2026-02-18T08:18:32.936 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test_image_replay_state cluster1 mirror test started 2026-02-18T08:18:32.936 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:18:32.936 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:18:32.936 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:18:32.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local test_state=started 2026-02-18T08:18:32.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local status_result 2026-02-18T08:18:32.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_state=stopped 2026-02-18T08:18:32.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-18T08:18:32.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -i state 2026-02-18T08:18:32.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_instance=cluster1 2026-02-18T08:18:32.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-18T08:18:32.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:18:32.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance=cluster1 2026-02-18T08:18:32.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local loop_instance 2026-02-18T08:18:32.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:18:32.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sleep 0 2026-02-18T08:18:32.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:18:32.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ seq 0 3 2026-02-18T08:18:32.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:18:32.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-18T08:18:32.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:0 2026-02-18T08:18:32.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-18T08:18:32.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:18:32.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:18:32.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:18:32.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-18T08:18:32.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:18:32.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:18:32.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:18:32.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:18:32.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-18T08:18:32.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:18:32.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:18:32.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:18:32.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:18:32.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:0 2026-02-18T08:18:32.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-18T08:18:32.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-18T08:18:32.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:18:32.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:0 2026-02-18T08:18:32.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T08:18:32.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-18T08:18:32.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:18:32.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:18:32.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-18T08:18:32.944 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-18T08:18:32.944 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-18T08:18:32.944 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=0 2026-02-18T08:18:32.944 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=0 2026-02-18T08:18:32.944 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:18:32.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:18:32.952 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:18:32.952 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:18:32.952 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-18T08:18:33.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:18:33.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status : get status for rbd mirror 2026-02-18T08:18:33.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:18:33.003 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:18:33.003 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test 2026-02-18T08:18:33.003 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:1 2026-02-18T08:18:33.003 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-18T08:18:33.004 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:18:33.004 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:18:33.004 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:18:33.004 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-18T08:18:33.004 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:18:33.004 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:18:33.004 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:18:33.004 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:18:33.004 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-18T08:18:33.004 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:18:33.004 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:18:33.004 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:18:33.005 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:18:33.005 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:1 2026-02-18T08:18:33.005 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-18T08:18:33.005 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-18T08:18:33.005 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:18:33.005 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:1 2026-02-18T08:18:33.005 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T08:18:33.005 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-18T08:18:33.005 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:18:33.005 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:18:33.005 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-18T08:18:33.005 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-18T08:18:33.006 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-18T08:18:33.006 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=1 2026-02-18T08:18:33.006 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=1 2026-02-18T08:18:33.006 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:18:33.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:18:33.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:18:33.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:18:33.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test 2026-02-18T08:18:33.062 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:18:33.062 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status : get status for rbd mirror 2026-02-18T08:18:33.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:18:33.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:18:33.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/test 2026-02-18T08:18:33.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:2 2026-02-18T08:18:33.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-18T08:18:33.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:18:33.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:18:33.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:18:33.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-18T08:18:33.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:18:33.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:18:33.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:18:33.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:18:33.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-18T08:18:33.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:18:33.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:18:33.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:18:33.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:18:33.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:2 2026-02-18T08:18:33.067 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-18T08:18:33.067 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-18T08:18:33.067 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:18:33.067 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:2 2026-02-18T08:18:33.067 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T08:18:33.067 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-18T08:18:33.067 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:18:33.067 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:18:33.067 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-18T08:18:33.067 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-18T08:18:33.067 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-18T08:18:33.067 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=2 2026-02-18T08:18:33.068 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=2 2026-02-18T08:18:33.068 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:18:33.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:18:33.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:18:33.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:18:33.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/test 2026-02-18T08:18:33.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:18:33.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status : get status for rbd mirror 2026-02-18T08:18:33.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:18:33.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:18:33.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/test 2026-02-18T08:18:33.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:3 2026-02-18T08:18:33.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-18T08:18:33.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:18:33.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:18:33.129 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:18:33.129 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-18T08:18:33.129 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:18:33.129 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:18:33.129 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:18:33.129 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:18:33.129 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-18T08:18:33.129 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:18:33.129 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:18:33.129 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:18:33.129 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:18:33.129 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:3 2026-02-18T08:18:33.130 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-18T08:18:33.130 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-18T08:18:33.130 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:18:33.130 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:3 2026-02-18T08:18:33.130 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T08:18:33.130 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-18T08:18:33.130 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:18:33.130 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:18:33.130 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-18T08:18:33.130 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-18T08:18:33.130 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-18T08:18:33.130 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=3 2026-02-18T08:18:33.131 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=3 2026-02-18T08:18:33.131 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:18:33.138 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:18:33.138 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:18:33.138 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:18:33.139 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/test 2026-02-18T08:18:33.195 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ return 0 2026-02-18T08:18:33.195 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ status_result=' "state": "Replaying"' 2026-02-18T08:18:33.196 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ echo ' "state": "Replaying"' 2026-02-18T08:18:33.196 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ grep -i Replaying 2026-02-18T08:18:33.197 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: "state": "Replaying" 2026-02-18T08:18:33.199 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ current_state=started 2026-02-18T08:18:33.199 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test started = started 2026-02-18T08:18:33.199 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:18:33.199 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror test 2026-02-18T08:18:33.199 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_cluster=cluster1 2026-02-18T08:18:33.199 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:18:33.199 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_pool=mirror 2026-02-18T08:18:33.199 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local remote_pool=mirror 2026-02-18T08:18:33.199 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:18:33.199 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' journal = journal ']' 2026-02-18T08:18:33.199 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror mirror test 2026-02-18T08:18:33.200 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_cluster=cluster1 2026-02-18T08:18:33.200 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:18:33.200 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_pool=mirror 2026-02-18T08:18:33.200 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local remote_pool=mirror 2026-02-18T08:18:33.200 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:18:33.200 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-02-18T08:18:33.200 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-02-18T08:18:33.200 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:18:33.200 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:18:33.200 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:18:33.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:18:33.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:18:33.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:18:33.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:18:33.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:18:33.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:18:33.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:18:33.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:18:33.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:18:33.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:18:33.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:18:33.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:18:33.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:18:33.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:18:33.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:18:33.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:18:33.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:18:33.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:18:33.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:18:33.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:18:33.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:18:33.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:18:33.405 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:18:33.405 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:18:33.405 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:18:33.405 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:18:33.405 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:18:33.405 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:18:33.405 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:18:33.405 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:18:33.405 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:18:33.405 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:18:33.405 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:18:33.405 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:18:33.406 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:18:33.406 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:18:33.406 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:18:33.406 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:18:33.406 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:18:33.406 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:18:33.406 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:18:33.406 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:18:33.406 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:18:33.406 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:18:33.406 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:18:33.406 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:18:33.407 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:18:33.407 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:18:33.407 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:18:33.414 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:18:33.414 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:18:33.414 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:18:33.414 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:18:33.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:18:33.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:18:33.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:18:33.474 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:18:33.474 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:18:33.474 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:18:33.474 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:18:33.474 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:18:33.474 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:18:33.474 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:18:33.474 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:18:33.475 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:18:33.475 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:18:33.475 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:18:33.475 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:18:33.475 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:18:33.475 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:18:33.475 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:18:33.475 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:18:33.475 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:18:33.475 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:18:33.475 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:18:33.475 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:18:33.476 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:18:33.476 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:18:33.476 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:18:33.476 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:18:33.476 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:18:33.476 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:18:33.476 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:18:33.476 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:18:33.476 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:18:33.476 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:18:33.476 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:18:33.476 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:18:33.484 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:18:33.485 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:18:33.485 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:18:33.485 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:18:33.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:18:33.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:18:33.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:18:33.545 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:18:33.545 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:18:33.545 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:18:33.545 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:18:33.545 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:18:33.545 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:18:33.545 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:18:33.545 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:18:33.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:18:33.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:18:33.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:18:33.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:18:33.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:18:33.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:18:33.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:18:33.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:18:33.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:18:33.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:18:33.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:18:33.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:18:33.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:18:33.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:18:33.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:18:33.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:18:33.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:18:33.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:18:33.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:18:33.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:18:33.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:18:33.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:18:33.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:18:33.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:18:33.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:18:33.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:18:33.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:18:33.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:18:33.605 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:18:33.605 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:18:33.605 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:18:33.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:18:33.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:18:33.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:18:33.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:18:33.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:18:33.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:18:33.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:18:33.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:18:33.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:18:33.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:18:33.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:18:33.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:18:33.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:18:33.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:18:33.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:18:33.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:18:33.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:18:33.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:18:33.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:18:33.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:18:33.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:18:33.610 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:18:33.610 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:18:33.610 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:18:33.610 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:18:33.610 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:18:33.610 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:18:33.610 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:18:33.610 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:18:33.610 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:18:33.610 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:18:33.610 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:18:33.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:18:33.618 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:18:33.618 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:18:33.618 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:18:33.686 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:18:33.687 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:18:33.687 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:18:33.687 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:18:33.687 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:18:33.687 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:18:33.687 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:18:33.687 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:18:33.687 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:18:33.688 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:18:33.688 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:18:33.688 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:18:33.688 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:18:33.688 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:18:33.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:18:33.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:18:33.717 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 63 2026-02-18T08:18:33.717 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 65 2026-02-18T08:18:33.717 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:18:33.717 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=261, tag_tid=18, entry_tid=1], [object_number=260, tag_tid=18, entry_tid=0], [object_number=262, tag_tid=17, entry_tid=30126], [object_number=263, tag_tid=17, entry_tid=30123]]], state=connected] 2026-02-18T08:18:33.718 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=255, tag_tid=17, entry_tid=21903], [object_number=254, tag_tid=17, entry_tid=21902], [object_number=253, tag_tid=17, entry_tid=21901], [object_number=252, tag_tid=17, entry_tid=21900]]], state=connected] 2026-02-18T08:18:33.720 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:18:33.721 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=261, tag_tid=18, entry_tid=1' 2026-02-18T08:18:33.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:18:33.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:18:33.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:18:33.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:18:33.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:18:33.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:18:33.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:18:33.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:18:33.724 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:18:33.724 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:18:33.724 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:18:33.724 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:18:33.724 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:18:33.725 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:18:33.725 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:18:33.754 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 63 2026-02-18T08:18:33.754 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 65 2026-02-18T08:18:33.754 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:18:33.754 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=261, tag_tid=18, entry_tid=1], [object_number=260, tag_tid=18, entry_tid=0], [object_number=262, tag_tid=17, entry_tid=30126], [object_number=263, tag_tid=17, entry_tid=30123]]], state=connected] 2026-02-18T08:18:33.754 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=255, tag_tid=17, entry_tid=21903], [object_number=254, tag_tid=17, entry_tid=21902], [object_number=253, tag_tid=17, entry_tid=21901], [object_number=252, tag_tid=17, entry_tid=21900]]], state=connected] 2026-02-18T08:18:33.756 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:18:33.759 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=255, tag_tid=17, entry_tid=21903' 2026-02-18T08:18:33.759 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=261, tag_tid=18, entry_tid=1' -a 'object_number=261, tag_tid=18, entry_tid=1' = 'object_number=255, tag_tid=17, entry_tid=21903' 2026-02-18T08:18:33.759 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=255, tag_tid=17, entry_tid=21903' '!=' '' 2026-02-18T08:18:33.759 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:18:33.759 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=255, tag_tid=17, entry_tid=21903' = '' 2026-02-18T08:18:33.759 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=255, tag_tid=17, entry_tid=21903' 2026-02-18T08:18:33.760 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=261, tag_tid=18, entry_tid=1' 2026-02-18T08:18:33.760 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:18:33.760 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:18:33.760 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=18 2026-02-18T08:18:33.761 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=255, tag_tid=17, entry_tid=21903' 2026-02-18T08:18:33.761 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:18:33.762 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:18:33.763 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=17 2026-02-18T08:18:33.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=261, tag_tid=18, entry_tid=1' 2026-02-18T08:18:33.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:18:33.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:18:33.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:18:33.766 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=255, tag_tid=17, entry_tid=21903' 2026-02-18T08:18:33.766 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:18:33.766 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:18:33.767 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=21903 2026-02-18T08:18:33.768 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 18 = 17 -a 1 -le 21903 2026-02-18T08:18:33.768 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:18:33.768 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:18:33.768 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:18:33.969 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:18:33.969 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:18:33.969 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:18:33.969 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:18:33.969 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:18:33.969 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:18:33.969 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:18:33.969 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:18:33.969 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:18:33.969 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:18:33.969 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:18:33.969 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:18:33.969 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:18:33.969 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:18:33.969 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:18:33.970 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:18:33.970 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:18:33.971 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:18:33.971 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:18:33.971 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:18:33.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:18:33.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:18:33.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:18:33.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:18:33.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:18:33.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:18:33.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:18:33.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:18:33.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:18:33.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:18:33.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:18:33.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:18:33.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:18:33.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:18:33.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:18:33.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:18:33.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:18:33.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:18:33.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:18:33.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:18:33.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:18:33.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:18:33.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:18:33.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:18:33.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:18:33.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:18:33.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:18:33.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:18:33.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:18:33.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:18:33.985 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:18:33.985 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:18:33.985 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:18:34.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:18:34.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:18:34.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:18:34.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:18:34.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:18:34.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:18:34.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:18:34.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:18:34.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:18:34.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:18:34.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:18:34.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:18:34.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:18:34.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:18:34.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:18:34.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:18:34.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:18:34.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:18:34.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:18:34.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:18:34.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:18:34.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:18:34.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:18:34.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:18:34.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:18:34.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:18:34.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:18:34.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:18:34.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:18:34.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:18:34.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:18:34.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:18:34.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:18:34.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:18:34.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:18:34.056 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:18:34.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:18:34.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:18:34.058 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:18:34.114 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:18:34.114 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:18:34.114 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:18:34.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:18:34.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:18:34.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:18:34.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:18:34.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:18:34.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:18:34.118 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:18:34.118 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:18:34.118 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:18:34.118 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:18:34.118 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:18:34.118 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:18:34.118 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:18:34.118 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:18:34.118 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:18:34.118 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:18:34.118 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:18:34.118 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:18:34.119 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:18:34.119 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:18:34.119 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:18:34.119 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:18:34.119 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:18:34.119 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:18:34.119 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:18:34.119 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:18:34.119 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:18:34.119 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:18:34.119 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:18:34.119 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:18:34.120 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:18:34.120 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:18:34.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:18:34.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:18:34.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:18:34.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:18:34.177 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:18:34.177 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:18:34.177 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:18:34.179 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:18:34.180 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:18:34.180 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:18:34.180 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:18:34.180 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:18:34.180 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:18:34.180 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:18:34.180 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:18:34.180 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:18:34.180 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:18:34.180 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:18:34.180 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:18:34.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:18:34.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:18:34.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:18:34.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:18:34.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:18:34.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:18:34.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:18:34.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:18:34.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:18:34.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:18:34.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:18:34.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:18:34.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:18:34.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:18:34.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:18:34.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:18:34.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:18:34.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:18:34.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:18:34.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:18:34.189 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:18:34.190 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:18:34.190 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:18:34.190 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:18:34.260 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:18:34.261 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:18:34.261 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:18:34.261 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:18:34.261 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:18:34.261 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:18:34.261 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:18:34.261 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:18:34.261 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:18:34.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:18:34.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:18:34.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:18:34.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:18:34.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:18:34.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:18:34.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:18:34.299 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 63 2026-02-18T08:18:34.299 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 65 2026-02-18T08:18:34.299 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:18:34.299 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=261, tag_tid=18, entry_tid=1], [object_number=260, tag_tid=18, entry_tid=0], [object_number=262, tag_tid=17, entry_tid=30126], [object_number=263, tag_tid=17, entry_tid=30123]]], state=connected] 2026-02-18T08:18:34.299 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=254, tag_tid=17, entry_tid=22258], [object_number=253, tag_tid=17, entry_tid=22257], [object_number=252, tag_tid=17, entry_tid=22256], [object_number=255, tag_tid=17, entry_tid=22255]]], state=connected] 2026-02-18T08:18:34.302 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:18:34.304 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=261, tag_tid=18, entry_tid=1' 2026-02-18T08:18:34.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:18:34.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:18:34.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:18:34.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:18:34.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:18:34.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:18:34.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:18:34.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:18:34.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:18:34.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:18:34.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:18:34.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:18:34.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:18:34.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:18:34.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:18:34.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 63 2026-02-18T08:18:34.344 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 65 2026-02-18T08:18:34.344 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:18:34.344 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=261, tag_tid=18, entry_tid=1], [object_number=260, tag_tid=18, entry_tid=0], [object_number=262, tag_tid=17, entry_tid=30126], [object_number=263, tag_tid=17, entry_tid=30123]]], state=connected] 2026-02-18T08:18:34.344 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=254, tag_tid=17, entry_tid=22258], [object_number=253, tag_tid=17, entry_tid=22257], [object_number=252, tag_tid=17, entry_tid=22256], [object_number=255, tag_tid=17, entry_tid=22255]]], state=connected] 2026-02-18T08:18:34.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:18:34.351 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=254, tag_tid=17, entry_tid=22258' 2026-02-18T08:18:34.351 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=261, tag_tid=18, entry_tid=1' -a 'object_number=261, tag_tid=18, entry_tid=1' = 'object_number=254, tag_tid=17, entry_tid=22258' 2026-02-18T08:18:34.351 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=254, tag_tid=17, entry_tid=22258' '!=' 'object_number=255, tag_tid=17, entry_tid=21903' 2026-02-18T08:18:34.351 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:18:34.351 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=254, tag_tid=17, entry_tid=22258' = 'object_number=255, tag_tid=17, entry_tid=21903' 2026-02-18T08:18:34.351 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=254, tag_tid=17, entry_tid=22258' 2026-02-18T08:18:34.351 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=261, tag_tid=18, entry_tid=1' 2026-02-18T08:18:34.352 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:18:34.352 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:18:34.352 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=18 2026-02-18T08:18:34.353 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=254, tag_tid=17, entry_tid=22258' 2026-02-18T08:18:34.353 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:18:34.353 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:18:34.354 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=17 2026-02-18T08:18:34.355 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=261, tag_tid=18, entry_tid=1' 2026-02-18T08:18:34.355 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:18:34.355 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:18:34.356 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:18:34.357 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=254, tag_tid=17, entry_tid=22258' 2026-02-18T08:18:34.357 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:18:34.357 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:18:34.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=22258 2026-02-18T08:18:34.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 18 = 17 -a 1 -le 22258 2026-02-18T08:18:34.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:18:34.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:18:34.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:18:34.561 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:18:34.561 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:18:34.561 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:18:34.561 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:18:34.561 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:18:34.561 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:18:34.561 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:18:34.561 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:18:34.562 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:18:34.562 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:18:34.562 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:18:34.562 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:18:34.562 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:18:34.562 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:18:34.562 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:18:34.565 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:18:34.565 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:18:34.565 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:18:34.565 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:18:34.565 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:18:34.565 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:18:34.565 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:18:34.565 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:18:34.565 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:18:34.565 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:18:34.566 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:18:34.566 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:18:34.566 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:18:34.566 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:18:34.566 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:18:34.566 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:18:34.566 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:18:34.566 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:18:34.566 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:18:34.566 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:18:34.566 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:18:34.566 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:18:34.567 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:18:34.567 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:18:34.567 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:18:34.567 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:18:34.567 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:18:34.567 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:18:34.567 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:18:34.567 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:18:34.567 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:18:34.567 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:18:34.567 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:18:34.567 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:18:34.575 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:18:34.576 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:18:34.576 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:18:34.576 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:18:34.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:18:34.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:18:34.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:18:34.628 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:18:34.628 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:18:34.629 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:18:34.629 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:18:34.629 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:18:34.629 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:18:34.629 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:18:34.629 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:18:34.629 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:18:34.629 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:18:34.629 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:18:34.629 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:18:34.629 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:18:34.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:18:34.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:18:34.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:18:34.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:18:34.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:18:34.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:18:34.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:18:34.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:18:34.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:18:34.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:18:34.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:18:34.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:18:34.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:18:34.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:18:34.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:18:34.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:18:34.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:18:34.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:18:34.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:18:34.640 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:18:34.641 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:18:34.641 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:18:34.641 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:18:34.697 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:18:34.697 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:18:34.697 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:18:34.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:18:34.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:18:34.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:18:34.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:18:34.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:18:34.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:18:34.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:18:34.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:18:34.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:18:34.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:18:34.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:18:34.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:18:34.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:18:34.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:18:34.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:18:34.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:18:34.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:18:34.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:18:34.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:18:34.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:18:34.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:18:34.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:18:34.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:18:34.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:18:34.700 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:18:34.700 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:18:34.700 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:18:34.700 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:18:34.700 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:18:34.700 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:18:34.700 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:18:34.700 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:18:34.709 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:18:34.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:18:34.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:18:34.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:18:34.760 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:18:34.760 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:18:34.760 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:18:34.763 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:18:34.763 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:18:34.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:18:34.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:18:34.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:18:34.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:18:34.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:18:34.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:18:34.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:18:34.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:18:34.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:18:34.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:18:34.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:18:34.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:18:34.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:18:34.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:18:34.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:18:34.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:18:34.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:18:34.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:18:34.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:18:34.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:18:34.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:18:34.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:18:34.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:18:34.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:18:34.766 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:18:34.766 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:18:34.766 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:18:34.766 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:18:34.766 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:18:34.766 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:18:34.774 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:18:34.774 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:18:34.774 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:18:34.774 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:18:34.841 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:18:34.842 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:18:34.842 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:18:34.842 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:18:34.842 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:18:34.842 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:18:34.842 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:18:34.842 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:18:34.842 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:18:34.843 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:18:34.843 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:18:34.843 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:18:34.843 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:18:34.843 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:18:34.844 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:18:34.844 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:18:34.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 64 2026-02-18T08:18:34.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 65 2026-02-18T08:18:34.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:18:34.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=261, tag_tid=18, entry_tid=1], [object_number=260, tag_tid=18, entry_tid=0], [object_number=262, tag_tid=17, entry_tid=30126], [object_number=263, tag_tid=17, entry_tid=30123]]], state=connected] 2026-02-18T08:18:34.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=259, tag_tid=17, entry_tid=22575], [object_number=258, tag_tid=17, entry_tid=22574], [object_number=257, tag_tid=17, entry_tid=22573], [object_number=256, tag_tid=17, entry_tid=22572]]], state=connected] 2026-02-18T08:18:34.874 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:18:34.875 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=261, tag_tid=18, entry_tid=1' 2026-02-18T08:18:34.877 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:18:34.877 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:18:34.877 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:18:34.877 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:18:34.877 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:18:34.877 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:18:34.877 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:18:34.877 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:18:34.878 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:18:34.878 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:18:34.878 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:18:34.878 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:18:34.879 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:18:34.879 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:18:34.879 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:18:34.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 64 2026-02-18T08:18:34.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 65 2026-02-18T08:18:34.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:18:34.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=261, tag_tid=18, entry_tid=1], [object_number=260, tag_tid=18, entry_tid=0], [object_number=262, tag_tid=17, entry_tid=30126], [object_number=263, tag_tid=17, entry_tid=30123]]], state=connected] 2026-02-18T08:18:34.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=259, tag_tid=17, entry_tid=22575], [object_number=258, tag_tid=17, entry_tid=22574], [object_number=257, tag_tid=17, entry_tid=22573], [object_number=256, tag_tid=17, entry_tid=22572]]], state=connected] 2026-02-18T08:18:34.909 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:18:34.911 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=259, tag_tid=17, entry_tid=22575' 2026-02-18T08:18:34.911 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=261, tag_tid=18, entry_tid=1' -a 'object_number=261, tag_tid=18, entry_tid=1' = 'object_number=259, tag_tid=17, entry_tid=22575' 2026-02-18T08:18:34.911 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=259, tag_tid=17, entry_tid=22575' '!=' 'object_number=254, tag_tid=17, entry_tid=22258' 2026-02-18T08:18:34.911 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:18:34.912 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=259, tag_tid=17, entry_tid=22575' = 'object_number=254, tag_tid=17, entry_tid=22258' 2026-02-18T08:18:34.912 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=259, tag_tid=17, entry_tid=22575' 2026-02-18T08:18:34.912 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=261, tag_tid=18, entry_tid=1' 2026-02-18T08:18:34.912 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:18:34.912 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:18:34.913 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=18 2026-02-18T08:18:34.914 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=259, tag_tid=17, entry_tid=22575' 2026-02-18T08:18:34.914 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:18:34.914 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:18:34.915 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=17 2026-02-18T08:18:34.916 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=261, tag_tid=18, entry_tid=1' 2026-02-18T08:18:34.916 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:18:34.916 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:18:34.918 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:18:34.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=259, tag_tid=17, entry_tid=22575' 2026-02-18T08:18:34.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:18:34.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:18:34.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=22575 2026-02-18T08:18:34.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 18 = 17 -a 1 -le 22575 2026-02-18T08:18:34.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:18:34.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:18:34.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:18:35.122 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:18:35.122 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:18:35.122 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:18:35.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:18:35.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:18:35.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:18:35.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:18:35.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:18:35.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:18:35.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:18:35.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:18:35.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:18:35.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:18:35.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:18:35.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:18:35.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:18:35.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:18:35.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:18:35.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:18:35.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:18:35.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:18:35.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:18:35.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:18:35.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:18:35.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:18:35.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:18:35.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:18:35.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:18:35.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:18:35.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:18:35.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:18:35.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:18:35.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:18:35.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:18:35.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:18:35.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:18:35.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:18:35.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:18:35.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:18:35.129 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:18:35.129 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:18:35.129 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:18:35.129 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:18:35.129 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:18:35.129 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:18:35.129 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:18:35.129 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:18:35.129 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:18:35.129 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:18:35.137 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:18:35.137 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:18:35.137 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:18:35.138 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:18:35.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:18:35.187 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:18:35.187 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:18:35.189 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:18:35.189 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:18:35.190 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:18:35.190 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:18:35.190 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:18:35.190 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:18:35.190 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:18:35.190 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:18:35.190 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:18:35.190 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:18:35.190 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:18:35.190 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:18:35.190 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:18:35.190 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:18:35.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:18:35.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:18:35.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:18:35.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:18:35.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:18:35.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:18:35.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:18:35.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:18:35.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:18:35.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:18:35.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:18:35.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:18:35.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:18:35.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:18:35.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:18:35.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:18:35.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:18:35.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:18:35.200 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:18:35.201 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:18:35.201 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:18:35.201 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:18:35.251 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:18:35.251 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:18:35.251 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:18:35.254 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:18:35.254 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:18:35.254 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:18:35.254 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:18:35.254 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:18:35.255 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:18:35.255 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:18:35.255 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:18:35.255 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:18:35.255 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:18:35.255 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:18:35.255 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:18:35.255 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:18:35.255 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:18:35.255 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:18:35.255 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:18:35.256 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:18:35.256 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:18:35.256 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:18:35.256 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:18:35.256 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:18:35.256 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:18:35.256 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:18:35.256 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:18:35.256 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:18:35.256 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:18:35.256 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:18:35.256 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:18:35.257 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:18:35.257 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:18:35.257 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:18:35.257 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:18:35.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:18:35.266 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:18:35.266 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:18:35.266 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:18:35.322 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:18:35.323 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:18:35.323 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:18:35.325 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:18:35.325 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:18:35.325 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:18:35.326 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:18:35.326 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:18:35.326 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:18:35.326 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:18:35.326 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:18:35.326 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:18:35.326 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:18:35.326 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:18:35.326 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:18:35.326 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:18:35.326 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:18:35.326 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:18:35.327 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:18:35.327 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:18:35.327 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:18:35.327 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:18:35.327 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:18:35.327 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:18:35.327 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:18:35.327 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:18:35.327 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:18:35.327 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:18:35.327 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:18:35.327 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:18:35.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:18:35.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:18:35.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:18:35.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:18:35.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:18:35.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:18:35.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:18:35.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:18:35.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:18:35.405 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:18:35.407 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:18:35.407 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:18:35.407 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:18:35.407 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:18:35.407 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:18:35.407 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:18:35.407 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:18:35.407 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:18:35.407 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:18:35.408 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:18:35.408 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:18:35.408 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:18:35.408 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:18:35.408 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:18:35.409 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:18:35.437 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 64 2026-02-18T08:18:35.437 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 65 2026-02-18T08:18:35.437 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:18:35.437 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=261, tag_tid=18, entry_tid=1], [object_number=260, tag_tid=18, entry_tid=0], [object_number=262, tag_tid=17, entry_tid=30126], [object_number=263, tag_tid=17, entry_tid=30123]]], state=connected] 2026-02-18T08:18:35.437 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=257, tag_tid=17, entry_tid=22949], [object_number=256, tag_tid=17, entry_tid=22948], [object_number=259, tag_tid=17, entry_tid=22947], [object_number=258, tag_tid=17, entry_tid=22946]]], state=connected] 2026-02-18T08:18:35.440 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:18:35.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=261, tag_tid=18, entry_tid=1' 2026-02-18T08:18:35.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:18:35.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:18:35.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:18:35.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:18:35.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:18:35.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:18:35.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:18:35.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:18:35.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:18:35.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:18:35.443 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:18:35.443 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:18:35.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:18:35.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:18:35.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:18:35.481 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 64 2026-02-18T08:18:35.482 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 65 2026-02-18T08:18:35.482 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:18:35.482 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=261, tag_tid=18, entry_tid=1], [object_number=260, tag_tid=18, entry_tid=0], [object_number=262, tag_tid=17, entry_tid=30126], [object_number=263, tag_tid=17, entry_tid=30123]]], state=connected] 2026-02-18T08:18:35.482 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=257, tag_tid=17, entry_tid=22949], [object_number=256, tag_tid=17, entry_tid=22948], [object_number=259, tag_tid=17, entry_tid=22947], [object_number=258, tag_tid=17, entry_tid=22946]]], state=connected] 2026-02-18T08:18:35.485 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:18:35.487 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=257, tag_tid=17, entry_tid=22949' 2026-02-18T08:18:35.487 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=261, tag_tid=18, entry_tid=1' -a 'object_number=261, tag_tid=18, entry_tid=1' = 'object_number=257, tag_tid=17, entry_tid=22949' 2026-02-18T08:18:35.487 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=257, tag_tid=17, entry_tid=22949' '!=' 'object_number=259, tag_tid=17, entry_tid=22575' 2026-02-18T08:18:35.487 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:18:35.487 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=257, tag_tid=17, entry_tid=22949' = 'object_number=259, tag_tid=17, entry_tid=22575' 2026-02-18T08:18:35.487 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=257, tag_tid=17, entry_tid=22949' 2026-02-18T08:18:35.488 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=261, tag_tid=18, entry_tid=1' 2026-02-18T08:18:35.488 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:18:35.488 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:18:35.489 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=18 2026-02-18T08:18:35.490 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=257, tag_tid=17, entry_tid=22949' 2026-02-18T08:18:35.490 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:18:35.490 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:18:35.491 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=17 2026-02-18T08:18:35.492 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=261, tag_tid=18, entry_tid=1' 2026-02-18T08:18:35.492 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:18:35.492 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:18:35.493 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:18:35.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=257, tag_tid=17, entry_tid=22949' 2026-02-18T08:18:35.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:18:35.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:18:35.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=22949 2026-02-18T08:18:35.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 18 = 17 -a 1 -le 22949 2026-02-18T08:18:35.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:18:35.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:18:35.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:18:35.697 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:18:35.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:18:35.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:18:35.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:18:35.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:18:35.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:18:35.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:18:35.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:18:35.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:18:35.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:18:35.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:18:35.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:18:35.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:18:35.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:18:35.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:18:35.702 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:18:35.702 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:18:35.702 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:18:35.702 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:18:35.702 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:18:35.702 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:18:35.702 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:18:35.702 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:18:35.702 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:18:35.702 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:18:35.703 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:18:35.703 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:18:35.703 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:18:35.703 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:18:35.703 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:18:35.703 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:18:35.703 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:18:35.703 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:18:35.703 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:18:35.703 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:18:35.703 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:18:35.703 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:18:35.704 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:18:35.704 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:18:35.704 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:18:35.704 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:18:35.704 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:18:35.704 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:18:35.704 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:18:35.704 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:18:35.704 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:18:35.704 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:18:35.704 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:18:35.705 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:18:35.712 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:18:35.713 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:18:35.713 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:18:35.713 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:18:35.770 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:18:35.770 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:18:35.770 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:18:35.773 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:18:35.773 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:18:35.773 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:18:35.773 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:18:35.773 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:18:35.773 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:18:35.774 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:18:35.774 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:18:35.774 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:18:35.774 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:18:35.774 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:18:35.774 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:18:35.774 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:18:35.774 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:18:35.774 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:18:35.774 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:18:35.774 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:18:35.774 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:18:35.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:18:35.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:18:35.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:18:35.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:18:35.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:18:35.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:18:35.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:18:35.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:18:35.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:18:35.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:18:35.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:18:35.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:18:35.776 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:18:35.776 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:18:35.785 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:18:35.785 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:18:35.785 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:18:35.785 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:18:35.842 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:18:35.842 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:18:35.842 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:18:35.845 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:18:35.845 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:18:35.845 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:18:35.845 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:18:35.845 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:18:35.845 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:18:35.845 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:18:35.845 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:18:35.846 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:18:35.846 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:18:35.846 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:18:35.846 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:18:35.846 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:18:35.846 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:18:35.846 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:18:35.846 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:18:35.846 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:18:35.846 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:18:35.846 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:18:35.847 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:18:35.847 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:18:35.847 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:18:35.847 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:18:35.847 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:18:35.847 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:18:35.847 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:18:35.847 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:18:35.847 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:18:35.847 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:18:35.847 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:18:35.848 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:18:35.848 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:18:35.856 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:18:35.856 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:18:35.856 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:18:35.857 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:18:35.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:18:35.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:18:35.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:18:35.910 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:18:35.910 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:18:35.911 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:18:35.911 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:18:35.911 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:18:35.911 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:18:35.911 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:18:35.911 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:18:35.911 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:18:35.911 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:18:35.911 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:18:35.911 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:18:35.911 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:18:35.912 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:18:35.912 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:18:35.912 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:18:35.912 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:18:35.912 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:18:35.912 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:18:35.912 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:18:35.912 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:18:35.912 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:18:35.912 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:18:35.912 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:18:35.912 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:18:35.913 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:18:35.913 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:18:35.913 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:18:35.913 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:18:35.913 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:18:35.913 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:18:35.913 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:18:35.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:18:35.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:18:35.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:18:35.922 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:18:35.990 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:18:35.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:18:35.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:18:35.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:18:35.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:18:35.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:18:35.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:18:35.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:18:35.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:18:35.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:18:35.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:18:35.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:18:35.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:18:35.993 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:18:35.993 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:18:35.993 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:18:36.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 64 2026-02-18T08:18:36.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 65 2026-02-18T08:18:36.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:18:36.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=261, tag_tid=18, entry_tid=1], [object_number=260, tag_tid=18, entry_tid=0], [object_number=262, tag_tid=17, entry_tid=30126], [object_number=263, tag_tid=17, entry_tid=30123]]], state=connected] 2026-02-18T08:18:36.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=256, tag_tid=17, entry_tid=23312], [object_number=259, tag_tid=17, entry_tid=23311], [object_number=258, tag_tid=17, entry_tid=23310], [object_number=257, tag_tid=17, entry_tid=23309]]], state=connected] 2026-02-18T08:18:36.035 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:18:36.037 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=261, tag_tid=18, entry_tid=1' 2026-02-18T08:18:36.038 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:18:36.038 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:18:36.038 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:18:36.039 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:18:36.039 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:18:36.039 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:18:36.039 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:18:36.039 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:18:36.039 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:18:36.039 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:18:36.039 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:18:36.039 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:18:36.040 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:18:36.040 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:18:36.041 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:18:36.067 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 64 2026-02-18T08:18:36.068 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 65 2026-02-18T08:18:36.068 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:18:36.068 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=261, tag_tid=18, entry_tid=1], [object_number=260, tag_tid=18, entry_tid=0], [object_number=262, tag_tid=17, entry_tid=30126], [object_number=263, tag_tid=17, entry_tid=30123]]], state=connected] 2026-02-18T08:18:36.068 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=256, tag_tid=17, entry_tid=23312], [object_number=259, tag_tid=17, entry_tid=23311], [object_number=258, tag_tid=17, entry_tid=23310], [object_number=257, tag_tid=17, entry_tid=23309]]], state=connected] 2026-02-18T08:18:36.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:18:36.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=256, tag_tid=17, entry_tid=23312' 2026-02-18T08:18:36.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=261, tag_tid=18, entry_tid=1' -a 'object_number=261, tag_tid=18, entry_tid=1' = 'object_number=256, tag_tid=17, entry_tid=23312' 2026-02-18T08:18:36.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=256, tag_tid=17, entry_tid=23312' '!=' 'object_number=257, tag_tid=17, entry_tid=22949' 2026-02-18T08:18:36.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:18:36.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=256, tag_tid=17, entry_tid=23312' = 'object_number=257, tag_tid=17, entry_tid=22949' 2026-02-18T08:18:36.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=256, tag_tid=17, entry_tid=23312' 2026-02-18T08:18:36.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=261, tag_tid=18, entry_tid=1' 2026-02-18T08:18:36.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:18:36.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:18:36.074 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=18 2026-02-18T08:18:36.075 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=256, tag_tid=17, entry_tid=23312' 2026-02-18T08:18:36.075 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:18:36.075 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:18:36.076 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=17 2026-02-18T08:18:36.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=261, tag_tid=18, entry_tid=1' 2026-02-18T08:18:36.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:18:36.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:18:36.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:18:36.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=256, tag_tid=17, entry_tid=23312' 2026-02-18T08:18:36.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:18:36.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:18:36.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=23312 2026-02-18T08:18:36.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 18 = 17 -a 1 -le 23312 2026-02-18T08:18:36.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:18:36.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:18:36.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:18:36.283 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:18:36.283 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:18:36.283 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:18:36.283 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:18:36.283 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:18:36.283 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:18:36.283 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:18:36.283 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:18:36.284 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:18:36.284 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:18:36.284 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:18:36.284 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:18:36.284 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:18:36.284 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:18:36.284 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:18:36.284 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:18:36.287 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:18:36.287 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:18:36.287 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:18:36.287 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:18:36.287 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:18:36.287 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:18:36.287 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:18:36.287 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:18:36.288 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:18:36.288 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:18:36.288 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:18:36.288 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:18:36.288 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:18:36.288 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:18:36.288 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:18:36.288 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:18:36.288 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:18:36.288 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:18:36.288 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:18:36.288 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:18:36.289 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:18:36.289 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:18:36.289 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:18:36.289 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:18:36.289 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:18:36.289 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:18:36.289 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:18:36.289 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:18:36.289 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:18:36.289 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:18:36.289 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:18:36.289 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:18:36.290 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:18:36.297 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:18:36.298 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:18:36.298 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:18:36.298 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:18:36.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:18:36.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:18:36.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:18:36.350 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:18:36.350 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:18:36.350 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:18:36.350 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:18:36.350 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:18:36.350 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:18:36.350 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:18:36.350 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:18:36.350 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:18:36.351 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:18:36.351 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:18:36.351 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:18:36.351 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:18:36.351 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:18:36.351 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:18:36.351 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:18:36.351 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:18:36.351 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:18:36.351 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:18:36.351 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:18:36.351 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:18:36.352 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:18:36.352 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:18:36.352 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:18:36.352 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:18:36.352 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:18:36.352 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:18:36.352 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:18:36.352 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:18:36.352 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:18:36.352 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:18:36.352 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:18:36.361 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:18:36.362 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:18:36.362 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:18:36.362 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:18:36.420 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:18:36.421 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:18:36.421 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:18:36.423 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:18:36.423 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:18:36.423 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:18:36.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:18:36.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:18:36.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:18:36.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:18:36.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:18:36.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:18:36.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:18:36.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:18:36.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:18:36.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:18:36.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:18:36.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:18:36.425 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:18:36.425 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:18:36.425 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:18:36.425 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:18:36.425 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:18:36.425 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:18:36.425 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:18:36.425 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:18:36.425 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:18:36.425 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:18:36.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:18:36.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:18:36.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:18:36.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:18:36.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:18:36.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:18:36.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:18:36.435 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:18:36.435 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:18:36.436 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:18:36.436 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:18:36.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:18:36.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:18:36.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:18:36.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:18:36.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:18:36.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:18:36.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:18:36.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:18:36.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:18:36.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:18:36.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:18:36.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:18:36.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:18:36.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:18:36.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:18:36.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:18:36.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:18:36.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:18:36.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:18:36.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:18:36.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:18:36.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:18:36.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:18:36.500 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:18:36.500 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:18:36.500 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:18:36.500 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:18:36.500 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:18:36.500 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:18:36.500 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:18:36.500 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:18:36.500 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:18:36.500 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:18:36.500 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:18:36.500 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:18:36.508 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:18:36.508 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:18:36.509 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:18:36.509 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:18:36.570 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:18:36.570 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:18:36.571 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:18:36.571 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:18:36.571 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:18:36.571 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:18:36.571 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:18:36.571 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:18:36.571 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:18:36.571 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:18:36.571 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:18:36.572 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:18:36.572 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:18:36.572 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:18:36.573 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:18:36.573 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:18:36.601 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 64 2026-02-18T08:18:36.601 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 65 2026-02-18T08:18:36.601 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:18:36.601 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=261, tag_tid=18, entry_tid=1], [object_number=260, tag_tid=18, entry_tid=0], [object_number=262, tag_tid=17, entry_tid=30126], [object_number=263, tag_tid=17, entry_tid=30123]]], state=connected] 2026-02-18T08:18:36.601 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=257, tag_tid=17, entry_tid=23673], [object_number=256, tag_tid=17, entry_tid=23672], [object_number=259, tag_tid=17, entry_tid=23671], [object_number=258, tag_tid=17, entry_tid=23670]]], state=connected] 2026-02-18T08:18:36.603 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:18:36.605 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=261, tag_tid=18, entry_tid=1' 2026-02-18T08:18:36.606 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:18:36.607 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:18:36.607 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:18:36.607 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:18:36.607 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:18:36.607 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:18:36.607 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:18:36.607 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:18:36.607 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:18:36.607 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:18:36.607 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:18:36.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:18:36.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:18:36.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:18:36.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:18:36.636 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 64 2026-02-18T08:18:36.637 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 65 2026-02-18T08:18:36.637 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:18:36.637 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=261, tag_tid=18, entry_tid=1], [object_number=260, tag_tid=18, entry_tid=0], [object_number=262, tag_tid=17, entry_tid=30126], [object_number=263, tag_tid=17, entry_tid=30123]]], state=connected] 2026-02-18T08:18:36.637 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=257, tag_tid=17, entry_tid=23673], [object_number=256, tag_tid=17, entry_tid=23672], [object_number=259, tag_tid=17, entry_tid=23671], [object_number=258, tag_tid=17, entry_tid=23670]]], state=connected] 2026-02-18T08:18:36.639 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:18:36.641 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=257, tag_tid=17, entry_tid=23673' 2026-02-18T08:18:36.641 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=261, tag_tid=18, entry_tid=1' -a 'object_number=261, tag_tid=18, entry_tid=1' = 'object_number=257, tag_tid=17, entry_tid=23673' 2026-02-18T08:18:36.641 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=257, tag_tid=17, entry_tid=23673' '!=' 'object_number=256, tag_tid=17, entry_tid=23312' 2026-02-18T08:18:36.641 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:18:36.641 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=257, tag_tid=17, entry_tid=23673' = 'object_number=256, tag_tid=17, entry_tid=23312' 2026-02-18T08:18:36.641 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=257, tag_tid=17, entry_tid=23673' 2026-02-18T08:18:36.642 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=261, tag_tid=18, entry_tid=1' 2026-02-18T08:18:36.642 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:18:36.642 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:18:36.643 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=18 2026-02-18T08:18:36.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=257, tag_tid=17, entry_tid=23673' 2026-02-18T08:18:36.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:18:36.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:18:36.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=17 2026-02-18T08:18:36.646 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=261, tag_tid=18, entry_tid=1' 2026-02-18T08:18:36.646 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:18:36.646 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:18:36.648 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:18:36.648 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=257, tag_tid=17, entry_tid=23673' 2026-02-18T08:18:36.649 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:18:36.649 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:18:36.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=23673 2026-02-18T08:18:36.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 18 = 17 -a 1 -le 23673 2026-02-18T08:18:36.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:18:36.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:18:36.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:18:36.852 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:18:36.852 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:18:36.852 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:18:36.853 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:18:36.853 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:18:36.853 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:18:36.853 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:18:36.853 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:18:36.853 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:18:36.853 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:18:36.853 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:18:36.853 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:18:36.853 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:18:36.853 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:18:36.854 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:18:36.856 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:18:36.857 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:18:36.857 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:18:36.857 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:18:36.857 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:18:36.857 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:18:36.857 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:18:36.857 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:18:36.857 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:18:36.857 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:18:36.857 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:18:36.857 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:18:36.858 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:18:36.858 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:18:36.858 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:18:36.858 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:18:36.858 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:18:36.858 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:18:36.858 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:18:36.858 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:18:36.858 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:18:36.858 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:18:36.858 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:18:36.858 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:18:36.858 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:18:36.859 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:18:36.859 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:18:36.859 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:18:36.859 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:18:36.859 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:18:36.859 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:18:36.859 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:18:36.859 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:18:36.859 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:18:36.866 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:18:36.867 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:18:36.867 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:18:36.867 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:18:36.917 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:18:36.917 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:18:36.917 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:18:36.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:18:36.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:18:36.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:18:36.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:18:36.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:18:36.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:18:36.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:18:36.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:18:36.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:18:36.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:18:36.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:18:36.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:18:36.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:18:36.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:18:36.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:18:36.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:18:36.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:18:36.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:18:36.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:18:36.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:18:36.922 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:18:36.922 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:18:36.922 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:18:36.922 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:18:36.922 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:18:36.922 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:18:36.922 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:18:36.922 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:18:36.922 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:18:36.922 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:18:36.922 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:18:36.922 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:18:36.930 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:18:36.931 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:18:36.931 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:18:36.931 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:18:36.990 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:18:36.990 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:18:36.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:18:36.993 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:18:36.994 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:18:36.994 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:18:36.994 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:18:36.994 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:18:36.994 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:18:36.994 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:18:36.994 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:18:36.994 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:18:36.994 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:18:36.994 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:18:36.994 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:18:36.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:18:36.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:18:36.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:18:36.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:18:36.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:18:36.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:18:36.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:18:36.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:18:36.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:18:36.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:18:36.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:18:36.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:18:36.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:18:36.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:18:36.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:18:36.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:18:36.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:18:36.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:18:36.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:18:36.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:18:37.004 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:18:37.005 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:18:37.005 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:18:37.005 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:18:37.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:18:37.055 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:18:37.055 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:18:37.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:18:37.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:18:37.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:18:37.058 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:18:37.058 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:18:37.058 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:18:37.058 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:18:37.058 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:18:37.058 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:18:37.058 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:18:37.058 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:18:37.058 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:18:37.058 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:18:37.058 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:18:37.059 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:18:37.059 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:18:37.059 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:18:37.059 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:18:37.059 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:18:37.059 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:18:37.059 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:18:37.059 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:18:37.059 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:18:37.059 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:18:37.059 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:18:37.059 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:18:37.059 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:18:37.060 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:18:37.060 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:18:37.060 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:18:37.060 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:18:37.060 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:18:37.068 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:18:37.069 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:18:37.069 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:18:37.069 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:18:37.129 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:18:37.130 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:18:37.130 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:18:37.130 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:18:37.130 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:18:37.131 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:18:37.131 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:18:37.131 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:18:37.131 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:18:37.131 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:18:37.131 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:18:37.131 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:18:37.131 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:18:37.132 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:18:37.132 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:18:37.132 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:18:37.162 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 64 2026-02-18T08:18:37.162 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 65 2026-02-18T08:18:37.162 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:18:37.162 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=261, tag_tid=18, entry_tid=1], [object_number=260, tag_tid=18, entry_tid=0], [object_number=262, tag_tid=17, entry_tid=30126], [object_number=263, tag_tid=17, entry_tid=30123]]], state=connected] 2026-02-18T08:18:37.162 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=258, tag_tid=17, entry_tid=24022], [object_number=257, tag_tid=17, entry_tid=24021], [object_number=256, tag_tid=17, entry_tid=24020], [object_number=259, tag_tid=17, entry_tid=24019]]], state=connected] 2026-02-18T08:18:37.165 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:18:37.166 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=261, tag_tid=18, entry_tid=1' 2026-02-18T08:18:37.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:18:37.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:18:37.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:18:37.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:18:37.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:18:37.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:18:37.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:18:37.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:18:37.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:18:37.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:18:37.169 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:18:37.169 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:18:37.169 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:18:37.170 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:18:37.170 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:18:37.199 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 64 2026-02-18T08:18:37.199 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 65 2026-02-18T08:18:37.199 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:18:37.199 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=261, tag_tid=18, entry_tid=1], [object_number=260, tag_tid=18, entry_tid=0], [object_number=262, tag_tid=17, entry_tid=30126], [object_number=263, tag_tid=17, entry_tid=30123]]], state=connected] 2026-02-18T08:18:37.199 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=258, tag_tid=17, entry_tid=24022], [object_number=257, tag_tid=17, entry_tid=24021], [object_number=256, tag_tid=17, entry_tid=24020], [object_number=259, tag_tid=17, entry_tid=24019]]], state=connected] 2026-02-18T08:18:37.202 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:18:37.204 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=258, tag_tid=17, entry_tid=24022' 2026-02-18T08:18:37.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=261, tag_tid=18, entry_tid=1' -a 'object_number=261, tag_tid=18, entry_tid=1' = 'object_number=258, tag_tid=17, entry_tid=24022' 2026-02-18T08:18:37.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=258, tag_tid=17, entry_tid=24022' '!=' 'object_number=257, tag_tid=17, entry_tid=23673' 2026-02-18T08:18:37.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:18:37.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=258, tag_tid=17, entry_tid=24022' = 'object_number=257, tag_tid=17, entry_tid=23673' 2026-02-18T08:18:37.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=258, tag_tid=17, entry_tid=24022' 2026-02-18T08:18:37.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=261, tag_tid=18, entry_tid=1' 2026-02-18T08:18:37.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:18:37.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:18:37.206 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=18 2026-02-18T08:18:37.207 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=258, tag_tid=17, entry_tid=24022' 2026-02-18T08:18:37.207 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:18:37.207 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:18:37.208 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=17 2026-02-18T08:18:37.209 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=261, tag_tid=18, entry_tid=1' 2026-02-18T08:18:37.209 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:18:37.209 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:18:37.210 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:18:37.211 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=258, tag_tid=17, entry_tid=24022' 2026-02-18T08:18:37.211 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:18:37.212 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:18:37.213 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=24022 2026-02-18T08:18:37.213 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 18 = 17 -a 1 -le 24022 2026-02-18T08:18:37.213 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:18:37.213 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:18:37.213 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:18:37.415 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:18:37.415 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:18:37.415 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:18:37.415 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:18:37.415 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:18:37.415 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:18:37.416 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:18:37.416 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:18:37.416 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:18:37.416 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:18:37.416 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:18:37.416 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:18:37.416 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:18:37.416 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:18:37.416 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:18:37.416 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:18:37.419 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:18:37.419 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:18:37.419 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:18:37.419 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:18:37.419 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:18:37.419 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:18:37.419 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:18:37.419 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:18:37.420 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:18:37.420 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:18:37.420 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:18:37.420 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:18:37.420 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:18:37.420 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:18:37.420 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:18:37.420 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:18:37.420 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:18:37.420 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:18:37.420 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:18:37.420 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:18:37.421 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:18:37.421 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:18:37.421 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:18:37.421 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:18:37.421 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:18:37.421 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:18:37.421 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:18:37.421 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:18:37.421 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:18:37.421 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:18:37.421 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:18:37.421 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:18:37.422 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:18:37.430 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:18:37.430 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:18:37.430 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:18:37.430 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:18:37.487 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:18:37.487 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:18:37.487 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:18:37.490 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:18:37.490 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:18:37.490 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:18:37.490 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:18:37.490 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:18:37.490 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:18:37.490 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:18:37.490 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:18:37.490 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:18:37.490 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:18:37.490 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:18:37.491 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:18:37.491 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:18:37.491 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:18:37.491 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:18:37.491 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:18:37.491 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:18:37.491 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:18:37.491 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:18:37.491 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:18:37.491 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:18:37.491 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:18:37.491 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:18:37.492 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:18:37.492 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:18:37.492 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:18:37.492 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:18:37.492 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:18:37.492 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:18:37.492 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:18:37.492 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:18:37.492 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:18:37.501 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:18:37.501 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:18:37.502 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:18:37.502 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:18:37.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:18:37.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:18:37.551 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:18:37.553 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:18:37.553 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:18:37.553 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:18:37.553 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:18:37.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:18:37.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:18:37.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:18:37.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:18:37.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:18:37.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:18:37.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:18:37.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:18:37.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:18:37.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:18:37.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:18:37.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:18:37.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:18:37.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:18:37.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:18:37.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:18:37.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:18:37.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:18:37.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:18:37.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:18:37.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:18:37.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:18:37.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:18:37.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:18:37.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:18:37.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:18:37.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:18:37.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:18:37.565 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:18:37.565 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:18:37.566 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:18:37.566 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:18:37.623 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:18:37.623 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:18:37.623 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:18:37.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:18:37.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:18:37.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:18:37.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:18:37.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:18:37.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:18:37.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:18:37.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:18:37.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:18:37.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:18:37.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:18:37.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:18:37.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:18:37.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:18:37.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:18:37.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:18:37.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:18:37.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:18:37.628 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:18:37.628 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:18:37.628 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:18:37.628 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:18:37.628 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:18:37.628 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:18:37.628 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:18:37.628 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:18:37.628 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:18:37.628 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:18:37.628 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:18:37.628 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:18:37.628 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:18:37.629 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:18:37.636 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:18:37.637 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:18:37.637 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:18:37.637 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:18:37.706 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:18:37.707 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:18:37.707 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:18:37.707 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:18:37.707 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:18:37.707 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:18:37.707 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:18:37.707 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:18:37.707 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:18:37.707 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:18:37.707 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:18:37.708 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:18:37.708 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:18:37.708 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:18:37.709 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:18:37.709 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:18:37.738 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 64 2026-02-18T08:18:37.738 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 65 2026-02-18T08:18:37.738 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:18:37.738 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=261, tag_tid=18, entry_tid=1], [object_number=260, tag_tid=18, entry_tid=0], [object_number=262, tag_tid=17, entry_tid=30126], [object_number=263, tag_tid=17, entry_tid=30123]]], state=connected] 2026-02-18T08:18:37.738 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=256, tag_tid=17, entry_tid=24400], [object_number=259, tag_tid=17, entry_tid=24399], [object_number=258, tag_tid=17, entry_tid=24398], [object_number=257, tag_tid=17, entry_tid=24397]]], state=connected] 2026-02-18T08:18:37.740 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:18:37.742 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=261, tag_tid=18, entry_tid=1' 2026-02-18T08:18:37.743 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:18:37.743 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:18:37.743 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:18:37.744 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:18:37.744 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:18:37.744 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:18:37.744 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:18:37.744 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:18:37.744 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:18:37.744 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:18:37.744 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:18:37.744 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:18:37.745 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:18:37.745 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:18:37.745 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:18:37.780 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 64 2026-02-18T08:18:37.780 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 65 2026-02-18T08:18:37.780 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:18:37.780 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=261, tag_tid=18, entry_tid=1], [object_number=260, tag_tid=18, entry_tid=0], [object_number=262, tag_tid=17, entry_tid=30126], [object_number=263, tag_tid=17, entry_tid=30123]]], state=connected] 2026-02-18T08:18:37.780 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=256, tag_tid=17, entry_tid=24400], [object_number=259, tag_tid=17, entry_tid=24399], [object_number=258, tag_tid=17, entry_tid=24398], [object_number=257, tag_tid=17, entry_tid=24397]]], state=connected] 2026-02-18T08:18:37.784 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:18:37.786 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=256, tag_tid=17, entry_tid=24400' 2026-02-18T08:18:37.786 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=261, tag_tid=18, entry_tid=1' -a 'object_number=261, tag_tid=18, entry_tid=1' = 'object_number=256, tag_tid=17, entry_tid=24400' 2026-02-18T08:18:37.786 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=256, tag_tid=17, entry_tid=24400' '!=' 'object_number=258, tag_tid=17, entry_tid=24022' 2026-02-18T08:18:37.786 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:18:37.786 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=256, tag_tid=17, entry_tid=24400' = 'object_number=258, tag_tid=17, entry_tid=24022' 2026-02-18T08:18:37.786 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=256, tag_tid=17, entry_tid=24400' 2026-02-18T08:18:37.787 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=261, tag_tid=18, entry_tid=1' 2026-02-18T08:18:37.787 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:18:37.787 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:18:37.788 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=18 2026-02-18T08:18:37.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=256, tag_tid=17, entry_tid=24400' 2026-02-18T08:18:37.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:18:37.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:18:37.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=17 2026-02-18T08:18:37.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=261, tag_tid=18, entry_tid=1' 2026-02-18T08:18:37.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:18:37.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:18:37.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:18:37.793 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=256, tag_tid=17, entry_tid=24400' 2026-02-18T08:18:37.793 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:18:37.794 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:18:37.795 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=24400 2026-02-18T08:18:37.795 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 18 = 17 -a 1 -le 24400 2026-02-18T08:18:37.795 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:18:37.795 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:18:37.796 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:18:37.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:18:37.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:18:37.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:18:37.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:18:37.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:18:37.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:18:37.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:18:37.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:18:37.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:18:37.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:18:37.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:18:37.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:18:37.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:18:37.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:18:37.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:18:38.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:18:38.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:18:38.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:18:38.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:18:38.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:18:38.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:18:38.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:18:38.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:18:38.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:18:38.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:18:38.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:18:38.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:18:38.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:18:38.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:18:38.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:18:38.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:18:38.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:18:38.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:18:38.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:18:38.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:18:38.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:18:38.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:18:38.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:18:38.003 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:18:38.003 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:18:38.003 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:18:38.003 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:18:38.003 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:18:38.003 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:18:38.003 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:18:38.003 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:18:38.003 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:18:38.003 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:18:38.003 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:18:38.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:18:38.012 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:18:38.012 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:18:38.012 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:18:38.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:18:38.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:18:38.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:18:38.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:18:38.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:18:38.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:18:38.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:18:38.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:18:38.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:18:38.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:18:38.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:18:38.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:18:38.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:18:38.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:18:38.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:18:38.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:18:38.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:18:38.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:18:38.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:18:38.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:18:38.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:18:38.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:18:38.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:18:38.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:18:38.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:18:38.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:18:38.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:18:38.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:18:38.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:18:38.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:18:38.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:18:38.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:18:38.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:18:38.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:18:38.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:18:38.075 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:18:38.076 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:18:38.076 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:18:38.076 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:18:38.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:18:38.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:18:38.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:18:38.129 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:18:38.129 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:18:38.129 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:18:38.129 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:18:38.129 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:18:38.129 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:18:38.129 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:18:38.130 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:18:38.130 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:18:38.130 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:18:38.130 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:18:38.130 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:18:38.130 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:18:38.130 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:18:38.130 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:18:38.130 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:18:38.130 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:18:38.130 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:18:38.130 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:18:38.131 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:18:38.131 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:18:38.131 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:18:38.131 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:18:38.131 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:18:38.131 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:18:38.131 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:18:38.131 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:18:38.131 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:18:38.131 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:18:38.131 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:18:38.131 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:18:38.131 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:18:38.140 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:18:38.141 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:18:38.141 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:18:38.141 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:18:38.198 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:18:38.198 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:18:38.198 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:18:38.200 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:18:38.201 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:18:38.201 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:18:38.201 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:18:38.201 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:18:38.201 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:18:38.201 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:18:38.201 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:18:38.201 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:18:38.201 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:18:38.201 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:18:38.201 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:18:38.202 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:18:38.202 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:18:38.202 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:18:38.202 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:18:38.202 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:18:38.202 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:18:38.202 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:18:38.202 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:18:38.202 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:18:38.202 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:18:38.202 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:18:38.202 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:18:38.202 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:18:38.203 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:18:38.203 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:18:38.203 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:18:38.203 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:18:38.203 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:18:38.203 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:18:38.203 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:18:38.212 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:18:38.213 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:18:38.213 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:18:38.213 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:18:38.273 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:18:38.274 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:18:38.274 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:18:38.274 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:18:38.274 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:18:38.274 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:18:38.274 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:18:38.274 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:18:38.274 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:18:38.275 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:18:38.275 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:18:38.275 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:18:38.275 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:18:38.275 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:18:38.276 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:18:38.276 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:18:38.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 64 2026-02-18T08:18:38.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 65 2026-02-18T08:18:38.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:18:38.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=261, tag_tid=18, entry_tid=1], [object_number=260, tag_tid=18, entry_tid=0], [object_number=262, tag_tid=17, entry_tid=30126], [object_number=263, tag_tid=17, entry_tid=30123]]], state=connected] 2026-02-18T08:18:38.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=259, tag_tid=17, entry_tid=24771], [object_number=258, tag_tid=17, entry_tid=24770], [object_number=257, tag_tid=17, entry_tid=24769], [object_number=256, tag_tid=17, entry_tid=24768]]], state=connected] 2026-02-18T08:18:38.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:18:38.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=261, tag_tid=18, entry_tid=1' 2026-02-18T08:18:38.310 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:18:38.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:18:38.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:18:38.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:18:38.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:18:38.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:18:38.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:18:38.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:18:38.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:18:38.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:18:38.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:18:38.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:18:38.312 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:18:38.312 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:18:38.312 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:18:38.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 64 2026-02-18T08:18:38.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 65 2026-02-18T08:18:38.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:18:38.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=261, tag_tid=18, entry_tid=1], [object_number=260, tag_tid=18, entry_tid=0], [object_number=262, tag_tid=17, entry_tid=30126], [object_number=263, tag_tid=17, entry_tid=30123]]], state=connected] 2026-02-18T08:18:38.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=259, tag_tid=17, entry_tid=24771], [object_number=258, tag_tid=17, entry_tid=24770], [object_number=257, tag_tid=17, entry_tid=24769], [object_number=256, tag_tid=17, entry_tid=24768]]], state=connected] 2026-02-18T08:18:38.350 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:18:38.352 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=259, tag_tid=17, entry_tid=24771' 2026-02-18T08:18:38.352 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=261, tag_tid=18, entry_tid=1' -a 'object_number=261, tag_tid=18, entry_tid=1' = 'object_number=259, tag_tid=17, entry_tid=24771' 2026-02-18T08:18:38.352 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=259, tag_tid=17, entry_tid=24771' '!=' 'object_number=256, tag_tid=17, entry_tid=24400' 2026-02-18T08:18:38.352 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:18:38.352 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=259, tag_tid=17, entry_tid=24771' = 'object_number=256, tag_tid=17, entry_tid=24400' 2026-02-18T08:18:38.352 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=259, tag_tid=17, entry_tid=24771' 2026-02-18T08:18:38.353 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=261, tag_tid=18, entry_tid=1' 2026-02-18T08:18:38.353 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:18:38.353 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:18:38.354 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=18 2026-02-18T08:18:38.355 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=259, tag_tid=17, entry_tid=24771' 2026-02-18T08:18:38.355 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:18:38.355 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:18:38.356 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=17 2026-02-18T08:18:38.357 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=261, tag_tid=18, entry_tid=1' 2026-02-18T08:18:38.357 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:18:38.357 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:18:38.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:18:38.360 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=259, tag_tid=17, entry_tid=24771' 2026-02-18T08:18:38.360 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:18:38.360 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:18:38.361 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=24771 2026-02-18T08:18:38.361 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 18 = 17 -a 1 -le 24771 2026-02-18T08:18:38.362 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:18:38.362 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:18:38.362 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:18:38.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:18:38.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:18:38.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:18:38.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:18:38.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:18:38.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:18:38.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:18:38.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:18:38.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:18:38.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:18:38.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:18:38.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:18:38.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:18:38.565 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:18:38.565 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:18:38.565 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:18:38.565 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:18:38.567 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:18:38.567 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:18:38.568 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:18:38.568 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:18:38.568 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:18:38.568 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:18:38.568 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:18:38.568 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:18:38.568 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:18:38.568 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:18:38.568 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:18:38.568 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:18:38.568 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:18:38.569 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:18:38.569 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:18:38.569 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:18:38.569 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:18:38.569 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:18:38.569 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:18:38.569 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:18:38.569 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:18:38.569 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:18:38.569 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:18:38.569 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:18:38.569 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:18:38.570 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:18:38.570 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:18:38.570 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:18:38.570 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:18:38.570 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:18:38.570 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:18:38.570 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:18:38.578 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:18:38.578 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:18:38.578 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:18:38.578 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:18:38.628 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:18:38.628 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:18:38.628 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:18:38.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:18:38.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:18:38.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:18:38.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:18:38.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:18:38.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:18:38.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:18:38.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:18:38.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:18:38.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:18:38.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:18:38.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:18:38.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:18:38.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:18:38.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:18:38.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:18:38.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:18:38.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:18:38.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:18:38.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:18:38.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:18:38.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:18:38.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:18:38.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:18:38.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:18:38.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:18:38.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:18:38.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:18:38.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:18:38.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:18:38.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:18:38.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:18:38.642 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:18:38.643 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:18:38.643 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:18:38.643 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:18:38.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:18:38.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:18:38.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:18:38.693 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:18:38.693 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:18:38.693 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:18:38.693 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:18:38.694 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:18:38.694 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:18:38.694 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:18:38.694 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:18:38.694 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:18:38.694 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:18:38.694 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:18:38.694 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:18:38.694 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:18:38.694 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:18:38.694 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:18:38.694 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:18:38.695 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:18:38.695 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:18:38.695 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:18:38.695 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:18:38.695 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:18:38.695 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:18:38.695 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:18:38.695 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:18:38.695 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:18:38.695 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:18:38.695 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:18:38.695 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:18:38.696 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:18:38.696 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:18:38.696 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:18:38.696 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:18:38.705 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:18:38.706 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:18:38.706 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:18:38.706 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:18:38.763 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:18:38.763 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:18:38.763 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:18:38.766 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:18:38.766 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:18:38.766 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:18:38.766 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:18:38.766 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:18:38.766 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:18:38.766 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:18:38.767 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:18:38.767 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:18:38.767 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:18:38.767 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:18:38.767 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:18:38.767 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:18:38.767 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:18:38.767 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:18:38.767 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:18:38.767 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:18:38.767 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:18:38.767 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:18:38.768 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:18:38.768 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:18:38.768 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:18:38.768 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:18:38.768 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:18:38.768 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:18:38.768 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:18:38.768 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:18:38.768 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:18:38.768 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:18:38.768 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:18:38.768 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:18:38.769 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:18:38.777 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:18:38.778 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:18:38.778 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:18:38.778 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:18:38.837 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:18:38.838 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:18:38.839 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:18:38.839 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:18:38.839 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:18:38.839 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:18:38.839 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:18:38.839 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:18:38.839 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:18:38.839 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:18:38.839 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:18:38.839 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:18:38.840 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:18:38.840 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:18:38.841 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:18:38.841 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:18:38.876 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 64 2026-02-18T08:18:38.876 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 65 2026-02-18T08:18:38.877 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:18:38.877 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=261, tag_tid=18, entry_tid=1], [object_number=260, tag_tid=18, entry_tid=0], [object_number=262, tag_tid=17, entry_tid=30126], [object_number=263, tag_tid=17, entry_tid=30123]]], state=connected] 2026-02-18T08:18:38.877 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=257, tag_tid=17, entry_tid=25161], [object_number=256, tag_tid=17, entry_tid=25160], [object_number=259, tag_tid=17, entry_tid=25159], [object_number=258, tag_tid=17, entry_tid=25158]]], state=connected] 2026-02-18T08:18:38.880 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:18:38.881 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=261, tag_tid=18, entry_tid=1' 2026-02-18T08:18:38.883 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:18:38.883 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:18:38.883 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:18:38.883 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:18:38.883 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:18:38.883 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:18:38.883 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:18:38.883 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:18:38.883 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:18:38.884 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:18:38.884 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:18:38.884 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:18:38.884 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:18:38.885 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:18:38.885 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:18:39.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 64 2026-02-18T08:18:39.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 65 2026-02-18T08:18:39.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:18:39.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=261, tag_tid=18, entry_tid=1], [object_number=260, tag_tid=18, entry_tid=0], [object_number=262, tag_tid=17, entry_tid=30126], [object_number=263, tag_tid=17, entry_tid=30123]]], state=connected] 2026-02-18T08:18:39.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=257, tag_tid=17, entry_tid=25161], [object_number=256, tag_tid=17, entry_tid=25160], [object_number=259, tag_tid=17, entry_tid=25159], [object_number=258, tag_tid=17, entry_tid=25158]]], state=connected] 2026-02-18T08:18:39.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:18:39.131 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=257, tag_tid=17, entry_tid=25161' 2026-02-18T08:18:39.131 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=261, tag_tid=18, entry_tid=1' -a 'object_number=261, tag_tid=18, entry_tid=1' = 'object_number=257, tag_tid=17, entry_tid=25161' 2026-02-18T08:18:39.131 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=257, tag_tid=17, entry_tid=25161' '!=' 'object_number=259, tag_tid=17, entry_tid=24771' 2026-02-18T08:18:39.131 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:18:39.131 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=257, tag_tid=17, entry_tid=25161' = 'object_number=259, tag_tid=17, entry_tid=24771' 2026-02-18T08:18:39.131 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=257, tag_tid=17, entry_tid=25161' 2026-02-18T08:18:39.132 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=261, tag_tid=18, entry_tid=1' 2026-02-18T08:18:39.132 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:18:39.132 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:18:39.132 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=18 2026-02-18T08:18:39.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=257, tag_tid=17, entry_tid=25161' 2026-02-18T08:18:39.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:18:39.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:18:39.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=17 2026-02-18T08:18:39.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=261, tag_tid=18, entry_tid=1' 2026-02-18T08:18:39.136 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:18:39.136 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:18:39.137 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:18:39.138 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=257, tag_tid=17, entry_tid=25161' 2026-02-18T08:18:39.138 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:18:39.138 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:18:39.140 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=25161 2026-02-18T08:18:39.140 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 18 = 17 -a 1 -le 25161 2026-02-18T08:18:39.140 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:18:39.140 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:18:39.140 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:18:39.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:18:39.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:18:39.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:18:39.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:18:39.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:18:39.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:18:39.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:18:39.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:18:39.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:18:39.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:18:39.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:18:39.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:18:39.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:18:39.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:18:39.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:18:39.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:18:39.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:18:39.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:18:39.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:18:39.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:18:39.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:18:39.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:18:39.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:18:39.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:18:39.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:18:39.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:18:39.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:18:39.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:18:39.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:18:39.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:18:39.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:18:39.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:18:39.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:18:39.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:18:39.348 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:18:39.348 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:18:39.348 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:18:39.348 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:18:39.348 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:18:39.348 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:18:39.348 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:18:39.348 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:18:39.348 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:18:39.348 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:18:39.348 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:18:39.348 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:18:39.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:18:39.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:18:39.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:18:39.356 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:18:39.356 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:18:39.356 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:18:39.357 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:18:39.406 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:18:39.406 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:18:39.406 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:18:39.409 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:18:39.409 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:18:39.409 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:18:39.409 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:18:39.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:18:39.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:18:39.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:18:39.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:18:39.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:18:39.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:18:39.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:18:39.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:18:39.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:18:39.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:18:39.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:18:39.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:18:39.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:18:39.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:18:39.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:18:39.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:18:39.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:18:39.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:18:39.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:18:39.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:18:39.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:18:39.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:18:39.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:18:39.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:18:39.412 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:18:39.412 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:18:39.412 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:18:39.412 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:18:39.421 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:18:39.421 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:18:39.421 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:18:39.422 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:18:39.481 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:18:39.481 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:18:39.481 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:18:39.484 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:18:39.485 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:18:39.485 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:18:39.485 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:18:39.485 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:18:39.485 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:18:39.485 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:18:39.485 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:18:39.485 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:18:39.485 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:18:39.485 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:18:39.485 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:18:39.486 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:18:39.486 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:18:39.486 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:18:39.486 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:18:39.486 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:18:39.486 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:18:39.486 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:18:39.486 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:18:39.486 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:18:39.486 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:18:39.486 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:18:39.486 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:18:39.486 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:18:39.487 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:18:39.487 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:18:39.487 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:18:39.487 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:18:39.487 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:18:39.487 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:18:39.487 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:18:39.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:18:39.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:18:39.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:18:39.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:18:39.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:18:39.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:18:39.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:18:39.557 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:18:39.557 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:18:39.557 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:18:39.557 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:18:39.557 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:18:39.557 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:18:39.558 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:18:39.558 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:18:39.558 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:18:39.558 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:18:39.558 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:18:39.558 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:18:39.558 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:18:39.558 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:18:39.558 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:18:39.558 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:18:39.558 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:18:39.558 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:18:39.559 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:18:39.559 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:18:39.559 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:18:39.559 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:18:39.559 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:18:39.559 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:18:39.559 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:18:39.559 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:18:39.559 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:18:39.559 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:18:39.559 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:18:39.560 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:18:39.560 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:18:39.560 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:18:39.569 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:18:39.569 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:18:39.569 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:18:39.570 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:18:39.638 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:18:39.639 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:18:39.639 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:18:39.639 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:18:39.640 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:18:39.640 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:18:39.640 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:18:39.640 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:18:39.640 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:18:39.640 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:18:39.640 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:18:39.640 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:18:39.640 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:18:39.641 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:18:39.641 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:18:39.641 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:18:39.679 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 64 2026-02-18T08:18:39.680 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 65 2026-02-18T08:18:39.680 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:18:39.680 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=261, tag_tid=18, entry_tid=1], [object_number=260, tag_tid=18, entry_tid=0], [object_number=262, tag_tid=17, entry_tid=30126], [object_number=263, tag_tid=17, entry_tid=30123]]], state=connected] 2026-02-18T08:18:39.680 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=259, tag_tid=17, entry_tid=25683], [object_number=258, tag_tid=17, entry_tid=25682], [object_number=257, tag_tid=17, entry_tid=25681], [object_number=256, tag_tid=17, entry_tid=25680]]], state=connected] 2026-02-18T08:18:39.683 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:18:39.685 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=261, tag_tid=18, entry_tid=1' 2026-02-18T08:18:39.686 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:18:39.686 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:18:39.686 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:18:39.686 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:18:39.686 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:18:39.686 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:18:39.687 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:18:39.687 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:18:39.687 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:18:39.687 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:18:39.687 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:18:39.687 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:18:39.688 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:18:39.688 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:18:39.688 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:18:39.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 64 2026-02-18T08:18:39.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 65 2026-02-18T08:18:39.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:18:39.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=261, tag_tid=18, entry_tid=1], [object_number=260, tag_tid=18, entry_tid=0], [object_number=262, tag_tid=17, entry_tid=30126], [object_number=263, tag_tid=17, entry_tid=30123]]], state=connected] 2026-02-18T08:18:39.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=259, tag_tid=17, entry_tid=25683], [object_number=258, tag_tid=17, entry_tid=25682], [object_number=257, tag_tid=17, entry_tid=25681], [object_number=256, tag_tid=17, entry_tid=25680]]], state=connected] 2026-02-18T08:18:39.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:18:39.728 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=259, tag_tid=17, entry_tid=25683' 2026-02-18T08:18:39.728 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=261, tag_tid=18, entry_tid=1' -a 'object_number=261, tag_tid=18, entry_tid=1' = 'object_number=259, tag_tid=17, entry_tid=25683' 2026-02-18T08:18:39.728 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=259, tag_tid=17, entry_tid=25683' '!=' 'object_number=257, tag_tid=17, entry_tid=25161' 2026-02-18T08:18:39.729 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:18:39.729 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=259, tag_tid=17, entry_tid=25683' = 'object_number=257, tag_tid=17, entry_tid=25161' 2026-02-18T08:18:39.729 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=259, tag_tid=17, entry_tid=25683' 2026-02-18T08:18:39.729 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=261, tag_tid=18, entry_tid=1' 2026-02-18T08:18:39.729 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:18:39.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:18:39.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=18 2026-02-18T08:18:39.731 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=259, tag_tid=17, entry_tid=25683' 2026-02-18T08:18:39.731 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:18:39.731 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:18:39.732 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=17 2026-02-18T08:18:39.733 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=261, tag_tid=18, entry_tid=1' 2026-02-18T08:18:39.733 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:18:39.734 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:18:39.735 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:18:39.735 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=259, tag_tid=17, entry_tid=25683' 2026-02-18T08:18:39.736 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:18:39.736 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:18:39.738 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=25683 2026-02-18T08:18:39.738 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 18 = 17 -a 1 -le 25683 2026-02-18T08:18:39.738 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:18:39.738 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:18:39.738 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:18:39.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:18:39.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:18:39.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:18:39.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:18:39.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:18:39.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:18:39.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:18:39.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:18:39.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:18:39.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:18:39.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:18:39.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:18:39.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:18:39.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:18:39.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:18:39.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:18:39.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:18:39.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:18:39.944 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:18:39.944 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:18:39.944 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:18:39.944 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:18:39.944 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:18:39.944 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:18:39.944 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:18:39.944 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:18:39.944 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:18:39.944 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:18:39.944 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:18:39.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:18:39.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:18:39.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:18:39.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:18:39.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:18:39.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:18:39.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:18:39.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:18:39.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:18:39.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:18:39.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:18:39.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:18:39.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:18:39.946 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:18:39.946 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:18:39.946 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:18:39.946 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:18:39.946 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:18:39.946 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:18:39.946 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:18:39.954 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:18:39.954 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:18:39.955 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:18:39.955 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:18:40.003 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:18:40.003 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:18:40.003 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:18:40.006 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:18:40.006 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:18:40.006 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:18:40.007 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:18:40.007 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:18:40.007 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:18:40.007 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:18:40.007 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:18:40.007 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:18:40.007 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:18:40.007 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:18:40.007 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:18:40.007 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:18:40.007 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:18:40.007 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:18:40.008 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:18:40.008 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:18:40.008 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:18:40.008 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:18:40.008 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:18:40.008 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:18:40.008 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:18:40.008 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:18:40.008 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:18:40.008 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:18:40.008 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:18:40.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:18:40.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:18:40.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:18:40.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:18:40.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:18:40.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:18:40.018 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:18:40.018 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:18:40.018 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:18:40.018 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:18:40.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:18:40.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:18:40.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:18:40.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:18:40.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:18:40.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:18:40.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:18:40.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:18:40.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:18:40.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:18:40.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:18:40.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:18:40.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:18:40.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:18:40.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:18:40.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:18:40.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:18:40.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:18:40.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:18:40.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:18:40.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:18:40.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:18:40.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:18:40.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:18:40.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:18:40.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:18:40.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:18:40.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:18:40.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:18:40.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:18:40.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:18:40.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:18:40.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:18:40.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:18:40.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:18:40.093 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:18:40.093 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:18:40.094 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:18:40.094 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:18:40.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:18:40.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:18:40.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:18:40.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:18:40.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:18:40.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:18:40.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:18:40.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:18:40.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:18:40.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:18:40.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:18:40.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:18:40.155 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:18:40.155 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:18:40.155 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:18:40.155 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:18:40.155 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:18:40.155 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:18:40.155 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:18:40.155 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:18:40.155 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:18:40.155 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:18:40.155 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:18:40.155 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:18:40.156 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:18:40.156 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:18:40.156 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:18:40.156 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:18:40.156 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:18:40.156 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:18:40.156 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:18:40.156 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:18:40.156 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:18:40.156 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:18:40.156 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:18:40.164 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:18:40.165 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:18:40.165 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:18:40.165 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:18:40.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:18:40.234 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:18:40.234 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:18:40.234 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:18:40.234 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:18:40.234 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:18:40.234 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:18:40.234 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:18:40.234 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:18:40.234 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:18:40.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:18:40.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:18:40.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:18:40.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:18:40.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:18:40.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:18:40.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 64 2026-02-18T08:18:40.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 65 2026-02-18T08:18:40.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:18:40.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=261, tag_tid=18, entry_tid=1], [object_number=260, tag_tid=18, entry_tid=0], [object_number=262, tag_tid=17, entry_tid=30126], [object_number=263, tag_tid=17, entry_tid=30123]]], state=connected] 2026-02-18T08:18:40.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=257, tag_tid=17, entry_tid=26045], [object_number=256, tag_tid=17, entry_tid=26044], [object_number=259, tag_tid=17, entry_tid=26043], [object_number=258, tag_tid=17, entry_tid=26042]]], state=connected] 2026-02-18T08:18:40.276 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:18:40.277 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=261, tag_tid=18, entry_tid=1' 2026-02-18T08:18:40.279 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:18:40.279 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:18:40.279 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:18:40.279 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:18:40.279 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:18:40.279 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:18:40.280 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:18:40.280 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:18:40.280 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:18:40.280 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:18:40.281 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:18:40.281 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:18:40.281 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:18:40.281 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:18:40.282 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:18:40.316 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 64 2026-02-18T08:18:40.316 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 65 2026-02-18T08:18:40.316 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:18:40.316 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=261, tag_tid=18, entry_tid=1], [object_number=260, tag_tid=18, entry_tid=0], [object_number=262, tag_tid=17, entry_tid=30126], [object_number=263, tag_tid=17, entry_tid=30123]]], state=connected] 2026-02-18T08:18:40.316 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=257, tag_tid=17, entry_tid=26045], [object_number=256, tag_tid=17, entry_tid=26044], [object_number=259, tag_tid=17, entry_tid=26043], [object_number=258, tag_tid=17, entry_tid=26042]]], state=connected] 2026-02-18T08:18:40.318 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:18:40.320 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=257, tag_tid=17, entry_tid=26045' 2026-02-18T08:18:40.321 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=261, tag_tid=18, entry_tid=1' -a 'object_number=261, tag_tid=18, entry_tid=1' = 'object_number=257, tag_tid=17, entry_tid=26045' 2026-02-18T08:18:40.321 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=257, tag_tid=17, entry_tid=26045' '!=' 'object_number=259, tag_tid=17, entry_tid=25683' 2026-02-18T08:18:40.321 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:18:40.321 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=257, tag_tid=17, entry_tid=26045' = 'object_number=259, tag_tid=17, entry_tid=25683' 2026-02-18T08:18:40.321 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=257, tag_tid=17, entry_tid=26045' 2026-02-18T08:18:40.321 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=261, tag_tid=18, entry_tid=1' 2026-02-18T08:18:40.321 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:18:40.321 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:18:40.322 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=18 2026-02-18T08:18:40.323 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=257, tag_tid=17, entry_tid=26045' 2026-02-18T08:18:40.323 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:18:40.323 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:18:40.324 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=17 2026-02-18T08:18:40.324 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=261, tag_tid=18, entry_tid=1' 2026-02-18T08:18:40.325 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:18:40.325 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:18:40.326 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:18:40.327 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=257, tag_tid=17, entry_tid=26045' 2026-02-18T08:18:40.327 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:18:40.327 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:18:40.329 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=26045 2026-02-18T08:18:40.329 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 18 = 17 -a 1 -le 26045 2026-02-18T08:18:40.329 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:18:40.329 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:18:40.329 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:18:40.530 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:18:40.530 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:18:40.530 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:18:40.530 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:18:40.531 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:18:40.531 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:18:40.531 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:18:40.531 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:18:40.531 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:18:40.531 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:18:40.531 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:18:40.531 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:18:40.531 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:18:40.531 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:18:40.531 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:18:40.531 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:18:40.531 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:18:40.532 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:18:40.532 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:18:40.532 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:18:40.532 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:18:40.532 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:18:40.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:18:40.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:18:40.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:18:40.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:18:40.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:18:40.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:18:40.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:18:40.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:18:40.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:18:40.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:18:40.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:18:40.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:18:40.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:18:40.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:18:40.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:18:40.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:18:40.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:18:40.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:18:40.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:18:40.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:18:40.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:18:40.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:18:40.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:18:40.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:18:40.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:18:40.534 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:18:40.534 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:18:40.545 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:18:40.545 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:18:40.545 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:18:40.545 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:18:40.594 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:18:40.594 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:18:40.594 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:18:40.596 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:18:40.596 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:18:40.596 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:18:40.596 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:18:40.596 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:18:40.596 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:18:40.596 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:18:40.596 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:18:40.596 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:18:40.596 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:18:40.596 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:18:40.596 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:18:40.596 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:18:40.596 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:18:40.596 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:18:40.596 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:18:40.596 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:18:40.596 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:18:40.596 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:18:40.596 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:18:40.596 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:18:40.597 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:18:40.597 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:18:40.597 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:18:40.597 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:18:40.597 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:18:40.597 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:18:40.597 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:18:40.597 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:18:40.597 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:18:40.597 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:18:40.597 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:18:40.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:18:40.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:18:40.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:18:40.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:18:40.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:18:40.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:18:40.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:18:40.658 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:18:40.658 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:18:40.658 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:18:40.658 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:18:40.658 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:18:40.658 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:18:40.658 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:18:40.658 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:18:40.658 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:18:40.658 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:18:40.658 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:18:40.658 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:18:40.658 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:18:40.658 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:18:40.658 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:18:40.658 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:18:40.659 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:18:40.659 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:18:40.659 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:18:40.659 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:18:40.659 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:18:40.659 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:18:40.659 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:18:40.659 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:18:40.659 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:18:40.659 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:18:40.659 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:18:40.659 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:18:40.659 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:18:40.659 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:18:40.659 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:18:40.659 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:18:40.670 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:18:40.670 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:18:40.670 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:18:40.670 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:18:40.718 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:18:40.718 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:18:40.718 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:18:40.719 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:18:40.719 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:18:40.719 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:18:40.719 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:18:40.719 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:18:40.719 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:18:40.719 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:18:40.719 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:18:40.719 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:18:40.719 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:18:40.719 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:18:40.719 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:18:40.719 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:18:40.720 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:18:40.720 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:18:40.720 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:18:40.720 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:18:40.720 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:18:40.720 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:18:40.720 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:18:40.720 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:18:40.720 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:18:40.720 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:18:40.720 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:18:40.720 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:18:40.720 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:18:40.720 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:18:40.720 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:18:40.720 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:18:40.720 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:18:40.720 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:18:40.720 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:18:40.731 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:18:40.732 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:18:40.732 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:18:40.732 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:18:40.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:18:40.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:18:40.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:18:40.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:18:40.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:18:40.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:18:40.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:18:40.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:18:40.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:18:40.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:18:40.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:18:40.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:18:40.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:18:40.794 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:18:40.794 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:18:40.794 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:18:40.821 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 64 2026-02-18T08:18:40.821 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 65 2026-02-18T08:18:40.821 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:18:40.821 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=261, tag_tid=18, entry_tid=1], [object_number=260, tag_tid=18, entry_tid=0], [object_number=262, tag_tid=17, entry_tid=30126], [object_number=263, tag_tid=17, entry_tid=30123]]], state=connected] 2026-02-18T08:18:40.821 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=257, tag_tid=17, entry_tid=26401], [object_number=256, tag_tid=17, entry_tid=26400], [object_number=259, tag_tid=17, entry_tid=26399], [object_number=258, tag_tid=17, entry_tid=26398]]], state=connected] 2026-02-18T08:18:40.824 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:18:40.826 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=261, tag_tid=18, entry_tid=1' 2026-02-18T08:18:40.826 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:18:40.826 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:18:40.826 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:18:40.826 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:18:40.826 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:18:40.826 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:18:40.826 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:18:40.826 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:18:40.826 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:18:40.826 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:18:40.827 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:18:40.827 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:18:40.828 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:18:40.828 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:18:40.828 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:18:40.855 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 64 2026-02-18T08:18:40.855 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 65 2026-02-18T08:18:40.855 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:18:40.856 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=261, tag_tid=18, entry_tid=1], [object_number=260, tag_tid=18, entry_tid=0], [object_number=262, tag_tid=17, entry_tid=30126], [object_number=263, tag_tid=17, entry_tid=30123]]], state=connected] 2026-02-18T08:18:40.856 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=257, tag_tid=17, entry_tid=26401], [object_number=256, tag_tid=17, entry_tid=26400], [object_number=259, tag_tid=17, entry_tid=26399], [object_number=258, tag_tid=17, entry_tid=26398]]], state=connected] 2026-02-18T08:18:40.858 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:18:40.860 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=257, tag_tid=17, entry_tid=26401' 2026-02-18T08:18:40.860 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=261, tag_tid=18, entry_tid=1' -a 'object_number=261, tag_tid=18, entry_tid=1' = 'object_number=257, tag_tid=17, entry_tid=26401' 2026-02-18T08:18:40.860 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=257, tag_tid=17, entry_tid=26401' '!=' 'object_number=257, tag_tid=17, entry_tid=26045' 2026-02-18T08:18:40.860 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:18:40.860 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=257, tag_tid=17, entry_tid=26401' = 'object_number=257, tag_tid=17, entry_tid=26045' 2026-02-18T08:18:40.860 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=257, tag_tid=17, entry_tid=26401' 2026-02-18T08:18:40.860 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=261, tag_tid=18, entry_tid=1' 2026-02-18T08:18:40.860 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:18:40.860 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:18:40.862 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=18 2026-02-18T08:18:40.862 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=257, tag_tid=17, entry_tid=26401' 2026-02-18T08:18:40.862 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:18:40.862 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:18:40.864 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=17 2026-02-18T08:18:40.864 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=261, tag_tid=18, entry_tid=1' 2026-02-18T08:18:40.865 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:18:40.865 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:18:40.866 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:18:40.867 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=257, tag_tid=17, entry_tid=26401' 2026-02-18T08:18:40.867 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:18:40.867 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:18:40.868 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=26401 2026-02-18T08:18:40.868 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 18 = 17 -a 1 -le 26401 2026-02-18T08:18:40.868 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:18:40.868 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:18:40.868 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:18:41.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:18:41.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:18:41.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:18:41.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:18:41.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:18:41.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:18:41.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:18:41.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:18:41.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:18:41.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:18:41.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:18:41.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:18:41.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:18:41.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:18:41.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:18:41.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:18:41.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:18:41.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:18:41.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:18:41.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:18:41.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:18:41.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:18:41.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:18:41.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:18:41.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:18:41.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:18:41.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:18:41.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:18:41.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:18:41.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:18:41.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:18:41.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:18:41.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:18:41.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:18:41.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:18:41.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:18:41.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:18:41.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:18:41.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:18:41.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:18:41.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:18:41.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:18:41.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:18:41.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:18:41.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:18:41.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:18:41.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:18:41.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:18:41.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:18:41.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:18:41.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:18:41.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:18:41.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:18:41.143 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:18:41.143 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:18:41.143 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:18:41.145 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:18:41.145 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:18:41.145 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:18:41.145 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:18:41.145 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:18:41.145 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:18:41.145 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:18:41.145 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:18:41.145 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:18:41.145 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:18:41.145 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:18:41.145 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:18:41.145 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:18:41.145 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:18:41.145 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:18:41.145 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:18:41.145 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:18:41.145 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:18:41.145 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:18:41.145 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:18:41.145 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:18:41.145 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:18:41.146 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:18:41.146 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:18:41.146 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:18:41.146 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:18:41.146 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:18:41.146 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:18:41.146 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:18:41.146 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:18:41.146 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:18:41.146 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:18:41.157 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:18:41.157 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:18:41.157 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:18:41.157 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:18:41.215 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:18:41.215 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:18:41.215 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:18:41.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:18:41.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:18:41.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:18:41.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:18:41.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:18:41.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:18:41.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:18:41.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:18:41.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:18:41.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:18:41.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:18:41.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:18:41.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:18:41.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:18:41.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:18:41.217 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:18:41.217 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:18:41.217 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:18:41.217 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:18:41.217 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:18:41.217 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:18:41.217 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:18:41.217 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:18:41.217 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:18:41.217 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:18:41.217 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:18:41.217 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:18:41.217 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:18:41.217 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:18:41.217 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:18:41.217 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:18:41.217 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:18:41.229 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:18:41.229 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:18:41.229 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:18:41.229 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:18:41.287 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:18:41.287 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:18:41.287 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:18:41.288 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:18:41.288 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:18:41.288 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:18:41.288 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:18:41.288 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:18:41.288 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:18:41.288 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:18:41.288 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:18:41.288 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:18:41.288 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:18:41.288 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:18:41.288 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:18:41.288 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:18:41.288 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:18:41.288 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:18:41.288 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:18:41.289 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:18:41.289 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:18:41.289 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:18:41.289 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:18:41.289 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:18:41.289 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:18:41.289 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:18:41.289 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:18:41.289 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:18:41.289 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:18:41.289 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:18:41.289 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:18:41.289 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:18:41.289 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:18:41.289 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:18:41.289 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:18:41.300 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:18:41.300 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:18:41.301 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:18:41.301 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:18:41.370 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:18:41.371 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:18:41.371 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:18:41.371 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:18:41.371 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:18:41.371 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:18:41.371 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:18:41.371 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:18:41.371 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:18:41.371 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:18:41.371 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:18:41.372 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:18:41.372 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:18:41.373 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:18:41.373 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:18:41.374 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:18:41.604 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 65 2026-02-18T08:18:41.604 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 65 2026-02-18T08:18:41.604 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:18:41.604 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=261, tag_tid=18, entry_tid=1], [object_number=260, tag_tid=18, entry_tid=0], [object_number=262, tag_tid=17, entry_tid=30126], [object_number=263, tag_tid=17, entry_tid=30123]]], state=connected] 2026-02-18T08:18:41.604 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=261, tag_tid=17, entry_tid=26705], [object_number=260, tag_tid=17, entry_tid=26704], [object_number=263, tag_tid=17, entry_tid=26703], [object_number=262, tag_tid=17, entry_tid=26702]]], state=connected] 2026-02-18T08:18:41.607 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:18:41.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=261, tag_tid=18, entry_tid=1' 2026-02-18T08:18:41.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:18:41.610 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:18:41.610 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:18:41.610 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:18:41.610 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:18:41.610 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:18:41.610 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:18:41.610 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:18:41.610 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:18:41.610 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:18:41.610 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:18:41.610 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:18:41.612 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:18:41.612 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:18:41.612 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:18:41.642 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 65 2026-02-18T08:18:41.642 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 65 2026-02-18T08:18:41.642 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:18:41.642 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=261, tag_tid=18, entry_tid=1], [object_number=260, tag_tid=18, entry_tid=0], [object_number=262, tag_tid=17, entry_tid=30126], [object_number=263, tag_tid=17, entry_tid=30123]]], state=connected] 2026-02-18T08:18:41.642 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=261, tag_tid=17, entry_tid=26705], [object_number=260, tag_tid=17, entry_tid=26704], [object_number=263, tag_tid=17, entry_tid=26703], [object_number=262, tag_tid=17, entry_tid=26702]]], state=connected] 2026-02-18T08:18:41.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:18:41.646 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=261, tag_tid=17, entry_tid=26705' 2026-02-18T08:18:41.646 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=261, tag_tid=18, entry_tid=1' -a 'object_number=261, tag_tid=18, entry_tid=1' = 'object_number=261, tag_tid=17, entry_tid=26705' 2026-02-18T08:18:41.646 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=261, tag_tid=17, entry_tid=26705' '!=' 'object_number=257, tag_tid=17, entry_tid=26401' 2026-02-18T08:18:41.646 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:18:41.647 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=261, tag_tid=17, entry_tid=26705' = 'object_number=257, tag_tid=17, entry_tid=26401' 2026-02-18T08:18:41.647 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=261, tag_tid=17, entry_tid=26705' 2026-02-18T08:18:41.647 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=261, tag_tid=18, entry_tid=1' 2026-02-18T08:18:41.647 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:18:41.647 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:18:41.648 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=18 2026-02-18T08:18:41.649 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=261, tag_tid=17, entry_tid=26705' 2026-02-18T08:18:41.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:18:41.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:18:41.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=17 2026-02-18T08:18:41.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=261, tag_tid=18, entry_tid=1' 2026-02-18T08:18:41.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:18:41.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:18:41.653 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:18:41.653 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=261, tag_tid=17, entry_tid=26705' 2026-02-18T08:18:41.654 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:18:41.654 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:18:41.655 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=26705 2026-02-18T08:18:41.655 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 18 = 17 -a 1 -le 26705 2026-02-18T08:18:41.655 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:18:41.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:18:41.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:18:41.857 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:18:41.858 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:18:41.858 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:18:41.858 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:18:41.858 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:18:41.858 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:18:41.858 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:18:41.858 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:18:41.858 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:18:41.858 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:18:41.859 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:18:41.859 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:18:41.859 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:18:41.859 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:18:41.859 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:18:41.859 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:18:41.859 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:18:41.861 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:18:41.861 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:18:41.861 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:18:41.862 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:18:41.862 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:18:41.862 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:18:41.862 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:18:41.862 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:18:41.862 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:18:41.862 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:18:41.862 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:18:41.862 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:18:41.862 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:18:41.862 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:18:41.862 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:18:41.863 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:18:41.863 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:18:41.863 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:18:41.863 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:18:41.863 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:18:41.863 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:18:41.863 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:18:41.863 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:18:41.863 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:18:41.863 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:18:41.863 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:18:41.863 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:18:41.864 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:18:41.864 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:18:41.864 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:18:41.864 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:18:41.864 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:18:41.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:18:41.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:18:41.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:18:41.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:18:41.922 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:18:41.922 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:18:41.922 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:18:41.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:18:41.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:18:41.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:18:41.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:18:41.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:18:41.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:18:41.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:18:41.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:18:41.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:18:41.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:18:41.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:18:41.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:18:41.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:18:41.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:18:41.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:18:41.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:18:41.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:18:41.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:18:41.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:18:41.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:18:41.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:18:41.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:18:41.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:18:41.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:18:41.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:18:41.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:18:41.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:18:41.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:18:41.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:18:41.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:18:41.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:18:41.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:18:41.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:18:41.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:18:41.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:18:41.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:18:41.986 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:18:41.986 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:18:41.986 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:18:41.988 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:18:41.989 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:18:41.989 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:18:41.989 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:18:41.989 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:18:41.989 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:18:41.989 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:18:41.989 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:18:41.989 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:18:41.989 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:18:41.989 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:18:41.989 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:18:41.989 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:18:41.990 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:18:41.990 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:18:41.990 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:18:41.990 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:18:41.990 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:18:41.990 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:18:41.990 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:18:41.990 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:18:41.990 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:18:41.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:18:41.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:18:41.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:18:41.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:18:41.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:18:41.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:18:41.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:18:41.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:18:41.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:18:41.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:18:42.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:18:42.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:18:42.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:18:42.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:18:42.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:18:42.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:18:42.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:18:42.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:18:42.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:18:42.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:18:42.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:18:42.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:18:42.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:18:42.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:18:42.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:18:42.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:18:42.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:18:42.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:18:42.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:18:42.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:18:42.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:18:42.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:18:42.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:18:42.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:18:42.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:18:42.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:18:42.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:18:42.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:18:42.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:18:42.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:18:42.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:18:42.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:18:42.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:18:42.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:18:42.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:18:42.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:18:42.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:18:42.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:18:42.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:18:42.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:18:42.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:18:42.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:18:42.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:18:42.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:18:42.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:18:42.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:18:42.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:18:42.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:18:42.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:18:42.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:18:42.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:18:42.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:18:42.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:18:42.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:18:42.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:18:42.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:18:42.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:18:42.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:18:42.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:18:42.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 65 2026-02-18T08:18:42.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 65 2026-02-18T08:18:42.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:18:42.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=261, tag_tid=18, entry_tid=1], [object_number=260, tag_tid=18, entry_tid=0], [object_number=262, tag_tid=17, entry_tid=30126], [object_number=263, tag_tid=17, entry_tid=30123]]], state=connected] 2026-02-18T08:18:42.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=263, tag_tid=17, entry_tid=27167], [object_number=262, tag_tid=17, entry_tid=27166], [object_number=261, tag_tid=17, entry_tid=27165], [object_number=260, tag_tid=17, entry_tid=27164]]], state=connected] 2026-02-18T08:18:42.156 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:18:42.158 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=261, tag_tid=18, entry_tid=1' 2026-02-18T08:18:42.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:18:42.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:18:42.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:18:42.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:18:42.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:18:42.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:18:42.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:18:42.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:18:42.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:18:42.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:18:42.161 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:18:42.161 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:18:42.161 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:18:42.162 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:18:42.162 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:18:42.188 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 65 2026-02-18T08:18:42.188 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 65 2026-02-18T08:18:42.188 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:18:42.188 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=261, tag_tid=18, entry_tid=1], [object_number=260, tag_tid=18, entry_tid=0], [object_number=262, tag_tid=17, entry_tid=30126], [object_number=263, tag_tid=17, entry_tid=30123]]], state=connected] 2026-02-18T08:18:42.189 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=263, tag_tid=17, entry_tid=27167], [object_number=262, tag_tid=17, entry_tid=27166], [object_number=261, tag_tid=17, entry_tid=27165], [object_number=260, tag_tid=17, entry_tid=27164]]], state=connected] 2026-02-18T08:18:42.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:18:42.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=263, tag_tid=17, entry_tid=27167' 2026-02-18T08:18:42.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=261, tag_tid=18, entry_tid=1' -a 'object_number=261, tag_tid=18, entry_tid=1' = 'object_number=263, tag_tid=17, entry_tid=27167' 2026-02-18T08:18:42.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=263, tag_tid=17, entry_tid=27167' '!=' 'object_number=261, tag_tid=17, entry_tid=26705' 2026-02-18T08:18:42.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:18:42.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=263, tag_tid=17, entry_tid=27167' = 'object_number=261, tag_tid=17, entry_tid=26705' 2026-02-18T08:18:42.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=263, tag_tid=17, entry_tid=27167' 2026-02-18T08:18:42.194 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=261, tag_tid=18, entry_tid=1' 2026-02-18T08:18:42.194 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:18:42.194 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:18:42.194 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=18 2026-02-18T08:18:42.195 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=263, tag_tid=17, entry_tid=27167' 2026-02-18T08:18:42.195 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:18:42.195 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:18:42.197 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=17 2026-02-18T08:18:42.197 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=261, tag_tid=18, entry_tid=1' 2026-02-18T08:18:42.198 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:18:42.198 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:18:42.199 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:18:42.200 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=263, tag_tid=17, entry_tid=27167' 2026-02-18T08:18:42.200 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:18:42.200 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:18:42.201 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=27167 2026-02-18T08:18:42.202 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 18 = 17 -a 1 -le 27167 2026-02-18T08:18:42.202 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:18:42.202 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:18:42.202 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:18:42.403 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:18:42.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:18:42.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:18:42.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:18:42.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:18:42.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:18:42.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:18:42.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:18:42.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:18:42.405 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:18:42.405 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:18:42.405 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:18:42.405 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:18:42.405 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:18:42.405 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:18:42.405 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:18:42.408 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:18:42.408 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:18:42.408 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:18:42.408 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:18:42.409 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:18:42.409 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:18:42.409 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:18:42.409 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:18:42.409 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:18:42.409 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:18:42.409 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:18:42.409 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:18:42.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:18:42.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:18:42.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:18:42.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:18:42.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:18:42.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:18:42.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:18:42.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:18:42.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:18:42.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:18:42.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:18:42.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:18:42.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:18:42.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:18:42.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:18:42.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:18:42.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:18:42.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:18:42.412 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:18:42.412 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:18:42.412 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:18:42.417 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:18:42.418 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:18:42.418 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:18:42.418 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:18:42.475 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:18:42.476 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:18:42.476 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:18:42.478 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:18:42.479 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:18:42.479 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:18:42.479 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:18:42.479 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:18:42.479 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:18:42.479 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:18:42.479 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:18:42.479 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:18:42.479 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:18:42.479 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:18:42.480 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:18:42.480 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:18:42.480 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:18:42.480 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:18:42.480 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:18:42.480 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:18:42.480 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:18:42.480 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:18:42.480 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:18:42.480 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:18:42.480 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:18:42.480 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:18:42.481 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:18:42.481 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:18:42.481 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:18:42.481 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:18:42.481 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:18:42.481 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:18:42.481 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:18:42.481 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:18:42.481 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:18:42.489 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:18:42.489 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:18:42.490 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:18:42.490 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:18:42.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:18:42.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:18:42.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:18:42.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:18:42.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:18:42.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:18:42.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:18:42.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:18:42.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:18:42.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:18:42.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:18:42.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:18:42.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:18:42.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:18:42.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:18:42.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:18:42.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:18:42.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:18:42.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:18:42.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:18:42.551 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:18:42.551 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:18:42.551 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:18:42.551 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:18:42.551 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:18:42.551 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:18:42.551 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:18:42.551 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:18:42.551 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:18:42.551 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:18:42.551 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:18:42.551 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:18:42.552 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:18:42.552 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:18:42.552 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:18:42.560 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:18:42.560 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:18:42.560 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:18:42.560 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:18:42.611 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:18:42.611 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:18:42.611 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:18:42.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:18:42.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:18:42.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:18:42.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:18:42.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:18:42.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:18:42.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:18:42.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:18:42.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:18:42.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:18:42.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:18:42.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:18:42.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:18:42.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:18:42.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:18:42.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:18:42.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:18:42.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:18:42.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:18:42.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:18:42.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:18:42.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:18:42.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:18:42.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:18:42.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:18:42.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:18:42.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:18:42.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:18:42.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:18:42.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:18:42.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:18:42.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:18:42.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:18:42.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:18:42.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:18:42.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:18:42.684 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:18:42.685 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:18:42.686 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:18:42.686 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:18:42.686 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:18:42.686 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:18:42.686 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:18:42.686 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:18:42.686 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:18:42.686 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:18:42.686 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:18:42.686 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:18:42.687 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:18:42.687 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:18:42.688 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:18:42.688 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:18:42.717 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 65 2026-02-18T08:18:42.717 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 65 2026-02-18T08:18:42.717 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:18:42.717 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=261, tag_tid=18, entry_tid=1], [object_number=260, tag_tid=18, entry_tid=0], [object_number=262, tag_tid=17, entry_tid=30126], [object_number=263, tag_tid=17, entry_tid=30123]]], state=connected] 2026-02-18T08:18:42.717 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=261, tag_tid=17, entry_tid=27505], [object_number=260, tag_tid=17, entry_tid=27504], [object_number=263, tag_tid=17, entry_tid=27503], [object_number=262, tag_tid=17, entry_tid=27502]]], state=connected] 2026-02-18T08:18:42.720 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:18:42.721 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=261, tag_tid=18, entry_tid=1' 2026-02-18T08:18:42.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:18:42.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:18:42.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:18:42.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:18:42.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:18:42.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:18:42.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:18:42.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:18:42.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:18:42.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:18:42.724 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:18:42.724 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:18:42.725 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:18:42.725 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:18:42.725 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:18:42.763 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 65 2026-02-18T08:18:42.763 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 65 2026-02-18T08:18:42.763 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:18:42.763 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=261, tag_tid=18, entry_tid=1], [object_number=260, tag_tid=18, entry_tid=0], [object_number=262, tag_tid=17, entry_tid=30126], [object_number=263, tag_tid=17, entry_tid=30123]]], state=connected] 2026-02-18T08:18:42.763 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=261, tag_tid=17, entry_tid=27505], [object_number=260, tag_tid=17, entry_tid=27504], [object_number=263, tag_tid=17, entry_tid=27503], [object_number=262, tag_tid=17, entry_tid=27502]]], state=connected] 2026-02-18T08:18:42.766 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:18:42.768 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=261, tag_tid=17, entry_tid=27505' 2026-02-18T08:18:42.768 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=261, tag_tid=18, entry_tid=1' -a 'object_number=261, tag_tid=18, entry_tid=1' = 'object_number=261, tag_tid=17, entry_tid=27505' 2026-02-18T08:18:42.768 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=261, tag_tid=17, entry_tid=27505' '!=' 'object_number=263, tag_tid=17, entry_tid=27167' 2026-02-18T08:18:42.768 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:18:42.768 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=261, tag_tid=17, entry_tid=27505' = 'object_number=263, tag_tid=17, entry_tid=27167' 2026-02-18T08:18:42.768 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=261, tag_tid=17, entry_tid=27505' 2026-02-18T08:18:42.769 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=261, tag_tid=18, entry_tid=1' 2026-02-18T08:18:42.769 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:18:42.769 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:18:42.770 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=18 2026-02-18T08:18:42.771 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=261, tag_tid=17, entry_tid=27505' 2026-02-18T08:18:42.771 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:18:42.771 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:18:42.772 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=17 2026-02-18T08:18:42.773 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=261, tag_tid=18, entry_tid=1' 2026-02-18T08:18:42.773 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:18:42.773 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:18:42.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:18:42.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=261, tag_tid=17, entry_tid=27505' 2026-02-18T08:18:42.776 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:18:42.776 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:18:42.777 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=27505 2026-02-18T08:18:42.777 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 18 = 17 -a 1 -le 27505 2026-02-18T08:18:42.777 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:18:42.777 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:18:42.777 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:18:42.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:18:42.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:18:42.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:18:42.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:18:42.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:18:42.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:18:42.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:18:42.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:18:42.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:18:42.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:18:42.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:18:42.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:18:42.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:18:42.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:18:42.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:18:42.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:18:42.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:18:42.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:18:42.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:18:42.985 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:18:42.985 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:18:42.985 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:18:42.985 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:18:42.985 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:18:42.985 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:18:42.985 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:18:42.985 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:18:42.985 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:18:42.985 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:18:42.985 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:18:42.986 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:18:42.986 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:18:42.986 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:18:42.986 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:18:42.986 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:18:42.986 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:18:42.986 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:18:42.986 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:18:42.986 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:18:42.986 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:18:42.986 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:18:42.986 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:18:42.986 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:18:42.987 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:18:42.987 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:18:42.987 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:18:42.987 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:18:42.987 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:18:42.987 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:18:42.994 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:18:42.994 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:18:42.994 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:18:42.994 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:18:43.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:18:43.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:18:43.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:18:43.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:18:43.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:18:43.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:18:43.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:18:43.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:18:43.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:18:43.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:18:43.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:18:43.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:18:43.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:18:43.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:18:43.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:18:43.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:18:43.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:18:43.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:18:43.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:18:43.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:18:43.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:18:43.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:18:43.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:18:43.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:18:43.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:18:43.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:18:43.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:18:43.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:18:43.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:18:43.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:18:43.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:18:43.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:18:43.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:18:43.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:18:43.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:18:43.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:18:43.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:18:43.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:18:43.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:18:43.106 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:18:43.106 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:18:43.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:18:43.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:18:43.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:18:43.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:18:43.110 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:18:43.110 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:18:43.110 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:18:43.110 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:18:43.110 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:18:43.110 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:18:43.110 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:18:43.110 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:18:43.110 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:18:43.110 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:18:43.110 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:18:43.110 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:18:43.110 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:18:43.111 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:18:43.111 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:18:43.111 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:18:43.111 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:18:43.111 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:18:43.111 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:18:43.111 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:18:43.111 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:18:43.111 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:18:43.111 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:18:43.111 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:18:43.111 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:18:43.112 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:18:43.112 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:18:43.112 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:18:43.112 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:18:43.120 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:18:43.121 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:18:43.121 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:18:43.121 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:18:43.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:18:43.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:18:43.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:18:43.171 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:18:43.171 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:18:43.171 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:18:43.171 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:18:43.171 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:18:43.171 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:18:43.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:18:43.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:18:43.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:18:43.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:18:43.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:18:43.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:18:43.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:18:43.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:18:43.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:18:43.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:18:43.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:18:43.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:18:43.173 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:18:43.173 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:18:43.173 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:18:43.173 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:18:43.173 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:18:43.173 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:18:43.173 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:18:43.173 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:18:43.173 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:18:43.173 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:18:43.173 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:18:43.173 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:18:43.174 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:18:43.174 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:18:43.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:18:43.183 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:18:43.183 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:18:43.183 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:18:43.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:18:43.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:18:43.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:18:43.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:18:43.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:18:43.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:18:43.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:18:43.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:18:43.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:18:43.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:18:43.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:18:43.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:18:43.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:18:43.244 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:18:43.244 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:18:43.244 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:18:43.273 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 65 2026-02-18T08:18:43.273 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 65 2026-02-18T08:18:43.273 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:18:43.273 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=261, tag_tid=18, entry_tid=1], [object_number=260, tag_tid=18, entry_tid=0], [object_number=262, tag_tid=17, entry_tid=30126], [object_number=263, tag_tid=17, entry_tid=30123]]], state=connected] 2026-02-18T08:18:43.273 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=262, tag_tid=17, entry_tid=27854], [object_number=261, tag_tid=17, entry_tid=27853], [object_number=260, tag_tid=17, entry_tid=27852], [object_number=263, tag_tid=17, entry_tid=27851]]], state=connected] 2026-02-18T08:18:43.276 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:18:43.277 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=261, tag_tid=18, entry_tid=1' 2026-02-18T08:18:43.278 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:18:43.278 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:18:43.278 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:18:43.279 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:18:43.279 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:18:43.279 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:18:43.279 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:18:43.279 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:18:43.279 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:18:43.279 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:18:43.280 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:18:43.280 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:18:43.280 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:18:43.281 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:18:43.281 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:18:43.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 65 2026-02-18T08:18:43.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 65 2026-02-18T08:18:43.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:18:43.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=261, tag_tid=18, entry_tid=1], [object_number=260, tag_tid=18, entry_tid=0], [object_number=262, tag_tid=17, entry_tid=30126], [object_number=263, tag_tid=17, entry_tid=30123]]], state=connected] 2026-02-18T08:18:43.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=262, tag_tid=17, entry_tid=27854], [object_number=261, tag_tid=17, entry_tid=27853], [object_number=260, tag_tid=17, entry_tid=27852], [object_number=263, tag_tid=17, entry_tid=27851]]], state=connected] 2026-02-18T08:18:43.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:18:43.312 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=262, tag_tid=17, entry_tid=27854' 2026-02-18T08:18:43.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=261, tag_tid=18, entry_tid=1' -a 'object_number=261, tag_tid=18, entry_tid=1' = 'object_number=262, tag_tid=17, entry_tid=27854' 2026-02-18T08:18:43.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=262, tag_tid=17, entry_tid=27854' '!=' 'object_number=261, tag_tid=17, entry_tid=27505' 2026-02-18T08:18:43.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:18:43.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=262, tag_tid=17, entry_tid=27854' = 'object_number=261, tag_tid=17, entry_tid=27505' 2026-02-18T08:18:43.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=262, tag_tid=17, entry_tid=27854' 2026-02-18T08:18:43.314 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=261, tag_tid=18, entry_tid=1' 2026-02-18T08:18:43.314 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:18:43.314 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:18:43.315 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=18 2026-02-18T08:18:43.315 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=262, tag_tid=17, entry_tid=27854' 2026-02-18T08:18:43.315 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:18:43.316 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:18:43.317 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=17 2026-02-18T08:18:43.318 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=261, tag_tid=18, entry_tid=1' 2026-02-18T08:18:43.318 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:18:43.318 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:18:43.319 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:18:43.320 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=262, tag_tid=17, entry_tid=27854' 2026-02-18T08:18:43.320 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:18:43.320 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:18:43.321 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=27854 2026-02-18T08:18:43.322 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 18 = 17 -a 1 -le 27854 2026-02-18T08:18:43.322 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:18:43.322 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:18:43.322 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:18:43.523 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:18:43.524 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:18:43.524 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:18:43.524 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:18:43.524 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:18:43.524 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:18:43.524 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:18:43.524 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:18:43.524 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:18:43.524 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:18:43.524 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:18:43.524 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:18:43.525 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:18:43.525 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:18:43.525 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:18:43.525 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:18:43.527 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:18:43.528 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:18:43.528 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:18:43.528 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:18:43.528 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:18:43.528 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:18:43.528 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:18:43.528 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:18:43.528 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:18:43.528 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:18:43.528 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:18:43.528 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:18:43.529 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:18:43.529 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:18:43.529 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:18:43.529 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:18:43.529 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:18:43.529 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:18:43.529 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:18:43.529 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:18:43.529 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:18:43.529 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:18:43.529 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:18:43.529 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:18:43.530 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:18:43.530 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:18:43.530 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:18:43.530 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:18:43.530 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:18:43.530 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:18:43.530 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:18:43.530 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:18:43.530 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:18:43.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:18:43.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:18:43.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:18:43.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:18:43.587 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:18:43.587 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:18:43.587 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:18:43.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:18:43.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:18:43.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:18:43.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:18:43.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:18:43.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:18:43.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:18:43.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:18:43.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:18:43.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:18:43.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:18:43.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:18:43.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:18:43.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:18:43.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:18:43.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:18:43.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:18:43.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:18:43.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:18:43.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:18:43.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:18:43.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:18:43.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:18:43.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:18:43.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:18:43.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:18:43.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:18:43.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:18:43.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:18:43.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:18:43.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:18:43.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:18:43.602 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:18:43.602 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:18:43.602 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:18:43.603 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:18:43.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:18:43.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:18:43.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:18:43.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:18:43.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:18:43.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:18:43.653 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:18:43.653 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:18:43.653 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:18:43.653 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:18:43.653 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:18:43.653 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:18:43.653 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:18:43.653 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:18:43.653 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:18:43.653 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:18:43.653 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:18:43.654 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:18:43.654 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:18:43.654 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:18:43.654 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:18:43.654 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:18:43.654 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:18:43.654 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:18:43.654 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:18:43.654 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:18:43.654 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:18:43.654 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:18:43.654 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:18:43.655 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:18:43.655 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:18:43.655 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:18:43.655 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:18:43.655 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:18:43.655 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:18:43.664 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:18:43.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:18:43.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:18:43.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:18:43.713 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:18:43.713 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:18:43.713 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:18:43.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:18:43.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:18:43.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:18:43.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:18:43.717 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:18:43.717 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:18:43.717 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:18:43.717 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:18:43.717 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:18:43.717 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:18:43.717 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:18:43.717 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:18:43.717 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:18:43.717 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:18:43.717 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:18:43.718 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:18:43.718 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:18:43.718 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:18:43.718 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:18:43.718 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:18:43.718 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:18:43.718 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:18:43.718 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:18:43.718 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:18:43.718 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:18:43.718 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:18:43.718 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:18:43.719 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:18:43.719 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:18:43.719 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:18:43.719 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:18:43.719 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:18:43.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:18:43.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:18:43.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:18:43.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:18:43.793 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:18:43.794 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:18:43.794 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:18:43.794 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:18:43.794 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:18:43.795 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:18:43.795 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:18:43.795 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:18:43.795 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:18:43.795 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:18:43.795 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:18:43.796 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:18:43.796 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:18:43.796 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:18:43.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:18:43.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:18:43.826 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 65 2026-02-18T08:18:43.826 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 65 2026-02-18T08:18:43.826 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:18:43.826 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=261, tag_tid=18, entry_tid=1], [object_number=260, tag_tid=18, entry_tid=0], [object_number=262, tag_tid=17, entry_tid=30126], [object_number=263, tag_tid=17, entry_tid=30123]]], state=connected] 2026-02-18T08:18:43.826 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=260, tag_tid=17, entry_tid=28180], [object_number=263, tag_tid=17, entry_tid=28179], [object_number=262, tag_tid=17, entry_tid=28178], [object_number=261, tag_tid=17, entry_tid=28177]]], state=connected] 2026-02-18T08:18:43.829 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:18:43.830 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=261, tag_tid=18, entry_tid=1' 2026-02-18T08:18:43.831 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:18:43.831 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:18:43.831 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:18:43.832 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:18:43.832 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:18:43.832 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:18:43.832 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:18:43.832 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:18:43.832 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:18:43.832 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:18:43.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:18:43.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:18:43.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:18:43.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:18:43.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:18:43.869 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 65 2026-02-18T08:18:43.869 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 65 2026-02-18T08:18:43.869 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:18:43.869 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=261, tag_tid=18, entry_tid=1], [object_number=260, tag_tid=18, entry_tid=0], [object_number=262, tag_tid=17, entry_tid=30126], [object_number=263, tag_tid=17, entry_tid=30123]]], state=connected] 2026-02-18T08:18:43.869 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=260, tag_tid=17, entry_tid=28180], [object_number=263, tag_tid=17, entry_tid=28179], [object_number=262, tag_tid=17, entry_tid=28178], [object_number=261, tag_tid=17, entry_tid=28177]]], state=connected] 2026-02-18T08:18:43.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:18:43.874 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=260, tag_tid=17, entry_tid=28180' 2026-02-18T08:18:43.874 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=261, tag_tid=18, entry_tid=1' -a 'object_number=261, tag_tid=18, entry_tid=1' = 'object_number=260, tag_tid=17, entry_tid=28180' 2026-02-18T08:18:43.874 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=260, tag_tid=17, entry_tid=28180' '!=' 'object_number=262, tag_tid=17, entry_tid=27854' 2026-02-18T08:18:43.874 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:18:43.874 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=260, tag_tid=17, entry_tid=28180' = 'object_number=262, tag_tid=17, entry_tid=27854' 2026-02-18T08:18:43.874 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=260, tag_tid=17, entry_tid=28180' 2026-02-18T08:18:43.875 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=261, tag_tid=18, entry_tid=1' 2026-02-18T08:18:43.875 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:18:43.875 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:18:43.876 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=18 2026-02-18T08:18:43.877 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=260, tag_tid=17, entry_tid=28180' 2026-02-18T08:18:43.877 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:18:43.877 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:18:43.878 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=17 2026-02-18T08:18:43.878 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=261, tag_tid=18, entry_tid=1' 2026-02-18T08:18:43.879 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:18:43.879 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:18:43.880 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:18:43.881 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=260, tag_tid=17, entry_tid=28180' 2026-02-18T08:18:43.881 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:18:43.881 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:18:43.883 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=28180 2026-02-18T08:18:43.883 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 18 = 17 -a 1 -le 28180 2026-02-18T08:18:43.883 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:18:43.883 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:18:43.883 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:18:44.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:18:44.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:18:44.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:18:44.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:18:44.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:18:44.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:18:44.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:18:44.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:18:44.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:18:44.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:18:44.086 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:18:44.086 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:18:44.086 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:18:44.086 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:18:44.086 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:18:44.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:18:44.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:18:44.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:18:44.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:18:44.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:18:44.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:18:44.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:18:44.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:18:44.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:18:44.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:18:44.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:18:44.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:18:44.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:18:44.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:18:44.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:18:44.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:18:44.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:18:44.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:18:44.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:18:44.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:18:44.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:18:44.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:18:44.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:18:44.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:18:44.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:18:44.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:18:44.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:18:44.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:18:44.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:18:44.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:18:44.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:18:44.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:18:44.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:18:44.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:18:44.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:18:44.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:18:44.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:18:44.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:18:44.148 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:18:44.148 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:18:44.148 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:18:44.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:18:44.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:18:44.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:18:44.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:18:44.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:18:44.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:18:44.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:18:44.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:18:44.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:18:44.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:18:44.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:18:44.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:18:44.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:18:44.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:18:44.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:18:44.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:18:44.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:18:44.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:18:44.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:18:44.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:18:44.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:18:44.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:18:44.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:18:44.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:18:44.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:18:44.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:18:44.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:18:44.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:18:44.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:18:44.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:18:44.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:18:44.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:18:44.161 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:18:44.162 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:18:44.162 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:18:44.162 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:18:44.212 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:18:44.212 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:18:44.212 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:18:44.214 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:18:44.214 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:18:44.215 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:18:44.215 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:18:44.215 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:18:44.215 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:18:44.215 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:18:44.215 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:18:44.215 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:18:44.215 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:18:44.215 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:18:44.215 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:18:44.215 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:18:44.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:18:44.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:18:44.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:18:44.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:18:44.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:18:44.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:18:44.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:18:44.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:18:44.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:18:44.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:18:44.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:18:44.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:18:44.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:18:44.217 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:18:44.217 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:18:44.217 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:18:44.217 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:18:44.217 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:18:44.217 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:18:44.225 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:18:44.225 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:18:44.225 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:18:44.225 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:18:44.274 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:18:44.274 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:18:44.274 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:18:44.277 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:18:44.277 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:18:44.277 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:18:44.277 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:18:44.277 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:18:44.277 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:18:44.278 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:18:44.278 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:18:44.278 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:18:44.278 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:18:44.278 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:18:44.278 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:18:44.278 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:18:44.278 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:18:44.278 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:18:44.278 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:18:44.278 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:18:44.278 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:18:44.278 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:18:44.279 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:18:44.279 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:18:44.279 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:18:44.279 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:18:44.279 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:18:44.279 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:18:44.279 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:18:44.279 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:18:44.279 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:18:44.279 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:18:44.279 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:18:44.279 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:18:44.280 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:18:44.288 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:18:44.289 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:18:44.289 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:18:44.289 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:18:44.348 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:18:44.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:18:44.350 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:18:44.350 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:18:44.350 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:18:44.350 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:18:44.350 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:18:44.350 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:18:44.350 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:18:44.350 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:18:44.350 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:18:44.351 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:18:44.351 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:18:44.351 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:18:44.352 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:18:44.352 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:18:44.381 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 65 2026-02-18T08:18:44.381 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 65 2026-02-18T08:18:44.381 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:18:44.381 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=261, tag_tid=18, entry_tid=1], [object_number=260, tag_tid=18, entry_tid=0], [object_number=262, tag_tid=17, entry_tid=30126], [object_number=263, tag_tid=17, entry_tid=30123]]], state=connected] 2026-02-18T08:18:44.382 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=260, tag_tid=17, entry_tid=28524], [object_number=263, tag_tid=17, entry_tid=28523], [object_number=262, tag_tid=17, entry_tid=28522], [object_number=261, tag_tid=17, entry_tid=28521]]], state=connected] 2026-02-18T08:18:44.384 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:18:44.385 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=261, tag_tid=18, entry_tid=1' 2026-02-18T08:18:44.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:18:44.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:18:44.387 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:18:44.387 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:18:44.387 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:18:44.387 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:18:44.387 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:18:44.387 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:18:44.387 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:18:44.387 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:18:44.387 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:18:44.388 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:18:44.388 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:18:44.389 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:18:44.389 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:18:44.415 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 65 2026-02-18T08:18:44.415 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 65 2026-02-18T08:18:44.415 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:18:44.416 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=261, tag_tid=18, entry_tid=1], [object_number=260, tag_tid=18, entry_tid=0], [object_number=262, tag_tid=17, entry_tid=30126], [object_number=263, tag_tid=17, entry_tid=30123]]], state=connected] 2026-02-18T08:18:44.416 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=260, tag_tid=17, entry_tid=28524], [object_number=263, tag_tid=17, entry_tid=28523], [object_number=262, tag_tid=17, entry_tid=28522], [object_number=261, tag_tid=17, entry_tid=28521]]], state=connected] 2026-02-18T08:18:44.418 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:18:44.420 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=260, tag_tid=17, entry_tid=28524' 2026-02-18T08:18:44.420 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=261, tag_tid=18, entry_tid=1' -a 'object_number=261, tag_tid=18, entry_tid=1' = 'object_number=260, tag_tid=17, entry_tid=28524' 2026-02-18T08:18:44.420 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=260, tag_tid=17, entry_tid=28524' '!=' 'object_number=260, tag_tid=17, entry_tid=28180' 2026-02-18T08:18:44.420 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:18:44.420 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=260, tag_tid=17, entry_tid=28524' = 'object_number=260, tag_tid=17, entry_tid=28180' 2026-02-18T08:18:44.420 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=260, tag_tid=17, entry_tid=28524' 2026-02-18T08:18:44.421 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=261, tag_tid=18, entry_tid=1' 2026-02-18T08:18:44.421 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:18:44.422 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:18:44.423 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=18 2026-02-18T08:18:44.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=260, tag_tid=17, entry_tid=28524' 2026-02-18T08:18:44.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:18:44.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:18:44.425 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=17 2026-02-18T08:18:44.425 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=261, tag_tid=18, entry_tid=1' 2026-02-18T08:18:44.425 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:18:44.425 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:18:44.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:18:44.427 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=260, tag_tid=17, entry_tid=28524' 2026-02-18T08:18:44.427 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:18:44.427 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:18:44.429 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=28524 2026-02-18T08:18:44.429 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 18 = 17 -a 1 -le 28524 2026-02-18T08:18:44.429 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:18:44.430 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:18:44.430 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:18:44.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:18:44.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:18:44.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:18:44.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:18:44.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:18:44.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:18:44.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:18:44.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:18:44.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:18:44.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:18:44.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:18:44.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:18:44.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:18:44.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:18:44.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:18:44.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:18:44.635 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:18:44.635 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:18:44.635 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:18:44.635 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:18:44.635 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:18:44.635 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:18:44.635 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:18:44.635 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:18:44.635 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:18:44.635 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:18:44.635 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:18:44.636 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:18:44.636 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:18:44.636 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:18:44.636 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:18:44.636 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:18:44.636 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:18:44.636 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:18:44.636 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:18:44.636 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:18:44.636 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:18:44.636 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:18:44.636 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:18:44.637 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:18:44.637 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:18:44.637 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:18:44.637 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:18:44.637 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:18:44.637 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:18:44.637 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:18:44.637 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:18:44.637 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:18:44.637 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:18:44.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:18:44.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:18:44.646 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:18:44.646 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:18:44.695 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:18:44.695 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:18:44.695 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:18:44.697 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:18:44.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:18:44.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:18:44.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:18:44.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:18:44.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:18:44.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:18:44.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:18:44.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:18:44.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:18:44.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:18:44.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:18:44.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:18:44.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:18:44.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:18:44.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:18:44.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:18:44.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:18:44.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:18:44.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:18:44.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:18:44.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:18:44.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:18:44.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:18:44.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:18:44.700 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:18:44.700 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:18:44.700 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:18:44.700 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:18:44.700 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:18:44.700 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:18:44.700 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:18:44.709 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:18:44.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:18:44.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:18:44.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:18:44.766 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:18:44.766 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:18:44.766 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:18:44.769 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:18:44.769 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:18:44.769 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:18:44.769 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:18:44.769 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:18:44.770 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:18:44.770 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:18:44.770 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:18:44.770 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:18:44.770 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:18:44.770 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:18:44.770 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:18:44.770 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:18:44.770 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:18:44.770 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:18:44.770 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:18:44.770 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:18:44.771 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:18:44.771 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:18:44.771 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:18:44.771 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:18:44.771 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:18:44.771 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:18:44.771 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:18:44.771 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:18:44.771 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:18:44.771 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:18:44.771 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:18:44.771 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:18:44.771 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:18:44.772 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:18:44.772 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:18:44.781 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:18:44.782 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:18:44.782 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:18:44.782 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:18:44.837 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:18:44.837 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:18:44.837 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:18:44.840 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:18:44.840 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:18:44.840 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:18:44.840 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:18:44.840 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:18:44.840 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:18:44.840 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:18:44.840 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:18:44.841 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:18:44.841 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:18:44.841 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:18:44.841 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:18:44.841 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:18:44.841 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:18:44.841 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:18:44.841 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:18:44.841 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:18:44.841 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:18:44.841 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:18:44.841 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:18:44.842 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:18:44.842 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:18:44.842 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:18:44.842 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:18:44.842 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:18:44.842 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:18:44.842 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:18:44.842 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:18:44.842 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:18:44.842 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:18:44.842 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:18:44.842 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:18:44.850 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:18:44.851 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:18:44.851 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:18:44.851 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:18:44.916 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:18:44.917 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:18:44.917 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:18:44.917 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:18:44.917 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:18:44.917 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:18:44.918 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:18:44.918 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:18:44.918 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:18:44.918 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:18:44.918 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:18:44.918 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:18:44.918 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:18:44.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:18:44.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:18:44.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:18:44.946 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 65 2026-02-18T08:18:44.947 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 65 2026-02-18T08:18:44.947 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:18:44.947 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=261, tag_tid=18, entry_tid=1], [object_number=260, tag_tid=18, entry_tid=0], [object_number=262, tag_tid=17, entry_tid=30126], [object_number=263, tag_tid=17, entry_tid=30123]]], state=connected] 2026-02-18T08:18:44.947 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=263, tag_tid=17, entry_tid=28867], [object_number=262, tag_tid=17, entry_tid=28866], [object_number=261, tag_tid=17, entry_tid=28865], [object_number=260, tag_tid=17, entry_tid=28864]]], state=connected] 2026-02-18T08:18:44.950 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:18:44.952 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=261, tag_tid=18, entry_tid=1' 2026-02-18T08:18:44.953 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:18:44.953 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:18:44.953 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:18:44.953 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:18:44.953 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:18:44.954 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:18:44.954 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:18:44.954 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:18:44.954 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:18:44.954 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:18:44.954 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:18:44.954 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:18:44.955 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:18:44.955 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:18:44.956 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:18:44.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 65 2026-02-18T08:18:44.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 65 2026-02-18T08:18:44.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:18:44.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=261, tag_tid=18, entry_tid=1], [object_number=260, tag_tid=18, entry_tid=0], [object_number=262, tag_tid=17, entry_tid=30126], [object_number=263, tag_tid=17, entry_tid=30123]]], state=connected] 2026-02-18T08:18:44.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=263, tag_tid=17, entry_tid=28867], [object_number=262, tag_tid=17, entry_tid=28866], [object_number=261, tag_tid=17, entry_tid=28865], [object_number=260, tag_tid=17, entry_tid=28864]]], state=connected] 2026-02-18T08:18:44.986 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:18:44.987 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=263, tag_tid=17, entry_tid=28867' 2026-02-18T08:18:44.988 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=261, tag_tid=18, entry_tid=1' -a 'object_number=261, tag_tid=18, entry_tid=1' = 'object_number=263, tag_tid=17, entry_tid=28867' 2026-02-18T08:18:44.988 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=263, tag_tid=17, entry_tid=28867' '!=' 'object_number=260, tag_tid=17, entry_tid=28524' 2026-02-18T08:18:44.988 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:18:44.988 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=263, tag_tid=17, entry_tid=28867' = 'object_number=260, tag_tid=17, entry_tid=28524' 2026-02-18T08:18:44.988 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=263, tag_tid=17, entry_tid=28867' 2026-02-18T08:18:44.989 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=261, tag_tid=18, entry_tid=1' 2026-02-18T08:18:44.989 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:18:44.989 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:18:44.989 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=18 2026-02-18T08:18:44.990 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=263, tag_tid=17, entry_tid=28867' 2026-02-18T08:18:44.990 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:18:44.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:18:44.992 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=17 2026-02-18T08:18:44.993 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=261, tag_tid=18, entry_tid=1' 2026-02-18T08:18:44.993 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:18:44.993 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:18:44.994 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:18:44.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=263, tag_tid=17, entry_tid=28867' 2026-02-18T08:18:44.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:18:44.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:18:44.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=28867 2026-02-18T08:18:44.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 18 = 17 -a 1 -le 28867 2026-02-18T08:18:44.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:18:44.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:18:44.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:18:45.198 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:18:45.198 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:18:45.198 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:18:45.198 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:18:45.199 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:18:45.199 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:18:45.199 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:18:45.199 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:18:45.199 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:18:45.199 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:18:45.199 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:18:45.199 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:18:45.199 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:18:45.199 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:18:45.199 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:18:45.202 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:18:45.202 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:18:45.202 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:18:45.202 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:18:45.202 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:18:45.203 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:18:45.203 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:18:45.203 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:18:45.203 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:18:45.203 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:18:45.203 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:18:45.203 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:18:45.203 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:18:45.203 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:18:45.203 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:18:45.203 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:18:45.203 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:18:45.204 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:18:45.204 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:18:45.204 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:18:45.204 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:18:45.204 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:18:45.204 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:18:45.204 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:18:45.204 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:18:45.204 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:18:45.204 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:18:45.204 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:18:45.204 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:18:45.204 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:18:45.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:18:45.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:18:45.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:18:45.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:18:45.213 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:18:45.213 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:18:45.214 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:18:45.214 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:18:45.270 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:18:45.270 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:18:45.270 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:18:45.273 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:18:45.273 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:18:45.273 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:18:45.273 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:18:45.273 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:18:45.273 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:18:45.273 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:18:45.274 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:18:45.274 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:18:45.274 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:18:45.274 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:18:45.274 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:18:45.274 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:18:45.274 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:18:45.274 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:18:45.274 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:18:45.274 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:18:45.274 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:18:45.275 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:18:45.275 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:18:45.275 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:18:45.275 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:18:45.275 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:18:45.275 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:18:45.275 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:18:45.275 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:18:45.275 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:18:45.275 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:18:45.275 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:18:45.275 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:18:45.275 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:18:45.276 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:18:45.285 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:18:45.286 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:18:45.286 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:18:45.286 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:18:45.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:18:45.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:18:45.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:18:45.344 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:18:45.344 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:18:45.344 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:18:45.344 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:18:45.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:18:45.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:18:45.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:18:45.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:18:45.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:18:45.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:18:45.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:18:45.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:18:45.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:18:45.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:18:45.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:18:45.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:18:45.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:18:45.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:18:45.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:18:45.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:18:45.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:18:45.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:18:45.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:18:45.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:18:45.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:18:45.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:18:45.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:18:45.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:18:45.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:18:45.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:18:45.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:18:45.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:18:45.354 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:18:45.355 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:18:45.355 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:18:45.355 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:18:45.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:18:45.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:18:45.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:18:45.413 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:18:45.414 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:18:45.414 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:18:45.414 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:18:45.414 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:18:45.414 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:18:45.414 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:18:45.414 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:18:45.414 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:18:45.414 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:18:45.414 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:18:45.414 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:18:45.415 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:18:45.415 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:18:45.415 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:18:45.415 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:18:45.415 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:18:45.415 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:18:45.415 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:18:45.415 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:18:45.415 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:18:45.415 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:18:45.415 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:18:45.415 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:18:45.415 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:18:45.416 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:18:45.416 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:18:45.416 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:18:45.416 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:18:45.416 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:18:45.416 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:18:45.416 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:18:45.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:18:45.425 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:18:45.425 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:18:45.425 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:18:45.491 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:18:45.492 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:18:45.492 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:18:45.493 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:18:45.493 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:18:45.493 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:18:45.493 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:18:45.493 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:18:45.493 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:18:45.493 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:18:45.493 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:18:45.493 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:18:45.493 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:18:45.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:18:45.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:18:45.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:18:45.531 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 65 2026-02-18T08:18:45.531 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 65 2026-02-18T08:18:45.531 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:18:45.532 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=261, tag_tid=18, entry_tid=1], [object_number=260, tag_tid=18, entry_tid=0], [object_number=262, tag_tid=17, entry_tid=30126], [object_number=263, tag_tid=17, entry_tid=30123]]], state=connected] 2026-02-18T08:18:45.532 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=262, tag_tid=17, entry_tid=29230], [object_number=261, tag_tid=17, entry_tid=29229], [object_number=260, tag_tid=17, entry_tid=29228], [object_number=263, tag_tid=17, entry_tid=29227]]], state=connected] 2026-02-18T08:18:45.535 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:18:45.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=261, tag_tid=18, entry_tid=1' 2026-02-18T08:18:45.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:18:45.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:18:45.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:18:45.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:18:45.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:18:45.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:18:45.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:18:45.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:18:45.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:18:45.539 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:18:45.539 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:18:45.539 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:18:45.540 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:18:45.540 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:18:45.540 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:18:45.567 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 65 2026-02-18T08:18:45.567 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 65 2026-02-18T08:18:45.568 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:18:45.568 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=261, tag_tid=18, entry_tid=1], [object_number=260, tag_tid=18, entry_tid=0], [object_number=262, tag_tid=17, entry_tid=30126], [object_number=263, tag_tid=17, entry_tid=30123]]], state=connected] 2026-02-18T08:18:45.568 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=262, tag_tid=17, entry_tid=29230], [object_number=261, tag_tid=17, entry_tid=29229], [object_number=260, tag_tid=17, entry_tid=29228], [object_number=263, tag_tid=17, entry_tid=29227]]], state=connected] 2026-02-18T08:18:45.570 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:18:45.572 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=262, tag_tid=17, entry_tid=29230' 2026-02-18T08:18:45.572 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=261, tag_tid=18, entry_tid=1' -a 'object_number=261, tag_tid=18, entry_tid=1' = 'object_number=262, tag_tid=17, entry_tid=29230' 2026-02-18T08:18:45.572 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=262, tag_tid=17, entry_tid=29230' '!=' 'object_number=263, tag_tid=17, entry_tid=28867' 2026-02-18T08:18:45.572 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:18:45.572 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=262, tag_tid=17, entry_tid=29230' = 'object_number=263, tag_tid=17, entry_tid=28867' 2026-02-18T08:18:45.572 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=262, tag_tid=17, entry_tid=29230' 2026-02-18T08:18:45.573 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=261, tag_tid=18, entry_tid=1' 2026-02-18T08:18:45.573 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:18:45.573 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:18:45.574 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=18 2026-02-18T08:18:45.575 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=262, tag_tid=17, entry_tid=29230' 2026-02-18T08:18:45.575 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:18:45.575 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:18:45.576 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=17 2026-02-18T08:18:45.577 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=261, tag_tid=18, entry_tid=1' 2026-02-18T08:18:45.577 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:18:45.577 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:18:45.578 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:18:45.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=262, tag_tid=17, entry_tid=29230' 2026-02-18T08:18:45.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:18:45.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:18:45.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=29230 2026-02-18T08:18:45.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 18 = 17 -a 1 -le 29230 2026-02-18T08:18:45.582 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:18:45.582 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:18:45.582 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:18:45.783 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:18:45.783 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:18:45.784 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:18:45.784 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:18:45.784 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:18:45.784 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:18:45.784 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:18:45.784 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:18:45.784 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:18:45.784 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:18:45.784 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:18:45.784 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:18:45.784 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:18:45.785 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:18:45.785 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:18:45.785 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:18:45.787 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:18:45.787 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:18:45.787 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:18:45.788 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:18:45.788 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:18:45.788 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:18:45.788 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:18:45.788 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:18:45.788 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:18:45.788 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:18:45.788 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:18:45.788 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:18:45.788 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:18:45.788 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:18:45.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:18:45.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:18:45.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:18:45.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:18:45.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:18:45.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:18:45.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:18:45.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:18:45.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:18:45.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:18:45.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:18:45.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:18:45.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:18:45.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:18:45.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:18:45.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:18:45.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:18:45.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:18:45.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:18:45.798 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:18:45.798 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:18:45.799 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:18:45.799 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:18:45.847 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:18:45.848 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:18:45.848 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:18:45.850 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:18:45.851 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:18:45.851 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:18:45.851 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:18:45.851 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:18:45.851 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:18:45.851 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:18:45.851 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:18:45.851 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:18:45.851 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:18:45.851 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:18:45.851 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:18:45.851 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:18:45.852 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:18:45.852 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:18:45.852 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:18:45.852 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:18:45.852 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:18:45.852 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:18:45.852 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:18:45.852 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:18:45.852 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:18:45.852 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:18:45.852 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:18:45.852 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:18:45.853 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:18:45.853 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:18:45.853 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:18:45.853 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:18:45.853 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:18:45.853 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:18:45.853 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:18:45.862 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:18:45.863 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:18:45.863 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:18:45.863 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:18:45.910 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:18:45.911 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:18:45.911 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:18:45.913 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:18:45.913 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:18:45.914 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:18:45.914 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:18:45.914 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:18:45.914 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:18:45.914 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:18:45.914 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:18:45.914 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:18:45.914 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:18:45.914 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:18:45.914 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:18:45.914 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:18:45.915 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:18:45.915 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:18:45.915 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:18:45.915 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:18:45.915 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:18:45.915 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:18:45.915 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:18:45.915 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:18:45.915 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:18:45.915 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:18:45.915 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:18:45.915 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:18:45.915 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:18:45.916 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:18:45.916 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:18:45.916 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:18:45.916 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:18:45.916 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:18:45.916 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:18:45.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:18:45.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:18:45.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:18:45.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:18:45.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:18:45.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:18:45.974 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:18:45.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:18:45.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:18:45.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:18:45.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:18:45.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:18:45.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:18:45.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:18:45.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:18:45.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:18:45.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:18:45.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:18:45.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:18:45.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:18:45.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:18:45.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:18:45.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:18:45.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:18:45.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:18:45.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:18:45.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:18:45.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:18:45.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:18:45.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:18:45.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:18:45.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:18:45.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:18:45.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:18:45.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:18:45.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:18:45.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:18:45.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:18:45.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:18:45.987 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:18:45.987 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:18:45.987 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:18:45.987 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:18:46.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:18:46.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:18:46.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:18:46.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:18:46.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:18:46.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:18:46.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:18:46.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:18:46.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:18:46.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:18:46.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:18:46.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:18:46.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:18:46.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:18:46.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:18:46.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:18:46.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 65 2026-02-18T08:18:46.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 65 2026-02-18T08:18:46.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:18:46.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=261, tag_tid=18, entry_tid=1], [object_number=260, tag_tid=18, entry_tid=0], [object_number=262, tag_tid=17, entry_tid=30126], [object_number=263, tag_tid=17, entry_tid=30123]]], state=connected] 2026-02-18T08:18:46.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=261, tag_tid=17, entry_tid=29605], [object_number=260, tag_tid=17, entry_tid=29604], [object_number=263, tag_tid=17, entry_tid=29603], [object_number=262, tag_tid=17, entry_tid=29602]]], state=connected] 2026-02-18T08:18:46.080 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:18:46.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=261, tag_tid=18, entry_tid=1' 2026-02-18T08:18:46.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:18:46.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:18:46.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:18:46.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:18:46.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:18:46.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:18:46.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:18:46.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:18:46.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:18:46.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:18:46.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:18:46.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:18:46.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:18:46.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:18:46.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:18:46.111 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 65 2026-02-18T08:18:46.112 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 65 2026-02-18T08:18:46.112 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:18:46.112 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=261, tag_tid=18, entry_tid=1], [object_number=260, tag_tid=18, entry_tid=0], [object_number=262, tag_tid=17, entry_tid=30126], [object_number=263, tag_tid=17, entry_tid=30123]]], state=connected] 2026-02-18T08:18:46.112 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=261, tag_tid=17, entry_tid=29605], [object_number=260, tag_tid=17, entry_tid=29604], [object_number=263, tag_tid=17, entry_tid=29603], [object_number=262, tag_tid=17, entry_tid=29602]]], state=connected] 2026-02-18T08:18:46.114 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:18:46.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=261, tag_tid=17, entry_tid=29605' 2026-02-18T08:18:46.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=261, tag_tid=18, entry_tid=1' -a 'object_number=261, tag_tid=18, entry_tid=1' = 'object_number=261, tag_tid=17, entry_tid=29605' 2026-02-18T08:18:46.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=261, tag_tid=17, entry_tid=29605' '!=' 'object_number=262, tag_tid=17, entry_tid=29230' 2026-02-18T08:18:46.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:18:46.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=261, tag_tid=17, entry_tid=29605' = 'object_number=262, tag_tid=17, entry_tid=29230' 2026-02-18T08:18:46.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=261, tag_tid=17, entry_tid=29605' 2026-02-18T08:18:46.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=261, tag_tid=18, entry_tid=1' 2026-02-18T08:18:46.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:18:46.118 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:18:46.118 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=18 2026-02-18T08:18:46.119 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=261, tag_tid=17, entry_tid=29605' 2026-02-18T08:18:46.119 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:18:46.119 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:18:46.121 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=17 2026-02-18T08:18:46.121 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=261, tag_tid=18, entry_tid=1' 2026-02-18T08:18:46.121 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:18:46.122 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:18:46.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:18:46.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=261, tag_tid=17, entry_tid=29605' 2026-02-18T08:18:46.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:18:46.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:18:46.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=29605 2026-02-18T08:18:46.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 18 = 17 -a 1 -le 29605 2026-02-18T08:18:46.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:18:46.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:18:46.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:18:46.327 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:18:46.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:18:46.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:18:46.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:18:46.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:18:46.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:18:46.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:18:46.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:18:46.329 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:18:46.329 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:18:46.329 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:18:46.329 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:18:46.329 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:18:46.329 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:18:46.329 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:18:46.329 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:18:46.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:18:46.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:18:46.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:18:46.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:18:46.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:18:46.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:18:46.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:18:46.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:18:46.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:18:46.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:18:46.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:18:46.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:18:46.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:18:46.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:18:46.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:18:46.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:18:46.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:18:46.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:18:46.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:18:46.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:18:46.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:18:46.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:18:46.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:18:46.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:18:46.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:18:46.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:18:46.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:18:46.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:18:46.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:18:46.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:18:46.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:18:46.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:18:46.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:18:46.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:18:46.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:18:46.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:18:46.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:18:46.393 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:18:46.393 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:18:46.393 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:18:46.396 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:18:46.397 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:18:46.397 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:18:46.397 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:18:46.397 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:18:46.397 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:18:46.397 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:18:46.397 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:18:46.397 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:18:46.397 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:18:46.397 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:18:46.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:18:46.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:18:46.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:18:46.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:18:46.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:18:46.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:18:46.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:18:46.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:18:46.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:18:46.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:18:46.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:18:46.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:18:46.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:18:46.399 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:18:46.399 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:18:46.399 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:18:46.399 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:18:46.399 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:18:46.399 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:18:46.399 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:18:46.399 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:18:46.409 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:18:46.409 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:18:46.409 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:18:46.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:18:46.460 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:18:46.460 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:18:46.460 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:18:46.462 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:18:46.463 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:18:46.463 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:18:46.463 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:18:46.463 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:18:46.463 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:18:46.463 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:18:46.463 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:18:46.463 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:18:46.463 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:18:46.463 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:18:46.463 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:18:46.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:18:46.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:18:46.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:18:46.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:18:46.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:18:46.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:18:46.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:18:46.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:18:46.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:18:46.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:18:46.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:18:46.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:18:46.465 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:18:46.465 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:18:46.465 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:18:46.465 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:18:46.465 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:18:46.465 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:18:46.465 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:18:46.465 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:18:46.473 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:18:46.474 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:18:46.474 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:18:46.474 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:18:46.524 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:18:46.524 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:18:46.524 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:18:46.527 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:18:46.527 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:18:46.527 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:18:46.527 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:18:46.527 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:18:46.527 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:18:46.527 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:18:46.528 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:18:46.528 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:18:46.528 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:18:46.528 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:18:46.528 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:18:46.528 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:18:46.528 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:18:46.528 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:18:46.528 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:18:46.528 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:18:46.528 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:18:46.528 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:18:46.529 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:18:46.529 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:18:46.529 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:18:46.529 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:18:46.529 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:18:46.529 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:18:46.529 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:18:46.529 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:18:46.529 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:18:46.529 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:18:46.529 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:18:46.529 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:18:46.529 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:18:46.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:18:46.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:18:46.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:18:46.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:18:46.599 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:18:46.600 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:18:46.600 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:18:46.601 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:18:46.601 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:18:46.601 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:18:46.601 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:18:46.601 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:18:46.601 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:18:46.601 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:18:46.601 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:18:46.602 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:18:46.602 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:18:46.602 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:18:46.603 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:18:46.603 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:18:46.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 65 2026-02-18T08:18:46.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 65 2026-02-18T08:18:46.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:18:46.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=261, tag_tid=18, entry_tid=1], [object_number=260, tag_tid=18, entry_tid=0], [object_number=262, tag_tid=17, entry_tid=30126], [object_number=263, tag_tid=17, entry_tid=30123]]], state=connected] 2026-02-18T08:18:46.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=261, tag_tid=17, entry_tid=29949], [object_number=260, tag_tid=17, entry_tid=29948], [object_number=263, tag_tid=17, entry_tid=29947], [object_number=262, tag_tid=17, entry_tid=29946]]], state=connected] 2026-02-18T08:18:46.837 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:18:46.838 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=261, tag_tid=18, entry_tid=1' 2026-02-18T08:18:46.839 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:18:46.839 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:18:46.840 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:18:46.840 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:18:46.840 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:18:46.840 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:18:46.840 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:18:46.840 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:18:46.840 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:18:46.840 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:18:46.841 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:18:46.841 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:18:46.842 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:18:46.842 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:18:46.842 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:18:46.868 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 65 2026-02-18T08:18:46.868 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 65 2026-02-18T08:18:46.868 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:18:46.868 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=261, tag_tid=18, entry_tid=1], [object_number=260, tag_tid=18, entry_tid=0], [object_number=262, tag_tid=17, entry_tid=30126], [object_number=263, tag_tid=17, entry_tid=30123]]], state=connected] 2026-02-18T08:18:46.869 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=261, tag_tid=17, entry_tid=29949], [object_number=260, tag_tid=17, entry_tid=29948], [object_number=263, tag_tid=17, entry_tid=29947], [object_number=262, tag_tid=17, entry_tid=29946]]], state=connected] 2026-02-18T08:18:46.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:18:46.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=261, tag_tid=17, entry_tid=29949' 2026-02-18T08:18:46.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=261, tag_tid=18, entry_tid=1' -a 'object_number=261, tag_tid=18, entry_tid=1' = 'object_number=261, tag_tid=17, entry_tid=29949' 2026-02-18T08:18:46.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=261, tag_tid=17, entry_tid=29949' '!=' 'object_number=261, tag_tid=17, entry_tid=29605' 2026-02-18T08:18:46.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:18:46.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=261, tag_tid=17, entry_tid=29949' = 'object_number=261, tag_tid=17, entry_tid=29605' 2026-02-18T08:18:46.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=261, tag_tid=17, entry_tid=29949' 2026-02-18T08:18:46.874 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=261, tag_tid=18, entry_tid=1' 2026-02-18T08:18:46.874 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:18:46.874 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:18:46.875 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=18 2026-02-18T08:18:46.876 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=261, tag_tid=17, entry_tid=29949' 2026-02-18T08:18:46.876 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:18:46.876 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:18:46.877 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=17 2026-02-18T08:18:46.878 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=261, tag_tid=18, entry_tid=1' 2026-02-18T08:18:46.878 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:18:46.878 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:18:46.879 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:18:46.880 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=261, tag_tid=17, entry_tid=29949' 2026-02-18T08:18:46.880 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:18:46.880 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:18:46.882 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=29949 2026-02-18T08:18:46.882 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 18 = 17 -a 1 -le 29949 2026-02-18T08:18:46.882 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:18:46.882 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:18:46.882 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:18:47.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:18:47.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:18:47.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:18:47.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:18:47.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:18:47.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:18:47.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:18:47.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:18:47.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:18:47.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:18:47.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:18:47.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:18:47.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:18:47.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:18:47.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:18:47.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:18:47.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:18:47.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:18:47.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:18:47.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:18:47.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:18:47.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:18:47.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:18:47.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:18:47.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:18:47.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:18:47.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:18:47.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:18:47.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:18:47.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:18:47.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:18:47.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:18:47.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:18:47.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:18:47.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:18:47.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:18:47.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:18:47.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:18:47.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:18:47.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:18:47.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:18:47.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:18:47.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:18:47.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:18:47.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:18:47.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:18:47.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:18:47.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:18:47.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:18:47.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:18:47.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:18:47.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:18:47.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:18:47.148 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:18:47.148 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:18:47.148 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:18:47.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:18:47.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:18:47.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:18:47.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:18:47.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:18:47.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:18:47.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:18:47.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:18:47.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:18:47.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:18:47.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:18:47.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:18:47.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:18:47.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:18:47.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:18:47.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:18:47.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:18:47.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:18:47.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:18:47.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:18:47.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:18:47.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:18:47.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:18:47.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:18:47.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:18:47.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:18:47.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:18:47.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:18:47.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:18:47.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:18:47.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:18:47.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:18:47.161 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:18:47.162 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:18:47.162 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:18:47.162 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:18:47.219 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:18:47.219 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:18:47.219 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:18:47.222 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:18:47.222 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:18:47.222 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:18:47.222 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:18:47.222 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:18:47.222 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:18:47.222 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:18:47.223 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:18:47.223 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:18:47.223 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:18:47.223 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:18:47.223 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:18:47.223 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:18:47.223 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:18:47.223 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:18:47.223 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:18:47.223 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:18:47.223 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:18:47.223 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:18:47.224 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:18:47.224 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:18:47.224 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:18:47.224 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:18:47.224 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:18:47.224 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:18:47.224 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:18:47.224 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:18:47.224 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:18:47.224 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:18:47.224 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:18:47.224 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:18:47.224 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:18:47.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:18:47.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:18:47.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:18:47.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:18:47.290 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:18:47.290 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:18:47.290 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:18:47.293 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:18:47.293 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:18:47.293 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:18:47.293 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:18:47.293 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:18:47.294 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:18:47.294 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:18:47.294 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:18:47.294 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:18:47.294 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:18:47.294 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:18:47.294 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:18:47.294 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:18:47.294 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:18:47.294 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:18:47.294 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:18:47.294 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:18:47.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:18:47.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:18:47.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:18:47.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:18:47.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:18:47.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:18:47.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:18:47.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:18:47.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:18:47.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:18:47.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:18:47.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:18:47.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:18:47.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:18:47.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:18:47.304 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:18:47.304 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:18:47.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:18:47.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:18:47.364 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:18:47.364 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:18:47.364 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:18:47.364 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:18:47.364 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:18:47.364 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:18:47.365 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:18:47.365 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:18:47.365 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:18:47.365 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:18:47.365 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:18:47.365 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:18:47.366 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:18:47.366 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:18:47.366 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:18:47.366 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:18:47.403 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 65 2026-02-18T08:18:47.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 65 2026-02-18T08:18:47.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:18:47.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=261, tag_tid=18, entry_tid=1], [object_number=260, tag_tid=18, entry_tid=0], [object_number=262, tag_tid=17, entry_tid=30126], [object_number=263, tag_tid=17, entry_tid=30123]]], state=connected] 2026-02-18T08:18:47.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=262, tag_tid=17, entry_tid=30126], [object_number=261, tag_tid=17, entry_tid=30125], [object_number=260, tag_tid=17, entry_tid=30124], [object_number=263, tag_tid=17, entry_tid=30123]]], state=connected] 2026-02-18T08:18:47.407 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:18:47.409 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=261, tag_tid=18, entry_tid=1' 2026-02-18T08:18:47.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:18:47.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:18:47.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:18:47.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:18:47.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:18:47.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:18:47.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:18:47.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:18:47.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:18:47.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:18:47.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:18:47.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:18:47.412 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:18:47.412 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:18:47.412 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:18:47.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 65 2026-02-18T08:18:47.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 65 2026-02-18T08:18:47.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:18:47.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=261, tag_tid=18, entry_tid=1], [object_number=260, tag_tid=18, entry_tid=0], [object_number=262, tag_tid=17, entry_tid=30126], [object_number=263, tag_tid=17, entry_tid=30123]]], state=connected] 2026-02-18T08:18:47.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=262, tag_tid=17, entry_tid=30126], [object_number=261, tag_tid=17, entry_tid=30125], [object_number=260, tag_tid=17, entry_tid=30124], [object_number=263, tag_tid=17, entry_tid=30123]]], state=connected] 2026-02-18T08:18:47.451 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:18:47.452 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=262, tag_tid=17, entry_tid=30126' 2026-02-18T08:18:47.452 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=261, tag_tid=18, entry_tid=1' -a 'object_number=261, tag_tid=18, entry_tid=1' = 'object_number=262, tag_tid=17, entry_tid=30126' 2026-02-18T08:18:47.452 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=262, tag_tid=17, entry_tid=30126' '!=' 'object_number=261, tag_tid=17, entry_tid=29949' 2026-02-18T08:18:47.453 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:18:47.453 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=262, tag_tid=17, entry_tid=30126' = 'object_number=261, tag_tid=17, entry_tid=29949' 2026-02-18T08:18:47.453 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=262, tag_tid=17, entry_tid=30126' 2026-02-18T08:18:47.453 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=261, tag_tid=18, entry_tid=1' 2026-02-18T08:18:47.453 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:18:47.454 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:18:47.455 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=18 2026-02-18T08:18:47.455 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=262, tag_tid=17, entry_tid=30126' 2026-02-18T08:18:47.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:18:47.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:18:47.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=17 2026-02-18T08:18:47.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=261, tag_tid=18, entry_tid=1' 2026-02-18T08:18:47.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:18:47.458 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:18:47.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:18:47.460 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=262, tag_tid=17, entry_tid=30126' 2026-02-18T08:18:47.460 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:18:47.460 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:18:47.462 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=30126 2026-02-18T08:18:47.462 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 18 = 17 -a 1 -le 30126 2026-02-18T08:18:47.462 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:18:47.462 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:18:47.462 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:18:47.663 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:18:47.664 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:18:47.664 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:18:47.664 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:18:47.664 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:18:47.664 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:18:47.664 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:18:47.664 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:18:47.664 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:18:47.664 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:18:47.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:18:47.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:18:47.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:18:47.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:18:47.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:18:47.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:18:47.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:18:47.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:18:47.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:18:47.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:18:47.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:18:47.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:18:47.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:18:47.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:18:47.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:18:47.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:18:47.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:18:47.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:18:47.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:18:47.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:18:47.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:18:47.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:18:47.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:18:47.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:18:47.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:18:47.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:18:47.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:18:47.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:18:47.670 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:18:47.670 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:18:47.670 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:18:47.670 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:18:47.670 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:18:47.670 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:18:47.670 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:18:47.670 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:18:47.670 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:18:47.670 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:18:47.670 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:18:47.678 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:18:47.678 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:18:47.678 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:18:47.678 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:18:47.736 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:18:47.736 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:18:47.736 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:18:47.739 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:18:47.739 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:18:47.739 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:18:47.739 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:18:47.739 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:18:47.739 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:18:47.739 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:18:47.740 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:18:47.740 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:18:47.740 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:18:47.740 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:18:47.740 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:18:47.740 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:18:47.740 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:18:47.740 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:18:47.740 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:18:47.740 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:18:47.740 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:18:47.740 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:18:47.740 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:18:47.741 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:18:47.741 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:18:47.741 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:18:47.741 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:18:47.741 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:18:47.741 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:18:47.741 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:18:47.741 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:18:47.741 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:18:47.741 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:18:47.741 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:18:47.741 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:18:47.750 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:18:47.750 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:18:47.750 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:18:47.751 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:18:47.800 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:18:47.800 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:18:47.800 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:18:47.803 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:18:47.803 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:18:47.803 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:18:47.803 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:18:47.803 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:18:47.803 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:18:47.804 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:18:47.804 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:18:47.804 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:18:47.804 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:18:47.804 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:18:47.804 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:18:47.804 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:18:47.804 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:18:47.804 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:18:47.804 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:18:47.804 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:18:47.805 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:18:47.805 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:18:47.805 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:18:47.805 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:18:47.805 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:18:47.805 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:18:47.805 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:18:47.805 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:18:47.805 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:18:47.805 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:18:47.805 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:18:47.805 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:18:47.806 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:18:47.806 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:18:47.806 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:18:47.813 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:18:47.814 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:18:47.814 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:18:47.814 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:18:47.864 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:18:47.865 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:18:47.865 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:18:47.868 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:18:47.868 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:18:47.868 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:18:47.868 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:18:47.868 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:18:47.868 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:18:47.869 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:18:47.869 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:18:47.869 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:18:47.869 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:18:47.869 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:18:47.869 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:18:47.869 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:18:47.869 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:18:47.869 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:18:47.869 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:18:47.869 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:18:47.869 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:18:47.869 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:18:47.870 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:18:47.870 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:18:47.870 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:18:47.870 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:18:47.870 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:18:47.870 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:18:47.870 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:18:47.870 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:18:47.870 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:18:47.870 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:18:47.870 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:18:47.870 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:18:47.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:18:47.878 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:18:47.878 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:18:47.878 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:18:47.879 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:18:47.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:18:47.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:18:47.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:18:47.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:18:47.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:18:47.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:18:47.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:18:47.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:18:47.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:18:47.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:18:47.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:18:47.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:18:47.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:18:47.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:18:47.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:18:47.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:18:47.970 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 65 2026-02-18T08:18:47.971 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 65 2026-02-18T08:18:47.971 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:18:47.971 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=261, tag_tid=18, entry_tid=1], [object_number=260, tag_tid=18, entry_tid=0], [object_number=262, tag_tid=17, entry_tid=30126], [object_number=263, tag_tid=17, entry_tid=30123]]], state=connected] 2026-02-18T08:18:47.971 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=262, tag_tid=17, entry_tid=30126], [object_number=261, tag_tid=17, entry_tid=30125], [object_number=260, tag_tid=17, entry_tid=30124], [object_number=263, tag_tid=17, entry_tid=30123]]], state=connected] 2026-02-18T08:18:47.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:18:47.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=261, tag_tid=18, entry_tid=1' 2026-02-18T08:18:47.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:18:47.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:18:47.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:18:47.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:18:47.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:18:47.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:18:47.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:18:47.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:18:47.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:18:47.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:18:47.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:18:47.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:18:47.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:18:47.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:18:47.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:18:48.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 65 2026-02-18T08:18:48.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 65 2026-02-18T08:18:48.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:18:48.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=261, tag_tid=18, entry_tid=1], [object_number=260, tag_tid=18, entry_tid=0], [object_number=262, tag_tid=17, entry_tid=30126], [object_number=263, tag_tid=17, entry_tid=30123]]], state=connected] 2026-02-18T08:18:48.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=262, tag_tid=17, entry_tid=30126], [object_number=261, tag_tid=17, entry_tid=30125], [object_number=260, tag_tid=17, entry_tid=30124], [object_number=263, tag_tid=17, entry_tid=30123]]], state=connected] 2026-02-18T08:18:48.018 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:18:48.019 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=262, tag_tid=17, entry_tid=30126' 2026-02-18T08:18:48.019 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=261, tag_tid=18, entry_tid=1' -a 'object_number=261, tag_tid=18, entry_tid=1' = 'object_number=262, tag_tid=17, entry_tid=30126' 2026-02-18T08:18:48.019 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=262, tag_tid=17, entry_tid=30126' '!=' 'object_number=262, tag_tid=17, entry_tid=30126' 2026-02-18T08:18:48.020 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:18:48.020 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.4 2026-02-18T08:18:48.421 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:18:48.422 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:18:48.422 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:18:48.422 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:18:48.422 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:18:48.422 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:18:48.422 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:18:48.422 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:18:48.422 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:18:48.422 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:18:48.422 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:18:48.423 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:18:48.423 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:18:48.423 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:18:48.423 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:18:48.423 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:18:48.425 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:18:48.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:18:48.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:18:48.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:18:48.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:18:48.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:18:48.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:18:48.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:18:48.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:18:48.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:18:48.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:18:48.427 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:18:48.427 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:18:48.427 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:18:48.427 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:18:48.427 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:18:48.427 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:18:48.427 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:18:48.427 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:18:48.427 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:18:48.427 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:18:48.427 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:18:48.427 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:18:48.427 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:18:48.428 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:18:48.428 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:18:48.428 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:18:48.428 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:18:48.428 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:18:48.428 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:18:48.428 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:18:48.428 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:18:48.428 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:18:48.436 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:18:48.437 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:18:48.437 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:18:48.437 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:18:48.486 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:18:48.486 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:18:48.487 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:18:48.489 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:18:48.489 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:18:48.489 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:18:48.489 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:18:48.490 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:18:48.490 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:18:48.490 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:18:48.490 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:18:48.490 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:18:48.490 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:18:48.490 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:18:48.490 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:18:48.490 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:18:48.490 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:18:48.490 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:18:48.490 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:18:48.491 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:18:48.491 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:18:48.491 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:18:48.491 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:18:48.491 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:18:48.491 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:18:48.491 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:18:48.491 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:18:48.491 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:18:48.491 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:18:48.491 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:18:48.491 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:18:48.492 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:18:48.492 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:18:48.492 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:18:48.492 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:18:48.500 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:18:48.501 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:18:48.501 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:18:48.501 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:18:48.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:18:48.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:18:48.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:18:48.553 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:18:48.553 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:18:48.553 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:18:48.553 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:18:48.553 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:18:48.553 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:18:48.553 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:18:48.553 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:18:48.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:18:48.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:18:48.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:18:48.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:18:48.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:18:48.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:18:48.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:18:48.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:18:48.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:18:48.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:18:48.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:18:48.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:18:48.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:18:48.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:18:48.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:18:48.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:18:48.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:18:48.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:18:48.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:18:48.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:18:48.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:18:48.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:18:48.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:18:48.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:18:48.565 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:18:48.565 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:18:48.565 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:18:48.565 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:18:48.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:18:48.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:18:48.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:18:48.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:18:48.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:18:48.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:18:48.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:18:48.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:18:48.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:18:48.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:18:48.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:18:48.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:18:48.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:18:48.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:18:48.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:18:48.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:18:48.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:18:48.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:18:48.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:18:48.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:18:48.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:18:48.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:18:48.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:18:48.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:18:48.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:18:48.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:18:48.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:18:48.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:18:48.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:18:48.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:18:48.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:18:48.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:18:48.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:18:48.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:18:48.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:18:48.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:18:48.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:18:48.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:18:48.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:18:48.697 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:18:48.697 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:18:48.697 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:18:48.697 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:18:48.697 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:18:48.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:18:48.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:18:48.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:18:48.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:18:48.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:18:48.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:18:48.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:18:48.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:18:48.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:18:48.700 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:18:48.700 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:18:48.737 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 65 2026-02-18T08:18:48.737 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 65 2026-02-18T08:18:48.738 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:18:48.738 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=261, tag_tid=18, entry_tid=1], [object_number=260, tag_tid=18, entry_tid=0], [object_number=262, tag_tid=17, entry_tid=30126], [object_number=263, tag_tid=17, entry_tid=30123]]], state=connected] 2026-02-18T08:18:48.738 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=262, tag_tid=17, entry_tid=30126], [object_number=261, tag_tid=17, entry_tid=30125], [object_number=260, tag_tid=17, entry_tid=30124], [object_number=263, tag_tid=17, entry_tid=30123]]], state=connected] 2026-02-18T08:18:48.741 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:18:48.742 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=261, tag_tid=18, entry_tid=1' 2026-02-18T08:18:48.745 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:18:48.745 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:18:48.745 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:18:48.745 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:18:48.745 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:18:48.746 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:18:48.746 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:18:48.746 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:18:48.746 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:18:48.746 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:18:48.746 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:18:48.746 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:18:48.747 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:18:48.747 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:18:48.748 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:18:48.773 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 65 2026-02-18T08:18:48.773 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 65 2026-02-18T08:18:48.774 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:18:48.774 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=261, tag_tid=18, entry_tid=1], [object_number=260, tag_tid=18, entry_tid=0], [object_number=262, tag_tid=17, entry_tid=30126], [object_number=263, tag_tid=17, entry_tid=30123]]], state=connected] 2026-02-18T08:18:48.774 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=262, tag_tid=17, entry_tid=30126], [object_number=261, tag_tid=17, entry_tid=30125], [object_number=260, tag_tid=17, entry_tid=30124], [object_number=263, tag_tid=17, entry_tid=30123]]], state=connected] 2026-02-18T08:18:48.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:18:48.777 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=262, tag_tid=17, entry_tid=30126' 2026-02-18T08:18:48.777 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=261, tag_tid=18, entry_tid=1' -a 'object_number=261, tag_tid=18, entry_tid=1' = 'object_number=262, tag_tid=17, entry_tid=30126' 2026-02-18T08:18:48.777 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=262, tag_tid=17, entry_tid=30126' '!=' 'object_number=262, tag_tid=17, entry_tid=30126' 2026-02-18T08:18:48.777 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:18:48.777 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.8 2026-02-18T08:18:49.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:18:49.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:18:49.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:18:49.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:18:49.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:18:49.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:18:49.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:18:49.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:18:49.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:18:49.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:18:49.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:18:49.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:18:49.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:18:49.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:18:49.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:18:49.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:18:49.584 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:18:49.584 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:18:49.584 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:18:49.584 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:18:49.584 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:18:49.584 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:18:49.584 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:18:49.585 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:18:49.585 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:18:49.585 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:18:49.585 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:18:49.585 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:18:49.585 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:18:49.585 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:18:49.585 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:18:49.585 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:18:49.585 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:18:49.585 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:18:49.585 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:18:49.586 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:18:49.586 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:18:49.586 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:18:49.586 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:18:49.586 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:18:49.586 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:18:49.586 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:18:49.586 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:18:49.586 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:18:49.586 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:18:49.586 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:18:49.586 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:18:49.587 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:18:49.587 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:18:49.594 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:18:49.594 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:18:49.594 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:18:49.594 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:18:49.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:18:49.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:18:49.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:18:49.653 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:18:49.653 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:18:49.654 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:18:49.654 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:18:49.654 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:18:49.654 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:18:49.654 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:18:49.654 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:18:49.654 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:18:49.654 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:18:49.654 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:18:49.654 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:18:49.654 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:18:49.654 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:18:49.655 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:18:49.655 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:18:49.655 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:18:49.655 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:18:49.655 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:18:49.655 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:18:49.655 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:18:49.655 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:18:49.655 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:18:49.655 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:18:49.655 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:18:49.655 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:18:49.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:18:49.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:18:49.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:18:49.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:18:49.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:18:49.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:18:49.664 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:18:49.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:18:49.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:18:49.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:18:49.721 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:18:49.722 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:18:49.722 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:18:49.724 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:18:49.724 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:18:49.724 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:18:49.724 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:18:49.725 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:18:49.725 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:18:49.725 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:18:49.725 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:18:49.725 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:18:49.725 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:18:49.725 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:18:49.725 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:18:49.725 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:18:49.725 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:18:49.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:18:49.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:18:49.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:18:49.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:18:49.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:18:49.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:18:49.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:18:49.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:18:49.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:18:49.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:18:49.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:18:49.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:18:49.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:18:49.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:18:49.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:18:49.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:18:49.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:18:49.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:18:49.735 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:18:49.736 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:18:49.736 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:18:49.736 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:18:49.785 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:18:49.785 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:18:49.785 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:18:49.788 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:18:49.788 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:18:49.788 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:18:49.788 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:18:49.788 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:18:49.788 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:18:49.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:18:49.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:18:49.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:18:49.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:18:49.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:18:49.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:18:49.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:18:49.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:18:49.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:18:49.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:18:49.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:18:49.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:18:49.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:18:49.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:18:49.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:18:49.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:18:49.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:18:49.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:18:49.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:18:49.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:18:49.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:18:49.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:18:49.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:18:49.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:18:49.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:18:49.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:18:49.798 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:18:49.798 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:18:49.798 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:18:49.798 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:18:49.855 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:18:49.856 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:18:49.856 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:18:49.856 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:18:49.857 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:18:49.857 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:18:49.857 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:18:49.857 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:18:49.857 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:18:49.857 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:18:49.857 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:18:49.857 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:18:49.857 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:18:49.858 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:18:49.858 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:18:49.858 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:18:49.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 65 2026-02-18T08:18:49.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 65 2026-02-18T08:18:49.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:18:49.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=261, tag_tid=18, entry_tid=1], [object_number=260, tag_tid=18, entry_tid=0], [object_number=262, tag_tid=17, entry_tid=30126], [object_number=263, tag_tid=17, entry_tid=30123]]], state=connected] 2026-02-18T08:18:49.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=262, tag_tid=17, entry_tid=30126], [object_number=261, tag_tid=17, entry_tid=30125], [object_number=260, tag_tid=17, entry_tid=30124], [object_number=263, tag_tid=17, entry_tid=30123]]], state=connected] 2026-02-18T08:18:49.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:18:49.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=261, tag_tid=18, entry_tid=1' 2026-02-18T08:18:49.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:18:49.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:18:49.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:18:49.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:18:49.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:18:49.894 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:18:49.894 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:18:49.894 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:18:49.894 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:18:49.894 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:18:49.894 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:18:49.895 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:18:49.895 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:18:49.895 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:18:49.896 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:18:49.923 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 65 2026-02-18T08:18:49.923 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 65 2026-02-18T08:18:49.923 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:18:49.923 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=261, tag_tid=18, entry_tid=1], [object_number=260, tag_tid=18, entry_tid=0], [object_number=262, tag_tid=17, entry_tid=30126], [object_number=263, tag_tid=17, entry_tid=30123]]], state=connected] 2026-02-18T08:18:49.924 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=262, tag_tid=17, entry_tid=30126], [object_number=261, tag_tid=17, entry_tid=30125], [object_number=260, tag_tid=17, entry_tid=30124], [object_number=263, tag_tid=17, entry_tid=30123]]], state=connected] 2026-02-18T08:18:49.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:18:49.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=262, tag_tid=17, entry_tid=30126' 2026-02-18T08:18:49.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=261, tag_tid=18, entry_tid=1' -a 'object_number=261, tag_tid=18, entry_tid=1' = 'object_number=262, tag_tid=17, entry_tid=30126' 2026-02-18T08:18:49.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=262, tag_tid=17, entry_tid=30126' '!=' 'object_number=262, tag_tid=17, entry_tid=30126' 2026-02-18T08:18:49.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:18:49.928 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1.6 2026-02-18T08:18:51.529 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:18:51.529 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:18:51.529 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:18:51.530 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:18:51.530 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:18:51.530 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:18:51.530 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:18:51.530 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:18:51.530 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:18:51.530 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:18:51.530 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:18:51.530 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:18:51.530 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:18:51.530 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:18:51.530 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:18:51.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:18:51.534 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:18:51.534 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:18:51.534 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:18:51.534 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:18:51.534 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:18:51.534 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:18:51.534 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:18:51.534 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:18:51.534 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:18:51.535 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:18:51.535 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:18:51.535 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:18:51.535 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:18:51.535 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:18:51.535 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:18:51.535 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:18:51.535 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:18:51.535 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:18:51.535 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:18:51.535 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:18:51.535 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:18:51.536 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:18:51.536 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:18:51.536 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:18:51.536 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:18:51.536 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:18:51.536 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:18:51.536 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:18:51.536 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:18:51.536 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:18:51.536 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:18:51.536 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:18:51.536 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:18:51.545 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:18:51.545 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:18:51.545 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:18:51.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:18:51.603 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:18:51.603 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:18:51.603 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:18:51.606 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:18:51.606 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:18:51.606 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:18:51.606 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:18:51.606 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:18:51.606 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:18:51.606 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:18:51.606 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:18:51.606 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:18:51.607 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:18:51.607 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:18:51.607 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:18:51.607 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:18:51.607 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:18:51.607 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:18:51.607 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:18:51.607 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:18:51.607 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:18:51.607 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:18:51.607 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:18:51.607 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:18:51.607 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:18:51.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:18:51.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:18:51.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:18:51.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:18:51.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:18:51.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:18:51.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:18:51.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:18:51.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:18:51.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:18:51.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:18:51.618 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:18:51.618 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:18:51.618 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:18:51.677 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:18:51.677 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:18:51.677 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:18:51.680 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:18:51.680 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:18:51.680 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:18:51.680 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:18:51.680 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:18:51.680 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:18:51.681 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:18:51.681 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:18:51.681 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:18:51.681 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:18:51.681 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:18:51.681 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:18:51.681 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:18:51.681 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:18:51.682 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:18:51.682 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:18:51.682 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:18:51.682 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:18:51.682 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:18:51.682 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:18:51.682 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:18:51.682 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:18:51.682 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:18:51.682 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:18:51.682 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:18:51.682 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:18:51.683 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:18:51.683 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:18:51.683 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:18:51.683 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:18:51.683 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:18:51.683 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:18:51.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:18:51.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:18:51.691 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:18:51.691 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:18:51.747 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:18:51.747 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:18:51.747 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:18:51.750 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:18:51.751 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:18:51.751 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:18:51.751 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:18:51.751 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:18:51.751 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:18:51.751 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:18:51.751 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:18:51.751 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:18:51.751 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:18:51.751 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:18:51.752 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:18:51.752 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:18:51.752 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:18:51.752 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:18:51.752 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:18:51.752 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:18:51.752 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:18:51.752 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:18:51.752 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:18:51.752 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:18:51.752 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:18:51.752 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:18:51.753 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:18:51.753 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:18:51.753 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:18:51.753 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:18:51.753 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:18:51.753 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:18:51.753 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:18:51.753 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:18:51.753 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:18:51.761 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:18:51.762 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:18:51.762 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:18:51.762 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:18:51.824 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:18:51.826 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:18:51.826 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:18:51.826 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:18:51.826 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:18:51.826 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:18:51.826 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:18:51.826 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:18:51.826 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:18:51.826 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:18:51.827 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:18:51.827 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:18:51.827 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:18:51.827 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:18:51.828 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:18:51.828 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:18:52.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 65 2026-02-18T08:18:52.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 65 2026-02-18T08:18:52.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:18:52.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=261, tag_tid=18, entry_tid=1], [object_number=260, tag_tid=18, entry_tid=0], [object_number=262, tag_tid=17, entry_tid=30126], [object_number=263, tag_tid=17, entry_tid=30123]]], state=connected] 2026-02-18T08:18:52.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=262, tag_tid=17, entry_tid=30126], [object_number=261, tag_tid=17, entry_tid=30125], [object_number=260, tag_tid=17, entry_tid=30124], [object_number=263, tag_tid=17, entry_tid=30123]]], state=connected] 2026-02-18T08:18:52.074 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:18:52.076 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=261, tag_tid=18, entry_tid=1' 2026-02-18T08:18:52.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:18:52.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:18:52.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:18:52.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:18:52.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:18:52.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:18:52.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:18:52.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:18:52.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:18:52.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:18:52.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:18:52.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:18:52.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:18:52.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:18:52.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:18:52.118 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 65 2026-02-18T08:18:52.118 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 65 2026-02-18T08:18:52.118 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:18:52.119 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=261, tag_tid=18, entry_tid=1], [object_number=260, tag_tid=18, entry_tid=0], [object_number=262, tag_tid=17, entry_tid=30126], [object_number=263, tag_tid=17, entry_tid=30123]]], state=connected] 2026-02-18T08:18:52.119 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=262, tag_tid=17, entry_tid=30126], [object_number=261, tag_tid=17, entry_tid=30125], [object_number=260, tag_tid=17, entry_tid=30124], [object_number=263, tag_tid=17, entry_tid=30123]]], state=connected] 2026-02-18T08:18:52.122 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:18:52.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=262, tag_tid=17, entry_tid=30126' 2026-02-18T08:18:52.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=261, tag_tid=18, entry_tid=1' -a 'object_number=261, tag_tid=18, entry_tid=1' = 'object_number=262, tag_tid=17, entry_tid=30126' 2026-02-18T08:18:52.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=262, tag_tid=17, entry_tid=30126' '!=' 'object_number=262, tag_tid=17, entry_tid=30126' 2026-02-18T08:18:52.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:18:52.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 2 2026-02-18T08:18:54.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:18:54.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:18:54.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:18:54.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:18:54.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:18:54.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:18:54.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:18:54.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:18:54.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:18:54.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:18:54.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:18:54.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:18:54.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:18:54.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:18:54.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:18:54.130 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:18:54.130 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:18:54.130 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:18:54.130 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:18:54.130 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:18:54.130 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:18:54.130 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:18:54.130 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:18:54.131 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:18:54.131 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:18:54.131 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:18:54.131 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:18:54.131 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:18:54.131 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:18:54.131 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:18:54.131 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:18:54.131 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:18:54.131 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:18:54.131 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:18:54.131 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:18:54.132 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:18:54.132 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:18:54.132 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:18:54.132 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:18:54.132 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:18:54.132 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:18:54.132 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:18:54.132 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:18:54.132 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:18:54.132 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:18:54.132 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:18:54.132 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:18:54.132 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:18:54.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:18:54.141 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:18:54.141 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:18:54.141 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:18:54.141 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:18:54.199 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:18:54.199 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:18:54.200 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:18:54.202 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:18:54.202 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:18:54.202 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:18:54.202 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:18:54.202 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:18:54.203 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:18:54.203 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:18:54.203 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:18:54.203 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:18:54.203 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:18:54.203 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:18:54.203 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:18:54.203 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:18:54.203 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:18:54.203 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:18:54.203 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:18:54.203 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:18:54.204 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:18:54.204 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:18:54.204 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:18:54.204 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:18:54.204 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:18:54.204 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:18:54.204 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:18:54.204 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:18:54.204 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:18:54.204 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:18:54.204 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:18:54.204 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:18:54.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:18:54.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:18:54.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:18:54.213 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:18:54.213 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:18:54.213 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:18:54.214 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:18:54.270 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:18:54.270 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:18:54.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:18:54.273 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:18:54.273 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:18:54.273 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:18:54.274 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:18:54.274 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:18:54.274 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:18:54.274 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:18:54.274 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:18:54.274 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:18:54.274 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:18:54.274 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:18:54.274 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:18:54.274 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:18:54.274 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:18:54.274 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:18:54.275 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:18:54.275 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:18:54.275 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:18:54.275 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:18:54.275 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:18:54.275 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:18:54.275 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:18:54.275 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:18:54.275 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:18:54.275 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:18:54.275 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:18:54.275 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:18:54.276 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:18:54.276 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:18:54.276 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:18:54.276 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:18:54.276 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:18:54.284 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:18:54.285 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:18:54.285 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:18:54.285 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:18:54.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:18:54.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:18:54.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:18:54.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:18:54.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:18:54.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:18:54.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:18:54.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:18:54.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:18:54.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:18:54.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:18:54.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:18:54.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:18:54.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:18:54.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:18:54.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:18:54.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:18:54.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:18:54.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:18:54.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:18:54.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:18:54.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:18:54.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:18:54.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:18:54.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:18:54.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:18:54.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:18:54.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:18:54.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:18:54.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:18:54.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:18:54.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:18:54.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:18:54.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:18:54.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:18:54.348 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:18:54.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:18:54.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:18:54.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:18:54.415 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:18:54.416 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:18:54.416 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:18:54.416 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:18:54.417 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:18:54.417 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:18:54.417 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:18:54.417 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:18:54.417 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:18:54.417 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:18:54.417 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:18:54.417 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:18:54.418 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:18:54.418 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:18:54.418 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:18:54.419 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:18:54.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 65 2026-02-18T08:18:54.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 65 2026-02-18T08:18:54.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:18:54.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=261, tag_tid=18, entry_tid=1], [object_number=260, tag_tid=18, entry_tid=0], [object_number=262, tag_tid=17, entry_tid=30126], [object_number=263, tag_tid=17, entry_tid=30123]]], state=connected] 2026-02-18T08:18:54.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=261, tag_tid=18, entry_tid=1], [object_number=260, tag_tid=18, entry_tid=0], [object_number=262, tag_tid=17, entry_tid=30126], [object_number=263, tag_tid=17, entry_tid=30123]]], state=connected] 2026-02-18T08:18:54.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:18:54.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=261, tag_tid=18, entry_tid=1' 2026-02-18T08:18:54.462 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:18:54.462 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:18:54.462 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:18:54.462 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:18:54.462 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:18:54.463 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:18:54.463 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:18:54.463 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:18:54.463 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:18:54.463 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:18:54.463 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:18:54.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:18:54.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:18:54.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:18:54.465 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:18:54.493 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 65 2026-02-18T08:18:54.493 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 65 2026-02-18T08:18:54.493 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:18:54.493 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=261, tag_tid=18, entry_tid=1], [object_number=260, tag_tid=18, entry_tid=0], [object_number=262, tag_tid=17, entry_tid=30126], [object_number=263, tag_tid=17, entry_tid=30123]]], state=connected] 2026-02-18T08:18:54.493 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=261, tag_tid=18, entry_tid=1], [object_number=260, tag_tid=18, entry_tid=0], [object_number=262, tag_tid=17, entry_tid=30126], [object_number=263, tag_tid=17, entry_tid=30123]]], state=connected] 2026-02-18T08:18:54.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:18:54.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=261, tag_tid=18, entry_tid=1' 2026-02-18T08:18:54.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=261, tag_tid=18, entry_tid=1' -a 'object_number=261, tag_tid=18, entry_tid=1' = 'object_number=261, tag_tid=18, entry_tid=1' 2026-02-18T08:18:54.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:18:54.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_snap_present cluster1 mirror test snap6 2026-02-18T08:18:54.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:18:54.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:18:54.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:18:54.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap6 2026-02-18T08:18:54.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-18T08:18:54.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-18T08:18:54.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1 2026-02-18T08:18:55.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster1 info mirror/test@snap6 2026-02-18T08:18:55.529 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:rbd image 'test': 2026-02-18T08:18:55.529 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: size 512 MiB in 128 objects 2026-02-18T08:18:55.529 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: order 22 (4 MiB objects) 2026-02-18T08:18:55.529 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: snapshot_count: 11 2026-02-18T08:18:55.529 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: id: 10a74aa8ebc0 2026-02-18T08:18:55.529 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: block_name_prefix: rbd_data.10a74aa8ebc0 2026-02-18T08:18:55.530 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: format: 2 2026-02-18T08:18:55.530 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: features: layering, exclusive-lock, journaling 2026-02-18T08:18:55.530 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: op_features: 2026-02-18T08:18:55.530 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: flags: 2026-02-18T08:18:55.530 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: create_timestamp: Wed Feb 18 08:10:48 2026 2026-02-18T08:18:55.530 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: access_timestamp: Wed Feb 18 08:17:50 2026 2026-02-18T08:18:55.530 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: modify_timestamp: Wed Feb 18 08:18:18 2026 2026-02-18T08:18:55.530 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: protected: False 2026-02-18T08:18:55.530 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: journal: 10a74aa8ebc0 2026-02-18T08:18:55.530 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring state: enabled 2026-02-18T08:18:55.530 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring mode: journal 2026-02-18T08:18:55.531 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring global id: 0b548823-d751-43df-a076-4efac2da4d3d 2026-02-18T08:18:55.531 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring primary: false 2026-02-18T08:18:55.532 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:18:55.532 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n snap5-clean ']' 2026-02-18T08:18:55.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ compare_image_snaps mirror test snap5-clean 2026-02-18T08:18:55.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:18:55.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:18:55.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap5-clean 2026-02-18T08:18:55.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local ret=0 2026-02-18T08:18:55.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local rmt_export=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.export 2026-02-18T08:18:55.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loc_export=/tmp/tmp.vybh1m9yJF/cluster1-mirror-test.export 2026-02-18T08:18:55.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.export /tmp/tmp.vybh1m9yJF/cluster1-mirror-test.export 2026-02-18T08:18:55.534 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 -p mirror export test@snap5-clean /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.export 2026-02-18T08:18:56.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-18T08:18:56.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster1 -p mirror export test@snap5-clean /tmp/tmp.vybh1m9yJF/cluster1-mirror-test.export 2026-02-18T08:18:56.628 INFO:tasks.rbd_mirror_thrash:kill cluster1.client.mirror.0 2026-02-18T08:18:56.629 INFO:tasks.rbd_mirror.cluster1.client.mirror.0:Sent signal 15 2026-02-18T08:18:56.629 INFO:tasks.rbd_mirror_thrash:waiting for 9.025050147492697 secs before reviving daemons 2026-02-18T08:18:56.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-18T08:18:56.635 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.trial163.stderr:2026-02-18T08:18:56.628+0000 7fcca8f96640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.0 (PID: 45256) UID: 1000 2026-02-18T08:18:56.646 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmp /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.export /tmp/tmp.vybh1m9yJF/cluster1-mirror-test.export 2026-02-18T08:18:56.841 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.export /tmp/tmp.vybh1m9yJF/cluster1-mirror-test.export 2026-02-18T08:18:56.967 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:18:56.968 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ compare_image_snaps mirror test snap6 2026-02-18T08:18:56.968 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:18:56.968 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:18:56.968 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap6 2026-02-18T08:18:56.968 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local ret=0 2026-02-18T08:18:56.968 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local rmt_export=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.export 2026-02-18T08:18:56.968 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loc_export=/tmp/tmp.vybh1m9yJF/cluster1-mirror-test.export 2026-02-18T08:18:56.968 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.export /tmp/tmp.vybh1m9yJF/cluster1-mirror-test.export 2026-02-18T08:18:56.969 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 -p mirror export test@snap6 /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.export 2026-02-18T08:18:57.681 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-18T08:18:57.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster1 -p mirror export test@snap6 /tmp/tmp.vybh1m9yJF/cluster1-mirror-test.export 2026-02-18T08:18:58.249 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-18T08:18:58.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmp /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.export /tmp/tmp.vybh1m9yJF/cluster1-mirror-test.export 2026-02-18T08:18:58.453 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.export /tmp/tmp.vybh1m9yJF/cluster1-mirror-test.export 2026-02-18T08:18:58.597 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:18:58.598 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ clean_snap_name=snap6-clean 2026-02-18T08:18:58.598 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_snap cluster2 mirror test snap6-clean 2026-02-18T08:18:58.598 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:18:58.598 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:18:58.598 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:18:58.598 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap6-clean 2026-02-18T08:18:58.598 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 -p mirror snap create test@snap6-clean --debug-rbd=20 --debug-journaler=20 2026-02-18T08:18:58.956 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 10` 2026-02-18T08:18:58.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ stress_write_image cluster2 mirror test 2026-02-18T08:18:58.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:18:58.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:18:58.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:18:58.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-18T08:18:58.958 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local duration=10 2026-02-18T08:18:58.959 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set +e 2026-02-18T08:18:58.959 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ timeout 10s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror test --debug-rbd=20 --debug-journaler=20 2026-02-18T08:18:59.475 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:100 IOs, 22188032 bytes 2026-02-18T08:19:00.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:200 IOs, 112595968 bytes 2026-02-18T08:19:00.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:300 IOs, 125712384 bytes 2026-02-18T08:19:01.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:400 IOs, 190757888 bytes 2026-02-18T08:19:02.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:500 IOs, 277089280 bytes 2026-02-18T08:19:02.844 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:600 IOs, 306383872 bytes 2026-02-18T08:19:03.683 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:700 IOs, 368635904 bytes 2026-02-18T08:19:03.876 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:800 IOs, 377565184 bytes 2026-02-18T08:19:03.930 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:900 IOs, 378175488 bytes 2026-02-18T08:19:04.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1000 IOs, 386128896 bytes 2026-02-18T08:19:04.203 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1100 IOs, 390572032 bytes 2026-02-18T08:19:05.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1200 IOs, 449927168 bytes 2026-02-18T08:19:05.422 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1300 IOs, 504809472 bytes 2026-02-18T08:19:05.655 INFO:tasks.rbd_mirror_thrash:waiting for cluster1.client.mirror.0 2026-02-18T08:19:05.655 INFO:teuthology.orchestra.run:waiting for 600 2026-02-18T08:19:05.655 INFO:tasks.rbd_mirror_thrash:reviving cluster1.client.mirror.0 2026-02-18T08:19:05.656 INFO:tasks.rbd_mirror.cluster1.client.mirror.0:Restarting daemon 2026-02-18T08:19:05.656 DEBUG:teuthology.orchestra.run.trial163:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.0 2026-02-18T08:19:05.658 INFO:tasks.rbd_mirror.cluster1.client.mirror.0:Started 2026-02-18T08:19:05.658 INFO:tasks.rbd_mirror_thrash:waiting for 72.30708145369017 secs before thrashing 2026-02-18T08:19:05.914 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1400 IOs, 528567296 bytes 2026-02-18T08:19:06.139 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1500 IOs, 544863232 bytes 2026-02-18T08:19:07.278 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1600 IOs, 627599360 bytes 2026-02-18T08:19:07.390 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1700 IOs, 630596608 bytes 2026-02-18T08:19:07.725 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1800 IOs, 648915968 bytes 2026-02-18T08:19:08.198 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1900 IOs, 709245952 bytes 2026-02-18T08:19:08.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:2000 IOs, 742676480 bytes 2026-02-18T08:19:08.967 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ error_code=124 2026-02-18T08:19:08.968 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set -e 2026-02-18T08:19:08.968 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 124 -eq 124 ']' 2026-02-18T08:19:08.968 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:19:08.968 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+replaying 2026-02-18T08:19:08.968 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:19:08.968 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:19:08.968 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:19:08.968 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local state_pattern=up+replaying 2026-02-18T08:19:08.969 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local description_pattern= 2026-02-18T08:19:08.969 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local service_pattern= 2026-02-18T08:19:08.969 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:19:08.969 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1 2026-02-18T08:19:09.968 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test_status_in_pool_dir cluster1 mirror test up+replaying '' '' 2026-02-18T08:19:09.968 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:19:09.968 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:19:09.968 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:19:09.968 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local state_pattern=up+replaying 2026-02-18T08:19:09.968 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local description_pattern= 2026-02-18T08:19:09.968 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local service_pattern= 2026-02-18T08:19:09.968 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local status 2026-02-18T08:19:09.968 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ CEPH_ARGS= 2026-02-18T08:19:09.968 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-18T08:19:10.007 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ status='test: 2026-02-18T08:19:10.007 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: global_id: 0b548823-d751-43df-a076-4efac2da4d3d 2026-02-18T08:19:10.007 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: state: up+replaying 2026-02-18T08:19:10.007 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: description: replaying, {"bytes_per_second":9780522.529999999,"entries_behind_primary":522,"entries_per_second":607.37,"non_primary_position":{"entry_tid":29605,"object_number":261,"tag_tid":17},"primary_position":{"entry_tid":1,"object_number":261,"tag_tid":18},"seconds_until_synced":0} 2026-02-18T08:19:10.007 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: service: mirror.3 on trial163 2026-02-18T08:19:10.007 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: last_update: 2026-02-18 08:18:46 2026-02-18T08:19:10.007 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: peer_sites: 2026-02-18T08:19:10.007 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: name: cluster2 2026-02-18T08:19:10.007 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: state: up+stopped 2026-02-18T08:19:10.007 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: description: local image is primary 2026-02-18T08:19:10.007 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: last_update: 2026-02-18 08:18:46' 2026-02-18T08:19:10.007 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ grep '^ state: .*up+replaying' 2026-02-18T08:19:10.008 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: state: up+replaying 2026-02-18T08:19:10.008 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ grep '^ description: .*' 2026-02-18T08:19:10.010 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: description: replaying, {"bytes_per_second":9780522.529999999,"entries_behind_primary":522,"entries_per_second":607.37,"non_primary_position":{"entry_tid":29605,"object_number":261,"tag_tid":17},"primary_position":{"entry_tid":1,"object_number":261,"tag_tid":18},"seconds_until_synced":0} 2026-02-18T08:19:10.010 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n '' ']' 2026-02-18T08:19:10.010 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ echo up+replaying 2026-02-18T08:19:10.010 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ grep '^up+' 2026-02-18T08:19:10.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:up+replaying 2026-02-18T08:19:10.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ grep 'service: *mirror..* on ' 2026-02-18T08:19:10.012 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: service: mirror.3 on trial163 2026-02-18T08:19:10.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local last_update 2026-02-18T08:19:10.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-18T08:19:10.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_update='2026-02-18 08:18:46' 2026-02-18T08:19:10.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+replaying '2026-02-18 08:18:46' 2026-02-18T08:19:10.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:19:10.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:19:10.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:19:10.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local state_pattern=up+replaying 2026-02-18T08:19:10.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'prev_last_update=2026-02-18 08:18:46' 2026-02-18T08:19:10.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local status 2026-02-18T08:19:10.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ CEPH_ARGS= 2026-02-18T08:19:10.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-18T08:19:10.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ status='WARNINGWARNINGOK142194268mirror.3trial16320.2.0-670-gadc19233falseOK43304333mirror.2trial16320.2.0-670-gadc19233falseOK43534356mirror.1trial16320.2.0-670-gadc19233trueOK4437mirror.0trial16320.2.0-670-gadc19233falseWARNINGnot reporting statustest0b548823-d751-43df-a076-4efac2da4d3dup+replayingreplaying, {"bytes_per_second":9780522.529999999,"entries_behind_primary":522,"entries_per_second":607.37,"non_primary_position":{"entry_tid":29605,"object_number":261,"tag_tid":17},"primary_position":{"entry_tid":1,"object_number":261,"tag_tid":18},"seconds_until_synced":0}42194268mirror.3trial1632026-02-18 08:18:46cluster2c5e4f85e-ec31-430b-a71d-56e000d0de0bup+stoppedlocal image is primary2026-02-18 08:18:46' 2026-02-18T08:19:10.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local last_update state 2026-02-18T08:19:10.058 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-18T08:19:10.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_update='2026-02-18 08:18:46' 2026-02-18T08:19:10.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-18T08:19:10.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ state=up+replaying 2026-02-18T08:19:10.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ echo up+replaying 2026-02-18T08:19:10.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ grep up+replaying 2026-02-18T08:19:10.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:up+replaying 2026-02-18T08:19:10.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:19:10.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:19:10.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ snap_name=snap7 2026-02-18T08:19:10.067 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_snap cluster2 mirror test snap7 2026-02-18T08:19:10.067 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:19:10.067 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:19:10.067 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:19:10.067 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap7 2026-02-18T08:19:10.068 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 -p mirror snap create test@snap7 --debug-rbd=20 --debug-journaler=20 2026-02-18T08:19:43.676 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_replay_started cluster1 mirror test 2026-02-18T08:19:43.677 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:19:43.677 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:19:43.677 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:19:43.677 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_replay_state cluster1 mirror test started 2026-02-18T08:19:43.677 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:19:43.677 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:19:43.677 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:19:43.677 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local state=started 2026-02-18T08:19:43.677 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-18T08:19:43.678 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:19:43.678 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1 2026-02-18T08:19:44.678 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test_image_replay_state cluster1 mirror test started 2026-02-18T08:19:44.678 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:19:44.678 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:19:44.678 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:19:44.678 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local test_state=started 2026-02-18T08:19:44.678 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local status_result 2026-02-18T08:19:44.678 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_state=stopped 2026-02-18T08:19:44.679 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-18T08:19:44.679 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_instance=cluster1 2026-02-18T08:19:44.680 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -i state 2026-02-18T08:19:44.680 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-18T08:19:44.680 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:19:44.680 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance=cluster1 2026-02-18T08:19:44.680 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local loop_instance 2026-02-18T08:19:44.680 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:19:44.680 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sleep 0 2026-02-18T08:19:44.680 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:19:44.683 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ seq 0 3 2026-02-18T08:19:44.683 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:19:44.683 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-18T08:19:44.683 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:0 2026-02-18T08:19:44.683 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-18T08:19:44.683 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:19:44.683 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:19:44.683 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:19:44.683 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-18T08:19:44.683 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:19:44.684 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:19:44.684 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:19:44.684 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:19:44.684 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-18T08:19:44.684 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:19:44.684 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:19:44.684 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:19:44.684 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:19:44.684 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:0 2026-02-18T08:19:44.684 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-18T08:19:44.684 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-18T08:19:44.684 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:19:44.685 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:0 2026-02-18T08:19:44.685 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T08:19:44.685 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-18T08:19:44.685 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:19:44.685 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:19:44.685 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-18T08:19:44.685 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-18T08:19:44.685 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-18T08:19:44.685 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=0 2026-02-18T08:19:44.685 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=0 2026-02-18T08:19:44.685 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:19:44.692 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:19:44.693 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:19:44.693 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:19:44.693 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-18T08:19:44.744 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:19:44.744 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status : get status for rbd mirror 2026-02-18T08:19:44.744 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:19:44.747 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:19:44.747 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test 2026-02-18T08:19:44.747 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:1 2026-02-18T08:19:44.747 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-18T08:19:44.747 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:19:44.747 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:19:44.747 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:19:44.747 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-18T08:19:44.747 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:19:44.747 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:19:44.748 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:19:44.748 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:19:44.748 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-18T08:19:44.748 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:19:44.748 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:19:44.748 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:19:44.748 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:19:44.748 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:1 2026-02-18T08:19:44.748 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-18T08:19:44.748 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-18T08:19:44.748 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:19:44.748 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:1 2026-02-18T08:19:44.749 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T08:19:44.749 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-18T08:19:44.749 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:19:44.749 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:19:44.749 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-18T08:19:44.749 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-18T08:19:44.749 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-18T08:19:44.749 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=1 2026-02-18T08:19:44.749 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=1 2026-02-18T08:19:44.749 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:19:44.756 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:19:44.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:19:44.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:19:44.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test 2026-02-18T08:19:44.807 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:19:44.807 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status : get status for rbd mirror 2026-02-18T08:19:44.808 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:19:44.810 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:19:44.810 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/test 2026-02-18T08:19:44.810 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:2 2026-02-18T08:19:44.810 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-18T08:19:44.811 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:19:44.811 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:19:44.811 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:19:44.811 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-18T08:19:44.811 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:19:44.811 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:19:44.811 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:19:44.811 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:19:44.811 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-18T08:19:44.811 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:19:44.811 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:19:44.811 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:19:44.812 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:19:44.812 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:2 2026-02-18T08:19:44.812 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-18T08:19:44.812 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-18T08:19:44.812 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:19:44.812 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:2 2026-02-18T08:19:44.812 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T08:19:44.812 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-18T08:19:44.812 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:19:44.812 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:19:44.812 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-18T08:19:44.812 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-18T08:19:44.813 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-18T08:19:44.813 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=2 2026-02-18T08:19:44.813 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=2 2026-02-18T08:19:44.813 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:19:44.821 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:19:44.821 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:19:44.821 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:19:44.821 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/test 2026-02-18T08:19:44.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:19:44.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status : get status for rbd mirror 2026-02-18T08:19:44.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:19:44.874 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:19:44.874 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/test 2026-02-18T08:19:44.874 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:3 2026-02-18T08:19:44.874 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-18T08:19:44.874 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:19:44.875 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:19:44.875 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:19:44.875 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-18T08:19:44.875 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:19:44.875 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:19:44.875 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:19:44.875 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:19:44.875 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-18T08:19:44.875 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:19:44.875 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:19:44.875 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:19:44.875 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:19:44.876 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:3 2026-02-18T08:19:44.876 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-18T08:19:44.876 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-18T08:19:44.876 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:19:44.876 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:3 2026-02-18T08:19:44.876 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T08:19:44.876 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-18T08:19:44.876 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:19:44.876 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:19:44.876 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-18T08:19:44.876 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-18T08:19:44.876 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-18T08:19:44.877 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=3 2026-02-18T08:19:44.877 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=3 2026-02-18T08:19:44.877 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:19:44.885 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:19:44.885 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:19:44.886 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:19:44.886 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/test 2026-02-18T08:19:44.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ return 0 2026-02-18T08:19:44.944 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ status_result=' "state": "Replaying"' 2026-02-18T08:19:44.944 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ echo ' "state": "Replaying"' 2026-02-18T08:19:44.944 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ grep -i Replaying 2026-02-18T08:19:44.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: "state": "Replaying" 2026-02-18T08:19:44.946 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ current_state=started 2026-02-18T08:19:44.947 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test started = started 2026-02-18T08:19:44.947 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:19:44.947 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror test 2026-02-18T08:19:44.947 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_cluster=cluster1 2026-02-18T08:19:44.947 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:19:44.947 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_pool=mirror 2026-02-18T08:19:44.947 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local remote_pool=mirror 2026-02-18T08:19:44.947 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:19:44.947 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' journal = journal ']' 2026-02-18T08:19:44.947 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror mirror test 2026-02-18T08:19:44.947 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_cluster=cluster1 2026-02-18T08:19:44.948 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:19:44.948 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_pool=mirror 2026-02-18T08:19:44.948 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local remote_pool=mirror 2026-02-18T08:19:44.948 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:19:44.948 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-02-18T08:19:44.948 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-02-18T08:19:44.948 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:19:44.948 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:19:44.948 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:19:45.147 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:19:45.147 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:19:45.147 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:19:45.147 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:19:45.147 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:19:45.148 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:19:45.148 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:19:45.148 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:19:45.148 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:19:45.148 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:19:45.148 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:19:45.148 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:19:45.148 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:19:45.148 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:19:45.148 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:19:45.148 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:19:45.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:19:45.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:19:45.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:19:45.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:19:45.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:19:45.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:19:45.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:19:45.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:19:45.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:19:45.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:19:45.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:19:45.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:19:45.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:19:45.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:19:45.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:19:45.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:19:45.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:19:45.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:19:45.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:19:45.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:19:45.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:19:45.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:19:45.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:19:45.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:19:45.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:19:45.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:19:45.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:19:45.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:19:45.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:19:45.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:19:45.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:19:45.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:19:45.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:19:45.161 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:19:45.162 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:19:45.162 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:19:45.162 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:19:45.212 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:19:45.212 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:19:45.212 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:19:45.214 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:19:45.215 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:19:45.215 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:19:45.215 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:19:45.215 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:19:45.215 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:19:45.215 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:19:45.215 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:19:45.215 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:19:45.215 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:19:45.215 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:19:45.215 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:19:45.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:19:45.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:19:45.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:19:45.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:19:45.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:19:45.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:19:45.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:19:45.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:19:45.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:19:45.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:19:45.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:19:45.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:19:45.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:19:45.217 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:19:45.217 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:19:45.217 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:19:45.217 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:19:45.217 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:19:45.217 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:19:45.217 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:19:45.225 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:19:45.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:19:45.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:19:45.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:19:45.273 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:19:45.274 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:19:45.274 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:19:45.276 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:19:45.276 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:19:45.276 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:19:45.276 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:19:45.277 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:19:45.277 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:19:45.277 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:19:45.277 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:19:45.277 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:19:45.277 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:19:45.277 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:19:45.277 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:19:45.277 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:19:45.277 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:19:45.277 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:19:45.277 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:19:45.277 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:19:45.278 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:19:45.278 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:19:45.278 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:19:45.278 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:19:45.278 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:19:45.278 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:19:45.278 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:19:45.278 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:19:45.278 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:19:45.278 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:19:45.278 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:19:45.278 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:19:45.279 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:19:45.279 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:19:45.279 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:19:45.286 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:19:45.286 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:19:45.286 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:19:45.286 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:19:45.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:19:45.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:19:45.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:19:45.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:19:45.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:19:45.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:19:45.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:19:45.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:19:45.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:19:45.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:19:45.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:19:45.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:19:45.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:19:45.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:19:45.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:19:45.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:19:45.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:19:45.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:19:45.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:19:45.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:19:45.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:19:45.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:19:45.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:19:45.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:19:45.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:19:45.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:19:45.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:19:45.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:19:45.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:19:45.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:19:45.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:19:45.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:19:45.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:19:45.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:19:45.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:19:45.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:19:45.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:19:45.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:19:45.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:19:45.408 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:19:45.409 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:19:45.409 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:19:45.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:19:45.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:19:45.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:19:45.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:19:45.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:19:45.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:19:45.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:19:45.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:19:45.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:19:45.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:19:45.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:19:45.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:19:45.412 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:19:45.448 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 72 2026-02-18T08:19:45.449 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 77 2026-02-18T08:19:45.449 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:19:45.449 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=309, tag_tid=21, entry_tid=1], [object_number=308, tag_tid=21, entry_tid=0], [object_number=310, tag_tid=20, entry_tid=46754], [object_number=311, tag_tid=20, entry_tid=46751]]], state=connected] 2026-02-18T08:19:45.449 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=291, tag_tid=20, entry_tid=26403], [object_number=290, tag_tid=20, entry_tid=26402], [object_number=289, tag_tid=20, entry_tid=26401], [object_number=288, tag_tid=20, entry_tid=26400]]], state=connected] 2026-02-18T08:19:45.452 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:19:45.454 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=309, tag_tid=21, entry_tid=1' 2026-02-18T08:19:45.455 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:19:45.455 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:19:45.455 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:19:45.455 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:19:45.455 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:19:45.455 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:19:45.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:19:45.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:19:45.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:19:45.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:19:45.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:19:45.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:19:45.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:19:45.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:19:45.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:19:45.493 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 72 2026-02-18T08:19:45.493 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 77 2026-02-18T08:19:45.493 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:19:45.493 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=309, tag_tid=21, entry_tid=1], [object_number=308, tag_tid=21, entry_tid=0], [object_number=310, tag_tid=20, entry_tid=46754], [object_number=311, tag_tid=20, entry_tid=46751]]], state=connected] 2026-02-18T08:19:45.493 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=291, tag_tid=20, entry_tid=26403], [object_number=290, tag_tid=20, entry_tid=26402], [object_number=289, tag_tid=20, entry_tid=26401], [object_number=288, tag_tid=20, entry_tid=26400]]], state=connected] 2026-02-18T08:19:45.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:19:45.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=291, tag_tid=20, entry_tid=26403' 2026-02-18T08:19:45.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=309, tag_tid=21, entry_tid=1' -a 'object_number=309, tag_tid=21, entry_tid=1' = 'object_number=291, tag_tid=20, entry_tid=26403' 2026-02-18T08:19:45.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=291, tag_tid=20, entry_tid=26403' '!=' '' 2026-02-18T08:19:45.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:19:45.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=291, tag_tid=20, entry_tid=26403' = '' 2026-02-18T08:19:45.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=291, tag_tid=20, entry_tid=26403' 2026-02-18T08:19:45.500 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=309, tag_tid=21, entry_tid=1' 2026-02-18T08:19:45.500 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:19:45.500 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:19:45.500 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=21 2026-02-18T08:19:45.501 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=291, tag_tid=20, entry_tid=26403' 2026-02-18T08:19:45.501 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:19:45.501 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:19:45.503 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=20 2026-02-18T08:19:45.503 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=309, tag_tid=21, entry_tid=1' 2026-02-18T08:19:45.504 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:19:45.504 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:19:45.505 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:19:45.506 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=291, tag_tid=20, entry_tid=26403' 2026-02-18T08:19:45.506 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:19:45.506 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:19:45.507 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=26403 2026-02-18T08:19:45.508 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 21 = 20 -a 1 -le 26403 2026-02-18T08:19:45.508 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:19:45.508 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:19:45.508 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:19:45.709 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:19:45.709 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:19:45.709 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:19:45.709 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:19:45.709 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:19:45.709 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:19:45.709 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:19:45.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:19:45.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:19:45.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:19:45.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:19:45.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:19:45.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:19:45.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:19:45.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:19:45.713 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:19:45.713 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:19:45.713 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:19:45.713 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:19:45.713 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:19:45.713 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:19:45.713 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:19:45.713 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:19:45.713 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:19:45.714 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:19:45.714 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:19:45.714 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:19:45.714 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:19:45.714 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:19:45.714 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:19:45.714 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:19:45.714 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:19:45.714 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:19:45.714 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:19:45.714 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:19:45.714 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:19:45.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:19:45.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:19:45.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:19:45.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:19:45.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:19:45.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:19:45.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:19:45.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:19:45.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:19:45.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:19:45.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:19:45.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:19:45.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:19:45.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:19:45.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:19:45.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:19:45.724 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:19:45.774 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:19:45.774 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:19:45.774 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:19:45.777 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:19:45.777 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:19:45.777 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:19:45.777 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:19:45.777 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:19:45.777 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:19:45.777 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:19:45.777 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:19:45.777 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:19:45.778 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:19:45.778 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:19:45.778 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:19:45.778 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:19:45.778 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:19:45.778 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:19:45.778 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:19:45.778 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:19:45.778 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:19:45.778 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:19:45.778 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:19:45.778 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:19:45.779 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:19:45.779 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:19:45.779 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:19:45.779 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:19:45.779 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:19:45.779 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:19:45.779 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:19:45.779 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:19:45.779 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:19:45.779 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:19:45.779 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:19:45.788 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:19:45.788 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:19:45.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:19:45.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:19:45.838 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:19:45.838 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:19:45.838 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:19:45.841 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:19:45.841 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:19:45.841 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:19:45.841 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:19:45.841 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:19:45.841 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:19:45.841 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:19:45.841 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:19:45.842 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:19:45.842 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:19:45.842 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:19:45.842 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:19:45.842 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:19:45.842 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:19:45.842 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:19:45.842 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:19:45.842 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:19:45.842 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:19:45.842 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:19:45.842 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:19:45.843 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:19:45.843 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:19:45.843 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:19:45.843 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:19:45.843 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:19:45.843 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:19:45.843 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:19:45.843 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:19:45.843 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:19:45.843 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:19:45.843 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:19:45.843 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:19:45.852 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:19:45.852 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:19:45.853 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:19:45.853 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:19:45.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:19:45.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:19:45.904 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:19:45.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:19:45.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:19:45.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:19:45.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:19:45.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:19:45.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:19:45.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:19:45.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:19:45.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:19:45.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:19:45.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:19:45.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:19:45.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:19:45.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:19:45.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:19:45.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:19:45.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:19:45.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:19:45.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:19:45.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:19:45.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:19:45.909 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:19:45.909 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:19:45.909 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:19:45.909 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:19:45.909 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:19:45.909 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:19:45.909 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:19:45.909 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:19:45.909 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:19:45.909 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:19:45.909 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:19:45.917 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:19:45.918 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:19:45.918 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:19:45.918 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:19:45.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:19:45.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:19:45.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:19:45.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:19:45.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:19:45.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:19:45.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:19:45.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:19:45.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:19:45.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:19:45.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:19:45.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:19:45.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:19:45.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:19:45.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:19:45.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:19:46.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 72 2026-02-18T08:19:46.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 77 2026-02-18T08:19:46.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:19:46.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=309, tag_tid=21, entry_tid=1], [object_number=308, tag_tid=21, entry_tid=0], [object_number=310, tag_tid=20, entry_tid=46754], [object_number=311, tag_tid=20, entry_tid=46751]]], state=connected] 2026-02-18T08:19:46.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=289, tag_tid=20, entry_tid=26789], [object_number=288, tag_tid=20, entry_tid=26788], [object_number=291, tag_tid=20, entry_tid=26787], [object_number=290, tag_tid=20, entry_tid=26786]]], state=connected] 2026-02-18T08:19:46.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:19:46.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=309, tag_tid=21, entry_tid=1' 2026-02-18T08:19:46.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:19:46.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:19:46.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:19:46.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:19:46.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:19:46.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:19:46.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:19:46.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:19:46.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:19:46.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:19:46.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:19:46.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:19:46.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:19:46.018 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:19:46.018 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:19:46.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 72 2026-02-18T08:19:46.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 77 2026-02-18T08:19:46.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:19:46.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=309, tag_tid=21, entry_tid=1], [object_number=308, tag_tid=21, entry_tid=0], [object_number=310, tag_tid=20, entry_tid=46754], [object_number=311, tag_tid=20, entry_tid=46751]]], state=connected] 2026-02-18T08:19:46.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=289, tag_tid=20, entry_tid=26789], [object_number=288, tag_tid=20, entry_tid=26788], [object_number=291, tag_tid=20, entry_tid=26787], [object_number=290, tag_tid=20, entry_tid=26786]]], state=connected] 2026-02-18T08:19:46.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:19:46.058 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=289, tag_tid=20, entry_tid=26789' 2026-02-18T08:19:46.058 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=309, tag_tid=21, entry_tid=1' -a 'object_number=309, tag_tid=21, entry_tid=1' = 'object_number=289, tag_tid=20, entry_tid=26789' 2026-02-18T08:19:46.058 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=289, tag_tid=20, entry_tid=26789' '!=' 'object_number=291, tag_tid=20, entry_tid=26403' 2026-02-18T08:19:46.058 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:19:46.058 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=289, tag_tid=20, entry_tid=26789' = 'object_number=291, tag_tid=20, entry_tid=26403' 2026-02-18T08:19:46.058 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=289, tag_tid=20, entry_tid=26789' 2026-02-18T08:19:46.059 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=309, tag_tid=21, entry_tid=1' 2026-02-18T08:19:46.059 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:19:46.059 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:19:46.060 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=21 2026-02-18T08:19:46.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=289, tag_tid=20, entry_tid=26789' 2026-02-18T08:19:46.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:19:46.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:19:46.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=20 2026-02-18T08:19:46.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=309, tag_tid=21, entry_tid=1' 2026-02-18T08:19:46.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:19:46.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:19:46.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:19:46.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=289, tag_tid=20, entry_tid=26789' 2026-02-18T08:19:46.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:19:46.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:19:46.067 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=26789 2026-02-18T08:19:46.067 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 21 = 20 -a 1 -le 26789 2026-02-18T08:19:46.067 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:19:46.067 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:19:46.067 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:19:46.270 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:19:46.270 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:19:46.270 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:19:46.270 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:19:46.270 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:19:46.270 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:19:46.270 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:19:46.270 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:19:46.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:19:46.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:19:46.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:19:46.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:19:46.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:19:46.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:19:46.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:19:46.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:19:46.274 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:19:46.274 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:19:46.274 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:19:46.274 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:19:46.274 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:19:46.274 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:19:46.274 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:19:46.275 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:19:46.275 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:19:46.275 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:19:46.275 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:19:46.275 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:19:46.275 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:19:46.275 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:19:46.275 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:19:46.275 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:19:46.275 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:19:46.275 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:19:46.275 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:19:46.276 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:19:46.276 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:19:46.276 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:19:46.276 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:19:46.276 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:19:46.276 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:19:46.276 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:19:46.276 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:19:46.276 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:19:46.276 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:19:46.276 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:19:46.276 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:19:46.277 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:19:46.277 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:19:46.283 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:19:46.284 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:19:46.284 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:19:46.284 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:19:46.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:19:46.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:19:46.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:19:46.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:19:46.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:19:46.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:19:46.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:19:46.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:19:46.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:19:46.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:19:46.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:19:46.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:19:46.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:19:46.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:19:46.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:19:46.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:19:46.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:19:46.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:19:46.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:19:46.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:19:46.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:19:46.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:19:46.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:19:46.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:19:46.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:19:46.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:19:46.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:19:46.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:19:46.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:19:46.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:19:46.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:19:46.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:19:46.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:19:46.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:19:46.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:19:46.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:19:46.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:19:46.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:19:46.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:19:46.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:19:46.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:19:46.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:19:46.403 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:19:46.403 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:19:46.403 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:19:46.403 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:19:46.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:19:46.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:19:46.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:19:46.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:19:46.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:19:46.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:19:46.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:19:46.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:19:46.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:19:46.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:19:46.405 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:19:46.405 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:19:46.405 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:19:46.405 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:19:46.405 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:19:46.405 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:19:46.405 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:19:46.405 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:19:46.405 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:19:46.405 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:19:46.405 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:19:46.406 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:19:46.406 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:19:46.406 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:19:46.406 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:19:46.406 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:19:46.406 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:19:46.406 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:19:46.413 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:19:46.414 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:19:46.414 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:19:46.414 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:19:46.463 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:19:46.463 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:19:46.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:19:46.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:19:46.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:19:46.467 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:19:46.467 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:19:46.467 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:19:46.467 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:19:46.467 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:19:46.467 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:19:46.467 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:19:46.467 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:19:46.467 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:19:46.467 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:19:46.467 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:19:46.468 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:19:46.468 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:19:46.468 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:19:46.468 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:19:46.468 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:19:46.468 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:19:46.468 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:19:46.468 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:19:46.468 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:19:46.468 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:19:46.468 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:19:46.468 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:19:46.468 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:19:46.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:19:46.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:19:46.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:19:46.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:19:46.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:19:46.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:19:46.477 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:19:46.477 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:19:46.477 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:19:46.477 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:19:46.540 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:19:46.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:19:46.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:19:46.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:19:46.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:19:46.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:19:46.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:19:46.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:19:46.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:19:46.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:19:46.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:19:46.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:19:46.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:19:46.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:19:46.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:19:46.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:19:46.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 72 2026-02-18T08:19:46.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 77 2026-02-18T08:19:46.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:19:46.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=309, tag_tid=21, entry_tid=1], [object_number=308, tag_tid=21, entry_tid=0], [object_number=310, tag_tid=20, entry_tid=46754], [object_number=311, tag_tid=20, entry_tid=46751]]], state=connected] 2026-02-18T08:19:46.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=290, tag_tid=20, entry_tid=27154], [object_number=289, tag_tid=20, entry_tid=27153], [object_number=288, tag_tid=20, entry_tid=27152], [object_number=291, tag_tid=20, entry_tid=27151]]], state=connected] 2026-02-18T08:19:46.584 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:19:46.585 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=309, tag_tid=21, entry_tid=1' 2026-02-18T08:19:46.586 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:19:46.586 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:19:46.586 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:19:46.586 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:19:46.586 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:19:46.586 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:19:46.586 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:19:46.586 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:19:46.587 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:19:46.587 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:19:46.587 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:19:46.587 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:19:46.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:19:46.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:19:46.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:19:46.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 72 2026-02-18T08:19:46.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 77 2026-02-18T08:19:46.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:19:46.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=309, tag_tid=21, entry_tid=1], [object_number=308, tag_tid=21, entry_tid=0], [object_number=310, tag_tid=20, entry_tid=46754], [object_number=311, tag_tid=20, entry_tid=46751]]], state=connected] 2026-02-18T08:19:46.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=290, tag_tid=20, entry_tid=27154], [object_number=289, tag_tid=20, entry_tid=27153], [object_number=288, tag_tid=20, entry_tid=27152], [object_number=291, tag_tid=20, entry_tid=27151]]], state=connected] 2026-02-18T08:19:46.619 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:19:46.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=290, tag_tid=20, entry_tid=27154' 2026-02-18T08:19:46.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=309, tag_tid=21, entry_tid=1' -a 'object_number=309, tag_tid=21, entry_tid=1' = 'object_number=290, tag_tid=20, entry_tid=27154' 2026-02-18T08:19:46.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=290, tag_tid=20, entry_tid=27154' '!=' 'object_number=289, tag_tid=20, entry_tid=26789' 2026-02-18T08:19:46.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:19:46.622 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=290, tag_tid=20, entry_tid=27154' = 'object_number=289, tag_tid=20, entry_tid=26789' 2026-02-18T08:19:46.622 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=290, tag_tid=20, entry_tid=27154' 2026-02-18T08:19:46.622 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=309, tag_tid=21, entry_tid=1' 2026-02-18T08:19:46.622 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:19:46.622 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:19:46.623 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=21 2026-02-18T08:19:46.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=290, tag_tid=20, entry_tid=27154' 2026-02-18T08:19:46.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:19:46.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:19:46.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=20 2026-02-18T08:19:46.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=309, tag_tid=21, entry_tid=1' 2026-02-18T08:19:46.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:19:46.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:19:46.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:19:46.628 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=290, tag_tid=20, entry_tid=27154' 2026-02-18T08:19:46.628 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:19:46.628 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:19:46.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=27154 2026-02-18T08:19:46.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 21 = 20 -a 1 -le 27154 2026-02-18T08:19:46.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:19:46.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:19:46.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:19:46.832 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:19:46.832 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:19:46.832 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:19:46.832 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:19:46.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:19:46.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:19:46.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:19:46.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:19:46.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:19:46.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:19:46.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:19:46.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:19:46.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:19:46.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:19:46.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:19:46.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:19:46.836 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:19:46.837 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:19:46.837 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:19:46.837 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:19:46.837 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:19:46.837 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:19:46.837 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:19:46.837 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:19:46.837 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:19:46.837 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:19:46.837 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:19:46.837 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:19:46.837 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:19:46.838 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:19:46.838 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:19:46.838 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:19:46.838 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:19:46.838 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:19:46.838 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:19:46.838 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:19:46.838 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:19:46.838 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:19:46.838 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:19:46.838 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:19:46.838 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:19:46.839 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:19:46.839 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:19:46.839 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:19:46.839 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:19:46.839 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:19:46.839 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:19:46.839 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:19:46.839 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:19:46.846 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:19:46.847 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:19:46.847 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:19:46.847 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:19:46.896 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:19:46.896 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:19:46.896 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:19:46.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:19:46.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:19:46.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:19:46.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:19:46.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:19:46.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:19:46.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:19:46.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:19:46.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:19:46.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:19:46.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:19:46.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:19:46.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:19:46.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:19:46.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:19:46.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:19:46.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:19:46.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:19:46.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:19:46.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:19:46.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:19:46.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:19:46.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:19:46.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:19:46.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:19:46.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:19:46.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:19:46.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:19:46.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:19:46.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:19:46.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:19:46.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:19:46.910 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:19:46.910 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:19:46.910 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:19:46.910 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:19:46.968 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:19:46.969 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:19:46.969 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:19:46.971 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:19:46.971 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:19:46.971 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:19:46.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:19:46.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:19:46.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:19:46.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:19:46.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:19:46.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:19:46.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:19:46.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:19:46.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:19:46.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:19:46.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:19:46.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:19:46.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:19:46.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:19:46.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:19:46.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:19:46.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:19:46.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:19:46.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:19:46.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:19:46.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:19:46.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:19:46.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:19:46.974 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:19:46.974 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:19:46.974 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:19:46.974 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:19:46.974 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:19:46.974 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:19:46.982 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:19:46.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:19:46.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:19:46.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:19:47.039 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:19:47.039 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:19:47.039 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:19:47.041 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:19:47.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:19:47.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:19:47.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:19:47.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:19:47.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:19:47.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:19:47.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:19:47.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:19:47.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:19:47.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:19:47.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:19:47.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:19:47.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:19:47.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:19:47.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:19:47.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:19:47.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:19:47.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:19:47.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:19:47.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:19:47.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:19:47.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:19:47.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:19:47.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:19:47.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:19:47.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:19:47.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:19:47.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:19:47.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:19:47.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:19:47.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:19:47.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:19:47.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:19:47.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:19:47.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:19:47.121 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:19:47.122 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:19:47.122 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:19:47.122 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:19:47.122 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:19:47.122 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:19:47.122 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:19:47.122 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:19:47.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:19:47.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:19:47.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:19:47.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:19:47.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:19:47.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:19:47.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:19:47.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:19:47.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 72 2026-02-18T08:19:47.161 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 77 2026-02-18T08:19:47.161 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:19:47.161 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=309, tag_tid=21, entry_tid=1], [object_number=308, tag_tid=21, entry_tid=0], [object_number=310, tag_tid=20, entry_tid=46754], [object_number=311, tag_tid=20, entry_tid=46751]]], state=connected] 2026-02-18T08:19:47.161 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=291, tag_tid=20, entry_tid=27555], [object_number=290, tag_tid=20, entry_tid=27554], [object_number=289, tag_tid=20, entry_tid=27553], [object_number=288, tag_tid=20, entry_tid=27552]]], state=connected] 2026-02-18T08:19:47.164 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:19:47.166 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=309, tag_tid=21, entry_tid=1' 2026-02-18T08:19:47.167 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:19:47.167 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:19:47.167 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:19:47.167 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:19:47.167 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:19:47.167 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:19:47.167 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:19:47.167 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:19:47.167 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:19:47.167 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:19:47.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:19:47.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:19:47.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:19:47.169 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:19:47.169 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:19:47.203 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 72 2026-02-18T08:19:47.203 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 77 2026-02-18T08:19:47.203 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:19:47.203 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=309, tag_tid=21, entry_tid=1], [object_number=308, tag_tid=21, entry_tid=0], [object_number=310, tag_tid=20, entry_tid=46754], [object_number=311, tag_tid=20, entry_tid=46751]]], state=connected] 2026-02-18T08:19:47.204 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=291, tag_tid=20, entry_tid=27555], [object_number=290, tag_tid=20, entry_tid=27554], [object_number=289, tag_tid=20, entry_tid=27553], [object_number=288, tag_tid=20, entry_tid=27552]]], state=connected] 2026-02-18T08:19:47.207 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:19:47.209 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=291, tag_tid=20, entry_tid=27555' 2026-02-18T08:19:47.209 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=309, tag_tid=21, entry_tid=1' -a 'object_number=309, tag_tid=21, entry_tid=1' = 'object_number=291, tag_tid=20, entry_tid=27555' 2026-02-18T08:19:47.209 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=291, tag_tid=20, entry_tid=27555' '!=' 'object_number=290, tag_tid=20, entry_tid=27154' 2026-02-18T08:19:47.209 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:19:47.209 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=291, tag_tid=20, entry_tid=27555' = 'object_number=290, tag_tid=20, entry_tid=27154' 2026-02-18T08:19:47.210 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=291, tag_tid=20, entry_tid=27555' 2026-02-18T08:19:47.210 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=309, tag_tid=21, entry_tid=1' 2026-02-18T08:19:47.210 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:19:47.210 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:19:47.211 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=21 2026-02-18T08:19:47.212 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=291, tag_tid=20, entry_tid=27555' 2026-02-18T08:19:47.212 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:19:47.212 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:19:47.214 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=20 2026-02-18T08:19:47.214 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=309, tag_tid=21, entry_tid=1' 2026-02-18T08:19:47.214 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:19:47.215 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:19:47.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:19:47.217 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=291, tag_tid=20, entry_tid=27555' 2026-02-18T08:19:47.217 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:19:47.217 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:19:47.218 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=27555 2026-02-18T08:19:47.219 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 21 = 20 -a 1 -le 27555 2026-02-18T08:19:47.219 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:19:47.219 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:19:47.219 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:19:47.420 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:19:47.421 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:19:47.421 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:19:47.421 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:19:47.421 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:19:47.421 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:19:47.421 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:19:47.421 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:19:47.421 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:19:47.421 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:19:47.422 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:19:47.422 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:19:47.422 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:19:47.422 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:19:47.422 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:19:47.422 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:19:47.422 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:19:47.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:19:47.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:19:47.425 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:19:47.425 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:19:47.425 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:19:47.425 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:19:47.425 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:19:47.425 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:19:47.425 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:19:47.425 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:19:47.425 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:19:47.425 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:19:47.425 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:19:47.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:19:47.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:19:47.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:19:47.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:19:47.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:19:47.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:19:47.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:19:47.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:19:47.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:19:47.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:19:47.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:19:47.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:19:47.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:19:47.427 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:19:47.427 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:19:47.427 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:19:47.427 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:19:47.427 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:19:47.427 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:19:47.434 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:19:47.434 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:19:47.435 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:19:47.435 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:19:47.492 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:19:47.493 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:19:47.493 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:19:47.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:19:47.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:19:47.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:19:47.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:19:47.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:19:47.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:19:47.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:19:47.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:19:47.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:19:47.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:19:47.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:19:47.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:19:47.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:19:47.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:19:47.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:19:47.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:19:47.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:19:47.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:19:47.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:19:47.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:19:47.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:19:47.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:19:47.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:19:47.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:19:47.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:19:47.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:19:47.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:19:47.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:19:47.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:19:47.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:19:47.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:19:47.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:19:47.506 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:19:47.506 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:19:47.506 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:19:47.507 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:19:47.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:19:47.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:19:47.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:19:47.558 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:19:47.558 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:19:47.559 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:19:47.559 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:19:47.559 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:19:47.559 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:19:47.559 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:19:47.559 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:19:47.559 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:19:47.559 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:19:47.559 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:19:47.559 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:19:47.559 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:19:47.560 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:19:47.560 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:19:47.560 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:19:47.560 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:19:47.560 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:19:47.560 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:19:47.560 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:19:47.560 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:19:47.560 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:19:47.560 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:19:47.560 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:19:47.560 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:19:47.560 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:19:47.561 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:19:47.561 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:19:47.561 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:19:47.561 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:19:47.561 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:19:47.561 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:19:47.569 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:19:47.570 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:19:47.570 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:19:47.570 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:19:47.620 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:19:47.620 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:19:47.620 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:19:47.623 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:19:47.623 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:19:47.623 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:19:47.623 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:19:47.623 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:19:47.623 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:19:47.623 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:19:47.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:19:47.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:19:47.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:19:47.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:19:47.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:19:47.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:19:47.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:19:47.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:19:47.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:19:47.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:19:47.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:19:47.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:19:47.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:19:47.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:19:47.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:19:47.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:19:47.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:19:47.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:19:47.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:19:47.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:19:47.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:19:47.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:19:47.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:19:47.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:19:47.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:19:47.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:19:47.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:19:47.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:19:47.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:19:47.694 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:19:47.695 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:19:47.695 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:19:47.695 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:19:47.696 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:19:47.696 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:19:47.696 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:19:47.696 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:19:47.696 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:19:47.696 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:19:47.696 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:19:47.697 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:19:47.697 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:19:47.697 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:19:47.697 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:19:47.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:19:47.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 72 2026-02-18T08:19:47.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 77 2026-02-18T08:19:47.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:19:47.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=309, tag_tid=21, entry_tid=1], [object_number=308, tag_tid=21, entry_tid=0], [object_number=310, tag_tid=20, entry_tid=46754], [object_number=311, tag_tid=20, entry_tid=46751]]], state=connected] 2026-02-18T08:19:47.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=290, tag_tid=20, entry_tid=27946], [object_number=289, tag_tid=20, entry_tid=27945], [object_number=288, tag_tid=20, entry_tid=27944], [object_number=291, tag_tid=20, entry_tid=27943]]], state=connected] 2026-02-18T08:19:47.728 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:19:47.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=309, tag_tid=21, entry_tid=1' 2026-02-18T08:19:47.731 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:19:47.731 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:19:47.731 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:19:47.731 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:19:47.731 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:19:47.732 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:19:47.732 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:19:47.732 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:19:47.732 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:19:47.732 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:19:47.732 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:19:47.733 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:19:47.733 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:19:47.733 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:19:47.734 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:19:47.966 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 72 2026-02-18T08:19:47.967 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 77 2026-02-18T08:19:47.967 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:19:47.967 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=309, tag_tid=21, entry_tid=1], [object_number=308, tag_tid=21, entry_tid=0], [object_number=310, tag_tid=20, entry_tid=46754], [object_number=311, tag_tid=20, entry_tid=46751]]], state=connected] 2026-02-18T08:19:47.967 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=290, tag_tid=20, entry_tid=27946], [object_number=289, tag_tid=20, entry_tid=27945], [object_number=288, tag_tid=20, entry_tid=27944], [object_number=291, tag_tid=20, entry_tid=27943]]], state=connected] 2026-02-18T08:19:47.970 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:19:47.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=290, tag_tid=20, entry_tid=27946' 2026-02-18T08:19:47.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=309, tag_tid=21, entry_tid=1' -a 'object_number=309, tag_tid=21, entry_tid=1' = 'object_number=290, tag_tid=20, entry_tid=27946' 2026-02-18T08:19:47.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=290, tag_tid=20, entry_tid=27946' '!=' 'object_number=291, tag_tid=20, entry_tid=27555' 2026-02-18T08:19:47.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:19:47.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=290, tag_tid=20, entry_tid=27946' = 'object_number=291, tag_tid=20, entry_tid=27555' 2026-02-18T08:19:47.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=290, tag_tid=20, entry_tid=27946' 2026-02-18T08:19:47.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=309, tag_tid=21, entry_tid=1' 2026-02-18T08:19:47.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:19:47.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:19:47.974 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=21 2026-02-18T08:19:47.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=290, tag_tid=20, entry_tid=27946' 2026-02-18T08:19:47.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:19:47.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:19:47.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=20 2026-02-18T08:19:47.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=309, tag_tid=21, entry_tid=1' 2026-02-18T08:19:47.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:19:47.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:19:47.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:19:47.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=290, tag_tid=20, entry_tid=27946' 2026-02-18T08:19:47.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:19:47.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:19:47.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=27946 2026-02-18T08:19:47.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 21 = 20 -a 1 -le 27946 2026-02-18T08:19:47.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:19:47.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:19:47.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:19:48.183 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:19:48.183 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:19:48.183 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:19:48.183 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:19:48.183 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:19:48.183 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:19:48.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:19:48.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:19:48.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:19:48.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:19:48.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:19:48.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:19:48.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:19:48.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:19:48.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:19:48.187 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:19:48.187 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:19:48.187 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:19:48.187 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:19:48.187 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:19:48.187 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:19:48.187 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:19:48.187 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:19:48.188 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:19:48.188 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:19:48.188 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:19:48.188 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:19:48.188 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:19:48.188 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:19:48.188 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:19:48.188 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:19:48.188 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:19:48.188 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:19:48.188 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:19:48.188 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:19:48.189 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:19:48.189 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:19:48.189 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:19:48.189 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:19:48.189 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:19:48.189 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:19:48.189 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:19:48.189 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:19:48.189 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:19:48.189 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:19:48.189 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:19:48.189 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:19:48.189 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:19:48.190 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:19:48.197 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:19:48.198 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:19:48.198 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:19:48.198 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:19:48.254 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:19:48.255 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:19:48.255 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:19:48.257 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:19:48.257 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:19:48.257 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:19:48.257 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:19:48.257 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:19:48.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:19:48.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:19:48.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:19:48.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:19:48.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:19:48.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:19:48.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:19:48.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:19:48.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:19:48.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:19:48.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:19:48.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:19:48.259 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:19:48.259 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:19:48.259 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:19:48.259 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:19:48.259 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:19:48.259 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:19:48.259 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:19:48.259 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:19:48.259 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:19:48.259 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:19:48.259 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:19:48.259 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:19:48.260 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:19:48.260 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:19:48.260 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:19:48.270 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:19:48.270 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:19:48.270 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:19:48.270 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:19:48.326 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:19:48.326 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:19:48.326 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:19:48.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:19:48.329 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:19:48.329 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:19:48.329 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:19:48.329 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:19:48.329 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:19:48.329 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:19:48.329 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:19:48.329 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:19:48.329 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:19:48.329 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:19:48.329 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:19:48.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:19:48.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:19:48.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:19:48.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:19:48.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:19:48.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:19:48.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:19:48.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:19:48.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:19:48.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:19:48.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:19:48.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:19:48.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:19:48.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:19:48.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:19:48.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:19:48.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:19:48.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:19:48.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:19:48.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:19:48.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:19:48.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:19:48.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:19:48.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:19:48.388 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:19:48.388 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:19:48.388 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:19:48.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:19:48.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:19:48.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:19:48.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:19:48.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:19:48.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:19:48.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:19:48.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:19:48.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:19:48.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:19:48.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:19:48.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:19:48.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:19:48.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:19:48.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:19:48.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:19:48.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:19:48.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:19:48.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:19:48.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:19:48.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:19:48.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:19:48.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:19:48.393 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:19:48.393 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:19:48.393 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:19:48.393 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:19:48.393 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:19:48.393 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:19:48.393 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:19:48.393 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:19:48.393 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:19:48.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:19:48.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:19:48.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:19:48.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:19:48.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:19:48.470 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:19:48.470 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:19:48.470 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:19:48.470 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:19:48.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:19:48.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:19:48.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:19:48.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:19:48.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:19:48.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:19:48.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:19:48.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:19:48.472 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:19:48.472 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:19:48.473 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:19:48.500 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 72 2026-02-18T08:19:48.501 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 77 2026-02-18T08:19:48.501 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:19:48.501 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=309, tag_tid=21, entry_tid=1], [object_number=308, tag_tid=21, entry_tid=0], [object_number=310, tag_tid=20, entry_tid=46754], [object_number=311, tag_tid=20, entry_tid=46751]]], state=connected] 2026-02-18T08:19:48.501 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=288, tag_tid=20, entry_tid=28412], [object_number=291, tag_tid=20, entry_tid=28411], [object_number=290, tag_tid=20, entry_tid=28410], [object_number=289, tag_tid=20, entry_tid=28409]]], state=connected] 2026-02-18T08:19:48.503 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:19:48.505 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=309, tag_tid=21, entry_tid=1' 2026-02-18T08:19:48.506 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:19:48.506 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:19:48.506 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:19:48.506 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:19:48.506 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:19:48.506 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:19:48.506 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:19:48.506 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:19:48.507 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:19:48.507 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:19:48.507 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:19:48.507 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:19:48.508 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:19:48.508 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:19:48.508 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:19:48.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 72 2026-02-18T08:19:48.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 77 2026-02-18T08:19:48.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:19:48.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=309, tag_tid=21, entry_tid=1], [object_number=308, tag_tid=21, entry_tid=0], [object_number=310, tag_tid=20, entry_tid=46754], [object_number=311, tag_tid=20, entry_tid=46751]]], state=connected] 2026-02-18T08:19:48.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=288, tag_tid=20, entry_tid=28412], [object_number=291, tag_tid=20, entry_tid=28411], [object_number=290, tag_tid=20, entry_tid=28410], [object_number=289, tag_tid=20, entry_tid=28409]]], state=connected] 2026-02-18T08:19:48.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:19:48.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=288, tag_tid=20, entry_tid=28412' 2026-02-18T08:19:48.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=309, tag_tid=21, entry_tid=1' -a 'object_number=309, tag_tid=21, entry_tid=1' = 'object_number=288, tag_tid=20, entry_tid=28412' 2026-02-18T08:19:48.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=288, tag_tid=20, entry_tid=28412' '!=' 'object_number=290, tag_tid=20, entry_tid=27946' 2026-02-18T08:19:48.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:19:48.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=288, tag_tid=20, entry_tid=28412' = 'object_number=290, tag_tid=20, entry_tid=27946' 2026-02-18T08:19:48.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=288, tag_tid=20, entry_tid=28412' 2026-02-18T08:19:48.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=309, tag_tid=21, entry_tid=1' 2026-02-18T08:19:48.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:19:48.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:19:48.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=21 2026-02-18T08:19:48.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=288, tag_tid=20, entry_tid=28412' 2026-02-18T08:19:48.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:19:48.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:19:48.551 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=20 2026-02-18T08:19:48.552 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=309, tag_tid=21, entry_tid=1' 2026-02-18T08:19:48.552 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:19:48.552 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:19:48.553 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:19:48.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=288, tag_tid=20, entry_tid=28412' 2026-02-18T08:19:48.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:19:48.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:19:48.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=28412 2026-02-18T08:19:48.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 21 = 20 -a 1 -le 28412 2026-02-18T08:19:48.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:19:48.557 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:19:48.557 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:19:48.758 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:19:48.758 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:19:48.758 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:19:48.759 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:19:48.759 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:19:48.759 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:19:48.759 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:19:48.759 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:19:48.759 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:19:48.759 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:19:48.759 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:19:48.759 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:19:48.759 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:19:48.760 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:19:48.760 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:19:48.760 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:19:48.760 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:19:48.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:19:48.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:19:48.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:19:48.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:19:48.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:19:48.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:19:48.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:19:48.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:19:48.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:19:48.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:19:48.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:19:48.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:19:48.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:19:48.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:19:48.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:19:48.766 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:19:48.766 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:19:48.766 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:19:48.766 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:19:48.766 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:19:48.766 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:19:48.766 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:19:48.766 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:19:48.766 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:19:48.766 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:19:48.766 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:19:48.766 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:19:48.767 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:19:48.767 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:19:48.767 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:19:48.767 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:19:48.767 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:19:48.773 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:19:48.773 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:19:48.773 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:19:48.773 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:19:48.823 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:19:48.824 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:19:48.824 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:19:48.827 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:19:48.827 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:19:48.827 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:19:48.827 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:19:48.828 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:19:48.828 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:19:48.828 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:19:48.828 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:19:48.828 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:19:48.828 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:19:48.828 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:19:48.828 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:19:48.828 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:19:48.828 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:19:48.828 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:19:48.828 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:19:48.829 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:19:48.829 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:19:48.829 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:19:48.829 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:19:48.829 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:19:48.829 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:19:48.829 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:19:48.829 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:19:48.829 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:19:48.829 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:19:48.829 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:19:48.829 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:19:48.830 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:19:48.830 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:19:48.830 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:19:48.830 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:19:48.838 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:19:48.838 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:19:48.838 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:19:48.839 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:19:48.853 INFO:tasks.rbd_mirror_thrash:kill cluster2.client.mirror.0 2026-02-18T08:19:48.853 INFO:tasks.rbd_mirror.cluster2.client.mirror.0:Sent signal 15 2026-02-18T08:19:48.853 INFO:tasks.rbd_mirror_thrash:waiting for 8.612922138284281 secs before reviving daemons 2026-02-18T08:19:48.854 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.trial163.stderr:2026-02-18T08:19:48.847+0000 7f92266e2640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.0 (PID: 16441) UID: 1000 2026-02-18T08:19:48.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:19:48.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:19:48.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:19:48.894 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:19:48.894 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:19:48.894 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:19:48.894 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:19:48.894 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:19:48.894 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:19:48.895 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:19:48.895 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:19:48.895 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:19:48.895 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:19:48.895 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:19:48.895 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:19:48.895 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:19:48.895 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:19:48.895 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:19:48.895 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:19:48.895 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:19:48.896 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:19:48.896 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:19:48.896 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:19:48.896 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:19:48.896 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:19:48.896 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:19:48.896 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:19:48.896 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:19:48.896 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:19:48.896 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:19:48.896 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:19:48.896 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:19:48.897 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:19:48.897 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:19:48.897 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:19:48.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:19:48.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:19:48.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:19:48.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:19:48.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:19:48.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:19:48.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:19:48.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:19:48.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:19:48.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:19:48.961 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:19:48.961 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:19:48.961 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:19:48.961 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:19:48.961 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:19:48.961 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:19:48.961 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:19:48.961 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:19:48.961 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:19:48.961 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:19:48.961 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:19:48.962 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:19:48.962 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:19:48.962 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:19:48.962 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:19:48.962 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:19:48.962 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:19:48.962 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:19:48.962 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:19:48.962 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:19:48.962 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:19:48.962 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:19:48.962 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:19:48.963 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:19:48.963 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:19:48.963 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:19:48.963 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:19:48.963 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:19:48.963 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:19:48.970 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:19:48.970 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:19:48.971 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:19:48.971 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:19:49.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:19:49.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:19:49.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:19:49.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:19:49.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:19:49.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:19:49.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:19:49.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:19:49.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:19:49.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:19:49.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:19:49.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:19:49.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:19:49.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:19:49.035 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:19:49.035 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:19:49.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 72 2026-02-18T08:19:49.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 77 2026-02-18T08:19:49.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:19:49.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=309, tag_tid=21, entry_tid=1], [object_number=308, tag_tid=21, entry_tid=0], [object_number=310, tag_tid=20, entry_tid=46754], [object_number=311, tag_tid=20, entry_tid=46751]]], state=connected] 2026-02-18T08:19:49.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=288, tag_tid=20, entry_tid=28736], [object_number=291, tag_tid=20, entry_tid=28735], [object_number=290, tag_tid=20, entry_tid=28734], [object_number=289, tag_tid=20, entry_tid=28733]]], state=connected] 2026-02-18T08:19:49.067 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:19:49.068 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=309, tag_tid=21, entry_tid=1' 2026-02-18T08:19:49.069 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:19:49.069 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:19:49.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:19:49.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:19:49.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:19:49.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:19:49.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:19:49.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:19:49.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:19:49.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:19:49.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:19:49.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:19:49.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:19:49.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:19:49.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:19:49.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 72 2026-02-18T08:19:49.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 77 2026-02-18T08:19:49.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:19:49.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=309, tag_tid=21, entry_tid=1], [object_number=308, tag_tid=21, entry_tid=0], [object_number=310, tag_tid=20, entry_tid=46754], [object_number=311, tag_tid=20, entry_tid=46751]]], state=connected] 2026-02-18T08:19:49.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=288, tag_tid=20, entry_tid=28736], [object_number=291, tag_tid=20, entry_tid=28735], [object_number=290, tag_tid=20, entry_tid=28734], [object_number=289, tag_tid=20, entry_tid=28733]]], state=connected] 2026-02-18T08:19:49.110 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:19:49.112 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=288, tag_tid=20, entry_tid=28736' 2026-02-18T08:19:49.112 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=309, tag_tid=21, entry_tid=1' -a 'object_number=309, tag_tid=21, entry_tid=1' = 'object_number=288, tag_tid=20, entry_tid=28736' 2026-02-18T08:19:49.112 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=288, tag_tid=20, entry_tid=28736' '!=' 'object_number=288, tag_tid=20, entry_tid=28412' 2026-02-18T08:19:49.113 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:19:49.113 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=288, tag_tid=20, entry_tid=28736' = 'object_number=288, tag_tid=20, entry_tid=28412' 2026-02-18T08:19:49.113 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=288, tag_tid=20, entry_tid=28736' 2026-02-18T08:19:49.113 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=309, tag_tid=21, entry_tid=1' 2026-02-18T08:19:49.114 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:19:49.114 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:19:49.114 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=21 2026-02-18T08:19:49.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=288, tag_tid=20, entry_tid=28736' 2026-02-18T08:19:49.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:19:49.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:19:49.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=20 2026-02-18T08:19:49.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=309, tag_tid=21, entry_tid=1' 2026-02-18T08:19:49.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:19:49.118 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:19:49.119 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:19:49.120 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=288, tag_tid=20, entry_tid=28736' 2026-02-18T08:19:49.120 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:19:49.120 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:19:49.121 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=28736 2026-02-18T08:19:49.121 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 21 = 20 -a 1 -le 28736 2026-02-18T08:19:49.121 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:19:49.121 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:19:49.122 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:19:49.323 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:19:49.323 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:19:49.323 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:19:49.324 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:19:49.324 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:19:49.324 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:19:49.324 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:19:49.324 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:19:49.324 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:19:49.324 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:19:49.324 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:19:49.324 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:19:49.324 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:19:49.324 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:19:49.325 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:19:49.325 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:19:49.327 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:19:49.327 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:19:49.327 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:19:49.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:19:49.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:19:49.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:19:49.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:19:49.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:19:49.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:19:49.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:19:49.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:19:49.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:19:49.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:19:49.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:19:49.329 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:19:49.329 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:19:49.329 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:19:49.329 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:19:49.329 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:19:49.329 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:19:49.329 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:19:49.329 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:19:49.329 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:19:49.329 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:19:49.329 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:19:49.329 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:19:49.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:19:49.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:19:49.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:19:49.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:19:49.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:19:49.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:19:49.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:19:49.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:19:49.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:19:49.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:19:49.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:19:49.388 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:19:49.388 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:19:49.388 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:19:49.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:19:49.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:19:49.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:19:49.393 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:19:49.393 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:19:49.393 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:19:49.393 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:19:49.393 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:19:49.393 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:19:49.393 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:19:49.393 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:19:49.393 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:19:49.393 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:19:49.394 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:19:49.394 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:19:49.394 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:19:49.394 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:19:49.394 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:19:49.394 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:19:49.394 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:19:49.394 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:19:49.394 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:19:49.395 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:19:49.395 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:19:49.395 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:19:49.395 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:19:49.395 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:19:49.395 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:19:49.395 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:19:49.396 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:19:49.396 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:19:49.396 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:19:49.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:19:49.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:19:49.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:19:49.403 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:19:49.452 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:19:49.452 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:19:49.452 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:19:49.455 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:19:49.455 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:19:49.455 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:19:49.455 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:19:49.455 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:19:49.455 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:19:49.455 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:19:49.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:19:49.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:19:49.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:19:49.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:19:49.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:19:49.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:19:49.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:19:49.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:19:49.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:19:49.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:19:49.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:19:49.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:19:49.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:19:49.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:19:49.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:19:49.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:19:49.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:19:49.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:19:49.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:19:49.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:19:49.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:19:49.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:19:49.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:19:49.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:19:49.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:19:49.465 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:19:49.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:19:49.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:19:49.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:19:49.516 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:19:49.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:19:49.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:19:49.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:19:49.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:19:49.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:19:49.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:19:49.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:19:49.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:19:49.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:19:49.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:19:49.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:19:49.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:19:49.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:19:49.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:19:49.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:19:49.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:19:49.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:19:49.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:19:49.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:19:49.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:19:49.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:19:49.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:19:49.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:19:49.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:19:49.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:19:49.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:19:49.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:19:49.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:19:49.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:19:49.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:19:49.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:19:49.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:19:49.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:19:49.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:19:49.529 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:19:49.530 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:19:49.530 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:19:49.530 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:19:49.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:19:49.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:19:49.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:19:49.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:19:49.594 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:19:49.594 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:19:49.594 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:19:49.594 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:19:49.594 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:19:49.594 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:19:49.594 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:19:49.594 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:19:49.594 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:19:49.595 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:19:49.595 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:19:49.595 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:19:49.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 72 2026-02-18T08:19:49.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 77 2026-02-18T08:19:49.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:19:49.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=309, tag_tid=21, entry_tid=1], [object_number=308, tag_tid=21, entry_tid=0], [object_number=310, tag_tid=20, entry_tid=46754], [object_number=311, tag_tid=20, entry_tid=46751]]], state=connected] 2026-02-18T08:19:49.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=289, tag_tid=20, entry_tid=29097], [object_number=288, tag_tid=20, entry_tid=29096], [object_number=291, tag_tid=20, entry_tid=29095], [object_number=290, tag_tid=20, entry_tid=29094]]], state=connected] 2026-02-18T08:19:49.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:19:49.628 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=309, tag_tid=21, entry_tid=1' 2026-02-18T08:19:49.629 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:19:49.629 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:19:49.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:19:49.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:19:49.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:19:49.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:19:49.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:19:49.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:19:49.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:19:49.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:19:49.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:19:49.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:19:49.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:19:49.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:19:49.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:19:49.660 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 72 2026-02-18T08:19:49.660 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 77 2026-02-18T08:19:49.660 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:19:49.660 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=309, tag_tid=21, entry_tid=1], [object_number=308, tag_tid=21, entry_tid=0], [object_number=310, tag_tid=20, entry_tid=46754], [object_number=311, tag_tid=20, entry_tid=46751]]], state=connected] 2026-02-18T08:19:49.660 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=289, tag_tid=20, entry_tid=29097], [object_number=288, tag_tid=20, entry_tid=29096], [object_number=291, tag_tid=20, entry_tid=29095], [object_number=290, tag_tid=20, entry_tid=29094]]], state=connected] 2026-02-18T08:19:49.663 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:19:49.664 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=289, tag_tid=20, entry_tid=29097' 2026-02-18T08:19:49.664 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=309, tag_tid=21, entry_tid=1' -a 'object_number=309, tag_tid=21, entry_tid=1' = 'object_number=289, tag_tid=20, entry_tid=29097' 2026-02-18T08:19:49.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=289, tag_tid=20, entry_tid=29097' '!=' 'object_number=288, tag_tid=20, entry_tid=28736' 2026-02-18T08:19:49.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:19:49.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=289, tag_tid=20, entry_tid=29097' = 'object_number=288, tag_tid=20, entry_tid=28736' 2026-02-18T08:19:49.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=289, tag_tid=20, entry_tid=29097' 2026-02-18T08:19:49.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=309, tag_tid=21, entry_tid=1' 2026-02-18T08:19:49.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:19:49.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:19:49.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=21 2026-02-18T08:19:49.667 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=289, tag_tid=20, entry_tid=29097' 2026-02-18T08:19:49.667 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:19:49.667 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:19:49.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=20 2026-02-18T08:19:49.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=309, tag_tid=21, entry_tid=1' 2026-02-18T08:19:49.670 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:19:49.670 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:19:49.671 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:19:49.672 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=289, tag_tid=20, entry_tid=29097' 2026-02-18T08:19:49.672 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:19:49.672 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:19:49.674 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=29097 2026-02-18T08:19:49.674 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 21 = 20 -a 1 -le 29097 2026-02-18T08:19:49.674 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:19:49.674 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:19:49.675 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:19:49.875 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:19:49.875 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:19:49.875 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:19:49.875 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:19:49.875 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:19:49.875 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:19:49.875 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:19:49.875 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:19:49.875 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:19:49.875 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:19:49.875 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:19:49.875 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:19:49.875 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:19:49.875 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:19:49.875 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:19:49.876 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:19:49.876 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:19:49.877 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:19:49.878 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:19:49.878 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:19:49.878 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:19:49.878 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:19:49.878 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:19:49.878 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:19:49.878 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:19:49.878 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:19:49.878 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:19:49.878 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:19:49.878 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:19:49.878 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:19:49.878 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:19:49.878 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:19:49.878 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:19:49.878 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:19:49.878 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:19:49.878 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:19:49.878 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:19:49.879 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:19:49.879 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:19:49.879 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:19:49.879 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:19:49.879 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:19:49.879 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:19:49.879 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:19:49.879 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:19:49.879 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:19:49.879 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:19:49.879 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:19:49.879 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:19:49.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:19:49.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:19:49.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:19:49.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:19:49.948 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:19:49.948 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:19:49.948 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:19:49.950 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:19:49.950 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:19:49.950 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:19:49.950 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:19:49.950 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:19:49.950 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:19:49.950 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:19:49.950 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:19:49.950 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:19:49.950 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:19:49.950 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:19:49.950 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:19:49.950 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:19:49.950 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:19:49.950 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:19:49.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:19:49.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:19:49.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:19:49.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:19:49.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:19:49.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:19:49.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:19:49.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:19:49.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:19:49.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:19:49.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:19:49.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:19:49.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:19:49.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:19:49.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:19:49.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:19:49.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:19:49.962 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:19:49.963 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:19:49.964 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:19:49.964 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:19:50.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:19:50.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:19:50.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:19:50.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:19:50.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:19:50.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:19:50.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:19:50.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:19:50.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:19:50.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:19:50.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:19:50.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:19:50.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:19:50.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:19:50.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:19:50.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:19:50.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:19:50.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:19:50.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:19:50.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:19:50.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:19:50.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:19:50.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:19:50.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:19:50.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:19:50.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:19:50.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:19:50.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:19:50.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:19:50.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:19:50.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:19:50.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:19:50.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:19:50.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:19:50.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:19:50.037 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:19:50.037 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:19:50.037 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:19:50.037 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:19:50.095 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:19:50.095 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:19:50.096 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:19:50.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:19:50.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:19:50.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:19:50.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:19:50.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:19:50.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:19:50.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:19:50.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:19:50.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:19:50.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:19:50.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:19:50.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:19:50.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:19:50.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:19:50.100 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:19:50.100 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:19:50.100 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:19:50.100 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:19:50.100 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:19:50.100 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:19:50.100 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:19:50.100 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:19:50.100 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:19:50.100 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:19:50.100 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:19:50.100 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:19:50.100 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:19:50.101 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:19:50.101 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:19:50.101 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:19:50.101 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:19:50.101 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:19:50.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:19:50.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:19:50.110 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:19:50.110 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:19:50.176 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:19:50.178 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:19:50.178 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:19:50.178 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:19:50.178 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:19:50.178 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:19:50.178 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:19:50.178 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:19:50.179 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:19:50.179 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:19:50.179 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:19:50.179 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:19:50.179 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:19:50.180 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:19:50.180 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:19:50.180 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:19:50.217 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 72 2026-02-18T08:19:50.217 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 77 2026-02-18T08:19:50.217 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:19:50.217 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=309, tag_tid=21, entry_tid=1], [object_number=308, tag_tid=21, entry_tid=0], [object_number=310, tag_tid=20, entry_tid=46754], [object_number=311, tag_tid=20, entry_tid=46751]]], state=connected] 2026-02-18T08:19:50.217 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=288, tag_tid=20, entry_tid=29436], [object_number=291, tag_tid=20, entry_tid=29435], [object_number=290, tag_tid=20, entry_tid=29434], [object_number=289, tag_tid=20, entry_tid=29433]]], state=connected] 2026-02-18T08:19:50.221 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:19:50.222 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=309, tag_tid=21, entry_tid=1' 2026-02-18T08:19:50.223 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:19:50.223 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:19:50.223 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:19:50.223 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:19:50.223 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:19:50.223 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:19:50.223 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:19:50.223 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:19:50.223 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:19:50.223 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:19:50.223 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:19:50.223 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:19:50.225 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:19:50.225 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:19:50.225 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:19:50.254 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 72 2026-02-18T08:19:50.254 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 77 2026-02-18T08:19:50.254 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:19:50.254 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=309, tag_tid=21, entry_tid=1], [object_number=308, tag_tid=21, entry_tid=0], [object_number=310, tag_tid=20, entry_tid=46754], [object_number=311, tag_tid=20, entry_tid=46751]]], state=connected] 2026-02-18T08:19:50.254 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=288, tag_tid=20, entry_tid=29436], [object_number=291, tag_tid=20, entry_tid=29435], [object_number=290, tag_tid=20, entry_tid=29434], [object_number=289, tag_tid=20, entry_tid=29433]]], state=connected] 2026-02-18T08:19:50.257 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:19:50.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=288, tag_tid=20, entry_tid=29436' 2026-02-18T08:19:50.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=309, tag_tid=21, entry_tid=1' -a 'object_number=309, tag_tid=21, entry_tid=1' = 'object_number=288, tag_tid=20, entry_tid=29436' 2026-02-18T08:19:50.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=288, tag_tid=20, entry_tid=29436' '!=' 'object_number=289, tag_tid=20, entry_tid=29097' 2026-02-18T08:19:50.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:19:50.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=288, tag_tid=20, entry_tid=29436' = 'object_number=289, tag_tid=20, entry_tid=29097' 2026-02-18T08:19:50.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=288, tag_tid=20, entry_tid=29436' 2026-02-18T08:19:50.259 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=309, tag_tid=21, entry_tid=1' 2026-02-18T08:19:50.259 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:19:50.259 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:19:50.260 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=21 2026-02-18T08:19:50.261 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=288, tag_tid=20, entry_tid=29436' 2026-02-18T08:19:50.261 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:19:50.261 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:19:50.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=20 2026-02-18T08:19:50.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=309, tag_tid=21, entry_tid=1' 2026-02-18T08:19:50.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:19:50.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:19:50.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:19:50.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=288, tag_tid=20, entry_tid=29436' 2026-02-18T08:19:50.266 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:19:50.266 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:19:50.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=29436 2026-02-18T08:19:50.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 21 = 20 -a 1 -le 29436 2026-02-18T08:19:50.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:19:50.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:19:50.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:19:50.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:19:50.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:19:50.470 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:19:50.470 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:19:50.470 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:19:50.470 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:19:50.470 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:19:50.470 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:19:50.470 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:19:50.470 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:19:50.470 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:19:50.470 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:19:50.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:19:50.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:19:50.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:19:50.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:19:50.474 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:19:50.474 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:19:50.475 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:19:50.475 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:19:50.475 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:19:50.475 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:19:50.475 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:19:50.475 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:19:50.476 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:19:50.476 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:19:50.476 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:19:50.476 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:19:50.476 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:19:50.476 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:19:50.476 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:19:50.476 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:19:50.476 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:19:50.476 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:19:50.476 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:19:50.476 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:19:50.477 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:19:50.477 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:19:50.477 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:19:50.477 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:19:50.477 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:19:50.477 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:19:50.478 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:19:50.478 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:19:50.478 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:19:50.478 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:19:50.478 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:19:50.478 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:19:50.478 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:19:50.484 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:19:50.484 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:19:50.484 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:19:50.484 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:19:50.534 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:19:50.534 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:19:50.534 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:19:50.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:19:50.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:19:50.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:19:50.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:19:50.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:19:50.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:19:50.539 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:19:50.539 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:19:50.539 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:19:50.539 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:19:50.539 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:19:50.539 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:19:50.539 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:19:50.539 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:19:50.539 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:19:50.539 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:19:50.539 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:19:50.540 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:19:50.540 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:19:50.540 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:19:50.540 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:19:50.540 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:19:50.540 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:19:50.540 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:19:50.540 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:19:50.540 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:19:50.540 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:19:50.540 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:19:50.540 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:19:50.540 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:19:50.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:19:50.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:19:50.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:19:50.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:19:50.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:19:50.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:19:50.600 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:19:50.600 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:19:50.600 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:19:50.604 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:19:50.604 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:19:50.604 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:19:50.604 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:19:50.605 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:19:50.605 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:19:50.605 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:19:50.605 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:19:50.605 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:19:50.605 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:19:50.605 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:19:50.605 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:19:50.606 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:19:50.606 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:19:50.606 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:19:50.606 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:19:50.606 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:19:50.606 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:19:50.606 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:19:50.606 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:19:50.607 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:19:50.607 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:19:50.607 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:19:50.607 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:19:50.607 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:19:50.607 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:19:50.607 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:19:50.607 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:19:50.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:19:50.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:19:50.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:19:50.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:19:50.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:19:50.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:19:50.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:19:50.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:19:50.664 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:19:50.664 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:19:50.664 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:19:50.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:19:50.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:19:50.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:19:50.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:19:50.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:19:50.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:19:50.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:19:50.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:19:50.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:19:50.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:19:50.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:19:50.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:19:50.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:19:50.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:19:50.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:19:50.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:19:50.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:19:50.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:19:50.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:19:50.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:19:50.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:19:50.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:19:50.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:19:50.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:19:50.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:19:50.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:19:50.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:19:50.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:19:50.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:19:50.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:19:50.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:19:50.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:19:50.677 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:19:50.677 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:19:50.677 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:19:50.677 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:19:50.739 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:19:50.739 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:19:50.739 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:19:50.740 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:19:50.740 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:19:50.740 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:19:50.740 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:19:50.740 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:19:50.740 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:19:50.740 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:19:50.740 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:19:50.741 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:19:50.741 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:19:50.742 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:19:50.742 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:19:50.742 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:19:50.770 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 73 2026-02-18T08:19:50.770 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 77 2026-02-18T08:19:50.770 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:19:50.771 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=309, tag_tid=21, entry_tid=1], [object_number=308, tag_tid=21, entry_tid=0], [object_number=310, tag_tid=20, entry_tid=46754], [object_number=311, tag_tid=20, entry_tid=46751]]], state=connected] 2026-02-18T08:19:50.771 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=293, tag_tid=20, entry_tid=29685], [object_number=292, tag_tid=20, entry_tid=29684], [object_number=295, tag_tid=20, entry_tid=29683], [object_number=294, tag_tid=20, entry_tid=29682]]], state=connected] 2026-02-18T08:19:50.773 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:19:50.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=309, tag_tid=21, entry_tid=1' 2026-02-18T08:19:50.776 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:19:50.776 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:19:50.776 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:19:50.776 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:19:50.776 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:19:50.776 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:19:50.776 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:19:50.776 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:19:50.776 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:19:50.777 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:19:50.777 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:19:50.777 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:19:50.777 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:19:50.778 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:19:50.778 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:19:50.806 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 73 2026-02-18T08:19:50.806 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 77 2026-02-18T08:19:50.807 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:19:50.807 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=309, tag_tid=21, entry_tid=1], [object_number=308, tag_tid=21, entry_tid=0], [object_number=310, tag_tid=20, entry_tid=46754], [object_number=311, tag_tid=20, entry_tid=46751]]], state=connected] 2026-02-18T08:19:50.807 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=293, tag_tid=20, entry_tid=29685], [object_number=292, tag_tid=20, entry_tid=29684], [object_number=295, tag_tid=20, entry_tid=29683], [object_number=294, tag_tid=20, entry_tid=29682]]], state=connected] 2026-02-18T08:19:50.809 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:19:50.810 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=293, tag_tid=20, entry_tid=29685' 2026-02-18T08:19:50.811 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=309, tag_tid=21, entry_tid=1' -a 'object_number=309, tag_tid=21, entry_tid=1' = 'object_number=293, tag_tid=20, entry_tid=29685' 2026-02-18T08:19:50.811 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=293, tag_tid=20, entry_tid=29685' '!=' 'object_number=288, tag_tid=20, entry_tid=29436' 2026-02-18T08:19:50.811 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:19:50.811 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=293, tag_tid=20, entry_tid=29685' = 'object_number=288, tag_tid=20, entry_tid=29436' 2026-02-18T08:19:50.811 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=293, tag_tid=20, entry_tid=29685' 2026-02-18T08:19:50.812 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=309, tag_tid=21, entry_tid=1' 2026-02-18T08:19:50.812 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:19:50.812 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:19:50.813 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=21 2026-02-18T08:19:50.813 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=293, tag_tid=20, entry_tid=29685' 2026-02-18T08:19:50.814 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:19:50.814 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:19:50.815 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=20 2026-02-18T08:19:50.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=309, tag_tid=21, entry_tid=1' 2026-02-18T08:19:50.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:19:50.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:19:50.817 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:19:50.818 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=293, tag_tid=20, entry_tid=29685' 2026-02-18T08:19:50.818 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:19:50.818 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:19:50.819 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=29685 2026-02-18T08:19:50.820 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 21 = 20 -a 1 -le 29685 2026-02-18T08:19:50.820 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:19:50.820 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:19:50.820 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:19:51.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:19:51.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:19:51.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:19:51.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:19:51.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:19:51.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:19:51.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:19:51.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:19:51.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:19:51.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:19:51.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:19:51.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:19:51.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:19:51.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:19:51.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:19:51.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:19:51.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:19:51.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:19:51.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:19:51.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:19:51.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:19:51.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:19:51.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:19:51.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:19:51.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:19:51.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:19:51.027 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:19:51.027 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:19:51.027 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:19:51.027 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:19:51.027 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:19:51.027 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:19:51.027 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:19:51.027 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:19:51.027 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:19:51.027 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:19:51.027 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:19:51.027 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:19:51.028 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:19:51.028 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:19:51.028 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:19:51.028 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:19:51.028 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:19:51.028 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:19:51.028 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:19:51.028 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:19:51.028 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:19:51.028 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:19:51.028 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:19:51.035 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:19:51.035 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:19:51.035 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:19:51.035 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:19:51.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:19:51.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:19:51.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:19:51.102 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:19:51.102 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:19:51.102 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:19:51.102 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:19:51.102 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:19:51.102 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:19:51.102 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:19:51.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:19:51.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:19:51.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:19:51.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:19:51.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:19:51.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:19:51.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:19:51.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:19:51.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:19:51.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:19:51.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:19:51.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:19:51.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:19:51.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:19:51.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:19:51.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:19:51.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:19:51.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:19:51.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:19:51.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:19:51.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:19:51.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:19:51.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:19:51.105 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:19:51.105 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:19:51.113 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:19:51.114 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:19:51.114 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:19:51.114 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:19:51.161 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:19:51.161 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:19:51.162 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:19:51.164 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:19:51.164 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:19:51.164 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:19:51.164 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:19:51.164 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:19:51.165 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:19:51.165 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:19:51.165 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:19:51.165 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:19:51.165 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:19:51.165 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:19:51.165 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:19:51.165 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:19:51.165 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:19:51.165 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:19:51.165 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:19:51.166 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:19:51.166 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:19:51.166 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:19:51.166 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:19:51.166 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:19:51.166 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:19:51.166 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:19:51.166 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:19:51.166 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:19:51.166 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:19:51.166 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:19:51.166 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:19:51.166 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:19:51.167 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:19:51.167 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:19:51.167 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:19:51.174 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:19:51.175 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:19:51.175 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:19:51.175 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:19:51.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:19:51.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:19:51.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:19:51.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:19:51.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:19:51.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:19:51.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:19:51.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:19:51.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:19:51.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:19:51.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:19:51.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:19:51.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:19:51.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:19:51.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:19:51.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:19:51.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:19:51.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:19:51.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:19:51.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:19:51.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:19:51.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:19:51.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:19:51.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:19:51.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:19:51.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:19:51.238 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:19:51.238 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:19:51.238 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:19:51.238 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:19:51.238 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:19:51.238 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:19:51.238 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:19:51.238 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:19:51.238 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:19:51.246 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:19:51.247 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:19:51.247 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:19:51.247 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:19:51.314 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:19:51.315 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:19:51.316 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:19:51.316 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:19:51.316 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:19:51.316 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:19:51.316 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:19:51.316 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:19:51.316 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:19:51.316 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:19:51.316 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:19:51.316 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:19:51.317 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:19:51.317 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:19:51.317 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:19:51.318 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:19:51.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 73 2026-02-18T08:19:51.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 77 2026-02-18T08:19:51.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:19:51.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=309, tag_tid=21, entry_tid=1], [object_number=308, tag_tid=21, entry_tid=0], [object_number=310, tag_tid=20, entry_tid=46754], [object_number=311, tag_tid=20, entry_tid=46751]]], state=connected] 2026-02-18T08:19:51.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=294, tag_tid=20, entry_tid=30026], [object_number=293, tag_tid=20, entry_tid=30025], [object_number=292, tag_tid=20, entry_tid=30024], [object_number=295, tag_tid=20, entry_tid=30023]]], state=connected] 2026-02-18T08:19:51.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:19:51.350 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=309, tag_tid=21, entry_tid=1' 2026-02-18T08:19:51.351 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:19:51.352 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:19:51.352 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:19:51.352 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:19:51.352 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:19:51.352 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:19:51.352 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:19:51.352 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:19:51.352 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:19:51.352 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:19:51.353 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:19:51.353 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:19:51.353 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:19:51.354 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:19:51.354 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:19:51.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 73 2026-02-18T08:19:51.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 77 2026-02-18T08:19:51.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:19:51.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=309, tag_tid=21, entry_tid=1], [object_number=308, tag_tid=21, entry_tid=0], [object_number=310, tag_tid=20, entry_tid=46754], [object_number=311, tag_tid=20, entry_tid=46751]]], state=connected] 2026-02-18T08:19:51.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=294, tag_tid=20, entry_tid=30026], [object_number=293, tag_tid=20, entry_tid=30025], [object_number=292, tag_tid=20, entry_tid=30024], [object_number=295, tag_tid=20, entry_tid=30023]]], state=connected] 2026-02-18T08:19:51.395 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:19:51.396 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=294, tag_tid=20, entry_tid=30026' 2026-02-18T08:19:51.396 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=309, tag_tid=21, entry_tid=1' -a 'object_number=309, tag_tid=21, entry_tid=1' = 'object_number=294, tag_tid=20, entry_tid=30026' 2026-02-18T08:19:51.397 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=294, tag_tid=20, entry_tid=30026' '!=' 'object_number=293, tag_tid=20, entry_tid=29685' 2026-02-18T08:19:51.397 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:19:51.397 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=294, tag_tid=20, entry_tid=30026' = 'object_number=293, tag_tid=20, entry_tid=29685' 2026-02-18T08:19:51.397 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=294, tag_tid=20, entry_tid=30026' 2026-02-18T08:19:51.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=309, tag_tid=21, entry_tid=1' 2026-02-18T08:19:51.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:19:51.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:19:51.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=21 2026-02-18T08:19:51.399 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=294, tag_tid=20, entry_tid=30026' 2026-02-18T08:19:51.399 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:19:51.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:19:51.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=20 2026-02-18T08:19:51.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=309, tag_tid=21, entry_tid=1' 2026-02-18T08:19:51.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:19:51.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:19:51.403 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:19:51.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=294, tag_tid=20, entry_tid=30026' 2026-02-18T08:19:51.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:19:51.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:19:51.406 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=30026 2026-02-18T08:19:51.406 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 21 = 20 -a 1 -le 30026 2026-02-18T08:19:51.406 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:19:51.406 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:19:51.406 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:19:51.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:19:51.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:19:51.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:19:51.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:19:51.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:19:51.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:19:51.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:19:51.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:19:51.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:19:51.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:19:51.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:19:51.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:19:51.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:19:51.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:19:51.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:19:51.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:19:51.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:19:51.612 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:19:51.612 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:19:51.612 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:19:51.612 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:19:51.612 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:19:51.612 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:19:51.612 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:19:51.612 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:19:51.612 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:19:51.612 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:19:51.612 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:19:51.613 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:19:51.613 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:19:51.613 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:19:51.613 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:19:51.613 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:19:51.613 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:19:51.613 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:19:51.613 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:19:51.613 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:19:51.613 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:19:51.613 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:19:51.613 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:19:51.613 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:19:51.613 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:19:51.613 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:19:51.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:19:51.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:19:51.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:19:51.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:19:51.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:19:51.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:19:51.622 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:19:51.622 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:19:51.622 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:19:51.623 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:19:51.671 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:19:51.671 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:19:51.671 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:19:51.674 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:19:51.674 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:19:51.675 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:19:51.675 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:19:51.675 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:19:51.675 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:19:51.675 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:19:51.675 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:19:51.675 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:19:51.675 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:19:51.675 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:19:51.675 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:19:51.675 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:19:51.676 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:19:51.676 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:19:51.676 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:19:51.676 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:19:51.676 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:19:51.676 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:19:51.676 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:19:51.676 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:19:51.676 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:19:51.676 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:19:51.676 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:19:51.677 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:19:51.677 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:19:51.677 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:19:51.677 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:19:51.677 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:19:51.677 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:19:51.677 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:19:51.677 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:19:51.686 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:19:51.686 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:19:51.686 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:19:51.686 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:19:51.734 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:19:51.734 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:19:51.735 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:19:51.738 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:19:51.738 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:19:51.738 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:19:51.738 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:19:51.738 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:19:51.738 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:19:51.738 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:19:51.738 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:19:51.738 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:19:51.739 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:19:51.739 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:19:51.739 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:19:51.739 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:19:51.739 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:19:51.739 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:19:51.739 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:19:51.739 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:19:51.739 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:19:51.739 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:19:51.739 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:19:51.739 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:19:51.739 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:19:51.740 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:19:51.740 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:19:51.740 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:19:51.740 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:19:51.740 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:19:51.740 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:19:51.740 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:19:51.740 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:19:51.740 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:19:51.740 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:19:51.749 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:19:51.749 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:19:51.749 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:19:51.749 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:19:51.798 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:19:51.798 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:19:51.799 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:19:51.801 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:19:51.802 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:19:51.802 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:19:51.802 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:19:51.802 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:19:51.802 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:19:51.802 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:19:51.802 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:19:51.802 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:19:51.802 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:19:51.802 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:19:51.802 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:19:51.803 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:19:51.803 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:19:51.803 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:19:51.803 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:19:51.803 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:19:51.803 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:19:51.803 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:19:51.803 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:19:51.803 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:19:51.803 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:19:51.803 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:19:51.803 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:19:51.804 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:19:51.804 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:19:51.804 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:19:51.804 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:19:51.804 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:19:51.804 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:19:51.804 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:19:51.804 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:19:51.813 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:19:51.813 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:19:51.813 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:19:51.813 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:19:51.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:19:51.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:19:51.874 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:19:51.874 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:19:51.874 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:19:51.874 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:19:51.874 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:19:51.874 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:19:51.874 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:19:51.874 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:19:51.874 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:19:51.874 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:19:51.874 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:19:51.875 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:19:51.875 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:19:51.876 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:19:51.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 73 2026-02-18T08:19:51.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 77 2026-02-18T08:19:51.904 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:19:51.904 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=309, tag_tid=21, entry_tid=1], [object_number=308, tag_tid=21, entry_tid=0], [object_number=310, tag_tid=20, entry_tid=46754], [object_number=311, tag_tid=20, entry_tid=46751]]], state=connected] 2026-02-18T08:19:51.904 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=295, tag_tid=20, entry_tid=30359], [object_number=294, tag_tid=20, entry_tid=30358], [object_number=293, tag_tid=20, entry_tid=30357], [object_number=292, tag_tid=20, entry_tid=30356]]], state=connected] 2026-02-18T08:19:51.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:19:51.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=309, tag_tid=21, entry_tid=1' 2026-02-18T08:19:51.909 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:19:51.909 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:19:51.909 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:19:51.909 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:19:51.909 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:19:51.909 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:19:51.909 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:19:51.909 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:19:51.909 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:19:51.910 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:19:51.910 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:19:51.910 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:19:51.911 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:19:51.911 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:19:51.911 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:19:51.949 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 73 2026-02-18T08:19:51.949 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 77 2026-02-18T08:19:51.949 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:19:51.949 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=309, tag_tid=21, entry_tid=1], [object_number=308, tag_tid=21, entry_tid=0], [object_number=310, tag_tid=20, entry_tid=46754], [object_number=311, tag_tid=20, entry_tid=46751]]], state=connected] 2026-02-18T08:19:51.949 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=295, tag_tid=20, entry_tid=30359], [object_number=294, tag_tid=20, entry_tid=30358], [object_number=293, tag_tid=20, entry_tid=30357], [object_number=292, tag_tid=20, entry_tid=30356]]], state=connected] 2026-02-18T08:19:51.952 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:19:51.954 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=295, tag_tid=20, entry_tid=30359' 2026-02-18T08:19:51.955 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=309, tag_tid=21, entry_tid=1' -a 'object_number=309, tag_tid=21, entry_tid=1' = 'object_number=295, tag_tid=20, entry_tid=30359' 2026-02-18T08:19:51.955 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=295, tag_tid=20, entry_tid=30359' '!=' 'object_number=294, tag_tid=20, entry_tid=30026' 2026-02-18T08:19:51.955 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:19:51.955 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=295, tag_tid=20, entry_tid=30359' = 'object_number=294, tag_tid=20, entry_tid=30026' 2026-02-18T08:19:51.955 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=295, tag_tid=20, entry_tid=30359' 2026-02-18T08:19:51.956 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=309, tag_tid=21, entry_tid=1' 2026-02-18T08:19:51.956 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:19:51.956 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:19:51.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=21 2026-02-18T08:19:51.958 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=295, tag_tid=20, entry_tid=30359' 2026-02-18T08:19:51.958 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:19:51.958 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:19:51.959 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=20 2026-02-18T08:19:51.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=309, tag_tid=21, entry_tid=1' 2026-02-18T08:19:51.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:19:51.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:19:51.961 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:19:51.962 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=295, tag_tid=20, entry_tid=30359' 2026-02-18T08:19:51.962 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:19:51.962 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:19:51.964 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=30359 2026-02-18T08:19:51.964 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 21 = 20 -a 1 -le 30359 2026-02-18T08:19:51.964 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:19:51.964 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:19:51.964 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:19:52.165 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:19:52.166 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:19:52.166 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:19:52.166 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:19:52.166 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:19:52.166 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:19:52.166 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:19:52.166 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:19:52.166 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:19:52.166 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:19:52.166 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:19:52.167 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:19:52.167 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:19:52.167 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:19:52.167 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:19:52.169 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:19:52.170 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:19:52.170 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:19:52.170 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:19:52.170 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:19:52.170 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:19:52.170 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:19:52.170 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:19:52.170 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:19:52.170 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:19:52.170 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:19:52.170 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:19:52.171 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:19:52.171 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:19:52.171 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:19:52.171 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:19:52.171 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:19:52.171 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:19:52.171 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:19:52.171 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:19:52.171 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:19:52.171 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:19:52.171 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:19:52.171 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:19:52.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:19:52.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:19:52.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:19:52.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:19:52.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:19:52.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:19:52.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:19:52.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:19:52.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:19:52.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:19:52.179 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:19:52.179 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:19:52.179 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:19:52.180 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:19:52.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:19:52.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:19:52.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:19:52.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:19:52.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:19:52.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:19:52.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:19:52.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:19:52.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:19:52.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:19:52.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:19:52.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:19:52.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:19:52.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:19:52.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:19:52.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:19:52.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:19:52.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:19:52.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:19:52.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:19:52.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:19:52.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:19:52.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:19:52.244 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:19:52.244 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:19:52.244 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:19:52.244 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:19:52.244 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:19:52.244 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:19:52.244 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:19:52.244 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:19:52.244 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:19:52.244 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:19:52.244 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:19:52.244 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:19:52.253 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:19:52.253 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:19:52.253 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:19:52.253 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:19:52.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:19:52.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:19:52.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:19:52.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:19:52.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:19:52.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:19:52.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:19:52.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:19:52.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:19:52.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:19:52.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:19:52.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:19:52.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:19:52.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:19:52.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:19:52.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:19:52.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:19:52.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:19:52.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:19:52.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:19:52.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:19:52.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:19:52.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:19:52.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:19:52.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:19:52.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:19:52.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:19:52.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:19:52.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:19:52.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:19:52.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:19:52.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:19:52.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:19:52.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:19:52.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:19:52.317 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:19:52.318 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:19:52.318 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:19:52.318 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:19:52.367 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:19:52.367 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:19:52.367 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:19:52.370 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:19:52.370 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:19:52.370 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:19:52.370 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:19:52.370 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:19:52.370 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:19:52.371 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:19:52.371 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:19:52.371 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:19:52.371 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:19:52.371 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:19:52.371 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:19:52.371 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:19:52.371 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:19:52.371 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:19:52.371 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:19:52.372 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:19:52.372 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:19:52.372 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:19:52.372 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:19:52.372 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:19:52.372 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:19:52.372 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:19:52.372 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:19:52.372 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:19:52.372 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:19:52.373 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:19:52.373 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:19:52.373 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:19:52.373 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:19:52.373 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:19:52.373 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:19:52.381 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:19:52.382 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:19:52.382 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:19:52.382 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:19:52.451 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:19:52.452 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:19:52.452 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:19:52.452 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:19:52.452 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:19:52.452 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:19:52.452 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:19:52.453 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:19:52.453 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:19:52.453 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:19:52.453 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:19:52.453 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:19:52.453 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:19:52.454 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:19:52.454 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:19:52.454 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:19:52.484 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 73 2026-02-18T08:19:52.484 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 77 2026-02-18T08:19:52.485 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:19:52.485 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=309, tag_tid=21, entry_tid=1], [object_number=308, tag_tid=21, entry_tid=0], [object_number=310, tag_tid=20, entry_tid=46754], [object_number=311, tag_tid=20, entry_tid=46751]]], state=connected] 2026-02-18T08:19:52.485 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=293, tag_tid=20, entry_tid=30713], [object_number=292, tag_tid=20, entry_tid=30712], [object_number=295, tag_tid=20, entry_tid=30711], [object_number=294, tag_tid=20, entry_tid=30710]]], state=connected] 2026-02-18T08:19:52.487 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:19:52.489 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=309, tag_tid=21, entry_tid=1' 2026-02-18T08:19:52.490 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:19:52.490 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:19:52.490 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:19:52.490 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:19:52.491 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:19:52.491 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:19:52.491 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:19:52.491 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:19:52.491 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:19:52.491 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:19:52.491 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:19:52.491 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:19:52.492 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:19:52.492 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:19:52.492 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:19:52.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 73 2026-02-18T08:19:52.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 77 2026-02-18T08:19:52.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:19:52.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=309, tag_tid=21, entry_tid=1], [object_number=308, tag_tid=21, entry_tid=0], [object_number=310, tag_tid=20, entry_tid=46754], [object_number=311, tag_tid=20, entry_tid=46751]]], state=connected] 2026-02-18T08:19:52.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=293, tag_tid=20, entry_tid=30713], [object_number=292, tag_tid=20, entry_tid=30712], [object_number=295, tag_tid=20, entry_tid=30711], [object_number=294, tag_tid=20, entry_tid=30710]]], state=connected] 2026-02-18T08:19:52.524 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:19:52.526 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=293, tag_tid=20, entry_tid=30713' 2026-02-18T08:19:52.526 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=309, tag_tid=21, entry_tid=1' -a 'object_number=309, tag_tid=21, entry_tid=1' = 'object_number=293, tag_tid=20, entry_tid=30713' 2026-02-18T08:19:52.526 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=293, tag_tid=20, entry_tid=30713' '!=' 'object_number=295, tag_tid=20, entry_tid=30359' 2026-02-18T08:19:52.526 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:19:52.527 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=293, tag_tid=20, entry_tid=30713' = 'object_number=295, tag_tid=20, entry_tid=30359' 2026-02-18T08:19:52.527 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=293, tag_tid=20, entry_tid=30713' 2026-02-18T08:19:52.527 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=309, tag_tid=21, entry_tid=1' 2026-02-18T08:19:52.527 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:19:52.528 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:19:52.528 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=21 2026-02-18T08:19:52.529 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=293, tag_tid=20, entry_tid=30713' 2026-02-18T08:19:52.529 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:19:52.529 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:19:52.530 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=20 2026-02-18T08:19:52.531 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=309, tag_tid=21, entry_tid=1' 2026-02-18T08:19:52.531 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:19:52.531 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:19:52.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:19:52.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=293, tag_tid=20, entry_tid=30713' 2026-02-18T08:19:52.534 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:19:52.534 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:19:52.535 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=30713 2026-02-18T08:19:52.535 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 21 = 20 -a 1 -le 30713 2026-02-18T08:19:52.535 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:19:52.536 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:19:52.536 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:19:52.736 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:19:52.736 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:19:52.736 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:19:52.736 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:19:52.736 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:19:52.736 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:19:52.736 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:19:52.736 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:19:52.737 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:19:52.737 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:19:52.737 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:19:52.737 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:19:52.737 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:19:52.737 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:19:52.737 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:19:52.737 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:19:52.738 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:19:52.740 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:19:52.740 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:19:52.741 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:19:52.741 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:19:52.741 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:19:52.741 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:19:52.741 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:19:52.741 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:19:52.741 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:19:52.741 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:19:52.741 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:19:52.741 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:19:52.741 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:19:52.742 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:19:52.742 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:19:52.742 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:19:52.742 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:19:52.742 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:19:52.742 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:19:52.742 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:19:52.742 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:19:52.742 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:19:52.742 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:19:52.742 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:19:52.742 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:19:52.743 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:19:52.743 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:19:52.743 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:19:52.743 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:19:52.743 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:19:52.743 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:19:52.743 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:19:52.750 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:19:52.751 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:19:52.751 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:19:52.751 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:19:52.811 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:19:52.811 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:19:52.811 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:19:52.814 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:19:52.814 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:19:52.814 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:19:52.814 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:19:52.814 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:19:52.814 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:19:52.815 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:19:52.815 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:19:52.815 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:19:52.815 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:19:52.815 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:19:52.815 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:19:52.815 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:19:52.815 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:19:52.815 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:19:52.815 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:19:52.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:19:52.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:19:52.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:19:52.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:19:52.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:19:52.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:19:52.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:19:52.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:19:52.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:19:52.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:19:52.817 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:19:52.817 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:19:52.817 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:19:52.817 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:19:52.817 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:19:52.817 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:19:52.825 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:19:52.825 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:19:52.826 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:19:52.826 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:19:52.875 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:19:52.875 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:19:52.876 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:19:52.879 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:19:52.879 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:19:52.879 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:19:52.879 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:19:52.879 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:19:52.879 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:19:52.879 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:19:52.879 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:19:52.880 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:19:52.880 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:19:52.880 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:19:52.880 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:19:52.880 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:19:52.880 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:19:52.880 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:19:52.880 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:19:52.880 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:19:52.880 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:19:52.881 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:19:52.881 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:19:52.881 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:19:52.881 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:19:52.881 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:19:52.881 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:19:52.881 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:19:52.881 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:19:52.881 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:19:52.881 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:19:52.882 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:19:52.882 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:19:52.882 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:19:52.882 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:19:52.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:19:52.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:19:52.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:19:52.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:19:52.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:19:52.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:19:52.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:19:52.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:19:52.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:19:52.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:19:52.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:19:52.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:19:52.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:19:52.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:19:52.944 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:19:52.944 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:19:52.944 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:19:52.944 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:19:52.944 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:19:52.944 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:19:52.944 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:19:52.944 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:19:52.944 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:19:52.944 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:19:52.944 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:19:52.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:19:52.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:19:52.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:19:52.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:19:52.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:19:52.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:19:52.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:19:52.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:19:52.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:19:52.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:19:52.946 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:19:52.946 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:19:52.946 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:19:52.946 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:19:52.953 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:19:52.954 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:19:52.954 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:19:52.954 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:19:53.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:19:53.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:19:53.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:19:53.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:19:53.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:19:53.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:19:53.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:19:53.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:19:53.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:19:53.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:19:53.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:19:53.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:19:53.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:19:53.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:19:53.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:19:53.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:19:53.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 73 2026-02-18T08:19:53.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 77 2026-02-18T08:19:53.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:19:53.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=309, tag_tid=21, entry_tid=1], [object_number=308, tag_tid=21, entry_tid=0], [object_number=310, tag_tid=20, entry_tid=46754], [object_number=311, tag_tid=20, entry_tid=46751]]], state=connected] 2026-02-18T08:19:53.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=292, tag_tid=20, entry_tid=31072], [object_number=295, tag_tid=20, entry_tid=31071], [object_number=294, tag_tid=20, entry_tid=31070], [object_number=293, tag_tid=20, entry_tid=31069]]], state=connected] 2026-02-18T08:19:53.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:19:53.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=309, tag_tid=21, entry_tid=1' 2026-02-18T08:19:53.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:19:53.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:19:53.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:19:53.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:19:53.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:19:53.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:19:53.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:19:53.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:19:53.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:19:53.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:19:53.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:19:53.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:19:53.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:19:53.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:19:53.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:19:53.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 73 2026-02-18T08:19:53.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 77 2026-02-18T08:19:53.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:19:53.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=309, tag_tid=21, entry_tid=1], [object_number=308, tag_tid=21, entry_tid=0], [object_number=310, tag_tid=20, entry_tid=46754], [object_number=311, tag_tid=20, entry_tid=46751]]], state=connected] 2026-02-18T08:19:53.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=292, tag_tid=20, entry_tid=31072], [object_number=295, tag_tid=20, entry_tid=31071], [object_number=294, tag_tid=20, entry_tid=31070], [object_number=293, tag_tid=20, entry_tid=31069]]], state=connected] 2026-02-18T08:19:53.093 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:19:53.095 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=292, tag_tid=20, entry_tid=31072' 2026-02-18T08:19:53.095 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=309, tag_tid=21, entry_tid=1' -a 'object_number=309, tag_tid=21, entry_tid=1' = 'object_number=292, tag_tid=20, entry_tid=31072' 2026-02-18T08:19:53.095 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=292, tag_tid=20, entry_tid=31072' '!=' 'object_number=293, tag_tid=20, entry_tid=30713' 2026-02-18T08:19:53.095 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:19:53.095 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=292, tag_tid=20, entry_tid=31072' = 'object_number=293, tag_tid=20, entry_tid=30713' 2026-02-18T08:19:53.095 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=292, tag_tid=20, entry_tid=31072' 2026-02-18T08:19:53.096 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=309, tag_tid=21, entry_tid=1' 2026-02-18T08:19:53.096 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:19:53.096 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:19:53.097 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=21 2026-02-18T08:19:53.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=292, tag_tid=20, entry_tid=31072' 2026-02-18T08:19:53.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:19:53.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:19:53.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=20 2026-02-18T08:19:53.100 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=309, tag_tid=21, entry_tid=1' 2026-02-18T08:19:53.100 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:19:53.100 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:19:53.101 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:19:53.102 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=292, tag_tid=20, entry_tid=31072' 2026-02-18T08:19:53.102 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:19:53.102 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:19:53.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=31072 2026-02-18T08:19:53.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 21 = 20 -a 1 -le 31072 2026-02-18T08:19:53.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:19:53.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:19:53.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:19:53.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:19:53.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:19:53.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:19:53.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:19:53.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:19:53.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:19:53.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:19:53.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:19:53.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:19:53.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:19:53.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:19:53.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:19:53.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:19:53.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:19:53.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:19:53.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:19:53.310 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:19:53.310 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:19:53.310 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:19:53.310 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:19:53.310 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:19:53.310 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:19:53.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:19:53.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:19:53.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:19:53.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:19:53.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:19:53.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:19:53.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:19:53.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:19:53.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:19:53.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:19:53.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:19:53.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:19:53.312 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:19:53.312 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:19:53.312 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:19:53.312 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:19:53.312 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:19:53.312 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:19:53.312 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:19:53.312 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:19:53.312 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:19:53.312 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:19:53.312 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:19:53.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:19:53.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:19:53.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:19:53.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:19:53.320 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:19:53.321 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:19:53.321 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:19:53.321 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:19:53.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:19:53.370 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:19:53.370 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:19:53.372 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:19:53.372 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:19:53.373 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:19:53.373 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:19:53.373 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:19:53.373 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:19:53.373 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:19:53.373 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:19:53.373 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:19:53.373 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:19:53.373 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:19:53.373 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:19:53.373 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:19:53.374 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:19:53.374 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:19:53.374 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:19:53.374 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:19:53.374 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:19:53.374 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:19:53.374 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:19:53.374 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:19:53.374 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:19:53.374 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:19:53.374 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:19:53.374 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:19:53.375 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:19:53.375 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:19:53.375 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:19:53.375 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:19:53.375 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:19:53.375 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:19:53.375 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:19:53.383 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:19:53.384 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:19:53.384 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:19:53.384 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:19:53.433 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:19:53.433 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:19:53.433 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:19:53.436 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:19:53.436 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:19:53.436 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:19:53.437 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:19:53.437 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:19:53.437 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:19:53.437 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:19:53.437 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:19:53.437 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:19:53.437 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:19:53.437 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:19:53.437 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:19:53.437 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:19:53.437 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:19:53.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:19:53.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:19:53.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:19:53.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:19:53.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:19:53.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:19:53.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:19:53.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:19:53.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:19:53.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:19:53.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:19:53.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:19:53.439 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:19:53.439 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:19:53.439 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:19:53.439 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:19:53.439 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:19:53.439 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:19:53.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:19:53.448 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:19:53.448 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:19:53.448 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:19:53.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:19:53.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:19:53.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:19:53.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:19:53.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:19:53.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:19:53.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:19:53.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:19:53.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:19:53.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:19:53.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:19:53.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:19:53.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:19:53.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:19:53.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:19:53.500 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:19:53.500 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:19:53.500 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:19:53.500 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:19:53.500 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:19:53.500 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:19:53.500 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:19:53.500 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:19:53.500 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:19:53.500 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:19:53.500 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:19:53.500 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:19:53.500 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:19:53.501 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:19:53.501 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:19:53.501 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:19:53.501 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:19:53.501 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:19:53.501 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:19:53.501 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:19:53.510 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:19:53.510 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:19:53.511 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:19:53.511 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:19:53.569 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:19:53.571 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:19:53.571 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:19:53.571 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:19:53.571 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:19:53.571 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:19:53.571 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:19:53.571 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:19:53.571 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:19:53.572 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:19:53.572 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:19:53.572 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:19:53.572 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:19:53.573 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:19:53.573 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:19:53.573 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:19:53.602 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 73 2026-02-18T08:19:53.602 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 77 2026-02-18T08:19:53.602 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:19:53.602 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=309, tag_tid=21, entry_tid=1], [object_number=308, tag_tid=21, entry_tid=0], [object_number=310, tag_tid=20, entry_tid=46754], [object_number=311, tag_tid=20, entry_tid=46751]]], state=connected] 2026-02-18T08:19:53.602 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=294, tag_tid=20, entry_tid=31438], [object_number=293, tag_tid=20, entry_tid=31437], [object_number=292, tag_tid=20, entry_tid=31436], [object_number=295, tag_tid=20, entry_tid=31435]]], state=connected] 2026-02-18T08:19:53.604 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:19:53.606 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=309, tag_tid=21, entry_tid=1' 2026-02-18T08:19:53.607 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:19:53.607 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:19:53.607 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:19:53.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:19:53.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:19:53.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:19:53.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:19:53.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:19:53.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:19:53.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:19:53.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:19:53.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:19:53.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:19:53.610 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:19:53.610 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:19:53.637 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 73 2026-02-18T08:19:53.637 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 77 2026-02-18T08:19:53.637 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:19:53.637 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=309, tag_tid=21, entry_tid=1], [object_number=308, tag_tid=21, entry_tid=0], [object_number=310, tag_tid=20, entry_tid=46754], [object_number=311, tag_tid=20, entry_tid=46751]]], state=connected] 2026-02-18T08:19:53.638 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=294, tag_tid=20, entry_tid=31438], [object_number=293, tag_tid=20, entry_tid=31437], [object_number=292, tag_tid=20, entry_tid=31436], [object_number=295, tag_tid=20, entry_tid=31435]]], state=connected] 2026-02-18T08:19:53.640 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:19:53.642 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=294, tag_tid=20, entry_tid=31438' 2026-02-18T08:19:53.642 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=309, tag_tid=21, entry_tid=1' -a 'object_number=309, tag_tid=21, entry_tid=1' = 'object_number=294, tag_tid=20, entry_tid=31438' 2026-02-18T08:19:53.642 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=294, tag_tid=20, entry_tid=31438' '!=' 'object_number=292, tag_tid=20, entry_tid=31072' 2026-02-18T08:19:53.642 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:19:53.642 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=294, tag_tid=20, entry_tid=31438' = 'object_number=292, tag_tid=20, entry_tid=31072' 2026-02-18T08:19:53.643 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=294, tag_tid=20, entry_tid=31438' 2026-02-18T08:19:53.643 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=309, tag_tid=21, entry_tid=1' 2026-02-18T08:19:53.643 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:19:53.643 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:19:53.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=21 2026-02-18T08:19:53.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=294, tag_tid=20, entry_tid=31438' 2026-02-18T08:19:53.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:19:53.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:19:53.646 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=20 2026-02-18T08:19:53.647 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=309, tag_tid=21, entry_tid=1' 2026-02-18T08:19:53.647 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:19:53.647 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:19:53.649 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:19:53.649 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=294, tag_tid=20, entry_tid=31438' 2026-02-18T08:19:53.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:19:53.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:19:53.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=31438 2026-02-18T08:19:53.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 21 = 20 -a 1 -le 31438 2026-02-18T08:19:53.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:19:53.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:19:53.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:19:53.853 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:19:53.853 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:19:53.854 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:19:53.854 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:19:53.854 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:19:53.854 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:19:53.854 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:19:53.854 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:19:53.854 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:19:53.854 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:19:53.854 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:19:53.854 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:19:53.855 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:19:53.855 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:19:53.855 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:19:53.855 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:19:53.857 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:19:53.858 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:19:53.858 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:19:53.858 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:19:53.858 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:19:53.858 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:19:53.858 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:19:53.858 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:19:53.858 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:19:53.859 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:19:53.859 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:19:53.859 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:19:53.859 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:19:53.859 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:19:53.859 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:19:53.859 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:19:53.859 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:19:53.859 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:19:53.859 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:19:53.859 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:19:53.859 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:19:53.860 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:19:53.860 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:19:53.860 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:19:53.860 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:19:53.860 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:19:53.860 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:19:53.860 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:19:53.860 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:19:53.860 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:19:53.860 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:19:53.860 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:19:53.860 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:19:53.867 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:19:53.868 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:19:53.868 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:19:53.868 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:19:53.916 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:19:53.916 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:19:53.916 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:19:53.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:19:53.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:19:53.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:19:53.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:19:53.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:19:53.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:19:53.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:19:53.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:19:53.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:19:53.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:19:53.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:19:53.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:19:53.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:19:53.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:19:53.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:19:53.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:19:53.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:19:53.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:19:53.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:19:53.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:19:53.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:19:53.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:19:53.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:19:53.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:19:53.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:19:53.922 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:19:53.922 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:19:53.922 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:19:53.922 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:19:53.922 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:19:53.922 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:19:53.922 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:19:53.930 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:19:53.930 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:19:53.931 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:19:53.931 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:19:53.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:19:53.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:19:53.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:19:53.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:19:53.982 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:19:53.982 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:19:53.982 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:19:53.982 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:19:53.982 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:19:53.982 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:19:53.982 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:19:53.982 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:19:53.982 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:19:53.982 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:19:53.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:19:53.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:19:53.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:19:53.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:19:53.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:19:53.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:19:53.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:19:53.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:19:53.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:19:53.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:19:53.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:19:53.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:19:53.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:19:53.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:19:53.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:19:53.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:19:53.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:19:53.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:19:53.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:19:53.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:19:53.985 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:19:53.993 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:19:53.994 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:19:53.994 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:19:53.994 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:19:54.041 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:19:54.041 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:19:54.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:19:54.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:19:54.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:19:54.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:19:54.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:19:54.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:19:54.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:19:54.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:19:54.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:19:54.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:19:54.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:19:54.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:19:54.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:19:54.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:19:54.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:19:54.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:19:54.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:19:54.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:19:54.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:19:54.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:19:54.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:19:54.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:19:54.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:19:54.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:19:54.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:19:54.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:19:54.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:19:54.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:19:54.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:19:54.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:19:54.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:19:54.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:19:54.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:19:54.055 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:19:54.055 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:19:54.055 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:19:54.055 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:19:54.122 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:19:54.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:19:54.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:19:54.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:19:54.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:19:54.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:19:54.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:19:54.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:19:54.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:19:54.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:19:54.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:19:54.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:19:54.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:19:54.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:19:54.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:19:54.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:19:54.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 73 2026-02-18T08:19:54.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 77 2026-02-18T08:19:54.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:19:54.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=309, tag_tid=21, entry_tid=1], [object_number=308, tag_tid=21, entry_tid=0], [object_number=310, tag_tid=20, entry_tid=46754], [object_number=311, tag_tid=20, entry_tid=46751]]], state=connected] 2026-02-18T08:19:54.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=293, tag_tid=20, entry_tid=31765], [object_number=292, tag_tid=20, entry_tid=31764], [object_number=295, tag_tid=20, entry_tid=31763], [object_number=294, tag_tid=20, entry_tid=31762]]], state=connected] 2026-02-18T08:19:54.156 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:19:54.158 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=309, tag_tid=21, entry_tid=1' 2026-02-18T08:19:54.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:19:54.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:19:54.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:19:54.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:19:54.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:19:54.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:19:54.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:19:54.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:19:54.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:19:54.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:19:54.161 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:19:54.161 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:19:54.161 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:19:54.161 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:19:54.162 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:19:54.189 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 73 2026-02-18T08:19:54.189 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 77 2026-02-18T08:19:54.189 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:19:54.189 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=309, tag_tid=21, entry_tid=1], [object_number=308, tag_tid=21, entry_tid=0], [object_number=310, tag_tid=20, entry_tid=46754], [object_number=311, tag_tid=20, entry_tid=46751]]], state=connected] 2026-02-18T08:19:54.189 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=293, tag_tid=20, entry_tid=31765], [object_number=292, tag_tid=20, entry_tid=31764], [object_number=295, tag_tid=20, entry_tid=31763], [object_number=294, tag_tid=20, entry_tid=31762]]], state=connected] 2026-02-18T08:19:54.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:19:54.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=293, tag_tid=20, entry_tid=31765' 2026-02-18T08:19:54.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=309, tag_tid=21, entry_tid=1' -a 'object_number=309, tag_tid=21, entry_tid=1' = 'object_number=293, tag_tid=20, entry_tid=31765' 2026-02-18T08:19:54.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=293, tag_tid=20, entry_tid=31765' '!=' 'object_number=294, tag_tid=20, entry_tid=31438' 2026-02-18T08:19:54.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:19:54.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=293, tag_tid=20, entry_tid=31765' = 'object_number=294, tag_tid=20, entry_tid=31438' 2026-02-18T08:19:54.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=293, tag_tid=20, entry_tid=31765' 2026-02-18T08:19:54.194 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=309, tag_tid=21, entry_tid=1' 2026-02-18T08:19:54.194 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:19:54.194 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:19:54.195 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=21 2026-02-18T08:19:54.196 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=293, tag_tid=20, entry_tid=31765' 2026-02-18T08:19:54.196 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:19:54.196 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:19:54.198 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=20 2026-02-18T08:19:54.198 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=309, tag_tid=21, entry_tid=1' 2026-02-18T08:19:54.199 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:19:54.199 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:19:54.200 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:19:54.201 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=293, tag_tid=20, entry_tid=31765' 2026-02-18T08:19:54.201 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:19:54.201 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:19:54.202 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=31765 2026-02-18T08:19:54.203 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 21 = 20 -a 1 -le 31765 2026-02-18T08:19:54.203 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:19:54.203 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:19:54.203 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:19:54.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:19:54.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:19:54.405 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:19:54.405 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:19:54.405 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:19:54.405 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:19:54.405 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:19:54.405 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:19:54.405 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:19:54.405 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:19:54.405 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:19:54.406 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:19:54.406 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:19:54.406 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:19:54.406 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:19:54.406 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:19:54.408 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:19:54.409 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:19:54.409 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:19:54.409 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:19:54.409 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:19:54.409 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:19:54.409 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:19:54.409 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:19:54.409 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:19:54.409 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:19:54.409 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:19:54.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:19:54.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:19:54.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:19:54.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:19:54.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:19:54.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:19:54.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:19:54.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:19:54.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:19:54.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:19:54.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:19:54.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:19:54.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:19:54.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:19:54.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:19:54.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:19:54.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:19:54.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:19:54.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:19:54.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:19:54.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:19:54.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:19:54.418 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:19:54.419 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:19:54.419 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:19:54.419 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:19:54.477 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:19:54.477 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:19:54.477 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:19:54.480 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:19:54.480 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:19:54.480 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:19:54.481 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:19:54.481 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:19:54.481 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:19:54.481 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:19:54.481 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:19:54.481 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:19:54.481 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:19:54.481 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:19:54.481 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:19:54.481 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:19:54.481 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:19:54.481 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:19:54.482 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:19:54.482 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:19:54.482 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:19:54.482 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:19:54.482 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:19:54.482 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:19:54.482 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:19:54.482 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:19:54.482 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:19:54.482 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:19:54.482 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:19:54.482 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:19:54.483 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:19:54.483 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:19:54.483 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:19:54.483 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:19:54.483 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:19:54.491 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:19:54.491 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:19:54.491 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:19:54.491 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:19:54.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:19:54.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:19:54.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:19:54.545 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:19:54.545 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:19:54.545 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:19:54.545 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:19:54.545 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:19:54.545 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:19:54.545 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:19:54.545 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:19:54.545 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:19:54.545 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:19:54.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:19:54.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:19:54.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:19:54.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:19:54.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:19:54.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:19:54.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:19:54.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:19:54.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:19:54.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:19:54.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:19:54.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:19:54.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:19:54.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:19:54.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:19:54.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:19:54.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:19:54.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:19:54.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:19:54.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:19:54.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:19:54.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:19:54.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:19:54.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:19:54.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:19:54.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:19:54.604 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:19:54.604 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:19:54.604 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:19:54.607 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:19:54.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:19:54.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:19:54.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:19:54.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:19:54.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:19:54.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:19:54.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:19:54.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:19:54.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:19:54.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:19:54.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:19:54.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:19:54.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:19:54.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:19:54.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:19:54.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:19:54.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:19:54.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:19:54.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:19:54.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:19:54.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:19:54.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:19:54.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:19:54.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:19:54.610 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:19:54.610 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:19:54.610 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:19:54.610 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:19:54.610 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:19:54.610 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:19:54.610 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:19:54.618 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:19:54.619 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:19:54.619 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:19:54.619 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:19:54.678 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:19:54.680 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:19:54.680 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:19:54.680 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:19:54.680 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:19:54.680 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:19:54.680 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:19:54.680 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:19:54.680 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:19:54.680 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:19:54.680 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:19:54.680 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:19:54.681 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:19:54.681 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:19:54.681 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:19:54.682 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:19:54.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 73 2026-02-18T08:19:54.711 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 77 2026-02-18T08:19:54.711 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:19:54.711 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=309, tag_tid=21, entry_tid=1], [object_number=308, tag_tid=21, entry_tid=0], [object_number=310, tag_tid=20, entry_tid=46754], [object_number=311, tag_tid=20, entry_tid=46751]]], state=connected] 2026-02-18T08:19:54.711 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=292, tag_tid=20, entry_tid=32112], [object_number=295, tag_tid=20, entry_tid=32111], [object_number=294, tag_tid=20, entry_tid=32110], [object_number=293, tag_tid=20, entry_tid=32109]]], state=connected] 2026-02-18T08:19:54.713 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:19:54.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=309, tag_tid=21, entry_tid=1' 2026-02-18T08:19:54.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:19:54.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:19:54.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:19:54.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:19:54.717 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:19:54.717 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:19:54.717 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:19:54.717 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:19:54.717 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:19:54.717 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:19:54.717 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:19:54.717 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:19:54.718 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:19:54.718 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:19:54.718 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:19:54.746 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 73 2026-02-18T08:19:54.747 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 77 2026-02-18T08:19:54.747 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:19:54.747 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=309, tag_tid=21, entry_tid=1], [object_number=308, tag_tid=21, entry_tid=0], [object_number=310, tag_tid=20, entry_tid=46754], [object_number=311, tag_tid=20, entry_tid=46751]]], state=connected] 2026-02-18T08:19:54.747 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=292, tag_tid=20, entry_tid=32112], [object_number=295, tag_tid=20, entry_tid=32111], [object_number=294, tag_tid=20, entry_tid=32110], [object_number=293, tag_tid=20, entry_tid=32109]]], state=connected] 2026-02-18T08:19:54.749 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:19:54.751 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=292, tag_tid=20, entry_tid=32112' 2026-02-18T08:19:54.751 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=309, tag_tid=21, entry_tid=1' -a 'object_number=309, tag_tid=21, entry_tid=1' = 'object_number=292, tag_tid=20, entry_tid=32112' 2026-02-18T08:19:54.752 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=292, tag_tid=20, entry_tid=32112' '!=' 'object_number=293, tag_tid=20, entry_tid=31765' 2026-02-18T08:19:54.752 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:19:54.752 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=292, tag_tid=20, entry_tid=32112' = 'object_number=293, tag_tid=20, entry_tid=31765' 2026-02-18T08:19:54.752 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=292, tag_tid=20, entry_tid=32112' 2026-02-18T08:19:54.753 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=309, tag_tid=21, entry_tid=1' 2026-02-18T08:19:54.753 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:19:54.753 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:19:54.754 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=21 2026-02-18T08:19:54.754 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=292, tag_tid=20, entry_tid=32112' 2026-02-18T08:19:54.754 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:19:54.754 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:19:54.755 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=20 2026-02-18T08:19:54.756 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=309, tag_tid=21, entry_tid=1' 2026-02-18T08:19:54.756 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:19:54.756 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:19:54.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:19:54.758 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=292, tag_tid=20, entry_tid=32112' 2026-02-18T08:19:54.758 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:19:54.759 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:19:54.760 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=32112 2026-02-18T08:19:54.760 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 21 = 20 -a 1 -le 32112 2026-02-18T08:19:54.760 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:19:54.760 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:19:54.760 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:19:54.962 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:19:54.962 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:19:54.962 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:19:54.962 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:19:54.962 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:19:54.962 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:19:54.963 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:19:54.963 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:19:54.963 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:19:54.963 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:19:54.963 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:19:54.963 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:19:54.963 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:19:54.963 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:19:54.963 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:19:54.963 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:19:54.966 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:19:54.966 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:19:54.966 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:19:54.966 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:19:54.967 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:19:54.967 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:19:54.967 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:19:54.967 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:19:54.967 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:19:54.967 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:19:54.967 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:19:54.967 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:19:54.967 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:19:54.967 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:19:54.967 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:19:54.968 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:19:54.968 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:19:54.968 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:19:54.968 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:19:54.968 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:19:54.968 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:19:54.968 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:19:54.968 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:19:54.968 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:19:54.968 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:19:54.968 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:19:54.968 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:19:54.969 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:19:54.969 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:19:54.969 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:19:54.969 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:19:54.969 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:19:54.969 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:19:54.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:19:54.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:19:54.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:19:54.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:19:55.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:19:55.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:19:55.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:19:55.027 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:19:55.028 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:19:55.028 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:19:55.028 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:19:55.028 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:19:55.028 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:19:55.028 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:19:55.028 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:19:55.028 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:19:55.028 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:19:55.028 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:19:55.029 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:19:55.029 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:19:55.029 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:19:55.029 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:19:55.029 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:19:55.029 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:19:55.029 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:19:55.029 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:19:55.029 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:19:55.029 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:19:55.029 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:19:55.029 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:19:55.029 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:19:55.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:19:55.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:19:55.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:19:55.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:19:55.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:19:55.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:19:55.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:19:55.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:19:55.039 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:19:55.039 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:19:55.039 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:19:55.039 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:19:55.086 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:19:55.086 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:19:55.086 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:19:55.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:19:55.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:19:55.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:19:55.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:19:55.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:19:55.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:19:55.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:19:55.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:19:55.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:19:55.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:19:55.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:19:55.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:19:55.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:19:55.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:19:55.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:19:55.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:19:55.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:19:55.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:19:55.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:19:55.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:19:55.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:19:55.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:19:55.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:19:55.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:19:55.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:19:55.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:19:55.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:19:55.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:19:55.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:19:55.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:19:55.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:19:55.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:19:55.100 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:19:55.101 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:19:55.101 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:19:55.101 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:19:55.147 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:19:55.148 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:19:55.148 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:19:55.150 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:19:55.150 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:19:55.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:19:55.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:19:55.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:19:55.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:19:55.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:19:55.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:19:55.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:19:55.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:19:55.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:19:55.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:19:55.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:19:55.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:19:55.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:19:55.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:19:55.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:19:55.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:19:55.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:19:55.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:19:55.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:19:55.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:19:55.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:19:55.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:19:55.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:19:55.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:19:55.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:19:55.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:19:55.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:19:55.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:19:55.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:19:55.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:19:55.162 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:19:55.162 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:19:55.162 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:19:55.162 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:19:55.220 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:19:55.221 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:19:55.222 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:19:55.222 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:19:55.222 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:19:55.222 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:19:55.222 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:19:55.222 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:19:55.222 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:19:55.222 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:19:55.222 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:19:55.222 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:19:55.223 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:19:55.223 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:19:55.224 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:19:55.224 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:19:55.260 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 73 2026-02-18T08:19:55.261 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 77 2026-02-18T08:19:55.261 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:19:55.261 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=309, tag_tid=21, entry_tid=1], [object_number=308, tag_tid=21, entry_tid=0], [object_number=310, tag_tid=20, entry_tid=46754], [object_number=311, tag_tid=20, entry_tid=46751]]], state=connected] 2026-02-18T08:19:55.261 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=295, tag_tid=20, entry_tid=32443], [object_number=294, tag_tid=20, entry_tid=32442], [object_number=293, tag_tid=20, entry_tid=32441], [object_number=292, tag_tid=20, entry_tid=32440]]], state=connected] 2026-02-18T08:19:55.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:19:55.266 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=309, tag_tid=21, entry_tid=1' 2026-02-18T08:19:55.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:19:55.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:19:55.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:19:55.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:19:55.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:19:55.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:19:55.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:19:55.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:19:55.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:19:55.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:19:55.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:19:55.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:19:55.269 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:19:55.269 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:19:55.269 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:19:55.297 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 73 2026-02-18T08:19:55.297 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 77 2026-02-18T08:19:55.297 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:19:55.297 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=309, tag_tid=21, entry_tid=1], [object_number=308, tag_tid=21, entry_tid=0], [object_number=310, tag_tid=20, entry_tid=46754], [object_number=311, tag_tid=20, entry_tid=46751]]], state=connected] 2026-02-18T08:19:55.297 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=295, tag_tid=20, entry_tid=32443], [object_number=294, tag_tid=20, entry_tid=32442], [object_number=293, tag_tid=20, entry_tid=32441], [object_number=292, tag_tid=20, entry_tid=32440]]], state=connected] 2026-02-18T08:19:55.300 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:19:55.301 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=295, tag_tid=20, entry_tid=32443' 2026-02-18T08:19:55.301 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=309, tag_tid=21, entry_tid=1' -a 'object_number=309, tag_tid=21, entry_tid=1' = 'object_number=295, tag_tid=20, entry_tid=32443' 2026-02-18T08:19:55.302 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=295, tag_tid=20, entry_tid=32443' '!=' 'object_number=292, tag_tid=20, entry_tid=32112' 2026-02-18T08:19:55.302 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:19:55.302 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=295, tag_tid=20, entry_tid=32443' = 'object_number=292, tag_tid=20, entry_tid=32112' 2026-02-18T08:19:55.302 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=295, tag_tid=20, entry_tid=32443' 2026-02-18T08:19:55.302 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=309, tag_tid=21, entry_tid=1' 2026-02-18T08:19:55.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:19:55.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:19:55.304 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=21 2026-02-18T08:19:55.304 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=295, tag_tid=20, entry_tid=32443' 2026-02-18T08:19:55.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:19:55.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:19:55.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=20 2026-02-18T08:19:55.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=309, tag_tid=21, entry_tid=1' 2026-02-18T08:19:55.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:19:55.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:19:55.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:19:55.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=295, tag_tid=20, entry_tid=32443' 2026-02-18T08:19:55.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:19:55.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:19:55.310 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=32443 2026-02-18T08:19:55.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 21 = 20 -a 1 -le 32443 2026-02-18T08:19:55.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:19:55.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:19:55.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:19:55.512 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:19:55.513 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:19:55.513 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:19:55.513 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:19:55.513 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:19:55.513 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:19:55.513 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:19:55.514 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:19:55.514 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:19:55.514 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:19:55.514 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:19:55.514 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:19:55.514 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:19:55.514 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:19:55.514 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:19:55.514 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:19:55.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:19:55.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:19:55.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:19:55.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:19:55.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:19:55.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:19:55.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:19:55.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:19:55.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:19:55.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:19:55.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:19:55.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:19:55.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:19:55.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:19:55.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:19:55.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:19:55.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:19:55.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:19:55.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:19:55.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:19:55.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:19:55.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:19:55.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:19:55.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:19:55.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:19:55.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:19:55.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:19:55.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:19:55.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:19:55.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:19:55.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:19:55.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:19:55.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:19:55.527 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:19:55.528 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:19:55.528 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:19:55.528 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:19:55.576 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:19:55.577 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:19:55.577 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:19:55.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:19:55.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:19:55.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:19:55.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:19:55.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:19:55.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:19:55.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:19:55.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:19:55.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:19:55.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:19:55.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:19:55.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:19:55.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:19:55.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:19:55.582 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:19:55.582 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:19:55.582 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:19:55.582 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:19:55.582 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:19:55.582 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:19:55.582 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:19:55.582 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:19:55.582 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:19:55.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:19:55.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:19:55.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:19:55.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:19:55.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:19:55.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:19:55.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:19:55.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:19:55.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:19:55.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:19:55.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:19:55.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:19:55.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:19:55.640 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:19:55.640 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:19:55.640 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:19:55.643 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:19:55.643 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:19:55.643 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:19:55.643 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:19:55.643 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:19:55.643 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:19:55.643 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:19:55.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:19:55.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:19:55.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:19:55.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:19:55.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:19:55.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:19:55.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:19:55.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:19:55.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:19:55.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:19:55.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:19:55.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:19:55.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:19:55.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:19:55.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:19:55.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:19:55.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:19:55.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:19:55.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:19:55.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:19:55.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:19:55.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:19:55.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:19:55.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:19:55.646 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:19:55.654 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:19:55.655 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:19:55.655 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:19:55.655 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:19:55.705 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:19:55.705 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:19:55.705 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:19:55.709 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:19:55.709 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:19:55.709 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:19:55.709 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:19:55.709 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:19:55.709 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:19:55.709 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:19:55.709 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:19:55.709 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:19:55.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:19:55.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:19:55.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:19:55.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:19:55.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:19:55.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:19:55.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:19:55.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:19:55.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:19:55.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:19:55.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:19:55.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:19:55.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:19:55.711 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:19:55.711 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:19:55.711 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:19:55.711 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:19:55.711 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:19:55.711 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:19:55.711 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:19:55.711 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:19:55.711 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:19:55.711 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:19:55.719 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:19:55.719 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:19:55.719 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:19:55.719 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:19:55.958 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:19:55.958 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:19:55.959 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:19:55.959 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:19:55.959 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:19:55.959 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:19:55.959 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:19:55.959 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:19:55.959 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:19:55.959 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:19:55.959 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:19:55.959 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:19:55.959 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:19:55.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:19:55.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:19:55.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:19:55.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 73 2026-02-18T08:19:55.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 77 2026-02-18T08:19:55.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:19:55.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=309, tag_tid=21, entry_tid=1], [object_number=308, tag_tid=21, entry_tid=0], [object_number=310, tag_tid=20, entry_tid=46754], [object_number=311, tag_tid=20, entry_tid=46751]]], state=connected] 2026-02-18T08:19:55.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=294, tag_tid=20, entry_tid=32758], [object_number=293, tag_tid=20, entry_tid=32757], [object_number=292, tag_tid=20, entry_tid=32756], [object_number=295, tag_tid=20, entry_tid=32755]]], state=connected] 2026-02-18T08:19:55.982 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:19:55.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=309, tag_tid=21, entry_tid=1' 2026-02-18T08:19:55.985 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:19:55.986 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:19:55.986 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:19:55.986 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:19:55.986 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:19:55.986 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:19:55.986 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:19:55.986 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:19:55.986 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:19:55.986 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:19:55.986 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:19:55.986 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:19:55.987 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:19:55.987 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:19:55.988 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:19:56.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 73 2026-02-18T08:19:56.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 77 2026-02-18T08:19:56.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:19:56.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=309, tag_tid=21, entry_tid=1], [object_number=308, tag_tid=21, entry_tid=0], [object_number=310, tag_tid=20, entry_tid=46754], [object_number=311, tag_tid=20, entry_tid=46751]]], state=connected] 2026-02-18T08:19:56.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=294, tag_tid=20, entry_tid=32758], [object_number=293, tag_tid=20, entry_tid=32757], [object_number=292, tag_tid=20, entry_tid=32756], [object_number=295, tag_tid=20, entry_tid=32755]]], state=connected] 2026-02-18T08:19:56.019 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:19:56.021 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=294, tag_tid=20, entry_tid=32758' 2026-02-18T08:19:56.021 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=309, tag_tid=21, entry_tid=1' -a 'object_number=309, tag_tid=21, entry_tid=1' = 'object_number=294, tag_tid=20, entry_tid=32758' 2026-02-18T08:19:56.021 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=294, tag_tid=20, entry_tid=32758' '!=' 'object_number=295, tag_tid=20, entry_tid=32443' 2026-02-18T08:19:56.021 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:19:56.021 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=294, tag_tid=20, entry_tid=32758' = 'object_number=295, tag_tid=20, entry_tid=32443' 2026-02-18T08:19:56.021 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=294, tag_tid=20, entry_tid=32758' 2026-02-18T08:19:56.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=309, tag_tid=21, entry_tid=1' 2026-02-18T08:19:56.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:19:56.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:19:56.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=21 2026-02-18T08:19:56.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=294, tag_tid=20, entry_tid=32758' 2026-02-18T08:19:56.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:19:56.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:19:56.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=20 2026-02-18T08:19:56.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=309, tag_tid=21, entry_tid=1' 2026-02-18T08:19:56.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:19:56.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:19:56.027 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:19:56.028 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=294, tag_tid=20, entry_tid=32758' 2026-02-18T08:19:56.028 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:19:56.029 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:19:56.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=32758 2026-02-18T08:19:56.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 21 = 20 -a 1 -le 32758 2026-02-18T08:19:56.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:19:56.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:19:56.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:19:56.231 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:19:56.231 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:19:56.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:19:56.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:19:56.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:19:56.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:19:56.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:19:56.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:19:56.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:19:56.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:19:56.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:19:56.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:19:56.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:19:56.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:19:56.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:19:56.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:19:56.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:19:56.234 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:19:56.234 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:19:56.234 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:19:56.234 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:19:56.234 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:19:56.234 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:19:56.234 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:19:56.234 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:19:56.234 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:19:56.234 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:19:56.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:19:56.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:19:56.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:19:56.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:19:56.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:19:56.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:19:56.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:19:56.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:19:56.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:19:56.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:19:56.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:19:56.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:19:56.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:19:56.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:19:56.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:19:56.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:19:56.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:19:56.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:19:56.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:19:56.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:19:56.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:19:56.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:19:56.246 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:19:56.246 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:19:56.246 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:19:56.246 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:19:56.297 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:19:56.298 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:19:56.298 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:19:56.302 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:19:56.302 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:19:56.302 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:19:56.302 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:19:56.302 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:19:56.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:19:56.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:19:56.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:19:56.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:19:56.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:19:56.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:19:56.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:19:56.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:19:56.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:19:56.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:19:56.304 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:19:56.304 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:19:56.304 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:19:56.304 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:19:56.304 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:19:56.304 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:19:56.304 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:19:56.304 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:19:56.304 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:19:56.304 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:19:56.304 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:19:56.304 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:19:56.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:19:56.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:19:56.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:19:56.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:19:56.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:19:56.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:19:56.312 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:19:56.312 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:19:56.312 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:19:56.360 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:19:56.360 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:19:56.360 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:19:56.362 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:19:56.362 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:19:56.362 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:19:56.362 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:19:56.362 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:19:56.362 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:19:56.362 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:19:56.362 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:19:56.362 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:19:56.362 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:19:56.362 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:19:56.363 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:19:56.363 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:19:56.363 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:19:56.363 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:19:56.363 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:19:56.363 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:19:56.363 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:19:56.363 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:19:56.363 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:19:56.363 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:19:56.363 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:19:56.363 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:19:56.363 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:19:56.363 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:19:56.363 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:19:56.363 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:19:56.363 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:19:56.363 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:19:56.363 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:19:56.363 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:19:56.364 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:19:56.374 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:19:56.375 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:19:56.375 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:19:56.376 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:19:56.423 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:19:56.423 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:19:56.423 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:19:56.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:19:56.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:19:56.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:19:56.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:19:56.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:19:56.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:19:56.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:19:56.427 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:19:56.427 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:19:56.427 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:19:56.427 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:19:56.427 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:19:56.427 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:19:56.427 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:19:56.427 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:19:56.427 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:19:56.427 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:19:56.427 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:19:56.427 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:19:56.427 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:19:56.427 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:19:56.428 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:19:56.428 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:19:56.428 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:19:56.428 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:19:56.428 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:19:56.428 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:19:56.428 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:19:56.428 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:19:56.428 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:19:56.428 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:19:56.428 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:19:56.436 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:19:56.437 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:19:56.437 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:19:56.437 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:19:56.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:19:56.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:19:56.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:19:56.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:19:56.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:19:56.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:19:56.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:19:56.500 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:19:56.500 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:19:56.500 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:19:56.500 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:19:56.501 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:19:56.501 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:19:56.501 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:19:56.501 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:19:56.501 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:19:56.539 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 73 2026-02-18T08:19:56.539 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 77 2026-02-18T08:19:56.539 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:19:56.539 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=309, tag_tid=21, entry_tid=1], [object_number=308, tag_tid=21, entry_tid=0], [object_number=310, tag_tid=20, entry_tid=46754], [object_number=311, tag_tid=20, entry_tid=46751]]], state=connected] 2026-02-18T08:19:56.539 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=294, tag_tid=20, entry_tid=33090], [object_number=293, tag_tid=20, entry_tid=33089], [object_number=292, tag_tid=20, entry_tid=33088], [object_number=295, tag_tid=20, entry_tid=33087]]], state=connected] 2026-02-18T08:19:56.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:19:56.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=309, tag_tid=21, entry_tid=1' 2026-02-18T08:19:56.545 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:19:56.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:19:56.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:19:56.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:19:56.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:19:56.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:19:56.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:19:56.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:19:56.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:19:56.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:19:56.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:19:56.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:19:56.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:19:56.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:19:56.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:19:56.576 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 73 2026-02-18T08:19:56.576 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 77 2026-02-18T08:19:56.576 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:19:56.576 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=309, tag_tid=21, entry_tid=1], [object_number=308, tag_tid=21, entry_tid=0], [object_number=310, tag_tid=20, entry_tid=46754], [object_number=311, tag_tid=20, entry_tid=46751]]], state=connected] 2026-02-18T08:19:56.576 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=294, tag_tid=20, entry_tid=33090], [object_number=293, tag_tid=20, entry_tid=33089], [object_number=292, tag_tid=20, entry_tid=33088], [object_number=295, tag_tid=20, entry_tid=33087]]], state=connected] 2026-02-18T08:19:56.578 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:19:56.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=294, tag_tid=20, entry_tid=33090' 2026-02-18T08:19:56.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=309, tag_tid=21, entry_tid=1' -a 'object_number=309, tag_tid=21, entry_tid=1' = 'object_number=294, tag_tid=20, entry_tid=33090' 2026-02-18T08:19:56.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=294, tag_tid=20, entry_tid=33090' '!=' 'object_number=294, tag_tid=20, entry_tid=32758' 2026-02-18T08:19:56.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:19:56.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=294, tag_tid=20, entry_tid=33090' = 'object_number=294, tag_tid=20, entry_tid=32758' 2026-02-18T08:19:56.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=294, tag_tid=20, entry_tid=33090' 2026-02-18T08:19:56.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=309, tag_tid=21, entry_tid=1' 2026-02-18T08:19:56.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:19:56.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:19:56.582 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=21 2026-02-18T08:19:56.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=294, tag_tid=20, entry_tid=33090' 2026-02-18T08:19:56.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:19:56.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:19:56.585 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=20 2026-02-18T08:19:56.585 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=309, tag_tid=21, entry_tid=1' 2026-02-18T08:19:56.586 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:19:56.586 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:19:56.587 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:19:56.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=294, tag_tid=20, entry_tid=33090' 2026-02-18T08:19:56.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:19:56.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:19:56.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=33090 2026-02-18T08:19:56.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 21 = 20 -a 1 -le 33090 2026-02-18T08:19:56.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:19:56.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:19:56.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:19:56.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:19:56.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:19:56.793 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:19:56.793 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:19:56.793 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:19:56.793 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:19:56.793 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:19:56.793 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:19:56.793 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:19:56.794 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:19:56.794 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:19:56.794 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:19:56.794 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:19:56.794 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:19:56.794 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:19:56.794 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:19:56.794 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:19:56.798 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:19:56.799 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:19:56.799 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:19:56.799 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:19:56.799 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:19:56.799 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:19:56.799 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:19:56.799 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:19:56.799 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:19:56.800 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:19:56.800 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:19:56.800 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:19:56.800 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:19:56.800 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:19:56.800 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:19:56.800 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:19:56.800 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:19:56.801 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:19:56.801 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:19:56.801 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:19:56.801 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:19:56.801 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:19:56.801 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:19:56.801 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:19:56.801 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:19:56.802 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:19:56.802 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:19:56.802 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:19:56.802 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:19:56.802 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:19:56.802 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:19:56.802 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:19:56.806 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:19:56.806 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:19:56.806 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:19:56.807 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:19:56.863 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:19:56.863 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:19:56.864 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:19:56.865 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:19:56.865 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:19:56.865 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:19:56.865 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:19:56.865 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:19:56.865 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:19:56.866 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:19:56.866 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:19:56.866 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:19:56.866 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:19:56.866 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:19:56.866 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:19:56.866 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:19:56.866 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:19:56.866 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:19:56.866 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:19:56.866 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:19:56.866 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:19:56.866 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:19:56.866 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:19:56.866 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:19:56.866 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:19:56.866 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:19:56.866 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:19:56.866 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:19:56.866 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:19:56.866 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:19:56.866 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:19:56.866 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:19:56.866 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:19:56.866 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:19:56.866 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:19:56.878 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:19:56.878 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:19:56.878 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:19:56.878 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:19:56.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:19:56.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:19:56.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:19:56.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:19:56.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:19:56.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:19:56.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:19:56.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:19:56.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:19:56.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:19:56.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:19:56.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:19:56.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:19:56.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:19:56.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:19:56.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:19:56.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:19:56.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:19:56.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:19:56.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:19:56.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:19:56.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:19:56.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:19:56.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:19:56.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:19:56.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:19:56.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:19:56.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:19:56.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:19:56.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:19:56.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:19:56.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:19:56.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:19:56.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:19:56.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:19:56.949 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:19:56.949 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:19:56.950 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:19:56.950 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:19:57.007 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:19:57.007 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:19:57.007 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:19:57.012 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:19:57.012 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:19:57.012 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:19:57.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:19:57.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:19:57.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:19:57.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:19:57.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:19:57.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:19:57.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:19:57.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:19:57.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:19:57.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:19:57.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:19:57.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:19:57.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:19:57.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:19:57.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:19:57.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:19:57.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:19:57.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:19:57.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:19:57.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:19:57.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:19:57.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:19:57.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:19:57.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:19:57.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:19:57.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:19:57.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:19:57.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:19:57.018 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:19:57.021 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:19:57.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:19:57.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:19:57.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:19:57.096 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:19:57.096 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:19:57.096 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:19:57.096 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:19:57.096 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:19:57.096 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:19:57.096 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:19:57.096 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:19:57.096 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:19:57.096 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:19:57.097 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:19:57.097 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:19:57.097 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:19:57.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:19:57.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:19:57.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:19:57.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 73 2026-02-18T08:19:57.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 77 2026-02-18T08:19:57.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:19:57.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=309, tag_tid=21, entry_tid=1], [object_number=308, tag_tid=21, entry_tid=0], [object_number=310, tag_tid=20, entry_tid=46754], [object_number=311, tag_tid=20, entry_tid=46751]]], state=connected] 2026-02-18T08:19:57.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=294, tag_tid=20, entry_tid=33438], [object_number=293, tag_tid=20, entry_tid=33437], [object_number=292, tag_tid=20, entry_tid=33436], [object_number=295, tag_tid=20, entry_tid=33435]]], state=connected] 2026-02-18T08:19:57.129 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:19:57.131 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=309, tag_tid=21, entry_tid=1' 2026-02-18T08:19:57.131 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:19:57.131 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:19:57.131 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:19:57.131 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:19:57.131 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:19:57.131 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:19:57.131 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:19:57.131 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:19:57.131 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:19:57.131 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:19:57.132 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:19:57.132 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:19:57.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:19:57.134 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:19:57.134 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:19:57.161 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 73 2026-02-18T08:19:57.161 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 77 2026-02-18T08:19:57.161 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:19:57.161 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=309, tag_tid=21, entry_tid=1], [object_number=308, tag_tid=21, entry_tid=0], [object_number=310, tag_tid=20, entry_tid=46754], [object_number=311, tag_tid=20, entry_tid=46751]]], state=connected] 2026-02-18T08:19:57.161 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=294, tag_tid=20, entry_tid=33438], [object_number=293, tag_tid=20, entry_tid=33437], [object_number=292, tag_tid=20, entry_tid=33436], [object_number=295, tag_tid=20, entry_tid=33435]]], state=connected] 2026-02-18T08:19:57.164 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:19:57.166 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=294, tag_tid=20, entry_tid=33438' 2026-02-18T08:19:57.166 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=309, tag_tid=21, entry_tid=1' -a 'object_number=309, tag_tid=21, entry_tid=1' = 'object_number=294, tag_tid=20, entry_tid=33438' 2026-02-18T08:19:57.166 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=294, tag_tid=20, entry_tid=33438' '!=' 'object_number=294, tag_tid=20, entry_tid=33090' 2026-02-18T08:19:57.166 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:19:57.166 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=294, tag_tid=20, entry_tid=33438' = 'object_number=294, tag_tid=20, entry_tid=33090' 2026-02-18T08:19:57.166 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=294, tag_tid=20, entry_tid=33438' 2026-02-18T08:19:57.167 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=309, tag_tid=21, entry_tid=1' 2026-02-18T08:19:57.167 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:19:57.167 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:19:57.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=21 2026-02-18T08:19:57.169 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=294, tag_tid=20, entry_tid=33438' 2026-02-18T08:19:57.169 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:19:57.169 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:19:57.171 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=20 2026-02-18T08:19:57.171 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=309, tag_tid=21, entry_tid=1' 2026-02-18T08:19:57.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:19:57.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:19:57.173 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:19:57.173 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=294, tag_tid=20, entry_tid=33438' 2026-02-18T08:19:57.174 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:19:57.174 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:19:57.175 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=33438 2026-02-18T08:19:57.175 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 21 = 20 -a 1 -le 33438 2026-02-18T08:19:57.175 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:19:57.175 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:19:57.175 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:19:57.383 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:19:57.384 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:19:57.384 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:19:57.384 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:19:57.384 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:19:57.385 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:19:57.385 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:19:57.385 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:19:57.385 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:19:57.385 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:19:57.385 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:19:57.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:19:57.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:19:57.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:19:57.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:19:57.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:19:57.387 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:19:57.387 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:19:57.387 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:19:57.387 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:19:57.387 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:19:57.387 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:19:57.388 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:19:57.388 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:19:57.388 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:19:57.388 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:19:57.388 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:19:57.389 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:19:57.389 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:19:57.389 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:19:57.389 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:19:57.389 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:19:57.389 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:19:57.390 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:19:57.390 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:19:57.390 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:19:57.390 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:19:57.390 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:19:57.390 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:19:57.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:19:57.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:19:57.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:19:57.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:19:57.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:19:57.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:19:57.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:19:57.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:19:57.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:19:57.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:19:57.394 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:19:57.394 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:19:57.394 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:19:57.394 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:19:57.454 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:19:57.454 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:19:57.454 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:19:57.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:19:57.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:19:57.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:19:57.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:19:57.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:19:57.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:19:57.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:19:57.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:19:57.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:19:57.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:19:57.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:19:57.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:19:57.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:19:57.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:19:57.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:19:57.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:19:57.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:19:57.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:19:57.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:19:57.458 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:19:57.458 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:19:57.458 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:19:57.458 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:19:57.458 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:19:57.458 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:19:57.458 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:19:57.458 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:19:57.458 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:19:57.458 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:19:57.458 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:19:57.458 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:19:57.458 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:19:57.466 INFO:tasks.rbd_mirror_thrash:waiting for cluster2.client.mirror.0 2026-02-18T08:19:57.467 INFO:teuthology.orchestra.run:waiting for 600 2026-02-18T08:19:57.467 INFO:tasks.rbd_mirror_thrash:reviving cluster2.client.mirror.0 2026-02-18T08:19:57.467 INFO:tasks.rbd_mirror.cluster2.client.mirror.0:Restarting daemon 2026-02-18T08:19:57.467 DEBUG:teuthology.orchestra.run.trial163:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.0 2026-02-18T08:19:57.468 INFO:tasks.rbd_mirror.cluster2.client.mirror.0:Started 2026-02-18T08:19:57.468 INFO:tasks.rbd_mirror_thrash:waiting for 71.98590451179294 secs before thrashing 2026-02-18T08:19:57.468 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:19:57.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:19:57.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:19:57.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:19:57.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:19:57.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:19:57.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:19:57.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:19:57.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:19:57.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:19:57.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:19:57.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:19:57.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:19:57.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:19:57.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:19:57.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:19:57.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:19:57.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:19:57.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:19:57.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:19:57.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:19:57.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:19:57.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:19:57.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:19:57.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:19:57.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:19:57.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:19:57.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:19:57.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:19:57.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:19:57.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:19:57.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:19:57.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:19:57.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:19:57.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:19:57.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:19:57.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:19:57.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:19:57.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:19:57.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:19:57.534 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:19:57.534 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:19:57.534 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:19:57.584 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:19:57.585 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:19:57.585 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:19:57.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:19:57.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:19:57.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:19:57.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:19:57.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:19:57.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:19:57.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:19:57.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:19:57.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:19:57.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:19:57.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:19:57.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:19:57.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:19:57.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:19:57.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:19:57.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:19:57.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:19:57.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:19:57.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:19:57.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:19:57.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:19:57.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:19:57.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:19:57.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:19:57.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:19:57.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:19:57.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:19:57.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:19:57.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:19:57.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:19:57.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:19:57.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:19:57.599 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:19:57.599 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:19:57.599 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:19:57.599 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:19:57.661 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:19:57.661 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:19:57.662 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:19:57.662 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:19:57.662 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:19:57.662 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:19:57.662 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:19:57.662 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:19:57.662 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:19:57.662 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:19:57.662 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:19:57.663 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:19:57.664 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:19:57.664 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:19:57.664 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:19:57.664 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:19:57.691 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 73 2026-02-18T08:19:57.692 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 77 2026-02-18T08:19:57.692 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:19:57.692 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=309, tag_tid=21, entry_tid=1], [object_number=308, tag_tid=21, entry_tid=0], [object_number=310, tag_tid=20, entry_tid=46754], [object_number=311, tag_tid=20, entry_tid=46751]]], state=connected] 2026-02-18T08:19:57.692 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=296, tag_tid=20, entry_tid=33708], [object_number=299, tag_tid=20, entry_tid=33707], [object_number=298, tag_tid=20, entry_tid=33706], [object_number=293, tag_tid=20, entry_tid=33705]]], state=connected] 2026-02-18T08:19:57.694 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:19:57.696 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=309, tag_tid=21, entry_tid=1' 2026-02-18T08:19:57.697 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:19:57.697 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:19:57.697 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:19:57.697 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:19:57.697 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:19:57.697 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:19:57.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:19:57.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:19:57.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:19:57.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:19:57.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:19:57.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:19:57.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:19:57.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:19:57.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:19:57.738 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 73 2026-02-18T08:19:57.739 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 77 2026-02-18T08:19:57.739 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:19:57.739 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=309, tag_tid=21, entry_tid=1], [object_number=308, tag_tid=21, entry_tid=0], [object_number=310, tag_tid=20, entry_tid=46754], [object_number=311, tag_tid=20, entry_tid=46751]]], state=connected] 2026-02-18T08:19:57.739 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=296, tag_tid=20, entry_tid=33708], [object_number=299, tag_tid=20, entry_tid=33707], [object_number=298, tag_tid=20, entry_tid=33706], [object_number=293, tag_tid=20, entry_tid=33705]]], state=connected] 2026-02-18T08:19:57.743 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:19:57.745 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=296, tag_tid=20, entry_tid=33708' 2026-02-18T08:19:57.745 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=309, tag_tid=21, entry_tid=1' -a 'object_number=309, tag_tid=21, entry_tid=1' = 'object_number=296, tag_tid=20, entry_tid=33708' 2026-02-18T08:19:57.745 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=296, tag_tid=20, entry_tid=33708' '!=' 'object_number=294, tag_tid=20, entry_tid=33438' 2026-02-18T08:19:57.746 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:19:57.746 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=296, tag_tid=20, entry_tid=33708' = 'object_number=294, tag_tid=20, entry_tid=33438' 2026-02-18T08:19:57.746 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=296, tag_tid=20, entry_tid=33708' 2026-02-18T08:19:57.746 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=309, tag_tid=21, entry_tid=1' 2026-02-18T08:19:57.746 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:19:57.747 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:19:57.748 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=21 2026-02-18T08:19:57.748 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=296, tag_tid=20, entry_tid=33708' 2026-02-18T08:19:57.748 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:19:57.749 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:19:57.750 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=20 2026-02-18T08:19:57.750 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=309, tag_tid=21, entry_tid=1' 2026-02-18T08:19:57.750 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:19:57.751 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:19:57.751 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:19:57.752 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=296, tag_tid=20, entry_tid=33708' 2026-02-18T08:19:57.752 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:19:57.753 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:19:57.754 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=33708 2026-02-18T08:19:57.754 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 21 = 20 -a 1 -le 33708 2026-02-18T08:19:57.754 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:19:57.754 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:19:57.754 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:19:57.955 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:19:57.956 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:19:57.956 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:19:57.956 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:19:57.956 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:19:57.956 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:19:57.956 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:19:57.956 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:19:57.956 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:19:57.956 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:19:57.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:19:57.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:19:57.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:19:57.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:19:57.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:19:57.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:19:57.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:19:57.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:19:57.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:19:57.961 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:19:57.961 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:19:57.961 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:19:57.961 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:19:57.961 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:19:57.961 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:19:57.961 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:19:57.961 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:19:57.961 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:19:57.961 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:19:57.962 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:19:57.962 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:19:57.962 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:19:57.962 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:19:57.962 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:19:57.962 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:19:57.963 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:19:57.963 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:19:57.963 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:19:57.963 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:19:57.963 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:19:57.963 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:19:57.963 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:19:57.963 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:19:57.964 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:19:57.964 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:19:57.964 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:19:57.964 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:19:57.964 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:19:57.964 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:19:57.970 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:19:57.970 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:19:57.970 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:19:57.971 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:19:58.028 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:19:58.028 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:19:58.028 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:19:58.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:19:58.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:19:58.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:19:58.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:19:58.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:19:58.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:19:58.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:19:58.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:19:58.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:19:58.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:19:58.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:19:58.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:19:58.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:19:58.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:19:58.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:19:58.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:19:58.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:19:58.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:19:58.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:19:58.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:19:58.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:19:58.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:19:58.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:19:58.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:19:58.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:19:58.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:19:58.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:19:58.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:19:58.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:19:58.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:19:58.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:19:58.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:19:58.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:19:58.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:19:58.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:19:58.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:19:58.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:19:58.100 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:19:58.100 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:19:58.102 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:19:58.102 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:19:58.102 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:19:58.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:19:58.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:19:58.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:19:58.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:19:58.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:19:58.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:19:58.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:19:58.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:19:58.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:19:58.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:19:58.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:19:58.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:19:58.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:19:58.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:19:58.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:19:58.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:19:58.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:19:58.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:19:58.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:19:58.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:19:58.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:19:58.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:19:58.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:19:58.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:19:58.105 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:19:58.105 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:19:58.105 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:19:58.105 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:19:58.105 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:19:58.113 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:19:58.114 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:19:58.114 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:19:58.114 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:19:58.171 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:19:58.171 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:19:58.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:19:58.174 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:19:58.175 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:19:58.175 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:19:58.175 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:19:58.175 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:19:58.175 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:19:58.175 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:19:58.175 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:19:58.175 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:19:58.175 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:19:58.175 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:19:58.176 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:19:58.176 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:19:58.176 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:19:58.176 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:19:58.176 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:19:58.176 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:19:58.176 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:19:58.176 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:19:58.176 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:19:58.176 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:19:58.177 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:19:58.177 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:19:58.177 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:19:58.177 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:19:58.177 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:19:58.177 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:19:58.177 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:19:58.177 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:19:58.177 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:19:58.177 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:19:58.177 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:19:58.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:19:58.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:19:58.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:19:58.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:19:58.253 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:19:58.254 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:19:58.254 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:19:58.254 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:19:58.254 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:19:58.254 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:19:58.254 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:19:58.255 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:19:58.255 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:19:58.255 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:19:58.255 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:19:58.255 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:19:58.255 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:19:58.256 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:19:58.256 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:19:58.257 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:19:58.286 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 74 2026-02-18T08:19:58.287 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 77 2026-02-18T08:19:58.287 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:19:58.287 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=309, tag_tid=21, entry_tid=1], [object_number=308, tag_tid=21, entry_tid=0], [object_number=310, tag_tid=20, entry_tid=46754], [object_number=311, tag_tid=20, entry_tid=46751]]], state=connected] 2026-02-18T08:19:58.287 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=296, tag_tid=20, entry_tid=34088], [object_number=299, tag_tid=20, entry_tid=34087], [object_number=298, tag_tid=20, entry_tid=34086], [object_number=297, tag_tid=20, entry_tid=34085]]], state=connected] 2026-02-18T08:19:58.289 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:19:58.290 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=309, tag_tid=21, entry_tid=1' 2026-02-18T08:19:58.292 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:19:58.292 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:19:58.292 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:19:58.292 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:19:58.292 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:19:58.292 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:19:58.292 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:19:58.292 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:19:58.293 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:19:58.293 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:19:58.293 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:19:58.293 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:19:58.294 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:19:58.294 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:19:58.294 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:19:58.321 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 74 2026-02-18T08:19:58.322 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 77 2026-02-18T08:19:58.322 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:19:58.322 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=309, tag_tid=21, entry_tid=1], [object_number=308, tag_tid=21, entry_tid=0], [object_number=310, tag_tid=20, entry_tid=46754], [object_number=311, tag_tid=20, entry_tid=46751]]], state=connected] 2026-02-18T08:19:58.322 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=296, tag_tid=20, entry_tid=34088], [object_number=299, tag_tid=20, entry_tid=34087], [object_number=298, tag_tid=20, entry_tid=34086], [object_number=297, tag_tid=20, entry_tid=34085]]], state=connected] 2026-02-18T08:19:58.324 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:19:58.326 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=296, tag_tid=20, entry_tid=34088' 2026-02-18T08:19:58.326 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=309, tag_tid=21, entry_tid=1' -a 'object_number=309, tag_tid=21, entry_tid=1' = 'object_number=296, tag_tid=20, entry_tid=34088' 2026-02-18T08:19:58.326 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=296, tag_tid=20, entry_tid=34088' '!=' 'object_number=296, tag_tid=20, entry_tid=33708' 2026-02-18T08:19:58.327 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:19:58.327 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=296, tag_tid=20, entry_tid=34088' = 'object_number=296, tag_tid=20, entry_tid=33708' 2026-02-18T08:19:58.327 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=296, tag_tid=20, entry_tid=34088' 2026-02-18T08:19:58.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=309, tag_tid=21, entry_tid=1' 2026-02-18T08:19:58.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:19:58.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:19:58.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=21 2026-02-18T08:19:58.329 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=296, tag_tid=20, entry_tid=34088' 2026-02-18T08:19:58.329 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:19:58.329 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:19:58.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=20 2026-02-18T08:19:58.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=309, tag_tid=21, entry_tid=1' 2026-02-18T08:19:58.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:19:58.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:19:58.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:19:58.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=296, tag_tid=20, entry_tid=34088' 2026-02-18T08:19:58.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:19:58.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:19:58.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=34088 2026-02-18T08:19:58.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 21 = 20 -a 1 -le 34088 2026-02-18T08:19:58.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:19:58.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:19:58.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:19:58.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:19:58.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:19:58.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:19:58.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:19:58.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:19:58.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:19:58.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:19:58.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:19:58.539 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:19:58.539 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:19:58.539 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:19:58.539 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:19:58.539 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:19:58.539 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:19:58.539 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:19:58.539 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:19:58.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:19:58.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:19:58.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:19:58.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:19:58.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:19:58.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:19:58.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:19:58.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:19:58.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:19:58.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:19:58.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:19:58.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:19:58.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:19:58.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:19:58.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:19:58.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:19:58.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:19:58.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:19:58.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:19:58.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:19:58.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:19:58.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:19:58.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:19:58.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:19:58.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:19:58.545 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:19:58.545 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:19:58.545 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:19:58.545 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:19:58.545 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:19:58.545 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:19:58.545 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:19:58.545 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:19:58.552 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:19:58.552 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:19:58.552 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:19:58.552 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:19:58.610 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:19:58.610 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:19:58.610 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:19:58.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:19:58.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:19:58.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:19:58.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:19:58.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:19:58.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:19:58.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:19:58.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:19:58.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:19:58.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:19:58.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:19:58.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:19:58.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:19:58.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:19:58.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:19:58.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:19:58.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:19:58.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:19:58.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:19:58.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:19:58.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:19:58.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:19:58.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:19:58.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:19:58.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:19:58.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:19:58.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:19:58.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:19:58.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:19:58.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:19:58.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:19:58.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:19:58.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:19:58.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:19:58.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:19:58.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:19:58.686 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:19:58.686 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:19:58.686 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:19:58.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:19:58.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:19:58.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:19:58.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:19:58.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:19:58.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:19:58.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:19:58.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:19:58.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:19:58.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:19:58.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:19:58.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:19:58.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:19:58.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:19:58.691 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:19:58.691 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:19:58.691 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:19:58.691 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:19:58.691 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:19:58.691 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:19:58.691 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:19:58.691 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:19:58.691 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:19:58.691 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:19:58.692 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:19:58.692 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:19:58.692 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:19:58.692 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:19:58.692 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:19:58.692 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:19:58.692 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:19:58.692 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:19:58.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:19:58.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:19:58.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:19:58.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:19:58.747 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:19:58.747 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:19:58.747 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:19:58.750 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:19:58.750 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:19:58.750 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:19:58.750 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:19:58.750 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:19:58.750 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:19:58.750 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:19:58.751 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:19:58.751 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:19:58.751 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:19:58.751 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:19:58.751 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:19:58.751 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:19:58.751 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:19:58.751 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:19:58.751 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:19:58.751 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:19:58.752 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:19:58.752 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:19:58.752 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:19:58.752 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:19:58.752 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:19:58.752 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:19:58.752 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:19:58.752 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:19:58.752 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:19:58.752 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:19:58.753 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:19:58.753 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:19:58.753 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:19:58.753 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:19:58.753 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:19:58.761 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:19:58.761 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:19:58.761 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:19:58.761 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:19:58.819 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:19:58.821 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:19:58.821 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:19:58.821 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:19:58.821 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:19:58.821 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:19:58.822 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:19:58.822 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:19:58.822 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:19:58.822 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:19:58.822 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:19:58.822 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:19:58.822 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:19:58.823 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:19:58.823 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:19:58.823 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:19:58.850 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 74 2026-02-18T08:19:58.850 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 77 2026-02-18T08:19:58.850 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:19:58.850 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=309, tag_tid=21, entry_tid=1], [object_number=308, tag_tid=21, entry_tid=0], [object_number=310, tag_tid=20, entry_tid=46754], [object_number=311, tag_tid=20, entry_tid=46751]]], state=connected] 2026-02-18T08:19:58.851 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=297, tag_tid=20, entry_tid=34297], [object_number=296, tag_tid=20, entry_tid=34296], [object_number=299, tag_tid=20, entry_tid=34295], [object_number=298, tag_tid=20, entry_tid=34294]]], state=connected] 2026-02-18T08:19:58.853 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:19:58.854 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=309, tag_tid=21, entry_tid=1' 2026-02-18T08:19:58.856 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:19:58.856 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:19:58.856 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:19:58.856 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:19:58.857 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:19:58.857 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:19:58.857 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:19:58.857 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:19:58.857 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:19:58.857 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:19:58.857 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:19:58.857 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:19:58.858 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:19:58.858 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:19:58.858 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:19:58.885 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 74 2026-02-18T08:19:58.885 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 77 2026-02-18T08:19:58.885 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:19:58.885 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=309, tag_tid=21, entry_tid=1], [object_number=308, tag_tid=21, entry_tid=0], [object_number=310, tag_tid=20, entry_tid=46754], [object_number=311, tag_tid=20, entry_tid=46751]]], state=connected] 2026-02-18T08:19:58.885 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=297, tag_tid=20, entry_tid=34297], [object_number=296, tag_tid=20, entry_tid=34296], [object_number=299, tag_tid=20, entry_tid=34295], [object_number=298, tag_tid=20, entry_tid=34294]]], state=connected] 2026-02-18T08:19:58.887 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:19:58.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=297, tag_tid=20, entry_tid=34297' 2026-02-18T08:19:58.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=309, tag_tid=21, entry_tid=1' -a 'object_number=309, tag_tid=21, entry_tid=1' = 'object_number=297, tag_tid=20, entry_tid=34297' 2026-02-18T08:19:58.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=297, tag_tid=20, entry_tid=34297' '!=' 'object_number=296, tag_tid=20, entry_tid=34088' 2026-02-18T08:19:58.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:19:58.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=297, tag_tid=20, entry_tid=34297' = 'object_number=296, tag_tid=20, entry_tid=34088' 2026-02-18T08:19:58.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=297, tag_tid=20, entry_tid=34297' 2026-02-18T08:19:58.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=309, tag_tid=21, entry_tid=1' 2026-02-18T08:19:58.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:19:58.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:19:58.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=21 2026-02-18T08:19:58.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=297, tag_tid=20, entry_tid=34297' 2026-02-18T08:19:58.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:19:58.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:19:58.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=20 2026-02-18T08:19:58.894 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=309, tag_tid=21, entry_tid=1' 2026-02-18T08:19:58.894 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:19:58.894 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:19:58.895 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:19:58.896 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=297, tag_tid=20, entry_tid=34297' 2026-02-18T08:19:58.896 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:19:58.896 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:19:58.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=34297 2026-02-18T08:19:58.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 21 = 20 -a 1 -le 34297 2026-02-18T08:19:58.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:19:58.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:19:58.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:19:59.100 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:19:59.100 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:19:59.100 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:19:59.101 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:19:59.101 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:19:59.101 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:19:59.101 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:19:59.101 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:19:59.101 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:19:59.101 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:19:59.101 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:19:59.101 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:19:59.102 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:19:59.102 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:19:59.102 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:19:59.102 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:19:59.102 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:19:59.105 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:19:59.105 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:19:59.105 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:19:59.105 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:19:59.105 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:19:59.105 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:19:59.105 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:19:59.105 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:19:59.105 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:19:59.106 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:19:59.106 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:19:59.106 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:19:59.106 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:19:59.106 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:19:59.106 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:19:59.106 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:19:59.106 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:19:59.106 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:19:59.106 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:19:59.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:19:59.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:19:59.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:19:59.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:19:59.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:19:59.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:19:59.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:19:59.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:19:59.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:19:59.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:19:59.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:19:59.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:19:59.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:19:59.114 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:19:59.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:19:59.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:19:59.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:19:59.164 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:19:59.164 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:19:59.164 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:19:59.167 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:19:59.167 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:19:59.167 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:19:59.167 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:19:59.167 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:19:59.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:19:59.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:19:59.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:19:59.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:19:59.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:19:59.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:19:59.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:19:59.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:19:59.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:19:59.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:19:59.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:19:59.169 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:19:59.169 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:19:59.169 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:19:59.169 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:19:59.169 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:19:59.169 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:19:59.169 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:19:59.169 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:19:59.169 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:19:59.169 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:19:59.169 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:19:59.169 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:19:59.170 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:19:59.170 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:19:59.170 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:19:59.170 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:19:59.178 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:19:59.179 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:19:59.179 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:19:59.179 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:19:59.227 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:19:59.227 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:19:59.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:19:59.230 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:19:59.230 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:19:59.230 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:19:59.230 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:19:59.231 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:19:59.231 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:19:59.231 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:19:59.231 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:19:59.231 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:19:59.231 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:19:59.231 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:19:59.231 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:19:59.231 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:19:59.231 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:19:59.231 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:19:59.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:19:59.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:19:59.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:19:59.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:19:59.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:19:59.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:19:59.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:19:59.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:19:59.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:19:59.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:19:59.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:19:59.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:19:59.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:19:59.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:19:59.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:19:59.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:19:59.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:19:59.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:19:59.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:19:59.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:19:59.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:19:59.298 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:19:59.299 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:19:59.299 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:19:59.301 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:19:59.301 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:19:59.301 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:19:59.301 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:19:59.301 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:19:59.302 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:19:59.302 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:19:59.302 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:19:59.302 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:19:59.302 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:19:59.302 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:19:59.302 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:19:59.302 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:19:59.302 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:19:59.302 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:19:59.302 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:19:59.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:19:59.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:19:59.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:19:59.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:19:59.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:19:59.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:19:59.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:19:59.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:19:59.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:19:59.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:19:59.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:19:59.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:19:59.304 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:19:59.304 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:19:59.304 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:19:59.304 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:19:59.312 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:19:59.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:19:59.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:19:59.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:19:59.372 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:19:59.373 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:19:59.373 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:19:59.373 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:19:59.373 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:19:59.374 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:19:59.374 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:19:59.374 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:19:59.374 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:19:59.374 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:19:59.374 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:19:59.374 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:19:59.374 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:19:59.375 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:19:59.375 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:19:59.375 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:19:59.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 74 2026-02-18T08:19:59.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 77 2026-02-18T08:19:59.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:19:59.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=309, tag_tid=21, entry_tid=1], [object_number=308, tag_tid=21, entry_tid=0], [object_number=310, tag_tid=20, entry_tid=46754], [object_number=311, tag_tid=20, entry_tid=46751]]], state=connected] 2026-02-18T08:19:59.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=296, tag_tid=20, entry_tid=34652], [object_number=299, tag_tid=20, entry_tid=34651], [object_number=298, tag_tid=20, entry_tid=34650], [object_number=297, tag_tid=20, entry_tid=34649]]], state=connected] 2026-02-18T08:19:59.415 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:19:59.416 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=309, tag_tid=21, entry_tid=1' 2026-02-18T08:19:59.417 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:19:59.417 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:19:59.417 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:19:59.417 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:19:59.417 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:19:59.417 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:19:59.417 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:19:59.417 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:19:59.417 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:19:59.417 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:19:59.418 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:19:59.418 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:19:59.419 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:19:59.420 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:19:59.420 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:19:59.446 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 74 2026-02-18T08:19:59.446 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 77 2026-02-18T08:19:59.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:19:59.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=309, tag_tid=21, entry_tid=1], [object_number=308, tag_tid=21, entry_tid=0], [object_number=310, tag_tid=20, entry_tid=46754], [object_number=311, tag_tid=20, entry_tid=46751]]], state=connected] 2026-02-18T08:19:59.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=296, tag_tid=20, entry_tid=34652], [object_number=299, tag_tid=20, entry_tid=34651], [object_number=298, tag_tid=20, entry_tid=34650], [object_number=297, tag_tid=20, entry_tid=34649]]], state=connected] 2026-02-18T08:19:59.449 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:19:59.452 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=296, tag_tid=20, entry_tid=34652' 2026-02-18T08:19:59.452 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=309, tag_tid=21, entry_tid=1' -a 'object_number=309, tag_tid=21, entry_tid=1' = 'object_number=296, tag_tid=20, entry_tid=34652' 2026-02-18T08:19:59.452 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=296, tag_tid=20, entry_tid=34652' '!=' 'object_number=297, tag_tid=20, entry_tid=34297' 2026-02-18T08:19:59.452 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:19:59.452 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=296, tag_tid=20, entry_tid=34652' = 'object_number=297, tag_tid=20, entry_tid=34297' 2026-02-18T08:19:59.452 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=296, tag_tid=20, entry_tid=34652' 2026-02-18T08:19:59.452 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=309, tag_tid=21, entry_tid=1' 2026-02-18T08:19:59.452 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:19:59.453 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:19:59.453 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=21 2026-02-18T08:19:59.454 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=296, tag_tid=20, entry_tid=34652' 2026-02-18T08:19:59.454 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:19:59.454 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:19:59.455 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=20 2026-02-18T08:19:59.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=309, tag_tid=21, entry_tid=1' 2026-02-18T08:19:59.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:19:59.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:19:59.458 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:19:59.458 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=296, tag_tid=20, entry_tid=34652' 2026-02-18T08:19:59.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:19:59.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:19:59.460 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=34652 2026-02-18T08:19:59.460 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 21 = 20 -a 1 -le 34652 2026-02-18T08:19:59.460 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:19:59.460 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:19:59.460 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:19:59.661 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:19:59.662 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:19:59.662 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:19:59.662 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:19:59.662 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:19:59.662 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:19:59.662 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:19:59.662 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:19:59.662 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:19:59.662 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:19:59.662 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:19:59.662 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:19:59.662 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:19:59.662 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:19:59.663 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:19:59.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:19:59.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:19:59.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:19:59.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:19:59.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:19:59.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:19:59.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:19:59.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:19:59.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:19:59.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:19:59.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:19:59.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:19:59.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:19:59.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:19:59.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:19:59.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:19:59.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:19:59.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:19:59.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:19:59.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:19:59.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:19:59.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:19:59.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:19:59.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:19:59.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:19:59.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:19:59.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:19:59.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:19:59.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:19:59.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:19:59.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:19:59.667 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:19:59.667 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:19:59.667 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:19:59.678 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:19:59.678 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:19:59.678 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:19:59.678 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:19:59.725 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:19:59.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:19:59.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:19:59.728 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:19:59.728 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:19:59.728 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:19:59.728 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:19:59.728 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:19:59.728 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:19:59.728 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:19:59.729 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:19:59.729 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:19:59.729 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:19:59.729 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:19:59.729 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:19:59.729 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:19:59.729 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:19:59.729 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:19:59.729 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:19:59.729 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:19:59.729 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:19:59.729 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:19:59.729 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:19:59.729 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:19:59.729 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:19:59.729 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:19:59.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:19:59.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:19:59.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:19:59.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:19:59.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:19:59.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:19:59.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:19:59.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:19:59.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:19:59.738 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:19:59.739 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:19:59.739 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:19:59.739 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:19:59.786 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:19:59.786 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:19:59.786 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:19:59.788 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:19:59.788 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:19:59.788 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:19:59.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:19:59.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:19:59.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:19:59.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:19:59.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:19:59.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:19:59.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:19:59.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:19:59.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:19:59.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:19:59.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:19:59.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:19:59.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:19:59.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:19:59.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:19:59.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:19:59.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:19:59.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:19:59.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:19:59.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:19:59.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:19:59.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:19:59.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:19:59.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:19:59.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:19:59.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:19:59.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:19:59.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:19:59.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:19:59.800 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:19:59.800 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:19:59.801 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:19:59.801 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:19:59.847 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:19:59.848 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:19:59.848 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:19:59.850 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:19:59.850 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:19:59.851 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:19:59.851 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:19:59.851 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:19:59.851 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:19:59.851 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:19:59.851 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:19:59.851 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:19:59.851 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:19:59.851 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:19:59.852 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:19:59.852 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:19:59.852 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:19:59.852 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:19:59.852 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:19:59.852 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:19:59.852 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:19:59.852 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:19:59.852 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:19:59.852 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:19:59.852 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:19:59.853 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:19:59.853 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:19:59.853 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:19:59.853 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:19:59.853 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:19:59.853 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:19:59.853 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:19:59.853 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:19:59.853 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:19:59.853 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:19:59.861 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:19:59.862 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:19:59.862 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:19:59.862 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:19:59.922 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:19:59.922 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:19:59.922 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:19:59.922 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:19:59.923 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:19:59.923 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:19:59.923 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:19:59.923 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:19:59.923 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:19:59.923 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:19:59.923 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:19:59.923 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:19:59.923 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:19:59.924 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:19:59.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:19:59.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:19:59.952 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 74 2026-02-18T08:19:59.952 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 77 2026-02-18T08:19:59.952 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:19:59.952 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=309, tag_tid=21, entry_tid=1], [object_number=308, tag_tid=21, entry_tid=0], [object_number=310, tag_tid=20, entry_tid=46754], [object_number=311, tag_tid=20, entry_tid=46751]]], state=connected] 2026-02-18T08:19:59.952 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=299, tag_tid=20, entry_tid=34999], [object_number=298, tag_tid=20, entry_tid=34998], [object_number=297, tag_tid=20, entry_tid=34997], [object_number=296, tag_tid=20, entry_tid=34996]]], state=connected] 2026-02-18T08:19:59.954 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:19:59.956 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=309, tag_tid=21, entry_tid=1' 2026-02-18T08:19:59.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:19:59.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:19:59.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:19:59.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:19:59.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:19:59.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:19:59.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:19:59.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:19:59.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:19:59.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:19:59.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:19:59.958 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:19:59.959 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:19:59.959 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:19:59.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:19:59.987 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 74 2026-02-18T08:19:59.987 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 77 2026-02-18T08:19:59.987 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:19:59.987 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=309, tag_tid=21, entry_tid=1], [object_number=308, tag_tid=21, entry_tid=0], [object_number=310, tag_tid=20, entry_tid=46754], [object_number=311, tag_tid=20, entry_tid=46751]]], state=connected] 2026-02-18T08:19:59.987 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=299, tag_tid=20, entry_tid=34999], [object_number=298, tag_tid=20, entry_tid=34998], [object_number=297, tag_tid=20, entry_tid=34997], [object_number=296, tag_tid=20, entry_tid=34996]]], state=connected] 2026-02-18T08:19:59.990 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:19:59.992 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=299, tag_tid=20, entry_tid=34999' 2026-02-18T08:19:59.992 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=309, tag_tid=21, entry_tid=1' -a 'object_number=309, tag_tid=21, entry_tid=1' = 'object_number=299, tag_tid=20, entry_tid=34999' 2026-02-18T08:19:59.992 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=299, tag_tid=20, entry_tid=34999' '!=' 'object_number=296, tag_tid=20, entry_tid=34652' 2026-02-18T08:19:59.992 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:19:59.992 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=299, tag_tid=20, entry_tid=34999' = 'object_number=296, tag_tid=20, entry_tid=34652' 2026-02-18T08:19:59.992 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=299, tag_tid=20, entry_tid=34999' 2026-02-18T08:19:59.992 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=309, tag_tid=21, entry_tid=1' 2026-02-18T08:19:59.993 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:19:59.993 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:19:59.994 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=21 2026-02-18T08:19:59.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=299, tag_tid=20, entry_tid=34999' 2026-02-18T08:19:59.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:19:59.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:19:59.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=20 2026-02-18T08:19:59.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=309, tag_tid=21, entry_tid=1' 2026-02-18T08:19:59.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:19:59.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:19:59.999 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:19:59.999 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=299, tag_tid=20, entry_tid=34999' 2026-02-18T08:19:59.999 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:19:59.999 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:20:00.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=34999 2026-02-18T08:20:00.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 21 = 20 -a 1 -le 34999 2026-02-18T08:20:00.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:20:00.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:20:00.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:20:00.202 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:20:00.202 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:20:00.202 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:20:00.202 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:20:00.203 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:20:00.203 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:20:00.203 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:20:00.203 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:20:00.203 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:20:00.203 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:20:00.203 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:20:00.203 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:20:00.203 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:20:00.203 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:20:00.203 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:20:00.203 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:20:00.204 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:20:00.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:20:00.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:20:00.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:20:00.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:20:00.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:20:00.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:20:00.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:20:00.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:20:00.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:20:00.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:20:00.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:20:00.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:20:00.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:20:00.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:20:00.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:20:00.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:20:00.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:20:00.206 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:20:00.206 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:20:00.206 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:20:00.206 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:20:00.206 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:20:00.206 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:20:00.206 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:20:00.206 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:20:00.206 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:20:00.206 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:20:00.206 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:20:00.206 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:20:00.206 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:20:00.206 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:20:00.206 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:20:00.218 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:20:00.218 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:20:00.218 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:20:00.218 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:20:00.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:20:00.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:20:00.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:20:00.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:20:00.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:20:00.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:20:00.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:20:00.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:20:00.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:20:00.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:20:00.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:20:00.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:20:00.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:20:00.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:20:00.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:20:00.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:20:00.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:20:00.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:20:00.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:20:00.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:20:00.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:20:00.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:20:00.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:20:00.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:20:00.273 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:20:00.273 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:20:00.273 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:20:00.273 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:20:00.273 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:20:00.273 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:20:00.273 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:20:00.273 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:20:00.273 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:20:00.273 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:20:00.273 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:20:00.282 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:20:00.283 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:20:00.283 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:20:00.283 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:20:00.344 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:20:00.344 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:20:00.344 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:20:00.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:20:00.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:20:00.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:20:00.348 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:20:00.348 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:20:00.348 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:20:00.348 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:20:00.348 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:20:00.348 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:20:00.348 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:20:00.348 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:20:00.348 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:20:00.348 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:20:00.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:20:00.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:20:00.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:20:00.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:20:00.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:20:00.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:20:00.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:20:00.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:20:00.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:20:00.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:20:00.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:20:00.350 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:20:00.350 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:20:00.350 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:20:00.350 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:20:00.350 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:20:00.350 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:20:00.350 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:20:00.350 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:20:00.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:20:00.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:20:00.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:20:00.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:20:00.422 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:20:00.423 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:20:00.423 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:20:00.425 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:20:00.425 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:20:00.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:20:00.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:20:00.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:20:00.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:20:00.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:20:00.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:20:00.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:20:00.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:20:00.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:20:00.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:20:00.427 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:20:00.427 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:20:00.427 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:20:00.427 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:20:00.427 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:20:00.427 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:20:00.427 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:20:00.427 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:20:00.427 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:20:00.427 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:20:00.427 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:20:00.428 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:20:00.428 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:20:00.428 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:20:00.428 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:20:00.428 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:20:00.428 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:20:00.428 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:20:00.428 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:20:00.428 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:20:00.437 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:20:00.437 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:20:00.437 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:20:00.437 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:20:00.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:20:00.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:20:00.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:20:00.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:20:00.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:20:00.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:20:00.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:20:00.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:20:00.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:20:00.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:20:00.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:20:00.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:20:00.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:20:00.500 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:20:00.500 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:20:00.500 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:20:00.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 74 2026-02-18T08:20:00.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 77 2026-02-18T08:20:00.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:20:00.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=309, tag_tid=21, entry_tid=1], [object_number=308, tag_tid=21, entry_tid=0], [object_number=310, tag_tid=20, entry_tid=46754], [object_number=311, tag_tid=20, entry_tid=46751]]], state=connected] 2026-02-18T08:20:00.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=297, tag_tid=20, entry_tid=35377], [object_number=296, tag_tid=20, entry_tid=35376], [object_number=299, tag_tid=20, entry_tid=35375], [object_number=298, tag_tid=20, entry_tid=35374]]], state=connected] 2026-02-18T08:20:00.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:20:00.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=309, tag_tid=21, entry_tid=1' 2026-02-18T08:20:00.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:20:00.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:20:00.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:20:00.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:20:00.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:20:00.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:20:00.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:20:00.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:20:00.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:20:00.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:20:00.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:20:00.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:20:00.545 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:20:00.545 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:20:00.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:20:00.575 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 74 2026-02-18T08:20:00.575 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 77 2026-02-18T08:20:00.575 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:20:00.575 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=309, tag_tid=21, entry_tid=1], [object_number=308, tag_tid=21, entry_tid=0], [object_number=310, tag_tid=20, entry_tid=46754], [object_number=311, tag_tid=20, entry_tid=46751]]], state=connected] 2026-02-18T08:20:00.575 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=297, tag_tid=20, entry_tid=35377], [object_number=296, tag_tid=20, entry_tid=35376], [object_number=299, tag_tid=20, entry_tid=35375], [object_number=298, tag_tid=20, entry_tid=35374]]], state=connected] 2026-02-18T08:20:00.577 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:20:00.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=297, tag_tid=20, entry_tid=35377' 2026-02-18T08:20:00.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=309, tag_tid=21, entry_tid=1' -a 'object_number=309, tag_tid=21, entry_tid=1' = 'object_number=297, tag_tid=20, entry_tid=35377' 2026-02-18T08:20:00.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=297, tag_tid=20, entry_tid=35377' '!=' 'object_number=299, tag_tid=20, entry_tid=34999' 2026-02-18T08:20:00.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:20:00.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=297, tag_tid=20, entry_tid=35377' = 'object_number=299, tag_tid=20, entry_tid=34999' 2026-02-18T08:20:00.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=297, tag_tid=20, entry_tid=35377' 2026-02-18T08:20:00.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=309, tag_tid=21, entry_tid=1' 2026-02-18T08:20:00.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:20:00.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:20:00.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=21 2026-02-18T08:20:00.582 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=297, tag_tid=20, entry_tid=35377' 2026-02-18T08:20:00.582 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:20:00.582 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:20:00.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=20 2026-02-18T08:20:00.584 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=309, tag_tid=21, entry_tid=1' 2026-02-18T08:20:00.584 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:20:00.584 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:20:00.586 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:20:00.586 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=297, tag_tid=20, entry_tid=35377' 2026-02-18T08:20:00.586 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:20:00.587 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:20:00.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=35377 2026-02-18T08:20:00.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 21 = 20 -a 1 -le 35377 2026-02-18T08:20:00.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:20:00.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:20:00.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:20:00.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:20:00.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:20:00.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:20:00.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:20:00.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:20:00.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:20:00.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:20:00.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:20:00.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:20:00.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:20:00.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:20:00.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:20:00.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:20:00.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:20:00.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:20:00.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:20:00.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:20:00.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:20:00.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:20:00.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:20:00.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:20:00.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:20:00.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:20:00.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:20:00.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:20:00.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:20:00.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:20:00.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:20:00.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:20:00.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:20:00.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:20:00.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:20:00.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:20:00.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:20:00.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:20:00.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:20:00.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:20:00.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:20:00.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:20:00.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:20:00.793 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:20:00.793 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:20:00.793 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:20:00.793 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:20:00.793 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:20:00.793 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:20:00.793 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:20:00.793 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:20:00.793 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:20:00.805 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:20:00.805 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:20:00.805 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:20:00.805 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:20:00.853 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:20:00.854 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:20:00.854 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:20:00.856 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:20:00.856 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:20:00.856 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:20:00.856 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:20:00.857 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:20:00.857 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:20:00.857 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:20:00.857 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:20:00.857 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:20:00.857 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:20:00.857 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:20:00.857 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:20:00.857 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:20:00.857 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:20:00.857 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:20:00.858 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:20:00.858 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:20:00.858 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:20:00.858 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:20:00.858 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:20:00.858 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:20:00.858 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:20:00.858 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:20:00.858 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:20:00.858 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:20:00.858 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:20:00.858 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:20:00.858 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:20:00.859 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:20:00.859 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:20:00.859 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:20:00.859 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:20:00.867 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:20:00.867 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:20:00.867 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:20:00.867 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:20:00.916 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:20:00.916 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:20:00.916 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:20:00.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:20:00.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:20:00.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:20:00.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:20:00.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:20:00.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:20:00.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:20:00.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:20:00.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:20:00.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:20:00.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:20:00.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:20:00.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:20:00.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:20:00.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:20:00.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:20:00.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:20:00.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:20:00.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:20:00.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:20:00.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:20:00.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:20:00.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:20:00.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:20:00.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:20:00.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:20:00.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:20:00.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:20:00.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:20:00.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:20:00.922 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:20:00.922 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:20:00.930 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:20:00.930 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:20:00.930 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:20:00.930 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:20:00.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:20:00.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:20:00.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:20:00.982 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:20:00.982 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:20:00.982 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:20:00.982 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:20:00.982 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:20:00.982 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:20:00.982 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:20:00.982 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:20:00.982 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:20:00.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:20:00.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:20:00.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:20:00.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:20:00.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:20:00.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:20:00.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:20:00.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:20:00.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:20:00.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:20:00.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:20:00.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:20:00.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:20:00.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:20:00.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:20:00.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:20:00.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:20:00.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:20:00.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:20:00.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:20:00.985 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:20:00.985 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:20:00.985 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:20:00.992 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:20:00.993 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:20:00.993 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:20:00.993 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:20:01.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:20:01.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:20:01.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:20:01.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:20:01.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:20:01.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:20:01.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:20:01.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:20:01.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:20:01.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:20:01.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:20:01.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:20:01.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:20:01.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:20:01.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:20:01.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:20:01.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 74 2026-02-18T08:20:01.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 77 2026-02-18T08:20:01.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:20:01.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=309, tag_tid=21, entry_tid=1], [object_number=308, tag_tid=21, entry_tid=0], [object_number=310, tag_tid=20, entry_tid=46754], [object_number=311, tag_tid=20, entry_tid=46751]]], state=connected] 2026-02-18T08:20:01.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=299, tag_tid=20, entry_tid=35719], [object_number=298, tag_tid=20, entry_tid=35718], [object_number=297, tag_tid=20, entry_tid=35717], [object_number=296, tag_tid=20, entry_tid=35716]]], state=connected] 2026-02-18T08:20:01.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:20:01.086 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=309, tag_tid=21, entry_tid=1' 2026-02-18T08:20:01.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:20:01.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:20:01.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:20:01.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:20:01.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:20:01.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:20:01.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:20:01.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:20:01.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:20:01.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:20:01.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:20:01.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:20:01.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:20:01.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:20:01.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:20:01.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 74 2026-02-18T08:20:01.118 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 77 2026-02-18T08:20:01.118 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:20:01.118 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=309, tag_tid=21, entry_tid=1], [object_number=308, tag_tid=21, entry_tid=0], [object_number=310, tag_tid=20, entry_tid=46754], [object_number=311, tag_tid=20, entry_tid=46751]]], state=connected] 2026-02-18T08:20:01.118 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=299, tag_tid=20, entry_tid=35719], [object_number=298, tag_tid=20, entry_tid=35718], [object_number=297, tag_tid=20, entry_tid=35717], [object_number=296, tag_tid=20, entry_tid=35716]]], state=connected] 2026-02-18T08:20:01.120 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:20:01.121 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=299, tag_tid=20, entry_tid=35719' 2026-02-18T08:20:01.121 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=309, tag_tid=21, entry_tid=1' -a 'object_number=309, tag_tid=21, entry_tid=1' = 'object_number=299, tag_tid=20, entry_tid=35719' 2026-02-18T08:20:01.122 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=299, tag_tid=20, entry_tid=35719' '!=' 'object_number=297, tag_tid=20, entry_tid=35377' 2026-02-18T08:20:01.122 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:20:01.122 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=299, tag_tid=20, entry_tid=35719' = 'object_number=297, tag_tid=20, entry_tid=35377' 2026-02-18T08:20:01.122 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=299, tag_tid=20, entry_tid=35719' 2026-02-18T08:20:01.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=309, tag_tid=21, entry_tid=1' 2026-02-18T08:20:01.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:20:01.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:20:01.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=21 2026-02-18T08:20:01.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=299, tag_tid=20, entry_tid=35719' 2026-02-18T08:20:01.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:20:01.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:20:01.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=20 2026-02-18T08:20:01.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=309, tag_tid=21, entry_tid=1' 2026-02-18T08:20:01.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:20:01.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:20:01.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:20:01.129 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=299, tag_tid=20, entry_tid=35719' 2026-02-18T08:20:01.129 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:20:01.129 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:20:01.131 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=35719 2026-02-18T08:20:01.131 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 21 = 20 -a 1 -le 35719 2026-02-18T08:20:01.131 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:20:01.131 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:20:01.131 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:20:01.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:20:01.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:20:01.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:20:01.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:20:01.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:20:01.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:20:01.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:20:01.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:20:01.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:20:01.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:20:01.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:20:01.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:20:01.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:20:01.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:20:01.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:20:01.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:20:01.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:20:01.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:20:01.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:20:01.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:20:01.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:20:01.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:20:01.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:20:01.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:20:01.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:20:01.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:20:01.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:20:01.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:20:01.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:20:01.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:20:01.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:20:01.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:20:01.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:20:01.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:20:01.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:20:01.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:20:01.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:20:01.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:20:01.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:20:01.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:20:01.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:20:01.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:20:01.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:20:01.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:20:01.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:20:01.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:20:01.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:20:01.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:20:01.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:20:01.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:20:01.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:20:01.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:20:01.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:20:01.395 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:20:01.395 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:20:01.395 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:20:01.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:20:01.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:20:01.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:20:01.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:20:01.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:20:01.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:20:01.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:20:01.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:20:01.399 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:20:01.399 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:20:01.399 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:20:01.399 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:20:01.399 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:20:01.399 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:20:01.399 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:20:01.399 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:20:01.399 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:20:01.399 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:20:01.399 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:20:01.399 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:20:01.399 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:20:01.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:20:01.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:20:01.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:20:01.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:20:01.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:20:01.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:20:01.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:20:01.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:20:01.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:20:01.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:20:01.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:20:01.409 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:20:01.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:20:01.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:20:01.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:20:01.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:20:01.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:20:01.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:20:01.460 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:20:01.460 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:20:01.460 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:20:01.460 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:20:01.460 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:20:01.460 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:20:01.460 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:20:01.460 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:20:01.460 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:20:01.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:20:01.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:20:01.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:20:01.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:20:01.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:20:01.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:20:01.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:20:01.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:20:01.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:20:01.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:20:01.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:20:01.462 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:20:01.462 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:20:01.462 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:20:01.462 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:20:01.462 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:20:01.462 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:20:01.462 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:20:01.462 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:20:01.462 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:20:01.462 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:20:01.462 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:20:01.462 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:20:01.470 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:20:01.470 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:20:01.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:20:01.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:20:01.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:20:01.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:20:01.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:20:01.524 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:20:01.524 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:20:01.524 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:20:01.524 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:20:01.524 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:20:01.524 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:20:01.524 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:20:01.524 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:20:01.525 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:20:01.525 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:20:01.525 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:20:01.525 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:20:01.525 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:20:01.525 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:20:01.525 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:20:01.525 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:20:01.525 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:20:01.525 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:20:01.526 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:20:01.526 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:20:01.526 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:20:01.526 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:20:01.526 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:20:01.526 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:20:01.526 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:20:01.526 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:20:01.526 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:20:01.526 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:20:01.527 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:20:01.527 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:20:01.527 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:20:01.527 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:20:01.535 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:20:01.536 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:20:01.536 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:20:01.536 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:20:01.597 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:20:01.599 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:20:01.599 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:20:01.599 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:20:01.599 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:20:01.599 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:20:01.599 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:20:01.599 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:20:01.600 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:20:01.600 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:20:01.600 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:20:01.600 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:20:01.600 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:20:01.601 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:20:01.601 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:20:01.601 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:20:01.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 74 2026-02-18T08:20:01.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 77 2026-02-18T08:20:01.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:20:01.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=309, tag_tid=21, entry_tid=1], [object_number=308, tag_tid=21, entry_tid=0], [object_number=310, tag_tid=20, entry_tid=46754], [object_number=311, tag_tid=20, entry_tid=46751]]], state=connected] 2026-02-18T08:20:01.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=299, tag_tid=20, entry_tid=36067], [object_number=298, tag_tid=20, entry_tid=36066], [object_number=297, tag_tid=20, entry_tid=36065], [object_number=296, tag_tid=20, entry_tid=36064]]], state=connected] 2026-02-18T08:20:01.636 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:20:01.637 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=309, tag_tid=21, entry_tid=1' 2026-02-18T08:20:01.638 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:20:01.638 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:20:01.638 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:20:01.638 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:20:01.638 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:20:01.639 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:20:01.639 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:20:01.639 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:20:01.639 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:20:01.639 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:20:01.639 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:20:01.639 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:20:01.640 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:20:01.640 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:20:01.640 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:20:01.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 74 2026-02-18T08:20:01.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 77 2026-02-18T08:20:01.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:20:01.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=309, tag_tid=21, entry_tid=1], [object_number=308, tag_tid=21, entry_tid=0], [object_number=310, tag_tid=20, entry_tid=46754], [object_number=311, tag_tid=20, entry_tid=46751]]], state=connected] 2026-02-18T08:20:01.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=299, tag_tid=20, entry_tid=36067], [object_number=298, tag_tid=20, entry_tid=36066], [object_number=297, tag_tid=20, entry_tid=36065], [object_number=296, tag_tid=20, entry_tid=36064]]], state=connected] 2026-02-18T08:20:01.671 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:20:01.673 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=299, tag_tid=20, entry_tid=36067' 2026-02-18T08:20:01.673 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=309, tag_tid=21, entry_tid=1' -a 'object_number=309, tag_tid=21, entry_tid=1' = 'object_number=299, tag_tid=20, entry_tid=36067' 2026-02-18T08:20:01.674 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=299, tag_tid=20, entry_tid=36067' '!=' 'object_number=299, tag_tid=20, entry_tid=35719' 2026-02-18T08:20:01.674 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:20:01.674 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=299, tag_tid=20, entry_tid=36067' = 'object_number=299, tag_tid=20, entry_tid=35719' 2026-02-18T08:20:01.674 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=299, tag_tid=20, entry_tid=36067' 2026-02-18T08:20:01.674 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=309, tag_tid=21, entry_tid=1' 2026-02-18T08:20:01.675 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:20:01.675 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:20:01.675 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=21 2026-02-18T08:20:01.676 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=299, tag_tid=20, entry_tid=36067' 2026-02-18T08:20:01.676 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:20:01.676 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:20:01.677 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=20 2026-02-18T08:20:01.678 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=309, tag_tid=21, entry_tid=1' 2026-02-18T08:20:01.678 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:20:01.678 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:20:01.679 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:20:01.680 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=299, tag_tid=20, entry_tid=36067' 2026-02-18T08:20:01.680 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:20:01.681 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:20:01.682 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=36067 2026-02-18T08:20:01.682 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 21 = 20 -a 1 -le 36067 2026-02-18T08:20:01.682 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:20:01.683 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:20:01.683 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:20:01.884 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:20:01.884 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:20:01.884 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:20:01.885 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:20:01.885 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:20:01.885 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:20:01.885 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:20:01.885 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:20:01.885 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:20:01.885 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:20:01.885 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:20:01.885 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:20:01.885 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:20:01.886 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:20:01.886 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:20:01.886 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:20:01.886 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:20:01.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:20:01.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:20:01.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:20:01.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:20:01.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:20:01.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:20:01.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:20:01.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:20:01.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:20:01.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:20:01.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:20:01.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:20:01.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:20:01.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:20:01.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:20:01.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:20:01.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:20:01.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:20:01.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:20:01.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:20:01.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:20:01.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:20:01.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:20:01.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:20:01.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:20:01.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:20:01.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:20:01.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:20:01.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:20:01.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:20:01.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:20:01.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:20:01.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:20:01.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:20:01.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:20:01.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:20:01.958 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:20:01.959 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:20:01.959 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:20:01.962 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:20:01.962 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:20:01.962 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:20:01.962 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:20:01.962 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:20:01.962 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:20:01.962 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:20:01.962 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:20:01.963 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:20:01.963 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:20:01.963 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:20:01.963 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:20:01.963 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:20:01.963 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:20:01.963 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:20:01.963 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:20:01.963 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:20:01.963 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:20:01.963 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:20:01.963 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:20:01.963 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:20:01.964 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:20:01.964 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:20:01.964 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:20:01.964 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:20:01.964 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:20:01.964 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:20:01.964 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:20:01.964 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:20:01.964 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:20:01.964 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:20:01.964 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:20:01.974 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:20:01.974 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:20:01.974 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:20:01.974 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:20:02.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:20:02.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:20:02.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:20:02.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:20:02.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:20:02.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:20:02.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:20:02.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:20:02.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:20:02.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:20:02.035 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:20:02.035 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:20:02.035 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:20:02.035 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:20:02.035 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:20:02.035 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:20:02.035 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:20:02.035 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:20:02.035 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:20:02.035 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:20:02.036 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:20:02.036 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:20:02.036 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:20:02.036 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:20:02.036 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:20:02.036 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:20:02.036 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:20:02.036 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:20:02.036 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:20:02.036 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:20:02.037 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:20:02.037 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:20:02.037 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:20:02.037 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:20:02.037 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:20:02.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:20:02.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:20:02.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:20:02.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:20:02.093 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:20:02.093 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:20:02.093 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:20:02.096 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:20:02.096 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:20:02.096 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:20:02.096 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:20:02.097 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:20:02.097 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:20:02.097 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:20:02.097 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:20:02.097 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:20:02.097 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:20:02.097 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:20:02.097 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:20:02.097 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:20:02.097 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:20:02.097 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:20:02.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:20:02.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:20:02.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:20:02.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:20:02.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:20:02.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:20:02.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:20:02.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:20:02.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:20:02.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:20:02.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:20:02.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:20:02.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:20:02.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:20:02.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:20:02.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:20:02.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:20:02.106 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:20:02.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:20:02.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:20:02.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:20:02.167 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:20:02.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:20:02.169 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:20:02.169 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:20:02.169 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:20:02.169 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:20:02.169 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:20:02.169 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:20:02.169 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:20:02.169 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:20:02.169 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:20:02.169 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:20:02.170 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:20:02.170 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:20:02.170 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:20:02.171 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:20:02.207 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 74 2026-02-18T08:20:02.207 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 77 2026-02-18T08:20:02.208 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:20:02.208 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=309, tag_tid=21, entry_tid=1], [object_number=308, tag_tid=21, entry_tid=0], [object_number=310, tag_tid=20, entry_tid=46754], [object_number=311, tag_tid=20, entry_tid=46751]]], state=connected] 2026-02-18T08:20:02.208 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=298, tag_tid=20, entry_tid=36298], [object_number=297, tag_tid=20, entry_tid=36297], [object_number=296, tag_tid=20, entry_tid=36296], [object_number=299, tag_tid=20, entry_tid=36295]]], state=connected] 2026-02-18T08:20:02.211 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:20:02.212 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=309, tag_tid=21, entry_tid=1' 2026-02-18T08:20:02.214 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:20:02.214 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:20:02.214 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:20:02.214 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:20:02.214 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:20:02.214 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:20:02.214 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:20:02.214 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:20:02.214 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:20:02.215 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:20:02.215 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:20:02.215 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:20:02.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:20:02.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:20:02.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:20:02.253 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 74 2026-02-18T08:20:02.253 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 77 2026-02-18T08:20:02.253 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:20:02.253 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=309, tag_tid=21, entry_tid=1], [object_number=308, tag_tid=21, entry_tid=0], [object_number=310, tag_tid=20, entry_tid=46754], [object_number=311, tag_tid=20, entry_tid=46751]]], state=connected] 2026-02-18T08:20:02.254 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=298, tag_tid=20, entry_tid=36298], [object_number=297, tag_tid=20, entry_tid=36297], [object_number=296, tag_tid=20, entry_tid=36296], [object_number=299, tag_tid=20, entry_tid=36295]]], state=connected] 2026-02-18T08:20:02.257 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:20:02.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=298, tag_tid=20, entry_tid=36298' 2026-02-18T08:20:02.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=309, tag_tid=21, entry_tid=1' -a 'object_number=309, tag_tid=21, entry_tid=1' = 'object_number=298, tag_tid=20, entry_tid=36298' 2026-02-18T08:20:02.259 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=298, tag_tid=20, entry_tid=36298' '!=' 'object_number=299, tag_tid=20, entry_tid=36067' 2026-02-18T08:20:02.259 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:20:02.259 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=298, tag_tid=20, entry_tid=36298' = 'object_number=299, tag_tid=20, entry_tid=36067' 2026-02-18T08:20:02.259 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=298, tag_tid=20, entry_tid=36298' 2026-02-18T08:20:02.260 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=309, tag_tid=21, entry_tid=1' 2026-02-18T08:20:02.260 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:20:02.260 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:20:02.260 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=21 2026-02-18T08:20:02.261 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=298, tag_tid=20, entry_tid=36298' 2026-02-18T08:20:02.261 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:20:02.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:20:02.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=20 2026-02-18T08:20:02.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=309, tag_tid=21, entry_tid=1' 2026-02-18T08:20:02.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:20:02.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:20:02.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:20:02.266 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=298, tag_tid=20, entry_tid=36298' 2026-02-18T08:20:02.266 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:20:02.266 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:20:02.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=36298 2026-02-18T08:20:02.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 21 = 20 -a 1 -le 36298 2026-02-18T08:20:02.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:20:02.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:20:02.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:20:02.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:20:02.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:20:02.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:20:02.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:20:02.470 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:20:02.470 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:20:02.470 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:20:02.470 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:20:02.470 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:20:02.470 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:20:02.470 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:20:02.470 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:20:02.470 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:20:02.470 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:20:02.470 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:20:02.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:20:02.474 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:20:02.474 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:20:02.474 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:20:02.474 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:20:02.474 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:20:02.474 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:20:02.474 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:20:02.474 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:20:02.475 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:20:02.475 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:20:02.475 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:20:02.475 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:20:02.475 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:20:02.475 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:20:02.475 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:20:02.475 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:20:02.475 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:20:02.476 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:20:02.476 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:20:02.476 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:20:02.476 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:20:02.476 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:20:02.476 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:20:02.476 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:20:02.476 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:20:02.476 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:20:02.476 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:20:02.476 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:20:02.477 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:20:02.477 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:20:02.477 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:20:02.477 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:20:02.477 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:20:02.484 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:20:02.484 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:20:02.484 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:20:02.484 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:20:02.535 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:20:02.535 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:20:02.535 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:20:02.539 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:20:02.539 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:20:02.539 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:20:02.539 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:20:02.539 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:20:02.539 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:20:02.539 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:20:02.540 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:20:02.540 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:20:02.540 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:20:02.540 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:20:02.540 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:20:02.540 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:20:02.540 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:20:02.540 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:20:02.540 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:20:02.540 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:20:02.540 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:20:02.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:20:02.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:20:02.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:20:02.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:20:02.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:20:02.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:20:02.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:20:02.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:20:02.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:20:02.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:20:02.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:20:02.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:20:02.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:20:02.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:20:02.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:20:02.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:20:02.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:20:02.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:20:02.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:20:02.610 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:20:02.610 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:20:02.613 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:20:02.613 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:20:02.613 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:20:02.613 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:20:02.613 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:20:02.613 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:20:02.613 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:20:02.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:20:02.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:20:02.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:20:02.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:20:02.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:20:02.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:20:02.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:20:02.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:20:02.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:20:02.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:20:02.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:20:02.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:20:02.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:20:02.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:20:02.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:20:02.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:20:02.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:20:02.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:20:02.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:20:02.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:20:02.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:20:02.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:20:02.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:20:02.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:20:02.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:20:02.622 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:20:02.623 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:20:02.623 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:20:02.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:20:02.673 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:20:02.673 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:20:02.673 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:20:02.675 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:20:02.676 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:20:02.676 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:20:02.676 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:20:02.676 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:20:02.676 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:20:02.676 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:20:02.676 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:20:02.676 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:20:02.677 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:20:02.677 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:20:02.677 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:20:02.677 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:20:02.677 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:20:02.677 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:20:02.677 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:20:02.677 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:20:02.677 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:20:02.677 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:20:02.677 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:20:02.677 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:20:02.678 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:20:02.678 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:20:02.678 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:20:02.678 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:20:02.678 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:20:02.678 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:20:02.678 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:20:02.678 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:20:02.678 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:20:02.678 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:20:02.679 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:20:02.686 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:20:02.687 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:20:02.687 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:20:02.687 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:20:02.748 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:20:02.750 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:20:02.750 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:20:02.750 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:20:02.750 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:20:02.750 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:20:02.750 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:20:02.750 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:20:02.750 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:20:02.751 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:20:02.751 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:20:02.751 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:20:02.751 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:20:02.752 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:20:02.752 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:20:02.752 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:20:02.788 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 74 2026-02-18T08:20:02.788 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 77 2026-02-18T08:20:02.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:20:02.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=309, tag_tid=21, entry_tid=1], [object_number=308, tag_tid=21, entry_tid=0], [object_number=310, tag_tid=20, entry_tid=46754], [object_number=311, tag_tid=20, entry_tid=46751]]], state=connected] 2026-02-18T08:20:02.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=299, tag_tid=20, entry_tid=36671], [object_number=298, tag_tid=20, entry_tid=36670], [object_number=297, tag_tid=20, entry_tid=36669], [object_number=296, tag_tid=20, entry_tid=36668]]], state=connected] 2026-02-18T08:20:02.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:20:02.794 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=309, tag_tid=21, entry_tid=1' 2026-02-18T08:20:02.795 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:20:02.795 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:20:02.795 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:20:02.795 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:20:02.795 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:20:02.795 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:20:02.796 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:20:02.796 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:20:02.796 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:20:02.796 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:20:02.796 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:20:02.796 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:20:02.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:20:02.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:20:02.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:20:02.825 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 74 2026-02-18T08:20:02.825 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 77 2026-02-18T08:20:02.826 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:20:02.826 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=309, tag_tid=21, entry_tid=1], [object_number=308, tag_tid=21, entry_tid=0], [object_number=310, tag_tid=20, entry_tid=46754], [object_number=311, tag_tid=20, entry_tid=46751]]], state=connected] 2026-02-18T08:20:02.826 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=299, tag_tid=20, entry_tid=36671], [object_number=298, tag_tid=20, entry_tid=36670], [object_number=297, tag_tid=20, entry_tid=36669], [object_number=296, tag_tid=20, entry_tid=36668]]], state=connected] 2026-02-18T08:20:02.828 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:20:02.830 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=299, tag_tid=20, entry_tid=36671' 2026-02-18T08:20:02.830 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=309, tag_tid=21, entry_tid=1' -a 'object_number=309, tag_tid=21, entry_tid=1' = 'object_number=299, tag_tid=20, entry_tid=36671' 2026-02-18T08:20:02.830 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=299, tag_tid=20, entry_tid=36671' '!=' 'object_number=298, tag_tid=20, entry_tid=36298' 2026-02-18T08:20:02.830 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:20:02.830 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=299, tag_tid=20, entry_tid=36671' = 'object_number=298, tag_tid=20, entry_tid=36298' 2026-02-18T08:20:02.831 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=299, tag_tid=20, entry_tid=36671' 2026-02-18T08:20:02.831 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=309, tag_tid=21, entry_tid=1' 2026-02-18T08:20:02.831 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:20:02.832 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:20:02.832 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=21 2026-02-18T08:20:02.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=299, tag_tid=20, entry_tid=36671' 2026-02-18T08:20:02.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:20:02.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:20:02.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=20 2026-02-18T08:20:02.835 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=309, tag_tid=21, entry_tid=1' 2026-02-18T08:20:02.835 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:20:02.835 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:20:02.837 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:20:02.838 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=299, tag_tid=20, entry_tid=36671' 2026-02-18T08:20:02.838 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:20:02.838 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:20:02.839 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=36671 2026-02-18T08:20:02.840 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 21 = 20 -a 1 -le 36671 2026-02-18T08:20:02.840 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:20:02.840 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:20:02.840 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:20:03.041 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:20:03.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:20:03.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:20:03.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:20:03.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:20:03.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:20:03.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:20:03.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:20:03.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:20:03.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:20:03.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:20:03.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:20:03.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:20:03.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:20:03.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:20:03.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:20:03.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:20:03.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:20:03.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:20:03.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:20:03.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:20:03.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:20:03.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:20:03.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:20:03.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:20:03.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:20:03.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:20:03.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:20:03.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:20:03.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:20:03.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:20:03.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:20:03.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:20:03.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:20:03.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:20:03.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:20:03.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:20:03.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:20:03.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:20:03.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:20:03.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:20:03.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:20:03.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:20:03.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:20:03.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:20:03.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:20:03.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:20:03.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:20:03.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:20:03.056 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:20:03.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:20:03.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:20:03.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:20:03.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:20:03.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:20:03.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:20:03.111 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:20:03.111 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:20:03.111 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:20:03.111 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:20:03.111 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:20:03.111 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:20:03.112 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:20:03.112 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:20:03.112 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:20:03.112 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:20:03.112 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:20:03.112 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:20:03.112 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:20:03.112 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:20:03.112 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:20:03.113 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:20:03.113 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:20:03.113 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:20:03.113 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:20:03.113 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:20:03.113 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:20:03.113 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:20:03.113 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:20:03.113 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:20:03.113 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:20:03.114 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:20:03.114 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:20:03.114 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:20:03.114 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:20:03.114 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:20:03.114 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:20:03.114 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:20:03.122 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:20:03.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:20:03.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:20:03.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:20:03.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:20:03.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:20:03.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:20:03.175 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:20:03.175 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:20:03.175 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:20:03.175 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:20:03.175 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:20:03.175 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:20:03.175 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:20:03.176 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:20:03.176 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:20:03.176 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:20:03.176 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:20:03.176 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:20:03.176 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:20:03.176 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:20:03.176 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:20:03.176 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:20:03.176 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:20:03.176 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:20:03.177 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:20:03.177 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:20:03.177 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:20:03.177 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:20:03.177 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:20:03.177 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:20:03.177 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:20:03.177 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:20:03.177 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:20:03.177 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:20:03.178 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:20:03.178 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:20:03.178 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:20:03.178 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:20:03.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:20:03.187 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:20:03.187 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:20:03.187 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:20:03.244 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:20:03.245 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:20:03.245 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:20:03.247 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:20:03.247 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:20:03.247 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:20:03.248 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:20:03.248 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:20:03.248 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:20:03.248 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:20:03.248 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:20:03.248 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:20:03.248 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:20:03.248 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:20:03.248 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:20:03.248 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:20:03.248 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:20:03.249 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:20:03.249 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:20:03.249 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:20:03.249 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:20:03.249 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:20:03.249 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:20:03.249 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:20:03.249 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:20:03.249 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:20:03.249 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:20:03.249 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:20:03.250 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:20:03.250 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:20:03.250 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:20:03.250 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:20:03.250 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:20:03.250 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:20:03.250 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:20:03.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:20:03.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:20:03.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:20:03.259 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:20:03.326 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:20:03.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:20:03.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:20:03.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:20:03.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:20:03.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:20:03.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:20:03.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:20:03.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:20:03.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:20:03.329 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:20:03.329 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:20:03.329 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:20:03.329 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:20:03.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:20:03.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:20:03.367 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 74 2026-02-18T08:20:03.367 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 77 2026-02-18T08:20:03.367 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:20:03.367 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=309, tag_tid=21, entry_tid=1], [object_number=308, tag_tid=21, entry_tid=0], [object_number=310, tag_tid=20, entry_tid=46754], [object_number=311, tag_tid=20, entry_tid=46751]]], state=connected] 2026-02-18T08:20:03.368 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=299, tag_tid=20, entry_tid=37003], [object_number=298, tag_tid=20, entry_tid=37002], [object_number=297, tag_tid=20, entry_tid=37001], [object_number=296, tag_tid=20, entry_tid=37000]]], state=connected] 2026-02-18T08:20:03.370 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:20:03.372 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=309, tag_tid=21, entry_tid=1' 2026-02-18T08:20:03.373 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:20:03.373 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:20:03.374 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:20:03.374 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:20:03.374 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:20:03.374 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:20:03.374 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:20:03.374 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:20:03.374 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:20:03.374 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:20:03.374 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:20:03.374 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:20:03.375 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:20:03.375 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:20:03.375 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:20:03.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 74 2026-02-18T08:20:03.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 77 2026-02-18T08:20:03.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:20:03.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=309, tag_tid=21, entry_tid=1], [object_number=308, tag_tid=21, entry_tid=0], [object_number=310, tag_tid=20, entry_tid=46754], [object_number=311, tag_tid=20, entry_tid=46751]]], state=connected] 2026-02-18T08:20:03.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=299, tag_tid=20, entry_tid=37003], [object_number=298, tag_tid=20, entry_tid=37002], [object_number=297, tag_tid=20, entry_tid=37001], [object_number=296, tag_tid=20, entry_tid=37000]]], state=connected] 2026-02-18T08:20:03.415 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:20:03.417 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=299, tag_tid=20, entry_tid=37003' 2026-02-18T08:20:03.417 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=309, tag_tid=21, entry_tid=1' -a 'object_number=309, tag_tid=21, entry_tid=1' = 'object_number=299, tag_tid=20, entry_tid=37003' 2026-02-18T08:20:03.417 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=299, tag_tid=20, entry_tid=37003' '!=' 'object_number=299, tag_tid=20, entry_tid=36671' 2026-02-18T08:20:03.417 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:20:03.417 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=299, tag_tid=20, entry_tid=37003' = 'object_number=299, tag_tid=20, entry_tid=36671' 2026-02-18T08:20:03.417 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=299, tag_tid=20, entry_tid=37003' 2026-02-18T08:20:03.418 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=309, tag_tid=21, entry_tid=1' 2026-02-18T08:20:03.418 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:20:03.418 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:20:03.419 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=21 2026-02-18T08:20:03.420 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=299, tag_tid=20, entry_tid=37003' 2026-02-18T08:20:03.420 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:20:03.420 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:20:03.421 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=20 2026-02-18T08:20:03.422 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=309, tag_tid=21, entry_tid=1' 2026-02-18T08:20:03.422 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:20:03.422 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:20:03.423 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:20:03.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=299, tag_tid=20, entry_tid=37003' 2026-02-18T08:20:03.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:20:03.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:20:03.425 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=37003 2026-02-18T08:20:03.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 21 = 20 -a 1 -le 37003 2026-02-18T08:20:03.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:20:03.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:20:03.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:20:03.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:20:03.628 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:20:03.628 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:20:03.628 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:20:03.628 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:20:03.628 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:20:03.628 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:20:03.628 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:20:03.628 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:20:03.628 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:20:03.628 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:20:03.629 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:20:03.629 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:20:03.629 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:20:03.629 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:20:03.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:20:03.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:20:03.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:20:03.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:20:03.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:20:03.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:20:03.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:20:03.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:20:03.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:20:03.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:20:03.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:20:03.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:20:03.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:20:03.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:20:03.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:20:03.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:20:03.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:20:03.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:20:03.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:20:03.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:20:03.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:20:03.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:20:03.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:20:03.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:20:03.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:20:03.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:20:03.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:20:03.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:20:03.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:20:03.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:20:03.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:20:03.635 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:20:03.635 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:20:03.635 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:20:03.642 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:20:03.643 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:20:03.643 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:20:03.643 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:20:03.701 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:20:03.701 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:20:03.701 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:20:03.703 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:20:03.704 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:20:03.704 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:20:03.704 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:20:03.704 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:20:03.704 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:20:03.704 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:20:03.704 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:20:03.704 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:20:03.704 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:20:03.704 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:20:03.705 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:20:03.705 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:20:03.705 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:20:03.705 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:20:03.705 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:20:03.705 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:20:03.705 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:20:03.705 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:20:03.705 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:20:03.705 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:20:03.705 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:20:03.706 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:20:03.706 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:20:03.706 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:20:03.706 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:20:03.706 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:20:03.706 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:20:03.706 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:20:03.706 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:20:03.706 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:20:03.706 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:20:03.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:20:03.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:20:03.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:20:03.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:20:03.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:20:03.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:20:03.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:20:03.767 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:20:03.767 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:20:03.767 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:20:03.767 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:20:03.767 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:20:03.767 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:20:03.768 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:20:03.768 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:20:03.768 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:20:03.768 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:20:03.768 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:20:03.768 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:20:03.768 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:20:03.768 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:20:03.768 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:20:03.768 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:20:03.768 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:20:03.768 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:20:03.769 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:20:03.769 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:20:03.769 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:20:03.769 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:20:03.769 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:20:03.769 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:20:03.769 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:20:03.769 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:20:03.769 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:20:03.769 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:20:03.769 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:20:03.769 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:20:03.770 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:20:03.770 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:20:03.778 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:20:03.779 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:20:03.779 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:20:03.779 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:20:03.828 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:20:03.828 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:20:03.828 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:20:03.831 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:20:03.831 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:20:03.831 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:20:03.831 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:20:03.831 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:20:03.831 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:20:03.831 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:20:03.831 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:20:03.832 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:20:03.832 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:20:03.832 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:20:03.832 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:20:03.832 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:20:03.832 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:20:03.832 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:20:03.832 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:20:03.832 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:20:03.832 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:20:03.832 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:20:03.832 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:20:03.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:20:03.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:20:03.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:20:03.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:20:03.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:20:03.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:20:03.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:20:03.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:20:03.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:20:03.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:20:03.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:20:03.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:20:03.842 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:20:03.842 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:20:03.842 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:20:03.842 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:20:03.909 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:20:03.911 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:20:03.911 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:20:03.911 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:20:03.911 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:20:03.911 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:20:03.911 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:20:03.911 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:20:03.911 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:20:03.911 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:20:03.911 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:20:03.912 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:20:03.912 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:20:03.912 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:20:03.913 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:20:03.913 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:20:03.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 74 2026-02-18T08:20:03.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 77 2026-02-18T08:20:03.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:20:03.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=309, tag_tid=21, entry_tid=1], [object_number=308, tag_tid=21, entry_tid=0], [object_number=310, tag_tid=20, entry_tid=46754], [object_number=311, tag_tid=20, entry_tid=46751]]], state=connected] 2026-02-18T08:20:03.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=298, tag_tid=20, entry_tid=37362], [object_number=297, tag_tid=20, entry_tid=37361], [object_number=296, tag_tid=20, entry_tid=37360], [object_number=299, tag_tid=20, entry_tid=37359]]], state=connected] 2026-02-18T08:20:03.944 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:20:03.946 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=309, tag_tid=21, entry_tid=1' 2026-02-18T08:20:03.947 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:20:03.947 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:20:03.947 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:20:03.947 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:20:03.947 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:20:03.947 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:20:03.948 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:20:03.948 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:20:03.948 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:20:03.948 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:20:03.948 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:20:03.948 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:20:03.949 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:20:03.949 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:20:03.949 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:20:03.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 74 2026-02-18T08:20:03.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 77 2026-02-18T08:20:03.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:20:03.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=309, tag_tid=21, entry_tid=1], [object_number=308, tag_tid=21, entry_tid=0], [object_number=310, tag_tid=20, entry_tid=46754], [object_number=311, tag_tid=20, entry_tid=46751]]], state=connected] 2026-02-18T08:20:03.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=298, tag_tid=20, entry_tid=37362], [object_number=297, tag_tid=20, entry_tid=37361], [object_number=296, tag_tid=20, entry_tid=37360], [object_number=299, tag_tid=20, entry_tid=37359]]], state=connected] 2026-02-18T08:20:03.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:20:03.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=298, tag_tid=20, entry_tid=37362' 2026-02-18T08:20:03.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=309, tag_tid=21, entry_tid=1' -a 'object_number=309, tag_tid=21, entry_tid=1' = 'object_number=298, tag_tid=20, entry_tid=37362' 2026-02-18T08:20:03.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=298, tag_tid=20, entry_tid=37362' '!=' 'object_number=299, tag_tid=20, entry_tid=37003' 2026-02-18T08:20:03.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:20:03.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=298, tag_tid=20, entry_tid=37362' = 'object_number=299, tag_tid=20, entry_tid=37003' 2026-02-18T08:20:03.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=298, tag_tid=20, entry_tid=37362' 2026-02-18T08:20:03.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=309, tag_tid=21, entry_tid=1' 2026-02-18T08:20:03.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:20:03.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:20:03.985 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=21 2026-02-18T08:20:03.986 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=298, tag_tid=20, entry_tid=37362' 2026-02-18T08:20:03.986 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:20:03.986 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:20:03.987 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=20 2026-02-18T08:20:03.988 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=309, tag_tid=21, entry_tid=1' 2026-02-18T08:20:03.988 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:20:03.988 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:20:03.989 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:20:03.990 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=298, tag_tid=20, entry_tid=37362' 2026-02-18T08:20:03.990 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:20:03.990 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:20:03.992 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=37362 2026-02-18T08:20:03.992 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 21 = 20 -a 1 -le 37362 2026-02-18T08:20:03.992 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:20:03.992 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:20:03.992 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:20:04.194 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:20:04.194 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:20:04.194 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:20:04.195 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:20:04.195 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:20:04.195 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:20:04.195 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:20:04.195 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:20:04.195 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:20:04.195 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:20:04.195 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:20:04.195 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:20:04.195 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:20:04.196 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:20:04.196 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:20:04.196 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:20:04.196 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:20:04.198 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:20:04.198 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:20:04.198 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:20:04.199 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:20:04.199 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:20:04.199 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:20:04.199 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:20:04.199 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:20:04.199 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:20:04.199 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:20:04.199 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:20:04.199 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:20:04.199 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:20:04.199 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:20:04.200 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:20:04.200 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:20:04.200 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:20:04.200 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:20:04.200 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:20:04.200 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:20:04.200 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:20:04.200 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:20:04.200 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:20:04.200 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:20:04.200 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:20:04.200 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:20:04.201 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:20:04.201 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:20:04.201 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:20:04.201 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:20:04.201 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:20:04.201 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:20:04.208 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:20:04.208 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:20:04.209 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:20:04.209 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:20:04.259 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:20:04.259 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:20:04.259 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:20:04.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:20:04.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:20:04.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:20:04.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:20:04.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:20:04.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:20:04.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:20:04.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:20:04.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:20:04.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:20:04.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:20:04.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:20:04.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:20:04.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:20:04.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:20:04.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:20:04.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:20:04.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:20:04.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:20:04.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:20:04.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:20:04.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:20:04.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:20:04.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:20:04.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:20:04.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:20:04.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:20:04.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:20:04.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:20:04.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:20:04.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:20:04.266 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:20:04.274 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:20:04.274 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:20:04.274 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:20:04.274 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:20:04.322 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:20:04.322 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:20:04.322 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:20:04.324 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:20:04.324 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:20:04.324 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:20:04.324 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:20:04.324 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:20:04.324 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:20:04.324 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:20:04.324 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:20:04.324 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:20:04.324 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:20:04.324 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:20:04.324 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:20:04.324 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:20:04.324 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:20:04.324 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:20:04.324 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:20:04.324 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:20:04.324 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:20:04.324 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:20:04.325 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:20:04.325 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:20:04.325 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:20:04.325 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:20:04.325 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:20:04.325 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:20:04.325 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:20:04.325 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:20:04.325 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:20:04.325 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:20:04.325 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:20:04.325 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:20:04.325 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:20:04.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:20:04.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:20:04.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:20:04.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:20:04.385 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:20:04.385 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:20:04.385 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:20:04.387 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:20:04.387 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:20:04.387 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:20:04.387 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:20:04.387 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:20:04.387 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:20:04.387 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:20:04.387 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:20:04.387 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:20:04.387 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:20:04.387 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:20:04.387 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:20:04.387 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:20:04.387 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:20:04.387 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:20:04.387 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:20:04.388 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:20:04.388 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:20:04.388 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:20:04.388 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:20:04.388 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:20:04.388 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:20:04.388 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:20:04.388 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:20:04.388 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:20:04.388 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:20:04.388 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:20:04.388 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:20:04.388 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:20:04.388 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:20:04.388 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:20:04.388 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:20:04.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:20:04.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:20:04.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:20:04.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:20:04.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:20:04.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:20:04.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:20:04.467 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:20:04.467 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:20:04.467 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:20:04.467 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:20:04.467 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:20:04.467 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:20:04.467 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:20:04.467 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:20:04.468 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:20:04.468 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:20:04.468 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:20:04.468 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:20:04.468 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:20:04.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 74 2026-02-18T08:20:04.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 77 2026-02-18T08:20:04.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:20:04.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=309, tag_tid=21, entry_tid=1], [object_number=308, tag_tid=21, entry_tid=0], [object_number=310, tag_tid=20, entry_tid=46754], [object_number=311, tag_tid=20, entry_tid=46751]]], state=connected] 2026-02-18T08:20:04.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=296, tag_tid=20, entry_tid=37708], [object_number=299, tag_tid=20, entry_tid=37707], [object_number=298, tag_tid=20, entry_tid=37706], [object_number=297, tag_tid=20, entry_tid=37705]]], state=connected] 2026-02-18T08:20:04.500 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:20:04.502 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=309, tag_tid=21, entry_tid=1' 2026-02-18T08:20:04.503 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:20:04.503 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:20:04.503 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:20:04.504 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:20:04.504 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:20:04.504 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:20:04.504 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:20:04.504 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:20:04.504 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:20:04.504 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:20:04.505 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:20:04.505 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:20:04.505 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:20:04.506 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:20:04.506 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:20:04.532 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 74 2026-02-18T08:20:04.532 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 77 2026-02-18T08:20:04.532 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:20:04.532 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=309, tag_tid=21, entry_tid=1], [object_number=308, tag_tid=21, entry_tid=0], [object_number=310, tag_tid=20, entry_tid=46754], [object_number=311, tag_tid=20, entry_tid=46751]]], state=connected] 2026-02-18T08:20:04.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=296, tag_tid=20, entry_tid=37708], [object_number=299, tag_tid=20, entry_tid=37707], [object_number=298, tag_tid=20, entry_tid=37706], [object_number=297, tag_tid=20, entry_tid=37705]]], state=connected] 2026-02-18T08:20:04.536 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:20:04.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=296, tag_tid=20, entry_tid=37708' 2026-02-18T08:20:04.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=309, tag_tid=21, entry_tid=1' -a 'object_number=309, tag_tid=21, entry_tid=1' = 'object_number=296, tag_tid=20, entry_tid=37708' 2026-02-18T08:20:04.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=296, tag_tid=20, entry_tid=37708' '!=' 'object_number=298, tag_tid=20, entry_tid=37362' 2026-02-18T08:20:04.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:20:04.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=296, tag_tid=20, entry_tid=37708' = 'object_number=298, tag_tid=20, entry_tid=37362' 2026-02-18T08:20:04.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=296, tag_tid=20, entry_tid=37708' 2026-02-18T08:20:04.539 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=309, tag_tid=21, entry_tid=1' 2026-02-18T08:20:04.539 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:20:04.539 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:20:04.539 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=21 2026-02-18T08:20:04.540 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=296, tag_tid=20, entry_tid=37708' 2026-02-18T08:20:04.540 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:20:04.540 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:20:04.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=20 2026-02-18T08:20:04.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=309, tag_tid=21, entry_tid=1' 2026-02-18T08:20:04.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:20:04.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:20:04.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:20:04.545 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=296, tag_tid=20, entry_tid=37708' 2026-02-18T08:20:04.545 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:20:04.545 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:20:04.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=37708 2026-02-18T08:20:04.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 21 = 20 -a 1 -le 37708 2026-02-18T08:20:04.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:20:04.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:20:04.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:20:04.748 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:20:04.749 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:20:04.749 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:20:04.749 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:20:04.749 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:20:04.749 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:20:04.749 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:20:04.749 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:20:04.749 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:20:04.750 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:20:04.750 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:20:04.750 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:20:04.750 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:20:04.750 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:20:04.750 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:20:04.750 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:20:04.753 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:20:04.753 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:20:04.754 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:20:04.754 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:20:04.754 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:20:04.754 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:20:04.754 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:20:04.754 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:20:04.754 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:20:04.754 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:20:04.754 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:20:04.755 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:20:04.755 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:20:04.755 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:20:04.755 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:20:04.755 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:20:04.755 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:20:04.755 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:20:04.755 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:20:04.755 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:20:04.755 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:20:04.755 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:20:04.756 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:20:04.756 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:20:04.756 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:20:04.756 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:20:04.756 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:20:04.756 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:20:04.756 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:20:04.756 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:20:04.756 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:20:04.756 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:20:04.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:20:04.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:20:04.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:20:04.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:20:04.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:20:04.814 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:20:04.814 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:20:04.814 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:20:04.818 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:20:04.818 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:20:04.818 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:20:04.818 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:20:04.818 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:20:04.818 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:20:04.819 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:20:04.819 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:20:04.819 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:20:04.819 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:20:04.819 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:20:04.819 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:20:04.819 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:20:04.819 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:20:04.819 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:20:04.819 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:20:04.819 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:20:04.820 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:20:04.820 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:20:04.820 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:20:04.820 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:20:04.820 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:20:04.820 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:20:04.820 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:20:04.820 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:20:04.820 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:20:04.820 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:20:04.820 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:20:04.821 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:20:04.821 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:20:04.821 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:20:04.821 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:20:04.830 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:20:04.831 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:20:04.831 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:20:04.831 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:20:04.882 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:20:04.883 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:20:04.883 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:20:04.886 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:20:04.886 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:20:04.887 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:20:04.887 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:20:04.887 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:20:04.887 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:20:04.887 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:20:04.887 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:20:04.887 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:20:04.887 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:20:04.887 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:20:04.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:20:04.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:20:04.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:20:04.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:20:04.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:20:04.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:20:04.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:20:04.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:20:04.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:20:04.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:20:04.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:20:04.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:20:04.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:20:04.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:20:04.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:20:04.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:20:04.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:20:04.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:20:04.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:20:04.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:20:04.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:20:04.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:20:04.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:20:04.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:20:04.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:20:04.956 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:20:04.956 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:20:04.956 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:20:04.958 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:20:04.958 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:20:04.958 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:20:04.959 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:20:04.959 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:20:04.959 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:20:04.959 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:20:04.959 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:20:04.959 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:20:04.959 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:20:04.959 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:20:04.959 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:20:04.959 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:20:04.959 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:20:04.959 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:20:04.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:20:04.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:20:04.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:20:04.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:20:04.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:20:04.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:20:04.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:20:04.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:20:04.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:20:04.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:20:04.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:20:04.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:20:04.961 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:20:04.961 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:20:04.961 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:20:04.961 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:20:04.961 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:20:04.969 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:20:04.970 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:20:04.970 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:20:04.970 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:20:05.039 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:20:05.040 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:20:05.040 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:20:05.041 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:20:05.041 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:20:05.041 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:20:05.041 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:20:05.041 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:20:05.041 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:20:05.041 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:20:05.041 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:20:05.041 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:20:05.041 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:20:05.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:20:05.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:20:05.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:20:05.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 75 2026-02-18T08:20:05.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 77 2026-02-18T08:20:05.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:20:05.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=309, tag_tid=21, entry_tid=1], [object_number=308, tag_tid=21, entry_tid=0], [object_number=310, tag_tid=20, entry_tid=46754], [object_number=311, tag_tid=20, entry_tid=46751]]], state=connected] 2026-02-18T08:20:05.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=303, tag_tid=20, entry_tid=37963], [object_number=302, tag_tid=20, entry_tid=37962], [object_number=301, tag_tid=20, entry_tid=37961], [object_number=300, tag_tid=20, entry_tid=37960]]], state=connected] 2026-02-18T08:20:05.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:20:05.075 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=309, tag_tid=21, entry_tid=1' 2026-02-18T08:20:05.076 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:20:05.076 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:20:05.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:20:05.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:20:05.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:20:05.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:20:05.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:20:05.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:20:05.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:20:05.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:20:05.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:20:05.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:20:05.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:20:05.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:20:05.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:20:05.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 75 2026-02-18T08:20:05.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 77 2026-02-18T08:20:05.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:20:05.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=309, tag_tid=21, entry_tid=1], [object_number=308, tag_tid=21, entry_tid=0], [object_number=310, tag_tid=20, entry_tid=46754], [object_number=311, tag_tid=20, entry_tid=46751]]], state=connected] 2026-02-18T08:20:05.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=303, tag_tid=20, entry_tid=37963], [object_number=302, tag_tid=20, entry_tid=37962], [object_number=301, tag_tid=20, entry_tid=37961], [object_number=300, tag_tid=20, entry_tid=37960]]], state=connected] 2026-02-18T08:20:05.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:20:05.111 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=303, tag_tid=20, entry_tid=37963' 2026-02-18T08:20:05.111 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=309, tag_tid=21, entry_tid=1' -a 'object_number=309, tag_tid=21, entry_tid=1' = 'object_number=303, tag_tid=20, entry_tid=37963' 2026-02-18T08:20:05.112 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=303, tag_tid=20, entry_tid=37963' '!=' 'object_number=296, tag_tid=20, entry_tid=37708' 2026-02-18T08:20:05.112 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:20:05.112 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=303, tag_tid=20, entry_tid=37963' = 'object_number=296, tag_tid=20, entry_tid=37708' 2026-02-18T08:20:05.112 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=303, tag_tid=20, entry_tid=37963' 2026-02-18T08:20:05.112 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=309, tag_tid=21, entry_tid=1' 2026-02-18T08:20:05.112 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:20:05.112 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:20:05.113 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=21 2026-02-18T08:20:05.114 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=303, tag_tid=20, entry_tid=37963' 2026-02-18T08:20:05.114 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:20:05.114 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:20:05.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=20 2026-02-18T08:20:05.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=309, tag_tid=21, entry_tid=1' 2026-02-18T08:20:05.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:20:05.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:20:05.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:20:05.118 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=303, tag_tid=20, entry_tid=37963' 2026-02-18T08:20:05.118 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:20:05.118 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:20:05.120 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=37963 2026-02-18T08:20:05.120 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 21 = 20 -a 1 -le 37963 2026-02-18T08:20:05.120 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:20:05.121 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:20:05.121 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:20:05.322 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:20:05.322 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:20:05.322 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:20:05.323 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:20:05.323 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:20:05.323 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:20:05.323 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:20:05.323 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:20:05.323 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:20:05.323 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:20:05.323 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:20:05.324 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:20:05.324 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:20:05.324 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:20:05.324 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:20:05.324 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:20:05.327 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:20:05.327 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:20:05.327 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:20:05.327 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:20:05.327 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:20:05.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:20:05.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:20:05.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:20:05.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:20:05.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:20:05.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:20:05.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:20:05.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:20:05.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:20:05.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:20:05.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:20:05.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:20:05.329 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:20:05.329 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:20:05.329 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:20:05.329 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:20:05.329 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:20:05.329 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:20:05.329 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:20:05.329 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:20:05.329 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:20:05.329 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:20:05.329 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:20:05.329 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:20:05.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:20:05.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:20:05.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:20:05.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:20:05.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:20:05.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:20:05.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:20:05.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:20:05.395 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:20:05.396 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:20:05.396 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:20:05.399 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:20:05.399 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:20:05.399 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:20:05.399 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:20:05.399 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:20:05.399 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:20:05.399 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:20:05.399 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:20:05.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:20:05.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:20:05.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:20:05.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:20:05.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:20:05.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:20:05.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:20:05.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:20:05.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:20:05.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:20:05.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:20:05.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:20:05.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:20:05.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:20:05.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:20:05.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:20:05.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:20:05.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:20:05.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:20:05.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:20:05.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:20:05.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:20:05.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:20:05.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:20:05.409 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:20:05.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:20:05.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:20:05.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:20:05.467 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:20:05.467 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:20:05.467 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:20:05.470 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:20:05.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:20:05.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:20:05.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:20:05.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:20:05.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:20:05.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:20:05.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:20:05.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:20:05.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:20:05.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:20:05.472 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:20:05.472 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:20:05.472 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:20:05.472 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:20:05.472 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:20:05.472 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:20:05.472 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:20:05.472 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:20:05.472 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:20:05.472 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:20:05.473 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:20:05.473 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:20:05.473 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:20:05.473 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:20:05.473 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:20:05.473 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:20:05.473 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:20:05.473 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:20:05.473 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:20:05.473 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:20:05.473 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:20:05.481 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:20:05.481 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:20:05.481 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:20:05.481 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:20:05.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:20:05.539 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:20:05.539 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:20:05.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:20:05.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:20:05.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:20:05.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:20:05.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:20:05.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:20:05.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:20:05.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:20:05.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:20:05.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:20:05.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:20:05.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:20:05.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:20:05.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:20:05.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:20:05.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:20:05.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:20:05.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:20:05.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:20:05.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:20:05.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:20:05.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:20:05.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:20:05.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:20:05.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:20:05.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:20:05.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:20:05.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:20:05.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:20:05.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:20:05.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:20:05.545 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:20:05.551 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:20:05.552 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:20:05.552 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:20:05.552 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:20:05.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:20:05.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:20:05.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:20:05.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:20:05.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:20:05.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:20:05.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:20:05.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:20:05.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:20:05.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:20:05.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:20:05.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:20:05.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:20:05.618 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:20:05.618 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:20:05.618 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:20:05.653 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 75 2026-02-18T08:20:05.653 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 77 2026-02-18T08:20:05.653 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:20:05.654 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=309, tag_tid=21, entry_tid=1], [object_number=308, tag_tid=21, entry_tid=0], [object_number=310, tag_tid=20, entry_tid=46754], [object_number=311, tag_tid=20, entry_tid=46751]]], state=connected] 2026-02-18T08:20:05.654 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=303, tag_tid=20, entry_tid=38291], [object_number=302, tag_tid=20, entry_tid=38290], [object_number=301, tag_tid=20, entry_tid=38289], [object_number=300, tag_tid=20, entry_tid=38288]]], state=connected] 2026-02-18T08:20:05.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:20:05.659 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=309, tag_tid=21, entry_tid=1' 2026-02-18T08:20:05.660 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:20:05.660 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:20:05.660 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:20:05.660 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:20:05.660 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:20:05.660 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:20:05.660 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:20:05.660 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:20:05.661 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:20:05.661 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:20:05.661 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:20:05.661 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:20:05.662 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:20:05.662 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:20:05.662 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:20:05.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 75 2026-02-18T08:20:05.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 77 2026-02-18T08:20:05.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:20:05.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=309, tag_tid=21, entry_tid=1], [object_number=308, tag_tid=21, entry_tid=0], [object_number=310, tag_tid=20, entry_tid=46754], [object_number=311, tag_tid=20, entry_tid=46751]]], state=connected] 2026-02-18T08:20:05.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=303, tag_tid=20, entry_tid=38291], [object_number=302, tag_tid=20, entry_tid=38290], [object_number=301, tag_tid=20, entry_tid=38289], [object_number=300, tag_tid=20, entry_tid=38288]]], state=connected] 2026-02-18T08:20:05.692 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:20:05.694 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=303, tag_tid=20, entry_tid=38291' 2026-02-18T08:20:05.694 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=309, tag_tid=21, entry_tid=1' -a 'object_number=309, tag_tid=21, entry_tid=1' = 'object_number=303, tag_tid=20, entry_tid=38291' 2026-02-18T08:20:05.694 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=303, tag_tid=20, entry_tid=38291' '!=' 'object_number=303, tag_tid=20, entry_tid=37963' 2026-02-18T08:20:05.694 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:20:05.694 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=303, tag_tid=20, entry_tid=38291' = 'object_number=303, tag_tid=20, entry_tid=37963' 2026-02-18T08:20:05.694 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=303, tag_tid=20, entry_tid=38291' 2026-02-18T08:20:05.695 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=309, tag_tid=21, entry_tid=1' 2026-02-18T08:20:05.695 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:20:05.695 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:20:05.696 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=21 2026-02-18T08:20:05.696 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=303, tag_tid=20, entry_tid=38291' 2026-02-18T08:20:05.697 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:20:05.697 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:20:05.697 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=20 2026-02-18T08:20:05.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=309, tag_tid=21, entry_tid=1' 2026-02-18T08:20:05.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:20:05.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:20:05.700 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:20:05.701 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=303, tag_tid=20, entry_tid=38291' 2026-02-18T08:20:05.701 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:20:05.701 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:20:05.703 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=38291 2026-02-18T08:20:05.703 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 21 = 20 -a 1 -le 38291 2026-02-18T08:20:05.703 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:20:05.703 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:20:05.703 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:20:05.904 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:20:05.904 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:20:05.905 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:20:05.905 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:20:05.905 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:20:05.905 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:20:05.905 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:20:05.905 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:20:05.905 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:20:05.905 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:20:05.905 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:20:05.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:20:05.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:20:05.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:20:05.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:20:05.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:20:05.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:20:05.909 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:20:05.909 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:20:05.909 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:20:05.909 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:20:05.909 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:20:05.909 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:20:05.909 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:20:05.909 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:20:05.909 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:20:05.909 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:20:05.909 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:20:05.910 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:20:05.910 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:20:05.910 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:20:05.910 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:20:05.910 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:20:05.910 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:20:05.910 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:20:05.910 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:20:05.910 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:20:05.910 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:20:05.911 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:20:05.911 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:20:05.911 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:20:05.911 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:20:05.911 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:20:05.911 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:20:05.911 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:20:05.911 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:20:05.911 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:20:05.911 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:20:05.912 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:20:05.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:20:05.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:20:05.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:20:05.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:20:05.968 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:20:05.968 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:20:05.968 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:20:05.971 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:20:05.971 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:20:05.971 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:20:05.971 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:20:05.971 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:20:05.971 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:20:05.971 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:20:05.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:20:05.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:20:05.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:20:05.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:20:05.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:20:05.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:20:05.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:20:05.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:20:05.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:20:05.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:20:05.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:20:05.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:20:05.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:20:05.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:20:05.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:20:05.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:20:05.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:20:05.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:20:05.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:20:05.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:20:05.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:20:05.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:20:05.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:20:05.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:20:05.974 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:20:05.982 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:20:05.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:20:05.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:20:05.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:20:06.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:20:06.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:20:06.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:20:06.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:20:06.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:20:06.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:20:06.035 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:20:06.035 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:20:06.035 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:20:06.035 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:20:06.035 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:20:06.035 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:20:06.035 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:20:06.035 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:20:06.035 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:20:06.035 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:20:06.036 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:20:06.036 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:20:06.036 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:20:06.036 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:20:06.036 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:20:06.036 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:20:06.036 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:20:06.036 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:20:06.036 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:20:06.036 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:20:06.037 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:20:06.037 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:20:06.037 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:20:06.037 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:20:06.037 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:20:06.037 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:20:06.037 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:20:06.037 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:20:06.037 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:20:06.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:20:06.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:20:06.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:20:06.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:20:06.095 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:20:06.095 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:20:06.095 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:20:06.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:20:06.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:20:06.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:20:06.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:20:06.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:20:06.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:20:06.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:20:06.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:20:06.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:20:06.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:20:06.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:20:06.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:20:06.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:20:06.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:20:06.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:20:06.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:20:06.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:20:06.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:20:06.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:20:06.100 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:20:06.100 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:20:06.100 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:20:06.100 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:20:06.100 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:20:06.100 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:20:06.100 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:20:06.100 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:20:06.100 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:20:06.100 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:20:06.100 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:20:06.100 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:20:06.101 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:20:06.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:20:06.110 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:20:06.110 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:20:06.110 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:20:06.170 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:20:06.171 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:20:06.171 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:20:06.171 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:20:06.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:20:06.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:20:06.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:20:06.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:20:06.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:20:06.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:20:06.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:20:06.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:20:06.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:20:06.173 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:20:06.173 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:20:06.174 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:20:06.201 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 75 2026-02-18T08:20:06.201 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 77 2026-02-18T08:20:06.201 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:20:06.201 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=309, tag_tid=21, entry_tid=1], [object_number=308, tag_tid=21, entry_tid=0], [object_number=310, tag_tid=20, entry_tid=46754], [object_number=311, tag_tid=20, entry_tid=46751]]], state=connected] 2026-02-18T08:20:06.201 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=300, tag_tid=20, entry_tid=38620], [object_number=303, tag_tid=20, entry_tid=38619], [object_number=302, tag_tid=20, entry_tid=38618], [object_number=301, tag_tid=20, entry_tid=38617]]], state=connected] 2026-02-18T08:20:06.204 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:20:06.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=309, tag_tid=21, entry_tid=1' 2026-02-18T08:20:06.206 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:20:06.206 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:20:06.206 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:20:06.206 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:20:06.207 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:20:06.207 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:20:06.207 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:20:06.207 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:20:06.207 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:20:06.207 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:20:06.208 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:20:06.208 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:20:06.208 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:20:06.209 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:20:06.209 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:20:06.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 75 2026-02-18T08:20:06.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 77 2026-02-18T08:20:06.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:20:06.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=309, tag_tid=21, entry_tid=1], [object_number=308, tag_tid=21, entry_tid=0], [object_number=310, tag_tid=20, entry_tid=46754], [object_number=311, tag_tid=20, entry_tid=46751]]], state=connected] 2026-02-18T08:20:06.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=300, tag_tid=20, entry_tid=38620], [object_number=303, tag_tid=20, entry_tid=38619], [object_number=302, tag_tid=20, entry_tid=38618], [object_number=301, tag_tid=20, entry_tid=38617]]], state=connected] 2026-02-18T08:20:06.238 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:20:06.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=300, tag_tid=20, entry_tid=38620' 2026-02-18T08:20:06.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=309, tag_tid=21, entry_tid=1' -a 'object_number=309, tag_tid=21, entry_tid=1' = 'object_number=300, tag_tid=20, entry_tid=38620' 2026-02-18T08:20:06.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=300, tag_tid=20, entry_tid=38620' '!=' 'object_number=303, tag_tid=20, entry_tid=38291' 2026-02-18T08:20:06.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:20:06.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=300, tag_tid=20, entry_tid=38620' = 'object_number=303, tag_tid=20, entry_tid=38291' 2026-02-18T08:20:06.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=300, tag_tid=20, entry_tid=38620' 2026-02-18T08:20:06.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=309, tag_tid=21, entry_tid=1' 2026-02-18T08:20:06.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:20:06.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:20:06.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=21 2026-02-18T08:20:06.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=300, tag_tid=20, entry_tid=38620' 2026-02-18T08:20:06.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:20:06.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:20:06.244 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=20 2026-02-18T08:20:06.245 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=309, tag_tid=21, entry_tid=1' 2026-02-18T08:20:06.245 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:20:06.245 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:20:06.247 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:20:06.247 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=300, tag_tid=20, entry_tid=38620' 2026-02-18T08:20:06.248 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:20:06.248 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:20:06.249 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=38620 2026-02-18T08:20:06.249 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 21 = 20 -a 1 -le 38620 2026-02-18T08:20:06.249 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:20:06.250 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:20:06.250 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:20:06.451 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:20:06.451 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:20:06.451 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:20:06.451 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:20:06.451 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:20:06.451 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:20:06.452 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:20:06.452 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:20:06.452 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:20:06.452 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:20:06.452 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:20:06.452 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:20:06.452 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:20:06.452 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:20:06.452 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:20:06.455 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:20:06.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:20:06.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:20:06.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:20:06.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:20:06.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:20:06.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:20:06.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:20:06.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:20:06.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:20:06.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:20:06.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:20:06.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:20:06.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:20:06.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:20:06.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:20:06.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:20:06.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:20:06.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:20:06.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:20:06.458 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:20:06.458 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:20:06.458 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:20:06.458 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:20:06.458 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:20:06.458 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:20:06.458 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:20:06.458 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:20:06.458 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:20:06.458 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:20:06.458 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:20:06.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:20:06.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:20:06.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:20:06.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:20:06.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:20:06.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:20:06.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:20:06.514 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:20:06.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:20:06.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:20:06.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:20:06.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:20:06.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:20:06.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:20:06.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:20:06.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:20:06.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:20:06.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:20:06.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:20:06.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:20:06.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:20:06.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:20:06.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:20:06.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:20:06.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:20:06.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:20:06.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:20:06.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:20:06.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:20:06.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:20:06.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:20:06.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:20:06.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:20:06.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:20:06.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:20:06.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:20:06.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:20:06.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:20:06.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:20:06.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:20:06.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:20:06.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:20:06.529 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:20:06.529 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:20:06.529 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:20:06.529 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:20:06.587 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:20:06.587 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:20:06.587 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:20:06.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:20:06.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:20:06.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:20:06.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:20:06.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:20:06.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:20:06.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:20:06.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:20:06.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:20:06.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:20:06.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:20:06.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:20:06.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:20:06.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:20:06.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:20:06.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:20:06.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:20:06.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:20:06.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:20:06.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:20:06.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:20:06.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:20:06.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:20:06.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:20:06.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:20:06.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:20:06.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:20:06.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:20:06.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:20:06.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:20:06.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:20:06.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:20:06.600 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:20:06.601 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:20:06.601 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:20:06.601 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:20:06.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:20:06.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:20:06.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:20:06.654 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:20:06.654 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:20:06.654 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:20:06.654 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:20:06.655 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:20:06.655 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:20:06.655 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:20:06.655 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:20:06.655 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:20:06.655 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:20:06.655 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:20:06.655 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:20:06.655 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:20:06.655 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:20:06.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:20:06.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:20:06.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:20:06.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:20:06.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:20:06.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:20:06.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:20:06.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:20:06.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:20:06.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:20:06.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:20:06.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:20:06.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:20:06.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:20:06.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:20:06.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:20:06.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:20:06.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:20:06.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:20:06.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:20:06.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:20:06.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:20:06.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:20:06.728 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:20:06.728 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:20:06.728 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:20:06.728 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:20:06.728 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:20:06.728 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:20:06.729 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:20:06.729 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:20:06.729 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:20:06.729 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:20:06.729 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:20:06.729 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:20:06.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:20:06.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:20:06.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:20:06.759 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 75 2026-02-18T08:20:06.759 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 77 2026-02-18T08:20:06.759 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:20:06.759 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=309, tag_tid=21, entry_tid=1], [object_number=308, tag_tid=21, entry_tid=0], [object_number=310, tag_tid=20, entry_tid=46754], [object_number=311, tag_tid=20, entry_tid=46751]]], state=connected] 2026-02-18T08:20:06.759 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=301, tag_tid=20, entry_tid=38949], [object_number=300, tag_tid=20, entry_tid=38948], [object_number=303, tag_tid=20, entry_tid=38947], [object_number=302, tag_tid=20, entry_tid=38946]]], state=connected] 2026-02-18T08:20:06.761 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:20:06.763 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=309, tag_tid=21, entry_tid=1' 2026-02-18T08:20:06.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:20:06.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:20:06.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:20:06.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:20:06.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:20:06.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:20:06.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:20:06.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:20:06.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:20:06.766 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:20:06.766 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:20:06.766 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:20:06.766 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:20:06.767 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:20:06.767 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:20:06.794 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 75 2026-02-18T08:20:06.794 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 77 2026-02-18T08:20:06.794 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:20:06.794 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=309, tag_tid=21, entry_tid=1], [object_number=308, tag_tid=21, entry_tid=0], [object_number=310, tag_tid=20, entry_tid=46754], [object_number=311, tag_tid=20, entry_tid=46751]]], state=connected] 2026-02-18T08:20:06.794 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=301, tag_tid=20, entry_tid=38949], [object_number=300, tag_tid=20, entry_tid=38948], [object_number=303, tag_tid=20, entry_tid=38947], [object_number=302, tag_tid=20, entry_tid=38946]]], state=connected] 2026-02-18T08:20:06.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:20:06.799 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=301, tag_tid=20, entry_tid=38949' 2026-02-18T08:20:06.799 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=309, tag_tid=21, entry_tid=1' -a 'object_number=309, tag_tid=21, entry_tid=1' = 'object_number=301, tag_tid=20, entry_tid=38949' 2026-02-18T08:20:06.800 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=301, tag_tid=20, entry_tid=38949' '!=' 'object_number=300, tag_tid=20, entry_tid=38620' 2026-02-18T08:20:06.800 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:20:06.800 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=301, tag_tid=20, entry_tid=38949' = 'object_number=300, tag_tid=20, entry_tid=38620' 2026-02-18T08:20:06.800 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=301, tag_tid=20, entry_tid=38949' 2026-02-18T08:20:06.800 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=309, tag_tid=21, entry_tid=1' 2026-02-18T08:20:06.800 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:20:06.800 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:20:06.801 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=21 2026-02-18T08:20:06.802 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=301, tag_tid=20, entry_tid=38949' 2026-02-18T08:20:06.802 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:20:06.802 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:20:06.803 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=20 2026-02-18T08:20:06.804 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=309, tag_tid=21, entry_tid=1' 2026-02-18T08:20:06.804 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:20:06.804 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:20:06.806 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:20:06.806 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=301, tag_tid=20, entry_tid=38949' 2026-02-18T08:20:06.807 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:20:06.807 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:20:06.808 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=38949 2026-02-18T08:20:06.808 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 21 = 20 -a 1 -le 38949 2026-02-18T08:20:06.808 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:20:06.809 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:20:06.809 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:20:07.010 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:20:07.010 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:20:07.010 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:20:07.010 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:20:07.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:20:07.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:20:07.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:20:07.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:20:07.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:20:07.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:20:07.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:20:07.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:20:07.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:20:07.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:20:07.012 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:20:07.012 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:20:07.012 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:20:07.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:20:07.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:20:07.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:20:07.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:20:07.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:20:07.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:20:07.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:20:07.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:20:07.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:20:07.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:20:07.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:20:07.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:20:07.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:20:07.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:20:07.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:20:07.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:20:07.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:20:07.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:20:07.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:20:07.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:20:07.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:20:07.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:20:07.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:20:07.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:20:07.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:20:07.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:20:07.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:20:07.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:20:07.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:20:07.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:20:07.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:20:07.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:20:07.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:20:07.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:20:07.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:20:07.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:20:07.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:20:07.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:20:07.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:20:07.075 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:20:07.075 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:20:07.076 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:20:07.076 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:20:07.076 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:20:07.076 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:20:07.076 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:20:07.076 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:20:07.076 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:20:07.076 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:20:07.076 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:20:07.076 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:20:07.076 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:20:07.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:20:07.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:20:07.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:20:07.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:20:07.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:20:07.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:20:07.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:20:07.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:20:07.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:20:07.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:20:07.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:20:07.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:20:07.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:20:07.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:20:07.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:20:07.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:20:07.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:20:07.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:20:07.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:20:07.086 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:20:07.086 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:20:07.086 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:20:07.086 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:20:07.143 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:20:07.143 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:20:07.144 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:20:07.146 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:20:07.146 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:20:07.147 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:20:07.147 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:20:07.147 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:20:07.147 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:20:07.147 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:20:07.147 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:20:07.147 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:20:07.147 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:20:07.147 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:20:07.147 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:20:07.147 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:20:07.148 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:20:07.148 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:20:07.148 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:20:07.148 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:20:07.148 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:20:07.148 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:20:07.148 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:20:07.148 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:20:07.148 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:20:07.148 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:20:07.149 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:20:07.149 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:20:07.149 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:20:07.149 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:20:07.149 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:20:07.149 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:20:07.149 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:20:07.149 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:20:07.149 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:20:07.157 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:20:07.157 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:20:07.157 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:20:07.158 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:20:07.206 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:20:07.206 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:20:07.206 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:20:07.209 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:20:07.209 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:20:07.209 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:20:07.209 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:20:07.210 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:20:07.210 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:20:07.210 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:20:07.210 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:20:07.210 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:20:07.210 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:20:07.210 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:20:07.210 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:20:07.210 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:20:07.211 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:20:07.211 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:20:07.211 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:20:07.211 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:20:07.211 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:20:07.211 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:20:07.211 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:20:07.211 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:20:07.211 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:20:07.211 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:20:07.211 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:20:07.212 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:20:07.212 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:20:07.212 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:20:07.212 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:20:07.212 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:20:07.212 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:20:07.212 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:20:07.212 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:20:07.220 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:20:07.221 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:20:07.221 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:20:07.221 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:20:07.279 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:20:07.280 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:20:07.280 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:20:07.280 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:20:07.280 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:20:07.280 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:20:07.280 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:20:07.281 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:20:07.281 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:20:07.281 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:20:07.281 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:20:07.281 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:20:07.281 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:20:07.282 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:20:07.282 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:20:07.282 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:20:07.318 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 75 2026-02-18T08:20:07.318 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 77 2026-02-18T08:20:07.318 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:20:07.319 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=309, tag_tid=21, entry_tid=1], [object_number=308, tag_tid=21, entry_tid=0], [object_number=310, tag_tid=20, entry_tid=46754], [object_number=311, tag_tid=20, entry_tid=46751]]], state=connected] 2026-02-18T08:20:07.319 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=303, tag_tid=20, entry_tid=39279], [object_number=302, tag_tid=20, entry_tid=39278], [object_number=301, tag_tid=20, entry_tid=39277], [object_number=300, tag_tid=20, entry_tid=39276]]], state=connected] 2026-02-18T08:20:07.322 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:20:07.324 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=309, tag_tid=21, entry_tid=1' 2026-02-18T08:20:07.325 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:20:07.325 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:20:07.325 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:20:07.325 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:20:07.325 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:20:07.325 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:20:07.326 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:20:07.326 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:20:07.326 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:20:07.326 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:20:07.326 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:20:07.326 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:20:07.327 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:20:07.327 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:20:07.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:20:07.354 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 75 2026-02-18T08:20:07.354 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 77 2026-02-18T08:20:07.354 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:20:07.354 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=309, tag_tid=21, entry_tid=1], [object_number=308, tag_tid=21, entry_tid=0], [object_number=310, tag_tid=20, entry_tid=46754], [object_number=311, tag_tid=20, entry_tid=46751]]], state=connected] 2026-02-18T08:20:07.354 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=303, tag_tid=20, entry_tid=39279], [object_number=302, tag_tid=20, entry_tid=39278], [object_number=301, tag_tid=20, entry_tid=39277], [object_number=300, tag_tid=20, entry_tid=39276]]], state=connected] 2026-02-18T08:20:07.356 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:20:07.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=303, tag_tid=20, entry_tid=39279' 2026-02-18T08:20:07.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=309, tag_tid=21, entry_tid=1' -a 'object_number=309, tag_tid=21, entry_tid=1' = 'object_number=303, tag_tid=20, entry_tid=39279' 2026-02-18T08:20:07.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=303, tag_tid=20, entry_tid=39279' '!=' 'object_number=301, tag_tid=20, entry_tid=38949' 2026-02-18T08:20:07.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:20:07.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=303, tag_tid=20, entry_tid=39279' = 'object_number=301, tag_tid=20, entry_tid=38949' 2026-02-18T08:20:07.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=303, tag_tid=20, entry_tid=39279' 2026-02-18T08:20:07.360 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=309, tag_tid=21, entry_tid=1' 2026-02-18T08:20:07.360 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:20:07.360 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:20:07.360 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=21 2026-02-18T08:20:07.361 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=303, tag_tid=20, entry_tid=39279' 2026-02-18T08:20:07.361 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:20:07.361 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:20:07.363 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=20 2026-02-18T08:20:07.363 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=309, tag_tid=21, entry_tid=1' 2026-02-18T08:20:07.364 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:20:07.364 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:20:07.365 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:20:07.366 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=303, tag_tid=20, entry_tid=39279' 2026-02-18T08:20:07.366 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:20:07.366 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:20:07.368 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=39279 2026-02-18T08:20:07.368 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 21 = 20 -a 1 -le 39279 2026-02-18T08:20:07.368 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:20:07.368 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:20:07.368 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:20:07.570 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:20:07.570 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:20:07.570 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:20:07.570 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:20:07.570 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:20:07.570 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:20:07.571 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:20:07.571 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:20:07.571 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:20:07.571 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:20:07.571 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:20:07.571 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:20:07.571 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:20:07.571 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:20:07.571 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:20:07.571 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:20:07.575 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:20:07.575 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:20:07.575 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:20:07.575 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:20:07.575 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:20:07.575 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:20:07.575 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:20:07.575 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:20:07.576 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:20:07.576 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:20:07.576 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:20:07.576 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:20:07.576 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:20:07.576 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:20:07.576 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:20:07.576 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:20:07.576 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:20:07.576 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:20:07.576 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:20:07.577 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:20:07.577 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:20:07.577 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:20:07.577 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:20:07.577 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:20:07.577 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:20:07.577 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:20:07.577 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:20:07.577 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:20:07.577 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:20:07.577 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:20:07.578 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:20:07.578 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:20:07.578 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:20:07.585 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:20:07.585 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:20:07.585 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:20:07.585 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:20:07.635 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:20:07.635 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:20:07.635 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:20:07.637 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:20:07.637 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:20:07.637 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:20:07.637 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:20:07.637 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:20:07.637 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:20:07.637 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:20:07.638 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:20:07.638 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:20:07.638 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:20:07.638 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:20:07.638 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:20:07.638 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:20:07.638 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:20:07.638 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:20:07.638 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:20:07.638 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:20:07.638 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:20:07.638 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:20:07.638 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:20:07.638 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:20:07.638 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:20:07.638 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:20:07.638 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:20:07.638 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:20:07.638 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:20:07.638 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:20:07.638 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:20:07.638 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:20:07.638 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:20:07.638 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:20:07.639 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:20:07.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:20:07.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:20:07.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:20:07.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:20:07.709 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:20:07.709 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:20:07.709 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:20:07.711 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:20:07.711 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:20:07.711 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:20:07.711 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:20:07.711 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:20:07.711 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:20:07.711 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:20:07.711 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:20:07.712 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:20:07.712 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:20:07.712 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:20:07.712 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:20:07.712 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:20:07.712 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:20:07.712 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:20:07.712 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:20:07.712 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:20:07.712 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:20:07.712 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:20:07.712 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:20:07.712 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:20:07.712 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:20:07.712 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:20:07.712 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:20:07.712 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:20:07.712 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:20:07.712 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:20:07.712 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:20:07.712 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:20:07.712 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:20:07.712 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:20:07.712 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:20:07.722 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:20:07.722 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:20:07.722 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:20:07.722 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:20:07.772 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:20:07.772 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:20:07.772 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:20:07.774 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:20:07.774 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:20:07.774 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:20:07.774 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:20:07.774 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:20:07.774 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:20:07.774 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:20:07.774 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:20:07.774 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:20:07.774 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:20:07.774 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:20:07.774 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:20:07.774 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:20:07.774 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:20:07.774 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:20:07.774 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:20:07.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:20:07.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:20:07.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:20:07.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:20:07.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:20:07.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:20:07.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:20:07.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:20:07.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:20:07.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:20:07.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:20:07.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:20:07.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:20:07.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:20:07.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:20:07.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:20:07.786 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:20:07.786 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:20:07.786 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:20:07.786 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:20:07.855 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:20:07.855 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:20:07.855 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:20:07.855 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:20:07.855 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:20:07.855 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:20:07.855 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:20:07.855 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:20:07.855 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:20:07.855 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:20:07.856 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:20:07.856 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:20:07.856 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:20:07.857 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:20:07.858 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:20:07.858 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:20:07.885 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 75 2026-02-18T08:20:07.885 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 77 2026-02-18T08:20:07.885 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:20:07.885 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=309, tag_tid=21, entry_tid=1], [object_number=308, tag_tid=21, entry_tid=0], [object_number=310, tag_tid=20, entry_tid=46754], [object_number=311, tag_tid=20, entry_tid=46751]]], state=connected] 2026-02-18T08:20:07.885 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=302, tag_tid=20, entry_tid=39638], [object_number=301, tag_tid=20, entry_tid=39637], [object_number=300, tag_tid=20, entry_tid=39636], [object_number=303, tag_tid=20, entry_tid=39635]]], state=connected] 2026-02-18T08:20:07.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:20:07.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=309, tag_tid=21, entry_tid=1' 2026-02-18T08:20:07.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:20:07.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:20:07.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:20:07.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:20:07.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:20:07.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:20:07.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:20:07.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:20:07.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:20:07.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:20:07.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:20:07.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:20:07.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:20:07.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:20:07.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:20:07.931 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 75 2026-02-18T08:20:07.931 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 77 2026-02-18T08:20:07.931 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:20:07.931 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=309, tag_tid=21, entry_tid=1], [object_number=308, tag_tid=21, entry_tid=0], [object_number=310, tag_tid=20, entry_tid=46754], [object_number=311, tag_tid=20, entry_tid=46751]]], state=connected] 2026-02-18T08:20:07.931 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=302, tag_tid=20, entry_tid=39638], [object_number=301, tag_tid=20, entry_tid=39637], [object_number=300, tag_tid=20, entry_tid=39636], [object_number=303, tag_tid=20, entry_tid=39635]]], state=connected] 2026-02-18T08:20:07.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:20:07.936 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=302, tag_tid=20, entry_tid=39638' 2026-02-18T08:20:07.936 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=309, tag_tid=21, entry_tid=1' -a 'object_number=309, tag_tid=21, entry_tid=1' = 'object_number=302, tag_tid=20, entry_tid=39638' 2026-02-18T08:20:07.936 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=302, tag_tid=20, entry_tid=39638' '!=' 'object_number=303, tag_tid=20, entry_tid=39279' 2026-02-18T08:20:07.936 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:20:07.936 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=302, tag_tid=20, entry_tid=39638' = 'object_number=303, tag_tid=20, entry_tid=39279' 2026-02-18T08:20:07.936 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=302, tag_tid=20, entry_tid=39638' 2026-02-18T08:20:07.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=309, tag_tid=21, entry_tid=1' 2026-02-18T08:20:07.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:20:07.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:20:07.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=21 2026-02-18T08:20:07.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=302, tag_tid=20, entry_tid=39638' 2026-02-18T08:20:07.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:20:07.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:20:07.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=20 2026-02-18T08:20:07.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=309, tag_tid=21, entry_tid=1' 2026-02-18T08:20:07.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:20:07.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:20:07.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:20:07.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=302, tag_tid=20, entry_tid=39638' 2026-02-18T08:20:07.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:20:07.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:20:07.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=39638 2026-02-18T08:20:07.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 21 = 20 -a 1 -le 39638 2026-02-18T08:20:07.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:20:07.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:20:07.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:20:08.146 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:20:08.146 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:20:08.147 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:20:08.147 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:20:08.147 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:20:08.147 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:20:08.147 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:20:08.147 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:20:08.147 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:20:08.147 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:20:08.147 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:20:08.147 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:20:08.147 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:20:08.147 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:20:08.147 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:20:08.147 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:20:08.148 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:20:08.149 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:20:08.149 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:20:08.149 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:20:08.149 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:20:08.149 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:20:08.149 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:20:08.149 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:20:08.149 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:20:08.149 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:20:08.149 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:20:08.149 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:20:08.149 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:20:08.149 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:20:08.149 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:20:08.149 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:20:08.149 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:20:08.150 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:20:08.150 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:20:08.150 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:20:08.150 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:20:08.150 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:20:08.150 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:20:08.150 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:20:08.150 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:20:08.150 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:20:08.150 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:20:08.150 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:20:08.150 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:20:08.150 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:20:08.150 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:20:08.150 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:20:08.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:20:08.162 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:20:08.162 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:20:08.162 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:20:08.162 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:20:08.219 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:20:08.219 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:20:08.219 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:20:08.221 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:20:08.221 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:20:08.221 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:20:08.221 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:20:08.221 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:20:08.221 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:20:08.221 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:20:08.221 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:20:08.221 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:20:08.221 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:20:08.221 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:20:08.221 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:20:08.221 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:20:08.221 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:20:08.221 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:20:08.221 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:20:08.222 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:20:08.222 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:20:08.222 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:20:08.222 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:20:08.222 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:20:08.222 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:20:08.222 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:20:08.222 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:20:08.222 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:20:08.222 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:20:08.222 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:20:08.222 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:20:08.222 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:20:08.222 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:20:08.223 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:20:08.223 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:20:08.234 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:20:08.234 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:20:08.234 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:20:08.234 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:20:08.293 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:20:08.293 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:20:08.293 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:20:08.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:20:08.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:20:08.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:20:08.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:20:08.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:20:08.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:20:08.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:20:08.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:20:08.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:20:08.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:20:08.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:20:08.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:20:08.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:20:08.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:20:08.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:20:08.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:20:08.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:20:08.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:20:08.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:20:08.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:20:08.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:20:08.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:20:08.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:20:08.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:20:08.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:20:08.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:20:08.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:20:08.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:20:08.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:20:08.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:20:08.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:20:08.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:20:08.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:20:08.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:20:08.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:20:08.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:20:08.356 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:20:08.357 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:20:08.357 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:20:08.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:20:08.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:20:08.360 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:20:08.360 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:20:08.360 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:20:08.360 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:20:08.360 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:20:08.360 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:20:08.360 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:20:08.360 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:20:08.360 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:20:08.360 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:20:08.360 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:20:08.361 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:20:08.361 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:20:08.361 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:20:08.361 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:20:08.361 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:20:08.361 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:20:08.361 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:20:08.361 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:20:08.361 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:20:08.361 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:20:08.361 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:20:08.361 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:20:08.362 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:20:08.362 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:20:08.362 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:20:08.362 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:20:08.362 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:20:08.362 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:20:08.362 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:20:08.370 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:20:08.371 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:20:08.371 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:20:08.371 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:20:08.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:20:08.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:20:08.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:20:08.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:20:08.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:20:08.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:20:08.443 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:20:08.443 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:20:08.443 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:20:08.443 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:20:08.443 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:20:08.443 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:20:08.443 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:20:08.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:20:08.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:20:08.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:20:08.483 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 75 2026-02-18T08:20:08.483 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 77 2026-02-18T08:20:08.483 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:20:08.484 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=309, tag_tid=21, entry_tid=1], [object_number=308, tag_tid=21, entry_tid=0], [object_number=310, tag_tid=20, entry_tid=46754], [object_number=311, tag_tid=20, entry_tid=46751]]], state=connected] 2026-02-18T08:20:08.484 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=303, tag_tid=20, entry_tid=39979], [object_number=302, tag_tid=20, entry_tid=39978], [object_number=301, tag_tid=20, entry_tid=39977], [object_number=300, tag_tid=20, entry_tid=39976]]], state=connected] 2026-02-18T08:20:08.486 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:20:08.488 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=309, tag_tid=21, entry_tid=1' 2026-02-18T08:20:08.489 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:20:08.489 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:20:08.489 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:20:08.489 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:20:08.489 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:20:08.490 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:20:08.490 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:20:08.490 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:20:08.490 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:20:08.490 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:20:08.490 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:20:08.491 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:20:08.491 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:20:08.492 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:20:08.492 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:20:08.528 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 75 2026-02-18T08:20:08.528 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 77 2026-02-18T08:20:08.528 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:20:08.528 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=309, tag_tid=21, entry_tid=1], [object_number=308, tag_tid=21, entry_tid=0], [object_number=310, tag_tid=20, entry_tid=46754], [object_number=311, tag_tid=20, entry_tid=46751]]], state=connected] 2026-02-18T08:20:08.528 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=303, tag_tid=20, entry_tid=39979], [object_number=302, tag_tid=20, entry_tid=39978], [object_number=301, tag_tid=20, entry_tid=39977], [object_number=300, tag_tid=20, entry_tid=39976]]], state=connected] 2026-02-18T08:20:08.532 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:20:08.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=303, tag_tid=20, entry_tid=39979' 2026-02-18T08:20:08.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=309, tag_tid=21, entry_tid=1' -a 'object_number=309, tag_tid=21, entry_tid=1' = 'object_number=303, tag_tid=20, entry_tid=39979' 2026-02-18T08:20:08.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=303, tag_tid=20, entry_tid=39979' '!=' 'object_number=302, tag_tid=20, entry_tid=39638' 2026-02-18T08:20:08.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:20:08.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=303, tag_tid=20, entry_tid=39979' = 'object_number=302, tag_tid=20, entry_tid=39638' 2026-02-18T08:20:08.534 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=303, tag_tid=20, entry_tid=39979' 2026-02-18T08:20:08.534 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=309, tag_tid=21, entry_tid=1' 2026-02-18T08:20:08.534 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:20:08.534 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:20:08.535 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=21 2026-02-18T08:20:08.536 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=303, tag_tid=20, entry_tid=39979' 2026-02-18T08:20:08.536 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:20:08.536 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:20:08.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=20 2026-02-18T08:20:08.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=309, tag_tid=21, entry_tid=1' 2026-02-18T08:20:08.539 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:20:08.539 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:20:08.540 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:20:08.540 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=303, tag_tid=20, entry_tid=39979' 2026-02-18T08:20:08.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:20:08.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:20:08.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=39979 2026-02-18T08:20:08.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 21 = 20 -a 1 -le 39979 2026-02-18T08:20:08.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:20:08.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:20:08.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:20:08.745 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:20:08.745 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:20:08.745 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:20:08.745 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:20:08.745 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:20:08.745 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:20:08.746 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:20:08.746 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:20:08.746 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:20:08.746 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:20:08.746 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:20:08.746 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:20:08.746 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:20:08.746 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:20:08.746 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:20:08.746 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:20:08.749 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:20:08.749 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:20:08.749 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:20:08.749 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:20:08.750 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:20:08.750 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:20:08.750 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:20:08.750 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:20:08.750 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:20:08.750 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:20:08.750 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:20:08.750 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:20:08.750 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:20:08.750 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:20:08.750 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:20:08.750 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:20:08.750 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:20:08.751 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:20:08.751 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:20:08.751 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:20:08.751 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:20:08.751 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:20:08.751 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:20:08.751 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:20:08.751 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:20:08.751 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:20:08.751 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:20:08.751 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:20:08.751 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:20:08.752 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:20:08.752 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:20:08.752 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:20:08.752 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:20:08.759 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:20:08.760 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:20:08.760 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:20:08.760 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:20:08.810 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:20:08.810 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:20:08.810 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:20:08.813 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:20:08.813 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:20:08.813 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:20:08.813 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:20:08.813 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:20:08.813 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:20:08.813 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:20:08.814 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:20:08.814 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:20:08.814 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:20:08.814 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:20:08.814 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:20:08.814 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:20:08.814 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:20:08.814 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:20:08.814 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:20:08.814 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:20:08.814 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:20:08.814 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:20:08.815 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:20:08.815 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:20:08.815 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:20:08.815 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:20:08.815 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:20:08.815 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:20:08.815 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:20:08.815 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:20:08.815 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:20:08.815 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:20:08.815 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:20:08.815 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:20:08.815 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:20:08.825 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:20:08.826 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:20:08.826 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:20:08.826 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:20:08.884 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:20:08.884 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:20:08.884 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:20:08.887 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:20:08.887 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:20:08.887 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:20:08.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:20:08.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:20:08.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:20:08.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:20:08.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:20:08.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:20:08.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:20:08.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:20:08.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:20:08.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:20:08.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:20:08.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:20:08.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:20:08.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:20:08.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:20:08.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:20:08.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:20:08.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:20:08.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:20:08.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:20:08.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:20:08.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:20:08.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:20:08.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:20:08.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:20:08.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:20:08.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:20:08.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:20:08.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:20:08.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:20:08.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:20:08.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:20:08.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:20:08.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:20:08.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:20:08.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:20:08.963 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:20:08.963 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:20:08.963 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:20:08.963 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:20:08.963 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:20:08.963 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:20:08.963 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:20:08.964 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:20:08.964 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:20:08.964 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:20:08.964 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:20:08.964 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:20:08.964 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:20:08.964 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:20:08.964 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:20:08.964 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:20:08.964 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:20:08.965 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:20:08.965 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:20:08.965 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:20:08.965 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:20:08.965 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:20:08.965 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:20:08.965 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:20:08.965 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:20:08.965 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:20:08.965 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:20:08.965 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:20:08.966 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:20:08.966 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:20:08.966 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:20:08.966 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:20:08.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:20:08.974 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:20:08.974 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:20:08.974 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:20:09.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:20:09.036 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:20:09.036 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:20:09.036 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:20:09.036 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:20:09.036 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:20:09.036 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:20:09.036 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:20:09.037 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:20:09.037 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:20:09.037 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:20:09.037 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:20:09.037 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:20:09.038 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:20:09.038 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:20:09.038 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:20:09.067 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 75 2026-02-18T08:20:09.067 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 77 2026-02-18T08:20:09.067 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:20:09.067 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=309, tag_tid=21, entry_tid=1], [object_number=308, tag_tid=21, entry_tid=0], [object_number=310, tag_tid=20, entry_tid=46754], [object_number=311, tag_tid=20, entry_tid=46751]]], state=connected] 2026-02-18T08:20:09.067 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=302, tag_tid=20, entry_tid=40366], [object_number=301, tag_tid=20, entry_tid=40365], [object_number=300, tag_tid=20, entry_tid=40364], [object_number=303, tag_tid=20, entry_tid=40363]]], state=connected] 2026-02-18T08:20:09.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:20:09.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=309, tag_tid=21, entry_tid=1' 2026-02-18T08:20:09.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:20:09.074 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:20:09.074 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:20:09.074 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:20:09.074 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:20:09.074 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:20:09.074 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:20:09.074 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:20:09.074 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:20:09.074 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:20:09.075 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:20:09.075 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:20:09.075 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:20:09.076 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:20:09.076 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:20:09.314 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 75 2026-02-18T08:20:09.314 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 77 2026-02-18T08:20:09.315 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:20:09.315 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=309, tag_tid=21, entry_tid=1], [object_number=308, tag_tid=21, entry_tid=0], [object_number=310, tag_tid=20, entry_tid=46754], [object_number=311, tag_tid=20, entry_tid=46751]]], state=connected] 2026-02-18T08:20:09.315 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=302, tag_tid=20, entry_tid=40366], [object_number=301, tag_tid=20, entry_tid=40365], [object_number=300, tag_tid=20, entry_tid=40364], [object_number=303, tag_tid=20, entry_tid=40363]]], state=connected] 2026-02-18T08:20:09.318 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:20:09.320 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=302, tag_tid=20, entry_tid=40366' 2026-02-18T08:20:09.320 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=309, tag_tid=21, entry_tid=1' -a 'object_number=309, tag_tid=21, entry_tid=1' = 'object_number=302, tag_tid=20, entry_tid=40366' 2026-02-18T08:20:09.321 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=302, tag_tid=20, entry_tid=40366' '!=' 'object_number=303, tag_tid=20, entry_tid=39979' 2026-02-18T08:20:09.321 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:20:09.321 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=302, tag_tid=20, entry_tid=40366' = 'object_number=303, tag_tid=20, entry_tid=39979' 2026-02-18T08:20:09.321 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=302, tag_tid=20, entry_tid=40366' 2026-02-18T08:20:09.321 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=309, tag_tid=21, entry_tid=1' 2026-02-18T08:20:09.321 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:20:09.321 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:20:09.322 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=21 2026-02-18T08:20:09.323 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=302, tag_tid=20, entry_tid=40366' 2026-02-18T08:20:09.323 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:20:09.323 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:20:09.324 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=20 2026-02-18T08:20:09.325 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=309, tag_tid=21, entry_tid=1' 2026-02-18T08:20:09.325 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:20:09.326 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:20:09.327 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:20:09.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=302, tag_tid=20, entry_tid=40366' 2026-02-18T08:20:09.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:20:09.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:20:09.329 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=40366 2026-02-18T08:20:09.329 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 21 = 20 -a 1 -le 40366 2026-02-18T08:20:09.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:20:09.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:20:09.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:20:09.531 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:20:09.532 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:20:09.532 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:20:09.532 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:20:09.532 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:20:09.532 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:20:09.532 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:20:09.532 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:20:09.532 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:20:09.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:20:09.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:20:09.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:20:09.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:20:09.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:20:09.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:20:09.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:20:09.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:20:09.536 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:20:09.536 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:20:09.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:20:09.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:20:09.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:20:09.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:20:09.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:20:09.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:20:09.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:20:09.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:20:09.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:20:09.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:20:09.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:20:09.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:20:09.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:20:09.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:20:09.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:20:09.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:20:09.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:20:09.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:20:09.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:20:09.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:20:09.539 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:20:09.539 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:20:09.539 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:20:09.539 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:20:09.539 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:20:09.539 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:20:09.539 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:20:09.539 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:20:09.539 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:20:09.539 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:20:09.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:20:09.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:20:09.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:20:09.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:20:09.596 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:20:09.596 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:20:09.596 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:20:09.600 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:20:09.600 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:20:09.600 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:20:09.600 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:20:09.600 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:20:09.600 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:20:09.600 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:20:09.600 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:20:09.600 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:20:09.601 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:20:09.601 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:20:09.601 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:20:09.601 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:20:09.601 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:20:09.601 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:20:09.601 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:20:09.601 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:20:09.601 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:20:09.601 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:20:09.601 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:20:09.602 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:20:09.602 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:20:09.602 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:20:09.602 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:20:09.602 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:20:09.602 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:20:09.602 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:20:09.602 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:20:09.602 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:20:09.602 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:20:09.603 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:20:09.603 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:20:09.610 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:20:09.611 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:20:09.611 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:20:09.611 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:20:09.660 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:20:09.661 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:20:09.661 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:20:09.664 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:20:09.664 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:20:09.664 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:20:09.664 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:20:09.664 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:20:09.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:20:09.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:20:09.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:20:09.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:20:09.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:20:09.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:20:09.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:20:09.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:20:09.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:20:09.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:20:09.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:20:09.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:20:09.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:20:09.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:20:09.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:20:09.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:20:09.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:20:09.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:20:09.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:20:09.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:20:09.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:20:09.667 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:20:09.667 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:20:09.667 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:20:09.667 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:20:09.667 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:20:09.667 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:20:09.674 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:20:09.675 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:20:09.675 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:20:09.675 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:20:09.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:20:09.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:20:09.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:20:09.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:20:09.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:20:09.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:20:09.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:20:09.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:20:09.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:20:09.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:20:09.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:20:09.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:20:09.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:20:09.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:20:09.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:20:09.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:20:09.728 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:20:09.728 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:20:09.728 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:20:09.728 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:20:09.728 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:20:09.728 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:20:09.728 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:20:09.728 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:20:09.728 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:20:09.728 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:20:09.729 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:20:09.729 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:20:09.729 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:20:09.729 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:20:09.729 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:20:09.729 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:20:09.729 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:20:09.729 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:20:09.729 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:20:09.736 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:20:09.737 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:20:09.737 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:20:09.737 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:20:09.798 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:20:09.799 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:20:09.799 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:20:09.799 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:20:09.800 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:20:09.800 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:20:09.800 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:20:09.800 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:20:09.800 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:20:09.800 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:20:09.800 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:20:09.800 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:20:09.800 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:20:09.801 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:20:09.801 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:20:09.801 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:20:09.829 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 75 2026-02-18T08:20:09.830 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 77 2026-02-18T08:20:09.830 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:20:09.830 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=309, tag_tid=21, entry_tid=1], [object_number=308, tag_tid=21, entry_tid=0], [object_number=310, tag_tid=20, entry_tid=46754], [object_number=311, tag_tid=20, entry_tid=46751]]], state=connected] 2026-02-18T08:20:09.830 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=302, tag_tid=20, entry_tid=40842], [object_number=301, tag_tid=20, entry_tid=40841], [object_number=300, tag_tid=20, entry_tid=40840], [object_number=303, tag_tid=20, entry_tid=40839]]], state=connected] 2026-02-18T08:20:09.832 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:20:09.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=309, tag_tid=21, entry_tid=1' 2026-02-18T08:20:09.835 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:20:09.835 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:20:09.835 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:20:09.835 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:20:09.836 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:20:09.836 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:20:09.836 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:20:09.836 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:20:09.836 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:20:09.836 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:20:09.837 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:20:09.837 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:20:09.837 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:20:09.838 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:20:09.838 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:20:09.867 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 75 2026-02-18T08:20:09.867 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 77 2026-02-18T08:20:09.867 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:20:09.868 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=309, tag_tid=21, entry_tid=1], [object_number=308, tag_tid=21, entry_tid=0], [object_number=310, tag_tid=20, entry_tid=46754], [object_number=311, tag_tid=20, entry_tid=46751]]], state=connected] 2026-02-18T08:20:09.868 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=302, tag_tid=20, entry_tid=40842], [object_number=301, tag_tid=20, entry_tid=40841], [object_number=300, tag_tid=20, entry_tid=40840], [object_number=303, tag_tid=20, entry_tid=40839]]], state=connected] 2026-02-18T08:20:09.870 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:20:09.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=302, tag_tid=20, entry_tid=40842' 2026-02-18T08:20:09.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=309, tag_tid=21, entry_tid=1' -a 'object_number=309, tag_tid=21, entry_tid=1' = 'object_number=302, tag_tid=20, entry_tid=40842' 2026-02-18T08:20:09.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=302, tag_tid=20, entry_tid=40842' '!=' 'object_number=302, tag_tid=20, entry_tid=40366' 2026-02-18T08:20:09.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:20:09.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=302, tag_tid=20, entry_tid=40842' = 'object_number=302, tag_tid=20, entry_tid=40366' 2026-02-18T08:20:09.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=302, tag_tid=20, entry_tid=40842' 2026-02-18T08:20:09.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=309, tag_tid=21, entry_tid=1' 2026-02-18T08:20:09.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:20:09.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:20:09.874 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=21 2026-02-18T08:20:09.875 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=302, tag_tid=20, entry_tid=40842' 2026-02-18T08:20:09.875 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:20:09.875 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:20:09.877 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=20 2026-02-18T08:20:09.877 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=309, tag_tid=21, entry_tid=1' 2026-02-18T08:20:09.877 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:20:09.878 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:20:09.879 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:20:09.880 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=302, tag_tid=20, entry_tid=40842' 2026-02-18T08:20:09.880 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:20:09.880 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:20:09.881 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=40842 2026-02-18T08:20:09.881 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 21 = 20 -a 1 -le 40842 2026-02-18T08:20:09.882 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:20:09.882 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:20:09.882 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:20:10.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:20:10.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:20:10.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:20:10.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:20:10.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:20:10.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:20:10.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:20:10.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:20:10.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:20:10.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:20:10.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:20:10.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:20:10.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:20:10.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:20:10.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:20:10.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:20:10.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:20:10.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:20:10.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:20:10.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:20:10.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:20:10.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:20:10.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:20:10.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:20:10.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:20:10.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:20:10.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:20:10.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:20:10.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:20:10.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:20:10.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:20:10.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:20:10.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:20:10.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:20:10.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:20:10.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:20:10.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:20:10.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:20:10.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:20:10.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:20:10.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:20:10.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:20:10.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:20:10.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:20:10.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:20:10.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:20:10.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:20:10.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:20:10.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:20:10.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:20:10.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:20:10.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:20:10.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:20:10.156 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:20:10.156 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:20:10.156 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:20:10.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:20:10.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:20:10.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:20:10.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:20:10.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:20:10.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:20:10.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:20:10.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:20:10.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:20:10.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:20:10.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:20:10.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:20:10.161 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:20:10.161 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:20:10.161 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:20:10.161 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:20:10.161 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:20:10.161 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:20:10.161 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:20:10.161 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:20:10.161 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:20:10.161 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:20:10.162 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:20:10.162 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:20:10.162 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:20:10.162 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:20:10.162 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:20:10.162 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:20:10.162 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:20:10.162 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:20:10.162 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:20:10.162 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:20:10.171 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:20:10.171 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:20:10.171 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:20:10.171 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:20:10.230 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:20:10.230 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:20:10.230 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:20:10.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:20:10.234 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:20:10.234 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:20:10.234 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:20:10.234 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:20:10.234 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:20:10.234 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:20:10.234 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:20:10.234 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:20:10.234 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:20:10.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:20:10.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:20:10.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:20:10.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:20:10.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:20:10.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:20:10.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:20:10.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:20:10.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:20:10.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:20:10.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:20:10.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:20:10.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:20:10.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:20:10.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:20:10.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:20:10.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:20:10.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:20:10.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:20:10.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:20:10.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:20:10.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:20:10.245 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:20:10.245 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:20:10.245 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:20:10.246 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:20:10.294 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:20:10.294 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:20:10.294 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:20:10.297 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:20:10.297 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:20:10.297 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:20:10.297 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:20:10.297 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:20:10.298 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:20:10.298 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:20:10.298 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:20:10.298 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:20:10.298 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:20:10.298 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:20:10.298 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:20:10.298 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:20:10.298 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:20:10.298 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:20:10.299 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:20:10.299 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:20:10.299 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:20:10.299 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:20:10.299 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:20:10.299 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:20:10.299 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:20:10.299 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:20:10.299 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:20:10.299 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:20:10.299 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:20:10.300 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:20:10.300 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:20:10.300 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:20:10.300 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:20:10.300 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:20:10.300 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:20:10.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:20:10.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:20:10.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:20:10.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:20:10.375 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:20:10.375 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:20:10.376 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:20:10.376 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:20:10.376 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:20:10.376 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:20:10.376 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:20:10.376 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:20:10.376 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:20:10.376 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:20:10.377 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:20:10.377 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:20:10.377 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:20:10.377 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:20:10.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:20:10.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:20:10.407 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 75 2026-02-18T08:20:10.407 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 77 2026-02-18T08:20:10.407 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:20:10.407 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=309, tag_tid=21, entry_tid=1], [object_number=308, tag_tid=21, entry_tid=0], [object_number=310, tag_tid=20, entry_tid=46754], [object_number=311, tag_tid=20, entry_tid=46751]]], state=connected] 2026-02-18T08:20:10.407 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=300, tag_tid=20, entry_tid=41220], [object_number=303, tag_tid=20, entry_tid=41219], [object_number=302, tag_tid=20, entry_tid=41218], [object_number=301, tag_tid=20, entry_tid=41217]]], state=connected] 2026-02-18T08:20:10.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:20:10.412 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=309, tag_tid=21, entry_tid=1' 2026-02-18T08:20:10.413 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:20:10.413 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:20:10.413 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:20:10.414 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:20:10.414 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:20:10.414 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:20:10.414 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:20:10.414 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:20:10.414 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:20:10.414 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:20:10.414 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:20:10.414 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:20:10.415 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:20:10.415 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:20:10.415 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:20:10.451 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 75 2026-02-18T08:20:10.451 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 77 2026-02-18T08:20:10.451 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:20:10.451 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=309, tag_tid=21, entry_tid=1], [object_number=308, tag_tid=21, entry_tid=0], [object_number=310, tag_tid=20, entry_tid=46754], [object_number=311, tag_tid=20, entry_tid=46751]]], state=connected] 2026-02-18T08:20:10.452 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=300, tag_tid=20, entry_tid=41220], [object_number=303, tag_tid=20, entry_tid=41219], [object_number=302, tag_tid=20, entry_tid=41218], [object_number=301, tag_tid=20, entry_tid=41217]]], state=connected] 2026-02-18T08:20:10.455 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:20:10.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=300, tag_tid=20, entry_tid=41220' 2026-02-18T08:20:10.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=309, tag_tid=21, entry_tid=1' -a 'object_number=309, tag_tid=21, entry_tid=1' = 'object_number=300, tag_tid=20, entry_tid=41220' 2026-02-18T08:20:10.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=300, tag_tid=20, entry_tid=41220' '!=' 'object_number=302, tag_tid=20, entry_tid=40842' 2026-02-18T08:20:10.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:20:10.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=300, tag_tid=20, entry_tid=41220' = 'object_number=302, tag_tid=20, entry_tid=40842' 2026-02-18T08:20:10.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=300, tag_tid=20, entry_tid=41220' 2026-02-18T08:20:10.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=309, tag_tid=21, entry_tid=1' 2026-02-18T08:20:10.458 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:20:10.458 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:20:10.458 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=21 2026-02-18T08:20:10.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=300, tag_tid=20, entry_tid=41220' 2026-02-18T08:20:10.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:20:10.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:20:10.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=20 2026-02-18T08:20:10.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=309, tag_tid=21, entry_tid=1' 2026-02-18T08:20:10.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:20:10.462 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:20:10.463 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:20:10.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=300, tag_tid=20, entry_tid=41220' 2026-02-18T08:20:10.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:20:10.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:20:10.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=41220 2026-02-18T08:20:10.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 21 = 20 -a 1 -le 41220 2026-02-18T08:20:10.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:20:10.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:20:10.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:20:10.667 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:20:10.667 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:20:10.667 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:20:10.667 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:20:10.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:20:10.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:20:10.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:20:10.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:20:10.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:20:10.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:20:10.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:20:10.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:20:10.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:20:10.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:20:10.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:20:10.671 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:20:10.671 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:20:10.671 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:20:10.671 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:20:10.671 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:20:10.672 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:20:10.672 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:20:10.672 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:20:10.672 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:20:10.672 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:20:10.672 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:20:10.672 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:20:10.672 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:20:10.672 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:20:10.672 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:20:10.672 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:20:10.673 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:20:10.673 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:20:10.673 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:20:10.673 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:20:10.673 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:20:10.673 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:20:10.673 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:20:10.673 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:20:10.673 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:20:10.673 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:20:10.673 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:20:10.673 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:20:10.674 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:20:10.674 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:20:10.674 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:20:10.674 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:20:10.674 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:20:10.674 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:20:10.682 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:20:10.682 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:20:10.682 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:20:10.683 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:20:10.732 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:20:10.732 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:20:10.732 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:20:10.735 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:20:10.735 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:20:10.735 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:20:10.735 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:20:10.735 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:20:10.735 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:20:10.735 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:20:10.736 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:20:10.736 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:20:10.736 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:20:10.736 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:20:10.736 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:20:10.736 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:20:10.736 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:20:10.736 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:20:10.736 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:20:10.736 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:20:10.736 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:20:10.736 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:20:10.737 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:20:10.737 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:20:10.737 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:20:10.737 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:20:10.737 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:20:10.737 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:20:10.737 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:20:10.737 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:20:10.737 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:20:10.737 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:20:10.737 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:20:10.737 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:20:10.737 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:20:10.746 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:20:10.747 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:20:10.747 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:20:10.747 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:20:10.796 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:20:10.796 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:20:10.796 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:20:10.798 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:20:10.799 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:20:10.799 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:20:10.799 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:20:10.799 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:20:10.799 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:20:10.799 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:20:10.799 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:20:10.799 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:20:10.799 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:20:10.799 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:20:10.799 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:20:10.800 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:20:10.800 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:20:10.800 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:20:10.800 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:20:10.800 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:20:10.800 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:20:10.800 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:20:10.800 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:20:10.800 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:20:10.800 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:20:10.800 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:20:10.800 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:20:10.801 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:20:10.801 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:20:10.801 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:20:10.801 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:20:10.801 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:20:10.801 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:20:10.801 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:20:10.801 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:20:10.810 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:20:10.811 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:20:10.811 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:20:10.811 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:20:10.859 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:20:10.859 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:20:10.859 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:20:10.862 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:20:10.862 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:20:10.862 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:20:10.862 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:20:10.862 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:20:10.862 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:20:10.862 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:20:10.862 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:20:10.863 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:20:10.863 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:20:10.863 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:20:10.863 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:20:10.863 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:20:10.863 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:20:10.863 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:20:10.863 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:20:10.863 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:20:10.863 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:20:10.863 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:20:10.863 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:20:10.864 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:20:10.864 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:20:10.864 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:20:10.864 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:20:10.864 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:20:10.864 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:20:10.864 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:20:10.864 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:20:10.864 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:20:10.864 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:20:10.864 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:20:10.864 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:20:10.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:20:10.874 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:20:10.874 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:20:10.874 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:20:10.933 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:20:10.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:20:10.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:20:10.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:20:10.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:20:10.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:20:10.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:20:10.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:20:10.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:20:10.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:20:10.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:20:10.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:20:10.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:20:10.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:20:10.936 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:20:10.936 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:20:10.963 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 75 2026-02-18T08:20:10.964 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 77 2026-02-18T08:20:10.964 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:20:10.964 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=309, tag_tid=21, entry_tid=1], [object_number=308, tag_tid=21, entry_tid=0], [object_number=310, tag_tid=20, entry_tid=46754], [object_number=311, tag_tid=20, entry_tid=46751]]], state=connected] 2026-02-18T08:20:10.964 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=301, tag_tid=20, entry_tid=41569], [object_number=300, tag_tid=20, entry_tid=41568], [object_number=303, tag_tid=20, entry_tid=41567], [object_number=302, tag_tid=20, entry_tid=41566]]], state=connected] 2026-02-18T08:20:10.966 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:20:10.968 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=309, tag_tid=21, entry_tid=1' 2026-02-18T08:20:10.969 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:20:10.969 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:20:10.969 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:20:10.969 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:20:10.970 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:20:10.970 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:20:10.970 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:20:10.970 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:20:10.970 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:20:10.970 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:20:10.970 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:20:10.971 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:20:10.971 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:20:10.971 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:20:10.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:20:11.007 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 75 2026-02-18T08:20:11.008 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 77 2026-02-18T08:20:11.008 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:20:11.008 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=309, tag_tid=21, entry_tid=1], [object_number=308, tag_tid=21, entry_tid=0], [object_number=310, tag_tid=20, entry_tid=46754], [object_number=311, tag_tid=20, entry_tid=46751]]], state=connected] 2026-02-18T08:20:11.008 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=301, tag_tid=20, entry_tid=41569], [object_number=300, tag_tid=20, entry_tid=41568], [object_number=303, tag_tid=20, entry_tid=41567], [object_number=302, tag_tid=20, entry_tid=41566]]], state=connected] 2026-02-18T08:20:11.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:20:11.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=301, tag_tid=20, entry_tid=41569' 2026-02-18T08:20:11.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=309, tag_tid=21, entry_tid=1' -a 'object_number=309, tag_tid=21, entry_tid=1' = 'object_number=301, tag_tid=20, entry_tid=41569' 2026-02-18T08:20:11.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=301, tag_tid=20, entry_tid=41569' '!=' 'object_number=300, tag_tid=20, entry_tid=41220' 2026-02-18T08:20:11.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:20:11.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=301, tag_tid=20, entry_tid=41569' = 'object_number=300, tag_tid=20, entry_tid=41220' 2026-02-18T08:20:11.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=301, tag_tid=20, entry_tid=41569' 2026-02-18T08:20:11.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=309, tag_tid=21, entry_tid=1' 2026-02-18T08:20:11.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:20:11.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:20:11.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=21 2026-02-18T08:20:11.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=301, tag_tid=20, entry_tid=41569' 2026-02-18T08:20:11.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:20:11.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:20:11.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=20 2026-02-18T08:20:11.018 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=309, tag_tid=21, entry_tid=1' 2026-02-18T08:20:11.018 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:20:11.018 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:20:11.019 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:20:11.020 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=301, tag_tid=20, entry_tid=41569' 2026-02-18T08:20:11.020 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:20:11.020 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:20:11.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=41569 2026-02-18T08:20:11.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 21 = 20 -a 1 -le 41569 2026-02-18T08:20:11.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:20:11.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:20:11.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:20:11.224 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:20:11.224 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:20:11.224 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:20:11.224 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:20:11.225 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:20:11.225 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:20:11.225 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:20:11.225 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:20:11.225 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:20:11.225 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:20:11.225 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:20:11.225 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:20:11.225 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:20:11.225 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:20:11.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:20:11.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:20:11.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:20:11.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:20:11.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:20:11.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:20:11.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:20:11.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:20:11.229 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:20:11.229 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:20:11.229 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:20:11.229 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:20:11.229 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:20:11.229 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:20:11.229 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:20:11.229 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:20:11.229 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:20:11.229 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:20:11.229 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:20:11.229 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:20:11.230 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:20:11.230 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:20:11.230 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:20:11.230 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:20:11.230 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:20:11.230 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:20:11.230 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:20:11.230 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:20:11.230 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:20:11.230 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:20:11.230 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:20:11.230 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:20:11.231 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:20:11.231 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:20:11.231 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:20:11.238 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:20:11.238 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:20:11.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:20:11.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:20:11.298 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:20:11.298 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:20:11.298 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:20:11.301 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:20:11.301 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:20:11.301 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:20:11.301 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:20:11.302 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:20:11.302 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:20:11.302 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:20:11.302 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:20:11.302 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:20:11.302 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:20:11.302 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:20:11.302 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:20:11.302 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:20:11.302 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:20:11.302 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:20:11.302 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:20:11.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:20:11.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:20:11.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:20:11.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:20:11.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:20:11.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:20:11.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:20:11.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:20:11.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:20:11.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:20:11.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:20:11.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:20:11.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:20:11.304 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:20:11.304 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:20:11.304 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:20:11.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:20:11.314 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:20:11.314 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:20:11.314 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:20:11.364 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:20:11.364 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:20:11.364 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:20:11.365 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:20:11.365 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:20:11.365 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:20:11.365 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:20:11.365 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:20:11.365 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:20:11.365 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:20:11.365 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:20:11.365 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:20:11.365 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:20:11.366 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:20:11.366 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:20:11.366 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:20:11.366 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:20:11.366 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:20:11.366 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:20:11.366 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:20:11.366 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:20:11.366 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:20:11.366 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:20:11.366 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:20:11.366 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:20:11.366 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:20:11.366 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:20:11.366 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:20:11.366 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:20:11.366 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:20:11.366 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:20:11.366 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:20:11.367 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:20:11.367 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:20:11.367 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:20:11.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:20:11.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:20:11.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:20:11.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:20:11.436 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:20:11.436 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:20:11.436 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:20:11.439 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:20:11.439 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:20:11.439 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:20:11.439 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:20:11.439 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:20:11.439 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:20:11.439 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:20:11.439 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:20:11.439 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:20:11.439 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:20:11.439 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:20:11.440 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:20:11.440 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:20:11.440 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:20:11.440 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:20:11.440 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:20:11.440 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:20:11.440 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:20:11.440 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:20:11.440 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:20:11.440 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:20:11.440 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:20:11.440 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:20:11.440 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:20:11.440 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:20:11.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:20:11.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:20:11.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:20:11.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:20:11.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:20:11.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:20:11.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:20:11.449 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:20:11.449 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:20:11.449 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:20:11.449 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:20:11.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:20:11.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:20:11.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:20:11.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:20:11.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:20:11.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:20:11.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:20:11.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:20:11.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:20:11.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:20:11.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:20:11.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:20:11.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:20:11.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:20:11.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:20:11.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:20:11.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 75 2026-02-18T08:20:11.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 77 2026-02-18T08:20:11.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:20:11.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=309, tag_tid=21, entry_tid=1], [object_number=308, tag_tid=21, entry_tid=0], [object_number=310, tag_tid=20, entry_tid=46754], [object_number=311, tag_tid=20, entry_tid=46751]]], state=connected] 2026-02-18T08:20:11.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=301, tag_tid=20, entry_tid=41917], [object_number=300, tag_tid=20, entry_tid=41916], [object_number=303, tag_tid=20, entry_tid=41915], [object_number=302, tag_tid=20, entry_tid=41914]]], state=connected] 2026-02-18T08:20:11.553 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:20:11.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=309, tag_tid=21, entry_tid=1' 2026-02-18T08:20:11.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:20:11.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:20:11.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:20:11.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:20:11.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:20:11.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:20:11.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:20:11.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:20:11.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:20:11.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:20:11.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:20:11.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:20:11.558 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:20:11.558 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:20:11.558 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:20:11.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 75 2026-02-18T08:20:11.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 77 2026-02-18T08:20:11.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:20:11.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=309, tag_tid=21, entry_tid=1], [object_number=308, tag_tid=21, entry_tid=0], [object_number=310, tag_tid=20, entry_tid=46754], [object_number=311, tag_tid=20, entry_tid=46751]]], state=connected] 2026-02-18T08:20:11.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=301, tag_tid=20, entry_tid=41917], [object_number=300, tag_tid=20, entry_tid=41916], [object_number=303, tag_tid=20, entry_tid=41915], [object_number=302, tag_tid=20, entry_tid=41914]]], state=connected] 2026-02-18T08:20:11.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:20:11.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=301, tag_tid=20, entry_tid=41917' 2026-02-18T08:20:11.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=309, tag_tid=21, entry_tid=1' -a 'object_number=309, tag_tid=21, entry_tid=1' = 'object_number=301, tag_tid=20, entry_tid=41917' 2026-02-18T08:20:11.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=301, tag_tid=20, entry_tid=41917' '!=' 'object_number=301, tag_tid=20, entry_tid=41569' 2026-02-18T08:20:11.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:20:11.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=301, tag_tid=20, entry_tid=41917' = 'object_number=301, tag_tid=20, entry_tid=41569' 2026-02-18T08:20:11.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=301, tag_tid=20, entry_tid=41917' 2026-02-18T08:20:11.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=309, tag_tid=21, entry_tid=1' 2026-02-18T08:20:11.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:20:11.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:20:11.594 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=21 2026-02-18T08:20:11.595 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=301, tag_tid=20, entry_tid=41917' 2026-02-18T08:20:11.595 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:20:11.595 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:20:11.596 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=20 2026-02-18T08:20:11.597 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=309, tag_tid=21, entry_tid=1' 2026-02-18T08:20:11.597 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:20:11.597 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:20:11.599 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:20:11.600 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=301, tag_tid=20, entry_tid=41917' 2026-02-18T08:20:11.600 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:20:11.600 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:20:11.601 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=41917 2026-02-18T08:20:11.601 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 21 = 20 -a 1 -le 41917 2026-02-18T08:20:11.601 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:20:11.601 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:20:11.601 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:20:11.803 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:20:11.804 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:20:11.804 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:20:11.804 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:20:11.804 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:20:11.804 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:20:11.804 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:20:11.804 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:20:11.804 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:20:11.804 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:20:11.805 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:20:11.805 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:20:11.805 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:20:11.805 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:20:11.805 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:20:11.805 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:20:11.808 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:20:11.808 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:20:11.808 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:20:11.808 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:20:11.808 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:20:11.808 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:20:11.808 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:20:11.808 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:20:11.808 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:20:11.808 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:20:11.809 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:20:11.809 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:20:11.809 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:20:11.809 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:20:11.809 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:20:11.809 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:20:11.809 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:20:11.809 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:20:11.809 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:20:11.809 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:20:11.809 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:20:11.809 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:20:11.809 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:20:11.810 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:20:11.810 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:20:11.810 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:20:11.810 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:20:11.810 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:20:11.810 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:20:11.810 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:20:11.810 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:20:11.810 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:20:11.810 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:20:11.818 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:20:11.818 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:20:11.818 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:20:11.819 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:20:11.867 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:20:11.867 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:20:11.867 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:20:11.870 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:20:11.870 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:20:11.870 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:20:11.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:20:11.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:20:11.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:20:11.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:20:11.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:20:11.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:20:11.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:20:11.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:20:11.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:20:11.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:20:11.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:20:11.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:20:11.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:20:11.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:20:11.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:20:11.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:20:11.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:20:11.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:20:11.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:20:11.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:20:11.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:20:11.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:20:11.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:20:11.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:20:11.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:20:11.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:20:11.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:20:11.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:20:11.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:20:11.882 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:20:11.882 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:20:11.882 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:20:11.882 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:20:11.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:20:11.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:20:11.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:20:11.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:20:11.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:20:11.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:20:11.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:20:11.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:20:11.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:20:11.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:20:11.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:20:11.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:20:11.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:20:11.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:20:11.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:20:11.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:20:11.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:20:11.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:20:11.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:20:11.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:20:11.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:20:11.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:20:11.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:20:11.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:20:11.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:20:11.944 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:20:11.944 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:20:11.944 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:20:11.944 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:20:11.944 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:20:11.944 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:20:11.944 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:20:11.944 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:20:11.944 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:20:11.944 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:20:11.953 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:20:11.954 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:20:11.954 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:20:11.954 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:20:12.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:20:12.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:20:12.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:20:12.005 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:20:12.005 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:20:12.005 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:20:12.005 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:20:12.005 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:20:12.005 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:20:12.005 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:20:12.005 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:20:12.005 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:20:12.005 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:20:12.005 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:20:12.006 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:20:12.006 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:20:12.006 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:20:12.006 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:20:12.006 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:20:12.006 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:20:12.006 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:20:12.006 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:20:12.006 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:20:12.006 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:20:12.006 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:20:12.006 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:20:12.007 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:20:12.007 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:20:12.007 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:20:12.007 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:20:12.007 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:20:12.007 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:20:12.007 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:20:12.007 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:20:12.007 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:20:12.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:20:12.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:20:12.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:20:12.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:20:12.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:20:12.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:20:12.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:20:12.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:20:12.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:20:12.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:20:12.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:20:12.080 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:20:12.080 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:20:12.080 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:20:12.080 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:20:12.080 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:20:12.080 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:20:12.080 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:20:12.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:20:12.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:20:12.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 76 2026-02-18T08:20:12.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 77 2026-02-18T08:20:12.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:20:12.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=309, tag_tid=21, entry_tid=1], [object_number=308, tag_tid=21, entry_tid=0], [object_number=310, tag_tid=20, entry_tid=46754], [object_number=311, tag_tid=20, entry_tid=46751]]], state=connected] 2026-02-18T08:20:12.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=306, tag_tid=20, entry_tid=42182], [object_number=305, tag_tid=20, entry_tid=42181], [object_number=304, tag_tid=20, entry_tid=42180], [object_number=307, tag_tid=20, entry_tid=42179]]], state=connected] 2026-02-18T08:20:12.111 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:20:12.113 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=309, tag_tid=21, entry_tid=1' 2026-02-18T08:20:12.114 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:20:12.114 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:20:12.114 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:20:12.114 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:20:12.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:20:12.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:20:12.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:20:12.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:20:12.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:20:12.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:20:12.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:20:12.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:20:12.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:20:12.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:20:12.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:20:12.146 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 76 2026-02-18T08:20:12.146 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 77 2026-02-18T08:20:12.146 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:20:12.146 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=309, tag_tid=21, entry_tid=1], [object_number=308, tag_tid=21, entry_tid=0], [object_number=310, tag_tid=20, entry_tid=46754], [object_number=311, tag_tid=20, entry_tid=46751]]], state=connected] 2026-02-18T08:20:12.147 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=306, tag_tid=20, entry_tid=42182], [object_number=305, tag_tid=20, entry_tid=42181], [object_number=304, tag_tid=20, entry_tid=42180], [object_number=307, tag_tid=20, entry_tid=42179]]], state=connected] 2026-02-18T08:20:12.149 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:20:12.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=306, tag_tid=20, entry_tid=42182' 2026-02-18T08:20:12.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=309, tag_tid=21, entry_tid=1' -a 'object_number=309, tag_tid=21, entry_tid=1' = 'object_number=306, tag_tid=20, entry_tid=42182' 2026-02-18T08:20:12.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=306, tag_tid=20, entry_tid=42182' '!=' 'object_number=301, tag_tid=20, entry_tid=41917' 2026-02-18T08:20:12.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:20:12.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=306, tag_tid=20, entry_tid=42182' = 'object_number=301, tag_tid=20, entry_tid=41917' 2026-02-18T08:20:12.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=306, tag_tid=20, entry_tid=42182' 2026-02-18T08:20:12.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=309, tag_tid=21, entry_tid=1' 2026-02-18T08:20:12.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:20:12.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:20:12.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=21 2026-02-18T08:20:12.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=306, tag_tid=20, entry_tid=42182' 2026-02-18T08:20:12.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:20:12.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:20:12.155 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=20 2026-02-18T08:20:12.156 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=309, tag_tid=21, entry_tid=1' 2026-02-18T08:20:12.156 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:20:12.157 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:20:12.158 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:20:12.158 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=306, tag_tid=20, entry_tid=42182' 2026-02-18T08:20:12.158 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:20:12.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:20:12.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=42182 2026-02-18T08:20:12.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 21 = 20 -a 1 -le 42182 2026-02-18T08:20:12.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:20:12.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:20:12.161 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:20:12.362 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:20:12.362 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:20:12.362 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:20:12.362 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:20:12.362 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:20:12.363 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:20:12.363 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:20:12.363 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:20:12.363 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:20:12.363 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:20:12.363 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:20:12.363 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:20:12.363 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:20:12.363 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:20:12.363 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:20:12.363 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:20:12.366 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:20:12.366 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:20:12.366 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:20:12.366 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:20:12.366 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:20:12.366 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:20:12.366 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:20:12.367 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:20:12.367 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:20:12.367 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:20:12.367 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:20:12.367 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:20:12.367 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:20:12.367 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:20:12.367 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:20:12.367 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:20:12.367 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:20:12.367 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:20:12.367 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:20:12.368 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:20:12.368 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:20:12.368 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:20:12.368 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:20:12.368 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:20:12.368 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:20:12.368 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:20:12.368 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:20:12.368 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:20:12.368 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:20:12.368 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:20:12.368 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:20:12.368 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:20:12.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:20:12.376 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:20:12.377 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:20:12.377 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:20:12.377 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:20:12.431 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:20:12.431 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:20:12.432 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:20:12.434 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:20:12.434 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:20:12.434 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:20:12.434 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:20:12.434 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:20:12.435 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:20:12.435 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:20:12.435 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:20:12.435 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:20:12.435 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:20:12.435 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:20:12.435 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:20:12.435 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:20:12.435 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:20:12.435 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:20:12.435 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:20:12.435 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:20:12.436 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:20:12.436 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:20:12.436 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:20:12.436 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:20:12.436 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:20:12.436 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:20:12.436 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:20:12.436 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:20:12.436 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:20:12.436 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:20:12.436 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:20:12.436 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:20:12.437 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:20:12.437 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:20:12.437 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:20:12.446 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:20:12.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:20:12.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:20:12.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:20:12.504 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:20:12.504 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:20:12.504 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:20:12.507 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:20:12.507 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:20:12.507 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:20:12.507 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:20:12.507 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:20:12.507 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:20:12.507 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:20:12.507 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:20:12.507 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:20:12.507 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:20:12.508 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:20:12.508 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:20:12.508 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:20:12.508 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:20:12.508 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:20:12.508 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:20:12.508 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:20:12.508 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:20:12.508 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:20:12.508 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:20:12.508 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:20:12.508 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:20:12.508 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:20:12.509 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:20:12.509 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:20:12.509 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:20:12.509 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:20:12.509 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:20:12.509 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:20:12.509 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:20:12.509 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:20:12.509 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:20:12.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:20:12.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:20:12.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:20:12.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:20:12.568 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:20:12.568 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:20:12.568 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:20:12.571 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:20:12.571 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:20:12.571 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:20:12.571 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:20:12.571 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:20:12.571 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:20:12.571 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:20:12.572 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:20:12.572 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:20:12.572 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:20:12.572 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:20:12.572 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:20:12.572 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:20:12.572 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:20:12.572 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:20:12.572 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:20:12.572 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:20:12.572 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:20:12.573 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:20:12.573 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:20:12.573 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:20:12.573 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:20:12.573 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:20:12.573 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:20:12.573 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:20:12.573 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:20:12.573 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:20:12.573 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:20:12.573 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:20:12.573 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:20:12.574 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:20:12.574 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:20:12.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:20:12.582 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:20:12.582 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:20:12.582 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:20:12.643 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:20:12.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:20:12.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:20:12.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:20:12.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:20:12.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:20:12.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:20:12.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:20:12.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:20:12.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:20:12.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:20:12.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:20:12.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:20:12.646 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:20:12.646 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:20:12.646 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:20:12.682 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 76 2026-02-18T08:20:12.682 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 77 2026-02-18T08:20:12.682 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:20:12.682 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=309, tag_tid=21, entry_tid=1], [object_number=308, tag_tid=21, entry_tid=0], [object_number=310, tag_tid=20, entry_tid=46754], [object_number=311, tag_tid=20, entry_tid=46751]]], state=connected] 2026-02-18T08:20:12.682 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=306, tag_tid=20, entry_tid=42570], [object_number=305, tag_tid=20, entry_tid=42569], [object_number=304, tag_tid=20, entry_tid=42568], [object_number=307, tag_tid=20, entry_tid=42567]]], state=connected] 2026-02-18T08:20:12.686 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:20:12.687 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=309, tag_tid=21, entry_tid=1' 2026-02-18T08:20:12.688 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:20:12.688 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:20:12.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:20:12.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:20:12.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:20:12.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:20:12.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:20:12.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:20:12.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:20:12.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:20:12.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:20:12.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:20:12.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:20:12.691 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:20:12.691 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:20:12.718 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 76 2026-02-18T08:20:12.718 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 77 2026-02-18T08:20:12.718 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:20:12.718 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=309, tag_tid=21, entry_tid=1], [object_number=308, tag_tid=21, entry_tid=0], [object_number=310, tag_tid=20, entry_tid=46754], [object_number=311, tag_tid=20, entry_tid=46751]]], state=connected] 2026-02-18T08:20:12.718 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=306, tag_tid=20, entry_tid=42570], [object_number=305, tag_tid=20, entry_tid=42569], [object_number=304, tag_tid=20, entry_tid=42568], [object_number=307, tag_tid=20, entry_tid=42567]]], state=connected] 2026-02-18T08:20:12.720 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:20:12.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=306, tag_tid=20, entry_tid=42570' 2026-02-18T08:20:12.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=309, tag_tid=21, entry_tid=1' -a 'object_number=309, tag_tid=21, entry_tid=1' = 'object_number=306, tag_tid=20, entry_tid=42570' 2026-02-18T08:20:12.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=306, tag_tid=20, entry_tid=42570' '!=' 'object_number=306, tag_tid=20, entry_tid=42182' 2026-02-18T08:20:12.724 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:20:12.724 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=306, tag_tid=20, entry_tid=42570' = 'object_number=306, tag_tid=20, entry_tid=42182' 2026-02-18T08:20:12.724 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=306, tag_tid=20, entry_tid=42570' 2026-02-18T08:20:12.724 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=309, tag_tid=21, entry_tid=1' 2026-02-18T08:20:12.724 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:20:12.724 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:20:12.724 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=21 2026-02-18T08:20:12.725 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=306, tag_tid=20, entry_tid=42570' 2026-02-18T08:20:12.725 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:20:12.725 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:20:12.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=20 2026-02-18T08:20:12.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=309, tag_tid=21, entry_tid=1' 2026-02-18T08:20:12.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:20:12.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:20:12.728 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:20:12.729 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=306, tag_tid=20, entry_tid=42570' 2026-02-18T08:20:12.729 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:20:12.729 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:20:12.731 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=42570 2026-02-18T08:20:12.731 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 21 = 20 -a 1 -le 42570 2026-02-18T08:20:12.731 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:20:12.731 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:20:12.731 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:20:12.932 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:20:12.932 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:20:12.932 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:20:12.932 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:20:12.932 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:20:12.932 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:20:12.932 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:20:12.932 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:20:12.932 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:20:12.932 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:20:12.932 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:20:12.932 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:20:12.932 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:20:12.932 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:20:12.932 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:20:12.933 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:20:12.933 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:20:12.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:20:12.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:20:12.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:20:12.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:20:12.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:20:12.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:20:12.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:20:12.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:20:12.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:20:12.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:20:12.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:20:12.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:20:12.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:20:12.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:20:12.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:20:12.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:20:12.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:20:12.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:20:12.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:20:12.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:20:12.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:20:12.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:20:12.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:20:12.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:20:12.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:20:12.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:20:12.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:20:12.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:20:12.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:20:12.936 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:20:12.936 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:20:12.936 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:20:12.946 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:20:12.946 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:20:12.946 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:20:12.946 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:20:12.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:20:12.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:20:12.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:20:13.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:20:13.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:20:13.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:20:13.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:20:13.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:20:13.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:20:13.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:20:13.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:20:13.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:20:13.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:20:13.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:20:13.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:20:13.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:20:13.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:20:13.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:20:13.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:20:13.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:20:13.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:20:13.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:20:13.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:20:13.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:20:13.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:20:13.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:20:13.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:20:13.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:20:13.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:20:13.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:20:13.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:20:13.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:20:13.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:20:13.003 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:20:13.003 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:20:13.010 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:20:13.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:20:13.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:20:13.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:20:13.060 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:20:13.060 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:20:13.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:20:13.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:20:13.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:20:13.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:20:13.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:20:13.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:20:13.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:20:13.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:20:13.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:20:13.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:20:13.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:20:13.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:20:13.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:20:13.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:20:13.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:20:13.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:20:13.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:20:13.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:20:13.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:20:13.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:20:13.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:20:13.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:20:13.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:20:13.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:20:13.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:20:13.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:20:13.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:20:13.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:20:13.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:20:13.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:20:13.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:20:13.067 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:20:13.067 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:20:13.075 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:20:13.075 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:20:13.076 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:20:13.076 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:20:13.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:20:13.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:20:13.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:20:13.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:20:13.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:20:13.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:20:13.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:20:13.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:20:13.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:20:13.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:20:13.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:20:13.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:20:13.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:20:13.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:20:13.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:20:13.129 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:20:13.129 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:20:13.129 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:20:13.129 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:20:13.129 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:20:13.129 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:20:13.129 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:20:13.129 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:20:13.129 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:20:13.129 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:20:13.129 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:20:13.129 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:20:13.130 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:20:13.130 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:20:13.130 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:20:13.130 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:20:13.130 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:20:13.130 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:20:13.130 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:20:13.130 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:20:13.138 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:20:13.139 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:20:13.139 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:20:13.139 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:20:13.199 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:20:13.200 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:20:13.200 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:20:13.201 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:20:13.201 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:20:13.201 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:20:13.201 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:20:13.201 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:20:13.201 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:20:13.201 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:20:13.201 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:20:13.201 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:20:13.201 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:20:13.202 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:20:13.202 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:20:13.202 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:20:13.238 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 76 2026-02-18T08:20:13.238 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 77 2026-02-18T08:20:13.238 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:20:13.238 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=309, tag_tid=21, entry_tid=1], [object_number=308, tag_tid=21, entry_tid=0], [object_number=310, tag_tid=20, entry_tid=46754], [object_number=311, tag_tid=20, entry_tid=46751]]], state=connected] 2026-02-18T08:20:13.238 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=306, tag_tid=20, entry_tid=42902], [object_number=305, tag_tid=20, entry_tid=42901], [object_number=304, tag_tid=20, entry_tid=42900], [object_number=307, tag_tid=20, entry_tid=42899]]], state=connected] 2026-02-18T08:20:13.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:20:13.244 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=309, tag_tid=21, entry_tid=1' 2026-02-18T08:20:13.245 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:20:13.245 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:20:13.245 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:20:13.245 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:20:13.245 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:20:13.245 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:20:13.245 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:20:13.246 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:20:13.246 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:20:13.246 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:20:13.246 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:20:13.247 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:20:13.247 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:20:13.247 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:20:13.247 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:20:13.283 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 76 2026-02-18T08:20:13.284 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 77 2026-02-18T08:20:13.284 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:20:13.284 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=309, tag_tid=21, entry_tid=1], [object_number=308, tag_tid=21, entry_tid=0], [object_number=310, tag_tid=20, entry_tid=46754], [object_number=311, tag_tid=20, entry_tid=46751]]], state=connected] 2026-02-18T08:20:13.284 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=306, tag_tid=20, entry_tid=42902], [object_number=305, tag_tid=20, entry_tid=42901], [object_number=304, tag_tid=20, entry_tid=42900], [object_number=307, tag_tid=20, entry_tid=42899]]], state=connected] 2026-02-18T08:20:13.287 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:20:13.289 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=306, tag_tid=20, entry_tid=42902' 2026-02-18T08:20:13.289 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=309, tag_tid=21, entry_tid=1' -a 'object_number=309, tag_tid=21, entry_tid=1' = 'object_number=306, tag_tid=20, entry_tid=42902' 2026-02-18T08:20:13.289 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=306, tag_tid=20, entry_tid=42902' '!=' 'object_number=306, tag_tid=20, entry_tid=42570' 2026-02-18T08:20:13.289 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:20:13.289 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=306, tag_tid=20, entry_tid=42902' = 'object_number=306, tag_tid=20, entry_tid=42570' 2026-02-18T08:20:13.290 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=306, tag_tid=20, entry_tid=42902' 2026-02-18T08:20:13.290 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=309, tag_tid=21, entry_tid=1' 2026-02-18T08:20:13.290 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:20:13.290 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:20:13.291 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=21 2026-02-18T08:20:13.292 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=306, tag_tid=20, entry_tid=42902' 2026-02-18T08:20:13.292 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:20:13.292 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:20:13.293 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=20 2026-02-18T08:20:13.294 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=309, tag_tid=21, entry_tid=1' 2026-02-18T08:20:13.294 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:20:13.294 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:20:13.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:20:13.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=306, tag_tid=20, entry_tid=42902' 2026-02-18T08:20:13.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:20:13.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:20:13.298 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=42902 2026-02-18T08:20:13.298 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 21 = 20 -a 1 -le 42902 2026-02-18T08:20:13.298 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:20:13.298 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:20:13.298 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:20:13.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:20:13.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:20:13.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:20:13.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:20:13.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:20:13.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:20:13.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:20:13.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:20:13.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:20:13.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:20:13.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:20:13.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:20:13.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:20:13.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:20:13.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:20:13.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:20:13.500 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:20:13.501 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:20:13.501 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:20:13.501 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:20:13.501 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:20:13.501 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:20:13.501 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:20:13.501 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:20:13.501 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:20:13.501 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:20:13.502 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:20:13.502 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:20:13.502 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:20:13.502 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:20:13.502 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:20:13.502 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:20:13.502 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:20:13.502 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:20:13.502 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:20:13.502 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:20:13.502 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:20:13.502 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:20:13.502 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:20:13.502 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:20:13.502 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:20:13.502 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:20:13.502 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:20:13.502 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:20:13.502 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:20:13.502 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:20:13.502 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:20:13.502 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:20:13.502 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:20:13.513 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:20:13.514 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:20:13.514 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:20:13.514 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:20:13.565 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:20:13.565 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:20:13.565 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:20:13.568 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:20:13.568 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:20:13.568 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:20:13.568 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:20:13.568 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:20:13.569 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:20:13.569 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:20:13.569 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:20:13.569 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:20:13.569 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:20:13.569 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:20:13.569 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:20:13.569 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:20:13.569 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:20:13.569 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:20:13.569 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:20:13.569 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:20:13.570 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:20:13.570 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:20:13.570 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:20:13.570 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:20:13.570 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:20:13.570 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:20:13.570 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:20:13.570 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:20:13.570 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:20:13.570 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:20:13.570 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:20:13.571 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:20:13.571 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:20:13.571 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:20:13.571 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:20:13.578 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:20:13.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:20:13.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:20:13.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:20:13.629 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:20:13.629 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:20:13.629 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:20:13.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:20:13.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:20:13.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:20:13.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:20:13.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:20:13.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:20:13.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:20:13.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:20:13.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:20:13.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:20:13.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:20:13.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:20:13.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:20:13.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:20:13.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:20:13.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:20:13.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:20:13.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:20:13.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:20:13.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:20:13.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:20:13.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:20:13.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:20:13.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:20:13.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:20:13.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:20:13.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:20:13.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:20:13.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:20:13.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:20:13.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:20:13.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:20:13.642 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:20:13.642 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:20:13.643 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:20:13.643 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:20:13.700 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:20:13.700 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:20:13.700 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:20:13.703 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:20:13.703 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:20:13.703 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:20:13.703 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:20:13.703 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:20:13.703 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:20:13.704 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:20:13.704 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:20:13.704 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:20:13.704 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:20:13.704 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:20:13.704 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:20:13.704 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:20:13.704 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:20:13.704 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:20:13.704 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:20:13.704 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:20:13.704 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:20:13.704 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:20:13.705 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:20:13.705 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:20:13.705 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:20:13.705 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:20:13.705 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:20:13.705 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:20:13.705 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:20:13.705 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:20:13.705 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:20:13.705 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:20:13.705 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:20:13.705 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:20:13.706 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:20:13.714 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:20:13.714 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:20:13.714 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:20:13.714 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:20:13.776 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:20:13.776 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:20:13.776 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:20:13.776 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:20:13.776 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:20:13.776 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:20:13.776 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:20:13.777 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:20:13.777 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:20:13.777 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:20:13.777 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:20:13.777 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:20:13.777 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:20:13.778 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:20:13.778 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:20:13.778 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:20:13.806 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 76 2026-02-18T08:20:13.806 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 77 2026-02-18T08:20:13.806 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:20:13.806 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=309, tag_tid=21, entry_tid=1], [object_number=308, tag_tid=21, entry_tid=0], [object_number=310, tag_tid=20, entry_tid=46754], [object_number=311, tag_tid=20, entry_tid=46751]]], state=connected] 2026-02-18T08:20:13.806 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=307, tag_tid=20, entry_tid=43279], [object_number=306, tag_tid=20, entry_tid=43278], [object_number=305, tag_tid=20, entry_tid=43277], [object_number=304, tag_tid=20, entry_tid=43276]]], state=connected] 2026-02-18T08:20:13.809 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:20:13.810 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=309, tag_tid=21, entry_tid=1' 2026-02-18T08:20:13.812 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:20:13.812 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:20:13.812 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:20:13.812 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:20:13.812 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:20:13.812 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:20:13.812 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:20:13.813 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:20:13.813 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:20:13.813 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:20:13.813 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:20:13.813 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:20:13.814 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:20:13.814 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:20:13.814 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:20:13.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 76 2026-02-18T08:20:13.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 77 2026-02-18T08:20:13.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:20:13.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=309, tag_tid=21, entry_tid=1], [object_number=308, tag_tid=21, entry_tid=0], [object_number=310, tag_tid=20, entry_tid=46754], [object_number=311, tag_tid=20, entry_tid=46751]]], state=connected] 2026-02-18T08:20:13.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=307, tag_tid=20, entry_tid=43279], [object_number=306, tag_tid=20, entry_tid=43278], [object_number=305, tag_tid=20, entry_tid=43277], [object_number=304, tag_tid=20, entry_tid=43276]]], state=connected] 2026-02-18T08:20:13.853 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:20:13.855 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=307, tag_tid=20, entry_tid=43279' 2026-02-18T08:20:13.855 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=309, tag_tid=21, entry_tid=1' -a 'object_number=309, tag_tid=21, entry_tid=1' = 'object_number=307, tag_tid=20, entry_tid=43279' 2026-02-18T08:20:13.855 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=307, tag_tid=20, entry_tid=43279' '!=' 'object_number=306, tag_tid=20, entry_tid=42902' 2026-02-18T08:20:13.855 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:20:13.855 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=307, tag_tid=20, entry_tid=43279' = 'object_number=306, tag_tid=20, entry_tid=42902' 2026-02-18T08:20:13.855 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=307, tag_tid=20, entry_tid=43279' 2026-02-18T08:20:13.856 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=309, tag_tid=21, entry_tid=1' 2026-02-18T08:20:13.856 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:20:13.856 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:20:13.857 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=21 2026-02-18T08:20:13.858 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=307, tag_tid=20, entry_tid=43279' 2026-02-18T08:20:13.858 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:20:13.858 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:20:13.859 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=20 2026-02-18T08:20:13.860 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=309, tag_tid=21, entry_tid=1' 2026-02-18T08:20:13.860 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:20:13.860 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:20:13.861 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:20:13.862 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=307, tag_tid=20, entry_tid=43279' 2026-02-18T08:20:13.862 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:20:13.862 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:20:13.863 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=43279 2026-02-18T08:20:13.864 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 21 = 20 -a 1 -le 43279 2026-02-18T08:20:13.864 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:20:13.864 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:20:13.864 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:20:14.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:20:14.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:20:14.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:20:14.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:20:14.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:20:14.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:20:14.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:20:14.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:20:14.067 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:20:14.067 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:20:14.067 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:20:14.067 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:20:14.067 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:20:14.067 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:20:14.067 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:20:14.067 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:20:14.067 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:20:14.069 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:20:14.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:20:14.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:20:14.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:20:14.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:20:14.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:20:14.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:20:14.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:20:14.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:20:14.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:20:14.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:20:14.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:20:14.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:20:14.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:20:14.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:20:14.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:20:14.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:20:14.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:20:14.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:20:14.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:20:14.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:20:14.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:20:14.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:20:14.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:20:14.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:20:14.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:20:14.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:20:14.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:20:14.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:20:14.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:20:14.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:20:14.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:20:14.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:20:14.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:20:14.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:20:14.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:20:14.129 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:20:14.129 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:20:14.129 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:20:14.132 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:20:14.132 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:20:14.132 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:20:14.132 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:20:14.132 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:20:14.132 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:20:14.132 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:20:14.132 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:20:14.132 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:20:14.132 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:20:14.132 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:20:14.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:20:14.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:20:14.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:20:14.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:20:14.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:20:14.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:20:14.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:20:14.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:20:14.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:20:14.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:20:14.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:20:14.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:20:14.134 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:20:14.134 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:20:14.134 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:20:14.134 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:20:14.134 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:20:14.134 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:20:14.134 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:20:14.134 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:20:14.134 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:20:14.143 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:20:14.143 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:20:14.143 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:20:14.143 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:20:14.200 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:20:14.200 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:20:14.200 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:20:14.203 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:20:14.204 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:20:14.204 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:20:14.204 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:20:14.204 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:20:14.204 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:20:14.204 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:20:14.204 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:20:14.204 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:20:14.204 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:20:14.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:20:14.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:20:14.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:20:14.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:20:14.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:20:14.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:20:14.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:20:14.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:20:14.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:20:14.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:20:14.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:20:14.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:20:14.206 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:20:14.206 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:20:14.206 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:20:14.206 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:20:14.206 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:20:14.206 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:20:14.206 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:20:14.206 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:20:14.206 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:20:14.206 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:20:14.214 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:20:14.214 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:20:14.214 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:20:14.215 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:20:14.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:20:14.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:20:14.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:20:14.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:20:14.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:20:14.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:20:14.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:20:14.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:20:14.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:20:14.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:20:14.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:20:14.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:20:14.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:20:14.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:20:14.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:20:14.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:20:14.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:20:14.266 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:20:14.266 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:20:14.266 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:20:14.266 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:20:14.266 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:20:14.266 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:20:14.266 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:20:14.266 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:20:14.266 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:20:14.266 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:20:14.266 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:20:14.266 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:20:14.266 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:20:14.266 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:20:14.266 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:20:14.266 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:20:14.266 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:20:14.266 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:20:14.277 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:20:14.277 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:20:14.277 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:20:14.277 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:20:14.352 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:20:14.352 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:20:14.352 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:20:14.352 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:20:14.352 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:20:14.353 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:20:14.353 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:20:14.353 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:20:14.353 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:20:14.353 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:20:14.353 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:20:14.353 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:20:14.353 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:20:14.354 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:20:14.354 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:20:14.354 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:20:14.389 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 76 2026-02-18T08:20:14.389 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 77 2026-02-18T08:20:14.389 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:20:14.389 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=309, tag_tid=21, entry_tid=1], [object_number=308, tag_tid=21, entry_tid=0], [object_number=310, tag_tid=20, entry_tid=46754], [object_number=311, tag_tid=20, entry_tid=46751]]], state=connected] 2026-02-18T08:20:14.389 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=304, tag_tid=20, entry_tid=43612], [object_number=307, tag_tid=20, entry_tid=43611], [object_number=306, tag_tid=20, entry_tid=43610], [object_number=305, tag_tid=20, entry_tid=43609]]], state=connected] 2026-02-18T08:20:14.393 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:20:14.394 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=309, tag_tid=21, entry_tid=1' 2026-02-18T08:20:14.395 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:20:14.395 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:20:14.395 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:20:14.395 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:20:14.395 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:20:14.395 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:20:14.395 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:20:14.396 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:20:14.396 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:20:14.396 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:20:14.396 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:20:14.396 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:20:14.397 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:20:14.397 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:20:14.397 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:20:14.427 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 76 2026-02-18T08:20:14.427 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 77 2026-02-18T08:20:14.427 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:20:14.427 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=309, tag_tid=21, entry_tid=1], [object_number=308, tag_tid=21, entry_tid=0], [object_number=310, tag_tid=20, entry_tid=46754], [object_number=311, tag_tid=20, entry_tid=46751]]], state=connected] 2026-02-18T08:20:14.427 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=304, tag_tid=20, entry_tid=43612], [object_number=307, tag_tid=20, entry_tid=43611], [object_number=306, tag_tid=20, entry_tid=43610], [object_number=305, tag_tid=20, entry_tid=43609]]], state=connected] 2026-02-18T08:20:14.430 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:20:14.432 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=304, tag_tid=20, entry_tid=43612' 2026-02-18T08:20:14.432 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=309, tag_tid=21, entry_tid=1' -a 'object_number=309, tag_tid=21, entry_tid=1' = 'object_number=304, tag_tid=20, entry_tid=43612' 2026-02-18T08:20:14.432 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=304, tag_tid=20, entry_tid=43612' '!=' 'object_number=307, tag_tid=20, entry_tid=43279' 2026-02-18T08:20:14.432 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:20:14.432 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=304, tag_tid=20, entry_tid=43612' = 'object_number=307, tag_tid=20, entry_tid=43279' 2026-02-18T08:20:14.433 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=304, tag_tid=20, entry_tid=43612' 2026-02-18T08:20:14.433 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=309, tag_tid=21, entry_tid=1' 2026-02-18T08:20:14.433 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:20:14.433 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:20:14.434 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=21 2026-02-18T08:20:14.435 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=304, tag_tid=20, entry_tid=43612' 2026-02-18T08:20:14.435 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:20:14.435 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:20:14.436 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=20 2026-02-18T08:20:14.437 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=309, tag_tid=21, entry_tid=1' 2026-02-18T08:20:14.437 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:20:14.437 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:20:14.439 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:20:14.439 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=304, tag_tid=20, entry_tid=43612' 2026-02-18T08:20:14.440 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:20:14.440 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:20:14.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=43612 2026-02-18T08:20:14.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 21 = 20 -a 1 -le 43612 2026-02-18T08:20:14.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:20:14.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:20:14.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:20:14.642 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:20:14.642 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:20:14.642 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:20:14.642 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:20:14.642 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:20:14.642 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:20:14.642 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:20:14.642 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:20:14.643 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:20:14.643 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:20:14.643 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:20:14.643 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:20:14.643 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:20:14.643 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:20:14.643 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:20:14.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:20:14.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:20:14.646 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:20:14.646 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:20:14.647 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:20:14.647 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:20:14.647 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:20:14.647 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:20:14.647 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:20:14.647 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:20:14.647 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:20:14.647 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:20:14.647 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:20:14.647 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:20:14.647 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:20:14.648 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:20:14.648 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:20:14.648 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:20:14.648 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:20:14.648 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:20:14.648 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:20:14.648 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:20:14.648 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:20:14.648 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:20:14.648 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:20:14.648 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:20:14.648 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:20:14.649 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:20:14.649 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:20:14.649 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:20:14.649 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:20:14.649 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:20:14.649 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:20:14.649 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:20:14.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:20:14.658 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:20:14.658 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:20:14.658 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:20:14.709 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:20:14.709 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:20:14.709 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:20:14.712 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:20:14.712 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:20:14.712 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:20:14.712 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:20:14.712 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:20:14.712 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:20:14.712 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:20:14.712 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:20:14.713 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:20:14.713 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:20:14.713 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:20:14.713 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:20:14.713 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:20:14.713 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:20:14.713 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:20:14.713 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:20:14.713 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:20:14.713 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:20:14.713 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:20:14.713 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:20:14.714 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:20:14.714 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:20:14.714 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:20:14.714 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:20:14.714 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:20:14.714 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:20:14.714 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:20:14.714 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:20:14.714 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:20:14.714 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:20:14.714 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:20:14.714 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:20:14.722 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:20:14.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:20:14.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:20:14.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:20:14.779 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:20:14.780 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:20:14.780 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:20:14.782 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:20:14.782 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:20:14.783 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:20:14.783 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:20:14.783 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:20:14.783 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:20:14.783 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:20:14.783 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:20:14.783 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:20:14.783 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:20:14.783 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:20:14.784 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:20:14.784 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:20:14.784 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:20:14.784 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:20:14.784 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:20:14.784 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:20:14.784 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:20:14.784 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:20:14.784 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:20:14.784 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:20:14.784 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:20:14.785 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:20:14.785 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:20:14.785 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:20:14.785 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:20:14.785 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:20:14.785 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:20:14.785 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:20:14.785 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:20:14.785 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:20:14.785 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:20:14.793 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:20:14.794 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:20:14.794 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:20:14.794 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:20:14.843 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:20:14.843 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:20:14.843 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:20:14.846 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:20:14.846 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:20:14.846 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:20:14.847 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:20:14.847 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:20:14.847 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:20:14.847 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:20:14.847 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:20:14.847 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:20:14.847 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:20:14.847 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:20:14.847 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:20:14.847 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:20:14.847 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:20:14.847 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:20:14.848 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:20:14.848 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:20:14.848 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:20:14.848 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:20:14.848 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:20:14.848 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:20:14.848 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:20:14.848 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:20:14.848 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:20:14.848 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:20:14.848 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:20:14.848 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:20:14.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:20:14.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:20:14.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:20:14.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:20:14.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:20:14.857 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:20:14.858 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:20:14.858 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:20:14.858 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:20:14.918 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:20:14.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:20:14.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:20:14.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:20:14.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:20:14.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:20:14.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:20:14.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:20:14.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:20:14.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:20:14.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:20:14.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:20:14.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:20:14.922 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:20:14.922 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:20:14.922 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:20:14.949 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 76 2026-02-18T08:20:14.949 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 77 2026-02-18T08:20:14.949 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:20:14.950 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=309, tag_tid=21, entry_tid=1], [object_number=308, tag_tid=21, entry_tid=0], [object_number=310, tag_tid=20, entry_tid=46754], [object_number=311, tag_tid=20, entry_tid=46751]]], state=connected] 2026-02-18T08:20:14.950 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=306, tag_tid=20, entry_tid=43958], [object_number=305, tag_tid=20, entry_tid=43957], [object_number=304, tag_tid=20, entry_tid=43956], [object_number=307, tag_tid=20, entry_tid=43955]]], state=connected] 2026-02-18T08:20:14.952 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:20:14.953 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=309, tag_tid=21, entry_tid=1' 2026-02-18T08:20:14.954 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:20:14.954 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:20:14.954 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:20:14.955 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:20:14.955 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:20:14.955 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:20:14.955 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:20:14.955 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:20:14.955 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:20:14.955 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:20:14.955 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:20:14.955 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:20:14.956 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:20:14.956 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:20:14.956 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:20:14.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 76 2026-02-18T08:20:14.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 77 2026-02-18T08:20:14.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:20:14.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=309, tag_tid=21, entry_tid=1], [object_number=308, tag_tid=21, entry_tid=0], [object_number=310, tag_tid=20, entry_tid=46754], [object_number=311, tag_tid=20, entry_tid=46751]]], state=connected] 2026-02-18T08:20:14.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=306, tag_tid=20, entry_tid=43958], [object_number=305, tag_tid=20, entry_tid=43957], [object_number=304, tag_tid=20, entry_tid=43956], [object_number=307, tag_tid=20, entry_tid=43955]]], state=connected] 2026-02-18T08:20:14.986 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:20:14.988 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=306, tag_tid=20, entry_tid=43958' 2026-02-18T08:20:14.988 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=309, tag_tid=21, entry_tid=1' -a 'object_number=309, tag_tid=21, entry_tid=1' = 'object_number=306, tag_tid=20, entry_tid=43958' 2026-02-18T08:20:14.988 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=306, tag_tid=20, entry_tid=43958' '!=' 'object_number=304, tag_tid=20, entry_tid=43612' 2026-02-18T08:20:14.989 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:20:14.989 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=306, tag_tid=20, entry_tid=43958' = 'object_number=304, tag_tid=20, entry_tid=43612' 2026-02-18T08:20:14.989 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=306, tag_tid=20, entry_tid=43958' 2026-02-18T08:20:14.989 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=309, tag_tid=21, entry_tid=1' 2026-02-18T08:20:14.990 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:20:14.990 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:20:14.990 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=21 2026-02-18T08:20:14.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=306, tag_tid=20, entry_tid=43958' 2026-02-18T08:20:14.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:20:14.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:20:14.993 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=20 2026-02-18T08:20:14.994 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=309, tag_tid=21, entry_tid=1' 2026-02-18T08:20:14.994 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:20:14.994 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:20:14.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:20:14.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=306, tag_tid=20, entry_tid=43958' 2026-02-18T08:20:14.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:20:14.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:20:14.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=43958 2026-02-18T08:20:14.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 21 = 20 -a 1 -le 43958 2026-02-18T08:20:14.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:20:14.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:20:14.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:20:15.198 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:20:15.198 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:20:15.198 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:20:15.198 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:20:15.198 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:20:15.198 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:20:15.198 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:20:15.198 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:20:15.198 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:20:15.198 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:20:15.199 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:20:15.199 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:20:15.199 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:20:15.199 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:20:15.199 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:20:15.199 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:20:15.199 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:20:15.200 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:20:15.200 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:20:15.200 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:20:15.200 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:20:15.200 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:20:15.200 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:20:15.200 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:20:15.200 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:20:15.200 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:20:15.200 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:20:15.201 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:20:15.201 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:20:15.201 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:20:15.201 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:20:15.201 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:20:15.201 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:20:15.201 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:20:15.201 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:20:15.201 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:20:15.201 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:20:15.201 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:20:15.201 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:20:15.201 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:20:15.201 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:20:15.201 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:20:15.201 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:20:15.201 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:20:15.201 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:20:15.201 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:20:15.201 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:20:15.201 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:20:15.201 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:20:15.214 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:20:15.214 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:20:15.214 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:20:15.215 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:20:15.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:20:15.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:20:15.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:20:15.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:20:15.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:20:15.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:20:15.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:20:15.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:20:15.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:20:15.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:20:15.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:20:15.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:20:15.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:20:15.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:20:15.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:20:15.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:20:15.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:20:15.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:20:15.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:20:15.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:20:15.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:20:15.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:20:15.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:20:15.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:20:15.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:20:15.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:20:15.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:20:15.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:20:15.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:20:15.269 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:20:15.269 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:20:15.269 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:20:15.269 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:20:15.269 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:20:15.269 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:20:15.278 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:20:15.278 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:20:15.279 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:20:15.279 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:20:15.327 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:20:15.327 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:20:15.327 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:20:15.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:20:15.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:20:15.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:20:15.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:20:15.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:20:15.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:20:15.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:20:15.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:20:15.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:20:15.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:20:15.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:20:15.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:20:15.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:20:15.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:20:15.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:20:15.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:20:15.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:20:15.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:20:15.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:20:15.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:20:15.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:20:15.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:20:15.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:20:15.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:20:15.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:20:15.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:20:15.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:20:15.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:20:15.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:20:15.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:20:15.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:20:15.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:20:15.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:20:15.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:20:15.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:20:15.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:20:15.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:20:15.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:20:15.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:20:15.394 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:20:15.394 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:20:15.394 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:20:15.394 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:20:15.394 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:20:15.394 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:20:15.394 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:20:15.394 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:20:15.395 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:20:15.395 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:20:15.395 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:20:15.395 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:20:15.395 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:20:15.395 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:20:15.396 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:20:15.396 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:20:15.396 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:20:15.396 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:20:15.396 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:20:15.396 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:20:15.396 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:20:15.397 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:20:15.397 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:20:15.397 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:20:15.397 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:20:15.397 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:20:15.397 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:20:15.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:20:15.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:20:15.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:20:15.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:20:15.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:20:15.405 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:20:15.405 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:20:15.406 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:20:15.406 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:20:15.465 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:20:15.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:20:15.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:20:15.467 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:20:15.467 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:20:15.467 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:20:15.467 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:20:15.467 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:20:15.467 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:20:15.467 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:20:15.467 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:20:15.467 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:20:15.467 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:20:15.468 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:20:15.468 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:20:15.468 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:20:15.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 76 2026-02-18T08:20:15.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 77 2026-02-18T08:20:15.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:20:15.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=309, tag_tid=21, entry_tid=1], [object_number=308, tag_tid=21, entry_tid=0], [object_number=310, tag_tid=20, entry_tid=46754], [object_number=311, tag_tid=20, entry_tid=46751]]], state=connected] 2026-02-18T08:20:15.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=304, tag_tid=20, entry_tid=44280], [object_number=307, tag_tid=20, entry_tid=44279], [object_number=306, tag_tid=20, entry_tid=44278], [object_number=305, tag_tid=20, entry_tid=44277]]], state=connected] 2026-02-18T08:20:15.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:20:15.501 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=309, tag_tid=21, entry_tid=1' 2026-02-18T08:20:15.502 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:20:15.502 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:20:15.502 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:20:15.503 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:20:15.503 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:20:15.503 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:20:15.503 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:20:15.503 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:20:15.503 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:20:15.503 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:20:15.503 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:20:15.503 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:20:15.504 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:20:15.504 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:20:15.504 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:20:15.530 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 76 2026-02-18T08:20:15.530 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 77 2026-02-18T08:20:15.530 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:20:15.531 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=309, tag_tid=21, entry_tid=1], [object_number=308, tag_tid=21, entry_tid=0], [object_number=310, tag_tid=20, entry_tid=46754], [object_number=311, tag_tid=20, entry_tid=46751]]], state=connected] 2026-02-18T08:20:15.531 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=304, tag_tid=20, entry_tid=44280], [object_number=307, tag_tid=20, entry_tid=44279], [object_number=306, tag_tid=20, entry_tid=44278], [object_number=305, tag_tid=20, entry_tid=44277]]], state=connected] 2026-02-18T08:20:15.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:20:15.534 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=304, tag_tid=20, entry_tid=44280' 2026-02-18T08:20:15.534 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=309, tag_tid=21, entry_tid=1' -a 'object_number=309, tag_tid=21, entry_tid=1' = 'object_number=304, tag_tid=20, entry_tid=44280' 2026-02-18T08:20:15.535 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=304, tag_tid=20, entry_tid=44280' '!=' 'object_number=306, tag_tid=20, entry_tid=43958' 2026-02-18T08:20:15.535 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:20:15.535 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=304, tag_tid=20, entry_tid=44280' = 'object_number=306, tag_tid=20, entry_tid=43958' 2026-02-18T08:20:15.535 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=304, tag_tid=20, entry_tid=44280' 2026-02-18T08:20:15.535 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=309, tag_tid=21, entry_tid=1' 2026-02-18T08:20:15.536 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:20:15.536 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:20:15.536 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=21 2026-02-18T08:20:15.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=304, tag_tid=20, entry_tid=44280' 2026-02-18T08:20:15.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:20:15.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:20:15.539 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=20 2026-02-18T08:20:15.539 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=309, tag_tid=21, entry_tid=1' 2026-02-18T08:20:15.540 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:20:15.540 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:20:15.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:20:15.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=304, tag_tid=20, entry_tid=44280' 2026-02-18T08:20:15.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:20:15.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:20:15.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=44280 2026-02-18T08:20:15.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 21 = 20 -a 1 -le 44280 2026-02-18T08:20:15.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:20:15.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:20:15.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:20:15.745 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:20:15.746 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:20:15.746 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:20:15.746 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:20:15.746 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:20:15.746 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:20:15.746 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:20:15.746 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:20:15.746 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:20:15.746 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:20:15.746 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:20:15.746 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:20:15.747 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:20:15.747 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:20:15.747 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:20:15.750 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:20:15.750 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:20:15.750 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:20:15.750 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:20:15.750 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:20:15.750 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:20:15.750 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:20:15.751 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:20:15.751 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:20:15.751 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:20:15.751 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:20:15.751 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:20:15.751 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:20:15.751 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:20:15.752 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:20:15.752 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:20:15.752 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:20:15.752 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:20:15.752 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:20:15.752 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:20:15.752 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:20:15.752 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:20:15.752 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:20:15.752 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:20:15.752 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:20:15.752 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:20:15.753 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:20:15.753 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:20:15.753 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:20:15.753 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:20:15.753 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:20:15.753 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:20:15.753 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:20:15.753 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:20:15.761 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:20:15.761 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:20:15.761 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:20:15.762 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:20:15.820 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:20:15.820 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:20:15.820 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:20:15.823 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:20:15.823 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:20:15.823 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:20:15.823 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:20:15.823 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:20:15.823 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:20:15.823 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:20:15.823 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:20:15.824 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:20:15.824 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:20:15.824 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:20:15.824 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:20:15.824 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:20:15.824 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:20:15.824 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:20:15.824 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:20:15.824 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:20:15.824 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:20:15.824 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:20:15.824 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:20:15.825 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:20:15.825 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:20:15.825 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:20:15.825 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:20:15.825 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:20:15.825 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:20:15.825 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:20:15.825 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:20:15.825 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:20:15.825 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:20:15.825 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:20:15.825 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:20:15.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:20:15.835 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:20:15.835 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:20:15.835 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:20:15.884 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:20:15.884 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:20:15.884 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:20:15.886 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:20:15.886 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:20:15.887 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:20:15.887 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:20:15.887 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:20:15.887 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:20:15.887 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:20:15.887 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:20:15.887 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:20:15.887 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:20:15.887 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:20:15.887 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:20:15.887 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:20:15.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:20:15.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:20:15.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:20:15.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:20:15.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:20:15.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:20:15.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:20:15.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:20:15.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:20:15.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:20:15.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:20:15.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:20:15.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:20:15.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:20:15.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:20:15.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:20:15.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:20:15.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:20:15.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:20:15.897 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:20:15.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:20:15.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:20:15.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:20:15.955 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:20:15.955 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:20:15.956 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:20:15.958 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:20:15.958 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:20:15.958 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:20:15.958 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:20:15.959 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:20:15.959 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:20:15.959 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:20:15.959 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:20:15.959 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:20:15.959 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:20:15.959 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:20:15.959 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:20:15.959 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:20:15.959 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:20:15.959 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:20:15.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:20:15.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:20:15.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:20:15.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:20:15.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:20:15.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:20:15.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:20:15.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:20:15.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:20:15.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:20:15.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:20:15.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:20:15.961 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:20:15.961 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:20:15.961 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:20:15.961 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:20:15.961 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:20:15.969 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:20:15.969 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:20:15.969 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:20:15.969 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:20:16.037 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:20:16.038 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:20:16.039 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:20:16.039 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:20:16.039 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:20:16.039 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:20:16.039 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:20:16.039 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:20:16.039 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:20:16.039 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:20:16.039 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:20:16.039 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:20:16.040 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:20:16.040 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:20:16.041 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:20:16.041 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:20:16.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 76 2026-02-18T08:20:16.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 77 2026-02-18T08:20:16.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:20:16.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=309, tag_tid=21, entry_tid=1], [object_number=308, tag_tid=21, entry_tid=0], [object_number=310, tag_tid=20, entry_tid=46754], [object_number=311, tag_tid=20, entry_tid=46751]]], state=connected] 2026-02-18T08:20:16.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=307, tag_tid=20, entry_tid=44503], [object_number=306, tag_tid=20, entry_tid=44502], [object_number=305, tag_tid=20, entry_tid=44501], [object_number=304, tag_tid=20, entry_tid=44500]]], state=connected] 2026-02-18T08:20:16.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:20:16.074 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=309, tag_tid=21, entry_tid=1' 2026-02-18T08:20:16.075 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:20:16.075 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:20:16.075 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:20:16.075 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:20:16.075 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:20:16.076 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:20:16.076 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:20:16.076 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:20:16.076 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:20:16.076 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:20:16.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:20:16.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:20:16.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:20:16.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:20:16.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:20:16.315 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 76 2026-02-18T08:20:16.315 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 77 2026-02-18T08:20:16.315 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:20:16.316 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=309, tag_tid=21, entry_tid=1], [object_number=308, tag_tid=21, entry_tid=0], [object_number=310, tag_tid=20, entry_tid=46754], [object_number=311, tag_tid=20, entry_tid=46751]]], state=connected] 2026-02-18T08:20:16.316 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=307, tag_tid=20, entry_tid=44503], [object_number=306, tag_tid=20, entry_tid=44502], [object_number=305, tag_tid=20, entry_tid=44501], [object_number=304, tag_tid=20, entry_tid=44500]]], state=connected] 2026-02-18T08:20:16.319 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:20:16.321 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=307, tag_tid=20, entry_tid=44503' 2026-02-18T08:20:16.321 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=309, tag_tid=21, entry_tid=1' -a 'object_number=309, tag_tid=21, entry_tid=1' = 'object_number=307, tag_tid=20, entry_tid=44503' 2026-02-18T08:20:16.321 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=307, tag_tid=20, entry_tid=44503' '!=' 'object_number=304, tag_tid=20, entry_tid=44280' 2026-02-18T08:20:16.321 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:20:16.321 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=307, tag_tid=20, entry_tid=44503' = 'object_number=304, tag_tid=20, entry_tid=44280' 2026-02-18T08:20:16.322 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=307, tag_tid=20, entry_tid=44503' 2026-02-18T08:20:16.322 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=309, tag_tid=21, entry_tid=1' 2026-02-18T08:20:16.322 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:20:16.322 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:20:16.323 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=21 2026-02-18T08:20:16.324 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=307, tag_tid=20, entry_tid=44503' 2026-02-18T08:20:16.324 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:20:16.324 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:20:16.325 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=20 2026-02-18T08:20:16.326 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=309, tag_tid=21, entry_tid=1' 2026-02-18T08:20:16.326 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:20:16.326 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:20:16.327 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:20:16.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=307, tag_tid=20, entry_tid=44503' 2026-02-18T08:20:16.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:20:16.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:20:16.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=44503 2026-02-18T08:20:16.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 21 = 20 -a 1 -le 44503 2026-02-18T08:20:16.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:20:16.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:20:16.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:20:16.531 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:20:16.531 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:20:16.531 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:20:16.531 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:20:16.531 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:20:16.531 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:20:16.531 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:20:16.531 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:20:16.531 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:20:16.531 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:20:16.531 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:20:16.531 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:20:16.531 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:20:16.531 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:20:16.531 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:20:16.532 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:20:16.532 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:20:16.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:20:16.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:20:16.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:20:16.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:20:16.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:20:16.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:20:16.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:20:16.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:20:16.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:20:16.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:20:16.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:20:16.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:20:16.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:20:16.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:20:16.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:20:16.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:20:16.534 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:20:16.534 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:20:16.534 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:20:16.534 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:20:16.534 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:20:16.534 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:20:16.534 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:20:16.534 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:20:16.534 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:20:16.534 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:20:16.534 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:20:16.534 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:20:16.534 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:20:16.534 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:20:16.534 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:20:16.534 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:20:16.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:20:16.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:20:16.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:20:16.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:20:16.606 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:20:16.606 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:20:16.606 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:20:16.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:20:16.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:20:16.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:20:16.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:20:16.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:20:16.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:20:16.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:20:16.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:20:16.610 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:20:16.610 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:20:16.610 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:20:16.610 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:20:16.610 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:20:16.610 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:20:16.610 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:20:16.610 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:20:16.610 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:20:16.610 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:20:16.611 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:20:16.611 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:20:16.611 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:20:16.611 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:20:16.611 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:20:16.611 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:20:16.611 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:20:16.611 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:20:16.611 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:20:16.611 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:20:16.611 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:20:16.612 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:20:16.612 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:20:16.612 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:20:16.619 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:20:16.619 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:20:16.620 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:20:16.620 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:20:16.679 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:20:16.679 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:20:16.679 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:20:16.682 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:20:16.682 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:20:16.683 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:20:16.683 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:20:16.683 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:20:16.683 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:20:16.683 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:20:16.683 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:20:16.683 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:20:16.683 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:20:16.683 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:20:16.683 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:20:16.683 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:20:16.684 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:20:16.684 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:20:16.684 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:20:16.684 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:20:16.684 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:20:16.684 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:20:16.684 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:20:16.684 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:20:16.684 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:20:16.684 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:20:16.684 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:20:16.684 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:20:16.685 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:20:16.685 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:20:16.685 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:20:16.685 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:20:16.685 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:20:16.685 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:20:16.685 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:20:16.693 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:20:16.693 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:20:16.693 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:20:16.693 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:20:16.754 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:20:16.754 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:20:16.754 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:20:16.756 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:20:16.756 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:20:16.756 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:20:16.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:20:16.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:20:16.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:20:16.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:20:16.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:20:16.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:20:16.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:20:16.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:20:16.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:20:16.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:20:16.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:20:16.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:20:16.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:20:16.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:20:16.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:20:16.758 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:20:16.758 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:20:16.758 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:20:16.758 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:20:16.758 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:20:16.758 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:20:16.758 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:20:16.758 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:20:16.758 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:20:16.758 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:20:16.758 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:20:16.758 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:20:16.758 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:20:16.758 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:20:16.766 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:20:16.767 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:20:16.767 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:20:16.767 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:20:16.836 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:20:16.837 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:20:16.837 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:20:16.837 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:20:16.837 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:20:16.837 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:20:16.837 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:20:16.837 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:20:16.837 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:20:16.837 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:20:16.838 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:20:16.838 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:20:16.838 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:20:16.839 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:20:16.840 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:20:16.840 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:20:16.869 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 76 2026-02-18T08:20:16.869 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 77 2026-02-18T08:20:16.869 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:20:16.869 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=309, tag_tid=21, entry_tid=1], [object_number=308, tag_tid=21, entry_tid=0], [object_number=310, tag_tid=20, entry_tid=46754], [object_number=311, tag_tid=20, entry_tid=46751]]], state=connected] 2026-02-18T08:20:16.869 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=307, tag_tid=20, entry_tid=44983], [object_number=306, tag_tid=20, entry_tid=44982], [object_number=305, tag_tid=20, entry_tid=44981], [object_number=304, tag_tid=20, entry_tid=44980]]], state=connected] 2026-02-18T08:20:16.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:20:16.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=309, tag_tid=21, entry_tid=1' 2026-02-18T08:20:16.875 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:20:16.875 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:20:16.875 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:20:16.875 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:20:16.875 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:20:16.876 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:20:16.876 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:20:16.876 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:20:16.876 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:20:16.876 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:20:16.876 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:20:16.877 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:20:16.878 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:20:16.878 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:20:16.878 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:20:16.912 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 76 2026-02-18T08:20:16.912 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 77 2026-02-18T08:20:16.912 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:20:16.912 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=309, tag_tid=21, entry_tid=1], [object_number=308, tag_tid=21, entry_tid=0], [object_number=310, tag_tid=20, entry_tid=46754], [object_number=311, tag_tid=20, entry_tid=46751]]], state=connected] 2026-02-18T08:20:16.912 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=307, tag_tid=20, entry_tid=44983], [object_number=306, tag_tid=20, entry_tid=44982], [object_number=305, tag_tid=20, entry_tid=44981], [object_number=304, tag_tid=20, entry_tid=44980]]], state=connected] 2026-02-18T08:20:16.916 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:20:16.917 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=307, tag_tid=20, entry_tid=44983' 2026-02-18T08:20:16.917 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=309, tag_tid=21, entry_tid=1' -a 'object_number=309, tag_tid=21, entry_tid=1' = 'object_number=307, tag_tid=20, entry_tid=44983' 2026-02-18T08:20:16.917 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=307, tag_tid=20, entry_tid=44983' '!=' 'object_number=307, tag_tid=20, entry_tid=44503' 2026-02-18T08:20:16.917 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:20:16.917 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=307, tag_tid=20, entry_tid=44983' = 'object_number=307, tag_tid=20, entry_tid=44503' 2026-02-18T08:20:16.917 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=307, tag_tid=20, entry_tid=44983' 2026-02-18T08:20:16.917 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=309, tag_tid=21, entry_tid=1' 2026-02-18T08:20:16.918 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:20:16.918 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:20:16.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=21 2026-02-18T08:20:16.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=307, tag_tid=20, entry_tid=44983' 2026-02-18T08:20:16.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:20:16.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:20:16.922 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=20 2026-02-18T08:20:16.922 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=309, tag_tid=21, entry_tid=1' 2026-02-18T08:20:16.922 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:20:16.922 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:20:16.924 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:20:16.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=307, tag_tid=20, entry_tid=44983' 2026-02-18T08:20:16.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:20:16.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:20:16.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=44983 2026-02-18T08:20:16.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 21 = 20 -a 1 -le 44983 2026-02-18T08:20:16.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:20:16.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:20:16.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:20:17.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:20:17.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:20:17.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:20:17.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:20:17.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:20:17.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:20:17.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:20:17.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:20:17.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:20:17.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:20:17.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:20:17.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:20:17.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:20:17.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:20:17.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:20:17.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:20:17.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:20:17.129 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:20:17.129 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:20:17.129 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:20:17.129 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:20:17.129 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:20:17.129 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:20:17.129 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:20:17.129 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:20:17.130 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:20:17.130 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:20:17.130 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:20:17.130 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:20:17.130 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:20:17.130 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:20:17.130 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:20:17.130 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:20:17.130 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:20:17.130 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:20:17.130 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:20:17.130 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:20:17.130 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:20:17.130 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:20:17.130 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:20:17.130 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:20:17.130 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:20:17.130 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:20:17.130 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:20:17.130 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:20:17.130 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:20:17.130 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:20:17.131 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:20:17.131 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:20:17.142 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:20:17.142 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:20:17.142 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:20:17.142 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:20:17.200 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:20:17.201 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:20:17.201 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:20:17.202 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:20:17.202 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:20:17.202 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:20:17.202 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:20:17.202 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:20:17.202 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:20:17.202 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:20:17.202 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:20:17.202 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:20:17.202 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:20:17.202 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:20:17.202 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:20:17.202 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:20:17.202 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:20:17.202 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:20:17.202 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:20:17.203 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:20:17.203 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:20:17.203 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:20:17.203 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:20:17.203 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:20:17.203 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:20:17.203 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:20:17.203 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:20:17.203 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:20:17.203 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:20:17.203 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:20:17.203 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:20:17.203 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:20:17.203 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:20:17.203 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:20:17.203 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:20:17.214 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:20:17.215 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:20:17.215 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:20:17.215 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:20:17.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:20:17.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:20:17.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:20:17.275 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:20:17.276 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:20:17.276 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:20:17.276 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:20:17.276 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:20:17.276 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:20:17.276 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:20:17.276 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:20:17.276 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:20:17.276 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:20:17.277 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:20:17.277 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:20:17.277 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:20:17.277 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:20:17.277 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:20:17.277 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:20:17.277 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:20:17.277 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:20:17.277 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:20:17.278 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:20:17.278 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:20:17.278 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:20:17.278 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:20:17.278 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:20:17.278 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:20:17.278 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:20:17.278 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:20:17.278 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:20:17.278 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:20:17.278 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:20:17.278 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:20:17.279 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:20:17.285 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:20:17.286 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:20:17.286 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:20:17.286 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:20:17.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:20:17.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:20:17.344 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:20:17.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:20:17.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:20:17.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:20:17.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:20:17.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:20:17.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:20:17.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:20:17.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:20:17.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:20:17.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:20:17.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:20:17.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:20:17.348 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:20:17.348 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:20:17.348 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:20:17.348 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:20:17.348 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:20:17.348 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:20:17.348 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:20:17.348 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:20:17.348 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:20:17.348 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:20:17.348 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:20:17.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:20:17.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:20:17.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:20:17.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:20:17.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:20:17.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:20:17.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:20:17.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:20:17.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:20:17.357 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:20:17.357 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:20:17.357 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:20:17.357 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:20:17.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:20:17.428 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:20:17.428 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:20:17.428 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:20:17.428 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:20:17.428 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:20:17.428 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:20:17.428 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:20:17.429 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:20:17.429 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:20:17.429 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:20:17.429 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:20:17.429 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:20:17.430 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:20:17.430 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:20:17.430 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:20:17.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 76 2026-02-18T08:20:17.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 77 2026-02-18T08:20:17.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:20:17.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=309, tag_tid=21, entry_tid=1], [object_number=308, tag_tid=21, entry_tid=0], [object_number=310, tag_tid=20, entry_tid=46754], [object_number=311, tag_tid=20, entry_tid=46751]]], state=connected] 2026-02-18T08:20:17.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=306, tag_tid=20, entry_tid=45374], [object_number=305, tag_tid=20, entry_tid=45373], [object_number=304, tag_tid=20, entry_tid=45372], [object_number=307, tag_tid=20, entry_tid=45371]]], state=connected] 2026-02-18T08:20:17.462 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:20:17.463 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=309, tag_tid=21, entry_tid=1' 2026-02-18T08:20:17.465 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:20:17.465 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:20:17.465 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:20:17.465 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:20:17.465 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:20:17.465 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:20:17.465 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:20:17.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:20:17.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:20:17.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:20:17.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:20:17.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:20:17.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:20:17.467 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:20:17.467 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:20:17.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 76 2026-02-18T08:20:17.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 77 2026-02-18T08:20:17.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:20:17.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=309, tag_tid=21, entry_tid=1], [object_number=308, tag_tid=21, entry_tid=0], [object_number=310, tag_tid=20, entry_tid=46754], [object_number=311, tag_tid=20, entry_tid=46751]]], state=connected] 2026-02-18T08:20:17.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=306, tag_tid=20, entry_tid=45374], [object_number=305, tag_tid=20, entry_tid=45373], [object_number=304, tag_tid=20, entry_tid=45372], [object_number=307, tag_tid=20, entry_tid=45371]]], state=connected] 2026-02-18T08:20:17.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:20:17.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=306, tag_tid=20, entry_tid=45374' 2026-02-18T08:20:17.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=309, tag_tid=21, entry_tid=1' -a 'object_number=309, tag_tid=21, entry_tid=1' = 'object_number=306, tag_tid=20, entry_tid=45374' 2026-02-18T08:20:17.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=306, tag_tid=20, entry_tid=45374' '!=' 'object_number=307, tag_tid=20, entry_tid=44983' 2026-02-18T08:20:17.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:20:17.500 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=306, tag_tid=20, entry_tid=45374' = 'object_number=307, tag_tid=20, entry_tid=44983' 2026-02-18T08:20:17.500 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=306, tag_tid=20, entry_tid=45374' 2026-02-18T08:20:17.500 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=309, tag_tid=21, entry_tid=1' 2026-02-18T08:20:17.500 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:20:17.501 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:20:17.501 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=21 2026-02-18T08:20:17.502 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=306, tag_tid=20, entry_tid=45374' 2026-02-18T08:20:17.502 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:20:17.502 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:20:17.504 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=20 2026-02-18T08:20:17.505 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=309, tag_tid=21, entry_tid=1' 2026-02-18T08:20:17.505 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:20:17.505 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:20:17.505 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:20:17.506 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=306, tag_tid=20, entry_tid=45374' 2026-02-18T08:20:17.506 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:20:17.506 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:20:17.508 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=45374 2026-02-18T08:20:17.508 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 21 = 20 -a 1 -le 45374 2026-02-18T08:20:17.508 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:20:17.508 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:20:17.508 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:20:17.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:20:17.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:20:17.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:20:17.711 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:20:17.711 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:20:17.711 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:20:17.711 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:20:17.711 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:20:17.711 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:20:17.711 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:20:17.711 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:20:17.711 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:20:17.712 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:20:17.712 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:20:17.712 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:20:17.712 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:20:17.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:20:17.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:20:17.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:20:17.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:20:17.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:20:17.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:20:17.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:20:17.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:20:17.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:20:17.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:20:17.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:20:17.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:20:17.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:20:17.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:20:17.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:20:17.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:20:17.717 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:20:17.717 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:20:17.717 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:20:17.717 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:20:17.717 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:20:17.717 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:20:17.717 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:20:17.717 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:20:17.717 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:20:17.717 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:20:17.717 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:20:17.718 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:20:17.718 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:20:17.718 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:20:17.718 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:20:17.718 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:20:17.718 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:20:17.725 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:20:17.725 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:20:17.725 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:20:17.725 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:20:17.777 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:20:17.777 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:20:17.777 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:20:17.780 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:20:17.780 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:20:17.780 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:20:17.781 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:20:17.781 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:20:17.781 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:20:17.781 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:20:17.781 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:20:17.781 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:20:17.781 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:20:17.781 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:20:17.781 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:20:17.781 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:20:17.781 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:20:17.782 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:20:17.782 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:20:17.782 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:20:17.782 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:20:17.782 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:20:17.782 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:20:17.782 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:20:17.782 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:20:17.782 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:20:17.782 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:20:17.783 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:20:17.783 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:20:17.783 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:20:17.783 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:20:17.783 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:20:17.783 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:20:17.783 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:20:17.783 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:20:17.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:20:17.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:20:17.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:20:17.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:20:17.848 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:20:17.848 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:20:17.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:20:17.852 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:20:17.852 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:20:17.852 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:20:17.853 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:20:17.853 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:20:17.853 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:20:17.853 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:20:17.853 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:20:17.853 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:20:17.853 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:20:17.853 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:20:17.853 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:20:17.853 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:20:17.853 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:20:17.854 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:20:17.854 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:20:17.854 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:20:17.854 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:20:17.854 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:20:17.854 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:20:17.854 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:20:17.854 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:20:17.854 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:20:17.854 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:20:17.854 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:20:17.855 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:20:17.855 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:20:17.855 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:20:17.855 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:20:17.855 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:20:17.855 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:20:17.855 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:20:17.863 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:20:17.864 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:20:17.864 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:20:17.865 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:20:17.913 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:20:17.914 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:20:17.914 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:20:17.917 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:20:17.917 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:20:17.917 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:20:17.917 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:20:17.917 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:20:17.917 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:20:17.918 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:20:17.918 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:20:17.918 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:20:17.918 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:20:17.918 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:20:17.918 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:20:17.918 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:20:17.918 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:20:17.918 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:20:17.918 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:20:17.918 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:20:17.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:20:17.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:20:17.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:20:17.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:20:17.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:20:17.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:20:17.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:20:17.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:20:17.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:20:17.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:20:17.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:20:17.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:20:17.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:20:17.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:20:17.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:20:17.929 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:20:17.929 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:20:17.930 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:20:17.930 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:20:17.966 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.0 with skip (0.9481653686596693) > weight (0.25) 2026-02-18T08:20:17.966 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.1 with skip (0.5691971492372866) > weight (0.25) 2026-02-18T08:20:17.966 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.2 with skip (0.36519423475618595) > weight (0.25) 2026-02-18T08:20:17.966 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.3 with skip (0.890317713133866) > weight (0.25) 2026-02-18T08:20:17.966 INFO:tasks.rbd_mirror_thrash:waiting for 69.0408500883764 secs before thrashing 2026-02-18T08:20:17.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:20:17.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:20:17.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:20:17.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:20:17.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:20:17.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:20:17.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:20:17.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:20:17.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:20:17.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:20:17.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:20:17.999 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:20:17.999 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:20:18.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:20:18.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:20:18.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:20:18.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 76 2026-02-18T08:20:18.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 77 2026-02-18T08:20:18.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:20:18.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=309, tag_tid=21, entry_tid=1], [object_number=308, tag_tid=21, entry_tid=0], [object_number=310, tag_tid=20, entry_tid=46754], [object_number=311, tag_tid=20, entry_tid=46751]]], state=connected] 2026-02-18T08:20:18.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=306, tag_tid=20, entry_tid=45722], [object_number=305, tag_tid=20, entry_tid=45721], [object_number=304, tag_tid=20, entry_tid=45720], [object_number=307, tag_tid=20, entry_tid=45719]]], state=connected] 2026-02-18T08:20:18.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:20:18.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=309, tag_tid=21, entry_tid=1' 2026-02-18T08:20:18.036 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:20:18.036 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:20:18.037 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:20:18.037 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:20:18.037 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:20:18.037 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:20:18.037 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:20:18.037 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:20:18.037 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:20:18.037 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:20:18.037 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:20:18.037 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:20:18.038 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:20:18.039 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:20:18.039 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:20:18.275 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 76 2026-02-18T08:20:18.276 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 77 2026-02-18T08:20:18.276 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:20:18.276 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=309, tag_tid=21, entry_tid=1], [object_number=308, tag_tid=21, entry_tid=0], [object_number=310, tag_tid=20, entry_tid=46754], [object_number=311, tag_tid=20, entry_tid=46751]]], state=connected] 2026-02-18T08:20:18.276 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=306, tag_tid=20, entry_tid=45722], [object_number=305, tag_tid=20, entry_tid=45721], [object_number=304, tag_tid=20, entry_tid=45720], [object_number=307, tag_tid=20, entry_tid=45719]]], state=connected] 2026-02-18T08:20:18.279 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:20:18.281 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=306, tag_tid=20, entry_tid=45722' 2026-02-18T08:20:18.281 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=309, tag_tid=21, entry_tid=1' -a 'object_number=309, tag_tid=21, entry_tid=1' = 'object_number=306, tag_tid=20, entry_tid=45722' 2026-02-18T08:20:18.281 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=306, tag_tid=20, entry_tid=45722' '!=' 'object_number=306, tag_tid=20, entry_tid=45374' 2026-02-18T08:20:18.281 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:20:18.281 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=306, tag_tid=20, entry_tid=45722' = 'object_number=306, tag_tid=20, entry_tid=45374' 2026-02-18T08:20:18.281 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=306, tag_tid=20, entry_tid=45722' 2026-02-18T08:20:18.282 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=309, tag_tid=21, entry_tid=1' 2026-02-18T08:20:18.282 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:20:18.282 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:20:18.283 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=21 2026-02-18T08:20:18.284 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=306, tag_tid=20, entry_tid=45722' 2026-02-18T08:20:18.284 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:20:18.284 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:20:18.285 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=20 2026-02-18T08:20:18.286 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=309, tag_tid=21, entry_tid=1' 2026-02-18T08:20:18.286 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:20:18.286 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:20:18.287 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:20:18.288 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=306, tag_tid=20, entry_tid=45722' 2026-02-18T08:20:18.288 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:20:18.288 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:20:18.290 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=45722 2026-02-18T08:20:18.290 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 21 = 20 -a 1 -le 45722 2026-02-18T08:20:18.290 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:20:18.290 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:20:18.290 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:20:18.492 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:20:18.492 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:20:18.492 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:20:18.492 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:20:18.493 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:20:18.493 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:20:18.493 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:20:18.493 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:20:18.493 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:20:18.493 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:20:18.493 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:20:18.493 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:20:18.493 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:20:18.493 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:20:18.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:20:18.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:20:18.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:20:18.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:20:18.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:20:18.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:20:18.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:20:18.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:20:18.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:20:18.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:20:18.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:20:18.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:20:18.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:20:18.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:20:18.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:20:18.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:20:18.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:20:18.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:20:18.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:20:18.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:20:18.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:20:18.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:20:18.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:20:18.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:20:18.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:20:18.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:20:18.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:20:18.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:20:18.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:20:18.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:20:18.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:20:18.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:20:18.500 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:20:18.500 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:20:18.500 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:20:18.506 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:20:18.507 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:20:18.507 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:20:18.507 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:20:18.557 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:20:18.558 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:20:18.558 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:20:18.561 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:20:18.561 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:20:18.561 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:20:18.561 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:20:18.561 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:20:18.561 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:20:18.561 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:20:18.561 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:20:18.561 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:20:18.562 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:20:18.562 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:20:18.562 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:20:18.562 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:20:18.562 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:20:18.562 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:20:18.562 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:20:18.562 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:20:18.562 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:20:18.562 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:20:18.562 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:20:18.562 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:20:18.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:20:18.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:20:18.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:20:18.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:20:18.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:20:18.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:20:18.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:20:18.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:20:18.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:20:18.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:20:18.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:20:18.571 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:20:18.572 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:20:18.572 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:20:18.572 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:20:18.629 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:20:18.629 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:20:18.629 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:20:18.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:20:18.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:20:18.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:20:18.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:20:18.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:20:18.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:20:18.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:20:18.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:20:18.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:20:18.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:20:18.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:20:18.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:20:18.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:20:18.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:20:18.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:20:18.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:20:18.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:20:18.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:20:18.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:20:18.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:20:18.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:20:18.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:20:18.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:20:18.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:20:18.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:20:18.635 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:20:18.635 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:20:18.635 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:20:18.635 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:20:18.635 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:20:18.635 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:20:18.635 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:20:18.642 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:20:18.643 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:20:18.643 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:20:18.643 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:20:18.691 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:20:18.692 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:20:18.692 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:20:18.694 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:20:18.694 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:20:18.694 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:20:18.695 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:20:18.695 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:20:18.695 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:20:18.695 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:20:18.695 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:20:18.695 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:20:18.695 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:20:18.695 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:20:18.695 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:20:18.695 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:20:18.695 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:20:18.695 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:20:18.696 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:20:18.696 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:20:18.696 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:20:18.696 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:20:18.696 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:20:18.696 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:20:18.696 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:20:18.696 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:20:18.696 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:20:18.696 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:20:18.696 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:20:18.696 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:20:18.697 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:20:18.697 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:20:18.697 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:20:18.697 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:20:18.697 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:20:18.705 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:20:18.706 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:20:18.706 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:20:18.706 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:20:18.774 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:20:18.774 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:20:18.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:20:18.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:20:18.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:20:18.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:20:18.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:20:18.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:20:18.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:20:18.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:20:18.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:20:18.776 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:20:18.776 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:20:18.777 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:20:18.777 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:20:18.777 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:20:19.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 76 2026-02-18T08:20:19.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 77 2026-02-18T08:20:19.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:20:19.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=309, tag_tid=21, entry_tid=1], [object_number=308, tag_tid=21, entry_tid=0], [object_number=310, tag_tid=20, entry_tid=46754], [object_number=311, tag_tid=20, entry_tid=46751]]], state=connected] 2026-02-18T08:20:19.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=306, tag_tid=20, entry_tid=46230], [object_number=305, tag_tid=20, entry_tid=46229], [object_number=304, tag_tid=20, entry_tid=46228], [object_number=307, tag_tid=20, entry_tid=46227]]], state=connected] 2026-02-18T08:20:19.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:20:19.019 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=309, tag_tid=21, entry_tid=1' 2026-02-18T08:20:19.020 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:20:19.020 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:20:19.020 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:20:19.020 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:20:19.020 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:20:19.021 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:20:19.021 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:20:19.021 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:20:19.021 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:20:19.021 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:20:19.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:20:19.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:20:19.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:20:19.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:20:19.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:20:19.059 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 76 2026-02-18T08:20:19.059 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 77 2026-02-18T08:20:19.059 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:20:19.059 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=309, tag_tid=21, entry_tid=1], [object_number=308, tag_tid=21, entry_tid=0], [object_number=310, tag_tid=20, entry_tid=46754], [object_number=311, tag_tid=20, entry_tid=46751]]], state=connected] 2026-02-18T08:20:19.059 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=306, tag_tid=20, entry_tid=46230], [object_number=305, tag_tid=20, entry_tid=46229], [object_number=304, tag_tid=20, entry_tid=46228], [object_number=307, tag_tid=20, entry_tid=46227]]], state=connected] 2026-02-18T08:20:19.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:20:19.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=306, tag_tid=20, entry_tid=46230' 2026-02-18T08:20:19.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=309, tag_tid=21, entry_tid=1' -a 'object_number=309, tag_tid=21, entry_tid=1' = 'object_number=306, tag_tid=20, entry_tid=46230' 2026-02-18T08:20:19.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=306, tag_tid=20, entry_tid=46230' '!=' 'object_number=306, tag_tid=20, entry_tid=45722' 2026-02-18T08:20:19.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:20:19.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=306, tag_tid=20, entry_tid=46230' = 'object_number=306, tag_tid=20, entry_tid=45722' 2026-02-18T08:20:19.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=306, tag_tid=20, entry_tid=46230' 2026-02-18T08:20:19.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=309, tag_tid=21, entry_tid=1' 2026-02-18T08:20:19.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:20:19.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:20:19.067 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=21 2026-02-18T08:20:19.067 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=306, tag_tid=20, entry_tid=46230' 2026-02-18T08:20:19.067 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:20:19.068 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:20:19.069 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=20 2026-02-18T08:20:19.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=309, tag_tid=21, entry_tid=1' 2026-02-18T08:20:19.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:20:19.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:20:19.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:20:19.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=306, tag_tid=20, entry_tid=46230' 2026-02-18T08:20:19.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:20:19.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:20:19.074 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=46230 2026-02-18T08:20:19.074 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 21 = 20 -a 1 -le 46230 2026-02-18T08:20:19.074 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:20:19.074 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:20:19.074 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:20:19.275 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:20:19.276 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:20:19.276 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:20:19.276 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:20:19.276 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:20:19.276 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:20:19.276 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:20:19.276 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:20:19.276 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:20:19.276 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:20:19.276 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:20:19.276 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:20:19.276 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:20:19.276 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:20:19.276 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:20:19.277 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:20:19.277 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:20:19.278 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:20:19.278 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:20:19.278 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:20:19.278 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:20:19.278 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:20:19.278 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:20:19.278 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:20:19.278 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:20:19.278 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:20:19.278 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:20:19.278 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:20:19.278 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:20:19.278 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:20:19.278 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:20:19.279 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:20:19.279 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:20:19.279 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:20:19.279 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:20:19.279 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:20:19.279 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:20:19.279 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:20:19.279 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:20:19.279 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:20:19.279 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:20:19.279 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:20:19.279 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:20:19.279 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:20:19.279 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:20:19.279 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:20:19.279 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:20:19.279 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:20:19.279 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:20:19.290 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:20:19.290 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:20:19.290 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:20:19.290 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:20:19.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:20:19.350 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:20:19.350 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:20:19.352 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:20:19.352 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:20:19.352 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:20:19.352 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:20:19.352 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:20:19.352 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:20:19.352 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:20:19.352 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:20:19.352 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:20:19.352 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:20:19.352 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:20:19.352 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:20:19.352 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:20:19.352 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:20:19.352 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:20:19.352 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:20:19.352 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:20:19.352 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:20:19.352 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:20:19.352 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:20:19.353 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:20:19.353 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:20:19.353 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:20:19.353 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:20:19.353 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:20:19.353 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:20:19.353 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:20:19.353 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:20:19.353 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:20:19.353 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:20:19.353 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:20:19.353 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:20:19.362 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:20:19.363 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:20:19.363 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:20:19.363 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:20:19.413 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:20:19.413 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:20:19.413 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:20:19.414 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:20:19.414 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:20:19.414 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:20:19.415 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:20:19.415 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:20:19.415 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:20:19.415 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:20:19.415 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:20:19.415 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:20:19.415 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:20:19.415 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:20:19.415 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:20:19.415 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:20:19.415 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:20:19.415 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:20:19.415 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:20:19.415 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:20:19.415 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:20:19.415 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:20:19.415 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:20:19.415 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:20:19.415 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:20:19.415 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:20:19.415 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:20:19.415 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:20:19.415 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:20:19.415 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:20:19.415 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:20:19.415 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:20:19.415 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:20:19.415 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:20:19.415 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:20:19.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:20:19.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:20:19.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:20:19.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:20:19.485 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:20:19.485 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:20:19.485 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:20:19.487 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:20:19.487 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:20:19.487 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:20:19.487 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:20:19.487 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:20:19.487 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:20:19.487 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:20:19.487 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:20:19.487 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:20:19.487 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:20:19.487 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:20:19.487 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:20:19.487 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:20:19.487 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:20:19.487 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:20:19.487 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:20:19.487 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:20:19.487 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:20:19.488 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:20:19.488 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:20:19.488 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:20:19.488 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:20:19.488 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:20:19.488 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:20:19.488 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:20:19.488 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:20:19.488 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:20:19.488 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:20:19.488 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:20:19.488 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:20:19.488 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:20:19.488 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:20:19.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:20:19.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:20:19.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:20:19.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:20:19.561 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:20:19.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:20:19.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:20:19.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:20:19.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:20:19.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:20:19.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:20:19.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:20:19.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:20:19.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:20:19.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:20:19.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:20:19.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:20:19.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:20:19.565 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:20:19.565 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:20:19.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 77 2026-02-18T08:20:19.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 77 2026-02-18T08:20:19.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:20:19.594 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=309, tag_tid=21, entry_tid=1], [object_number=308, tag_tid=21, entry_tid=0], [object_number=310, tag_tid=20, entry_tid=46754], [object_number=311, tag_tid=20, entry_tid=46751]]], state=connected] 2026-02-18T08:20:19.594 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=309, tag_tid=20, entry_tid=46729], [object_number=308, tag_tid=20, entry_tid=46728], [object_number=311, tag_tid=20, entry_tid=46727], [object_number=310, tag_tid=20, entry_tid=46726]]], state=connected] 2026-02-18T08:20:19.596 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:20:19.598 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=309, tag_tid=21, entry_tid=1' 2026-02-18T08:20:19.599 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:20:19.599 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:20:19.599 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:20:19.599 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:20:19.599 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:20:19.599 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:20:19.599 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:20:19.599 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:20:19.600 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:20:19.600 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:20:19.600 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:20:19.600 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:20:19.601 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:20:19.601 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:20:19.601 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:20:19.628 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 77 2026-02-18T08:20:19.628 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 77 2026-02-18T08:20:19.628 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:20:19.628 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=309, tag_tid=21, entry_tid=1], [object_number=308, tag_tid=21, entry_tid=0], [object_number=310, tag_tid=20, entry_tid=46754], [object_number=311, tag_tid=20, entry_tid=46751]]], state=connected] 2026-02-18T08:20:19.629 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=309, tag_tid=20, entry_tid=46729], [object_number=308, tag_tid=20, entry_tid=46728], [object_number=311, tag_tid=20, entry_tid=46727], [object_number=310, tag_tid=20, entry_tid=46726]]], state=connected] 2026-02-18T08:20:19.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:20:19.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=309, tag_tid=20, entry_tid=46729' 2026-02-18T08:20:19.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=309, tag_tid=21, entry_tid=1' -a 'object_number=309, tag_tid=21, entry_tid=1' = 'object_number=309, tag_tid=20, entry_tid=46729' 2026-02-18T08:20:19.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=309, tag_tid=20, entry_tid=46729' '!=' 'object_number=306, tag_tid=20, entry_tid=46230' 2026-02-18T08:20:19.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:20:19.635 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=309, tag_tid=20, entry_tid=46729' = 'object_number=306, tag_tid=20, entry_tid=46230' 2026-02-18T08:20:19.635 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=309, tag_tid=20, entry_tid=46729' 2026-02-18T08:20:19.635 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=309, tag_tid=21, entry_tid=1' 2026-02-18T08:20:19.636 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:20:19.636 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:20:19.636 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=21 2026-02-18T08:20:19.637 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=309, tag_tid=20, entry_tid=46729' 2026-02-18T08:20:19.637 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:20:19.637 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:20:19.639 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=20 2026-02-18T08:20:19.639 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=309, tag_tid=21, entry_tid=1' 2026-02-18T08:20:19.640 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:20:19.640 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:20:19.641 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:20:19.642 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=309, tag_tid=20, entry_tid=46729' 2026-02-18T08:20:19.642 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:20:19.642 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:20:19.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=46729 2026-02-18T08:20:19.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 21 = 20 -a 1 -le 46729 2026-02-18T08:20:19.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:20:19.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:20:19.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:20:19.846 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:20:19.846 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:20:19.846 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:20:19.846 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:20:19.847 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:20:19.847 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:20:19.847 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:20:19.847 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:20:19.847 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:20:19.847 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:20:19.847 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:20:19.847 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:20:19.847 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:20:19.847 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:20:19.847 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:20:19.848 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:20:19.848 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:20:19.851 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:20:19.851 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:20:19.851 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:20:19.851 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:20:19.851 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:20:19.851 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:20:19.851 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:20:19.851 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:20:19.851 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:20:19.851 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:20:19.852 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:20:19.852 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:20:19.852 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:20:19.852 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:20:19.852 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:20:19.852 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:20:19.852 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:20:19.852 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:20:19.852 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:20:19.852 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:20:19.853 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:20:19.853 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:20:19.853 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:20:19.853 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:20:19.853 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:20:19.853 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:20:19.853 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:20:19.853 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:20:19.853 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:20:19.853 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:20:19.854 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:20:19.854 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:20:19.859 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:20:19.859 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:20:19.860 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:20:19.860 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:20:19.909 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:20:19.909 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:20:19.909 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:20:19.912 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:20:19.912 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:20:19.912 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:20:19.912 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:20:19.912 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:20:19.912 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:20:19.912 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:20:19.912 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:20:19.912 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:20:19.913 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:20:19.913 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:20:19.913 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:20:19.913 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:20:19.913 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:20:19.913 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:20:19.913 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:20:19.913 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:20:19.913 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:20:19.913 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:20:19.914 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:20:19.914 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:20:19.914 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:20:19.914 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:20:19.914 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:20:19.914 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:20:19.914 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:20:19.914 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:20:19.914 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:20:19.914 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:20:19.914 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:20:19.915 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:20:19.915 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:20:19.922 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:20:19.923 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:20:19.923 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:20:19.923 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:20:19.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:20:19.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:20:19.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:20:19.982 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:20:19.982 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:20:19.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:20:19.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:20:19.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:20:19.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:20:19.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:20:19.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:20:19.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:20:19.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:20:19.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:20:19.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:20:19.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:20:19.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:20:19.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:20:19.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:20:19.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:20:19.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:20:19.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:20:19.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:20:19.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:20:19.985 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:20:19.985 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:20:19.985 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:20:19.985 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:20:19.985 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:20:19.985 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:20:19.985 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:20:19.985 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:20:19.985 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:20:19.985 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:20:19.985 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:20:19.993 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:20:19.993 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:20:19.993 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:20:19.993 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:20:20.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:20:20.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:20:20.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:20:20.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:20:20.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:20:20.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:20:20.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:20:20.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:20:20.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:20:20.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:20:20.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:20:20.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:20:20.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:20:20.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:20:20.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:20:20.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:20:20.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:20:20.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:20:20.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:20:20.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:20:20.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:20:20.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:20:20.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:20:20.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:20:20.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:20:20.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:20:20.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:20:20.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:20:20.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:20:20.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:20:20.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:20:20.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:20:20.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:20:20.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:20:20.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:20:20.062 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:20:20.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:20:20.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:20:20.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:20:20.131 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:20:20.132 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:20:20.132 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:20:20.132 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:20:20.132 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:20:20.132 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:20:20.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:20:20.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:20:20.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:20:20.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:20:20.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:20:20.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:20:20.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:20:20.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:20:20.134 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:20:20.134 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:20:20.170 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 77 2026-02-18T08:20:20.170 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 77 2026-02-18T08:20:20.171 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:20:20.171 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=309, tag_tid=21, entry_tid=1], [object_number=308, tag_tid=21, entry_tid=0], [object_number=310, tag_tid=20, entry_tid=46754], [object_number=311, tag_tid=20, entry_tid=46751]]], state=connected] 2026-02-18T08:20:20.171 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=310, tag_tid=20, entry_tid=46754], [object_number=309, tag_tid=20, entry_tid=46753], [object_number=308, tag_tid=20, entry_tid=46752], [object_number=311, tag_tid=20, entry_tid=46751]]], state=connected] 2026-02-18T08:20:20.173 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:20:20.175 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=309, tag_tid=21, entry_tid=1' 2026-02-18T08:20:20.176 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:20:20.176 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:20:20.176 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:20:20.177 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:20:20.177 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:20:20.177 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:20:20.177 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:20:20.177 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:20:20.177 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:20:20.177 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:20:20.178 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:20:20.178 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:20:20.178 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:20:20.179 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:20:20.179 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:20:20.215 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 77 2026-02-18T08:20:20.215 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 77 2026-02-18T08:20:20.215 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:20:20.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=309, tag_tid=21, entry_tid=1], [object_number=308, tag_tid=21, entry_tid=0], [object_number=310, tag_tid=20, entry_tid=46754], [object_number=311, tag_tid=20, entry_tid=46751]]], state=connected] 2026-02-18T08:20:20.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=310, tag_tid=20, entry_tid=46754], [object_number=309, tag_tid=20, entry_tid=46753], [object_number=308, tag_tid=20, entry_tid=46752], [object_number=311, tag_tid=20, entry_tid=46751]]], state=connected] 2026-02-18T08:20:20.219 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:20:20.220 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=310, tag_tid=20, entry_tid=46754' 2026-02-18T08:20:20.221 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=309, tag_tid=21, entry_tid=1' -a 'object_number=309, tag_tid=21, entry_tid=1' = 'object_number=310, tag_tid=20, entry_tid=46754' 2026-02-18T08:20:20.221 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=310, tag_tid=20, entry_tid=46754' '!=' 'object_number=309, tag_tid=20, entry_tid=46729' 2026-02-18T08:20:20.221 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:20:20.221 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=310, tag_tid=20, entry_tid=46754' = 'object_number=309, tag_tid=20, entry_tid=46729' 2026-02-18T08:20:20.221 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=310, tag_tid=20, entry_tid=46754' 2026-02-18T08:20:20.222 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=309, tag_tid=21, entry_tid=1' 2026-02-18T08:20:20.222 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:20:20.222 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:20:20.223 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=21 2026-02-18T08:20:20.224 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=310, tag_tid=20, entry_tid=46754' 2026-02-18T08:20:20.224 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:20:20.224 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:20:20.225 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=20 2026-02-18T08:20:20.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=309, tag_tid=21, entry_tid=1' 2026-02-18T08:20:20.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:20:20.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:20:20.227 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:20:20.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=310, tag_tid=20, entry_tid=46754' 2026-02-18T08:20:20.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:20:20.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:20:20.229 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=46754 2026-02-18T08:20:20.229 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 21 = 20 -a 1 -le 46754 2026-02-18T08:20:20.230 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:20:20.230 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:20:20.230 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:20:20.430 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:20:20.430 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:20:20.430 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:20:20.430 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:20:20.430 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:20:20.430 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:20:20.430 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:20:20.430 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:20:20.430 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:20:20.431 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:20:20.431 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:20:20.431 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:20:20.431 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:20:20.431 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:20:20.431 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:20:20.431 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:20:20.432 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:20:20.433 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:20:20.433 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:20:20.433 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:20:20.433 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:20:20.433 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:20:20.433 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:20:20.433 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:20:20.433 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:20:20.433 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:20:20.433 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:20:20.433 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:20:20.433 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:20:20.433 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:20:20.433 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:20:20.433 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:20:20.433 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:20:20.434 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:20:20.434 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:20:20.434 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:20:20.434 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:20:20.434 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:20:20.434 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:20:20.434 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:20:20.434 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:20:20.434 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:20:20.434 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:20:20.434 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:20:20.434 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:20:20.434 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:20:20.434 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:20:20.434 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:20:20.434 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:20:20.446 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:20:20.446 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:20:20.446 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:20:20.446 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:20:20.505 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:20:20.505 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:20:20.505 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:20:20.508 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:20:20.508 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:20:20.508 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:20:20.508 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:20:20.508 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:20:20.508 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:20:20.508 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:20:20.509 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:20:20.509 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:20:20.509 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:20:20.509 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:20:20.509 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:20:20.509 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:20:20.509 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:20:20.509 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:20:20.509 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:20:20.509 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:20:20.509 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:20:20.509 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:20:20.510 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:20:20.510 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:20:20.510 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:20:20.510 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:20:20.510 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:20:20.510 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:20:20.510 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:20:20.510 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:20:20.510 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:20:20.510 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:20:20.510 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:20:20.510 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:20:20.511 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:20:20.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:20:20.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:20:20.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:20:20.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:20:20.576 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:20:20.576 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:20:20.576 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:20:20.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:20:20.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:20:20.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:20:20.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:20:20.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:20:20.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:20:20.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:20:20.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:20:20.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:20:20.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:20:20.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:20:20.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:20:20.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:20:20.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:20:20.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:20:20.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:20:20.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:20:20.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:20:20.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:20:20.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:20:20.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:20:20.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:20:20.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:20:20.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:20:20.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:20:20.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:20:20.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:20:20.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:20:20.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:20:20.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:20:20.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:20:20.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:20:20.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:20:20.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:20:20.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:20:20.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:20:20.647 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:20:20.647 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:20:20.647 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:20:20.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:20:20.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:20:20.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:20:20.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:20:20.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:20:20.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:20:20.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:20:20.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:20:20.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:20:20.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:20:20.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:20:20.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:20:20.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:20:20.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:20:20.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:20:20.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:20:20.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:20:20.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:20:20.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:20:20.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:20:20.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:20:20.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:20:20.653 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:20:20.653 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:20:20.653 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:20:20.653 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:20:20.653 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:20:20.653 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:20:20.653 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:20:20.653 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:20:20.653 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:20:20.653 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:20:20.661 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:20:20.661 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:20:20.662 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:20:20.662 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:20:20.725 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:20:20.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:20:20.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:20:20.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:20:20.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:20:20.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:20:20.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:20:20.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:20:20.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:20:20.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:20:20.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:20:20.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:20:20.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:20:20.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:20:20.728 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:20:20.728 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:20:20.766 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 77 2026-02-18T08:20:20.766 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 77 2026-02-18T08:20:20.766 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:20:20.766 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=309, tag_tid=21, entry_tid=1], [object_number=308, tag_tid=21, entry_tid=0], [object_number=310, tag_tid=20, entry_tid=46754], [object_number=311, tag_tid=20, entry_tid=46751]]], state=connected] 2026-02-18T08:20:20.766 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=310, tag_tid=20, entry_tid=46754], [object_number=309, tag_tid=20, entry_tid=46753], [object_number=308, tag_tid=20, entry_tid=46752], [object_number=311, tag_tid=20, entry_tid=46751]]], state=connected] 2026-02-18T08:20:20.769 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:20:20.771 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=309, tag_tid=21, entry_tid=1' 2026-02-18T08:20:20.772 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:20:20.772 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:20:20.772 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:20:20.772 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:20:20.773 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:20:20.773 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:20:20.773 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:20:20.773 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:20:20.773 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:20:20.773 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:20:20.774 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:20:20.774 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:20:20.774 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:20:20.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:20:20.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:20:20.811 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 77 2026-02-18T08:20:20.811 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 77 2026-02-18T08:20:20.811 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:20:20.812 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=309, tag_tid=21, entry_tid=1], [object_number=308, tag_tid=21, entry_tid=0], [object_number=310, tag_tid=20, entry_tid=46754], [object_number=311, tag_tid=20, entry_tid=46751]]], state=connected] 2026-02-18T08:20:20.812 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=310, tag_tid=20, entry_tid=46754], [object_number=309, tag_tid=20, entry_tid=46753], [object_number=308, tag_tid=20, entry_tid=46752], [object_number=311, tag_tid=20, entry_tid=46751]]], state=connected] 2026-02-18T08:20:20.815 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:20:20.817 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=310, tag_tid=20, entry_tid=46754' 2026-02-18T08:20:20.817 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=309, tag_tid=21, entry_tid=1' -a 'object_number=309, tag_tid=21, entry_tid=1' = 'object_number=310, tag_tid=20, entry_tid=46754' 2026-02-18T08:20:20.817 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=310, tag_tid=20, entry_tid=46754' '!=' 'object_number=310, tag_tid=20, entry_tid=46754' 2026-02-18T08:20:20.818 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:20:20.818 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.4 2026-02-18T08:20:21.219 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:20:21.220 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:20:21.220 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:20:21.220 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:20:21.220 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:20:21.220 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:20:21.220 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:20:21.220 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:20:21.220 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:20:21.220 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:20:21.220 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:20:21.221 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:20:21.221 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:20:21.221 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:20:21.221 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:20:21.224 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:20:21.224 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:20:21.224 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:20:21.224 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:20:21.224 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:20:21.224 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:20:21.224 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:20:21.224 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:20:21.224 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:20:21.224 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:20:21.225 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:20:21.225 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:20:21.225 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:20:21.225 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:20:21.225 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:20:21.225 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:20:21.225 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:20:21.225 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:20:21.225 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:20:21.225 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:20:21.225 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:20:21.225 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:20:21.225 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:20:21.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:20:21.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:20:21.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:20:21.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:20:21.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:20:21.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:20:21.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:20:21.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:20:21.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:20:21.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:20:21.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:20:21.234 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:20:21.234 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:20:21.234 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:20:21.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:20:21.292 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:20:21.292 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:20:21.292 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:20:21.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:20:21.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:20:21.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:20:21.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:20:21.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:20:21.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:20:21.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:20:21.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:20:21.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:20:21.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:20:21.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:20:21.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:20:21.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:20:21.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:20:21.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:20:21.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:20:21.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:20:21.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:20:21.297 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:20:21.297 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:20:21.297 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:20:21.297 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:20:21.297 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:20:21.297 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:20:21.297 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:20:21.297 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:20:21.297 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:20:21.297 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:20:21.297 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:20:21.297 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:20:21.298 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:20:21.298 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:20:21.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:20:21.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:20:21.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:20:21.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:20:21.362 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:20:21.362 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:20:21.363 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:20:21.365 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:20:21.365 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:20:21.365 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:20:21.365 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:20:21.365 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:20:21.365 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:20:21.366 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:20:21.366 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:20:21.366 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:20:21.366 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:20:21.366 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:20:21.366 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:20:21.366 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:20:21.366 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:20:21.366 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:20:21.366 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:20:21.366 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:20:21.367 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:20:21.367 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:20:21.367 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:20:21.367 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:20:21.367 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:20:21.367 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:20:21.367 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:20:21.367 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:20:21.367 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:20:21.367 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:20:21.367 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:20:21.367 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:20:21.367 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:20:21.368 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:20:21.368 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:20:21.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:20:21.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:20:21.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:20:21.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:20:21.434 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:20:21.434 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:20:21.435 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:20:21.437 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:20:21.437 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:20:21.437 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:20:21.437 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:20:21.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:20:21.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:20:21.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:20:21.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:20:21.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:20:21.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:20:21.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:20:21.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:20:21.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:20:21.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:20:21.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:20:21.439 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:20:21.439 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:20:21.439 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:20:21.439 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:20:21.439 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:20:21.439 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:20:21.439 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:20:21.439 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:20:21.439 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:20:21.439 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:20:21.439 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:20:21.439 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:20:21.440 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:20:21.440 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:20:21.440 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:20:21.440 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:20:21.440 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:20:21.448 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:20:21.449 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:20:21.449 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:20:21.449 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:20:21.504 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:20:21.504 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:20:21.504 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:20:21.504 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:20:21.504 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:20:21.504 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:20:21.505 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:20:21.505 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:20:21.505 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:20:21.505 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:20:21.505 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:20:21.506 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:20:21.506 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:20:21.506 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:20:21.506 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:20:21.506 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:20:21.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 77 2026-02-18T08:20:21.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 77 2026-02-18T08:20:21.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:20:21.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=309, tag_tid=21, entry_tid=1], [object_number=308, tag_tid=21, entry_tid=0], [object_number=310, tag_tid=20, entry_tid=46754], [object_number=311, tag_tid=20, entry_tid=46751]]], state=connected] 2026-02-18T08:20:21.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=310, tag_tid=20, entry_tid=46754], [object_number=309, tag_tid=20, entry_tid=46753], [object_number=308, tag_tid=20, entry_tid=46752], [object_number=311, tag_tid=20, entry_tid=46751]]], state=connected] 2026-02-18T08:20:21.552 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:20:21.553 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=309, tag_tid=21, entry_tid=1' 2026-02-18T08:20:21.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:20:21.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:20:21.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:20:21.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:20:21.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:20:21.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:20:21.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:20:21.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:20:21.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:20:21.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:20:21.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:20:21.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:20:21.557 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:20:21.557 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:20:21.557 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:20:21.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 77 2026-02-18T08:20:21.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 77 2026-02-18T08:20:21.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:20:21.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=309, tag_tid=21, entry_tid=1], [object_number=308, tag_tid=21, entry_tid=0], [object_number=310, tag_tid=20, entry_tid=46754], [object_number=311, tag_tid=20, entry_tid=46751]]], state=connected] 2026-02-18T08:20:21.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=310, tag_tid=20, entry_tid=46754], [object_number=309, tag_tid=20, entry_tid=46753], [object_number=308, tag_tid=20, entry_tid=46752], [object_number=311, tag_tid=20, entry_tid=46751]]], state=connected] 2026-02-18T08:20:21.594 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:20:21.596 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=310, tag_tid=20, entry_tid=46754' 2026-02-18T08:20:21.596 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=309, tag_tid=21, entry_tid=1' -a 'object_number=309, tag_tid=21, entry_tid=1' = 'object_number=310, tag_tid=20, entry_tid=46754' 2026-02-18T08:20:21.596 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=310, tag_tid=20, entry_tid=46754' '!=' 'object_number=310, tag_tid=20, entry_tid=46754' 2026-02-18T08:20:21.596 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:20:21.596 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.8 2026-02-18T08:20:22.397 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:20:22.397 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:20:22.397 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:20:22.397 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:20:22.397 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:20:22.397 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:20:22.397 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:20:22.397 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:20:22.397 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:20:22.397 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:20:22.397 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:20:22.397 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:20:22.397 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:20:22.397 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:20:22.397 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:20:22.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:20:22.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:20:22.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:20:22.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:20:22.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:20:22.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:20:22.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:20:22.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:20:22.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:20:22.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:20:22.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:20:22.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:20:22.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:20:22.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:20:22.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:20:22.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:20:22.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:20:22.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:20:22.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:20:22.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:20:22.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:20:22.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:20:22.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:20:22.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:20:22.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:20:22.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:20:22.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:20:22.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:20:22.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:20:22.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:20:22.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:20:22.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:20:22.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:20:22.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:20:22.413 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:20:22.413 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:20:22.413 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:20:22.413 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:20:22.472 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:20:22.472 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:20:22.472 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:20:22.475 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:20:22.475 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:20:22.475 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:20:22.475 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:20:22.476 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:20:22.476 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:20:22.476 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:20:22.476 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:20:22.476 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:20:22.476 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:20:22.476 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:20:22.476 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:20:22.476 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:20:22.476 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:20:22.477 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:20:22.477 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:20:22.477 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:20:22.477 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:20:22.477 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:20:22.477 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:20:22.477 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:20:22.477 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:20:22.477 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:20:22.477 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:20:22.477 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:20:22.478 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:20:22.478 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:20:22.478 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:20:22.478 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:20:22.478 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:20:22.478 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:20:22.478 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:20:22.486 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:20:22.486 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:20:22.486 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:20:22.486 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:20:22.535 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:20:22.535 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:20:22.536 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:20:22.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:20:22.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:20:22.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:20:22.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:20:22.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:20:22.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:20:22.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:20:22.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:20:22.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:20:22.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:20:22.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:20:22.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:20:22.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:20:22.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:20:22.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:20:22.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:20:22.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:20:22.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:20:22.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:20:22.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:20:22.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:20:22.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:20:22.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:20:22.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:20:22.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:20:22.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:20:22.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:20:22.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:20:22.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:20:22.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:20:22.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:20:22.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:20:22.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:20:22.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:20:22.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:20:22.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:20:22.607 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:20:22.607 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:20:22.607 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:20:22.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:20:22.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:20:22.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:20:22.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:20:22.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:20:22.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:20:22.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:20:22.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:20:22.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:20:22.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:20:22.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:20:22.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:20:22.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:20:22.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:20:22.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:20:22.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:20:22.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:20:22.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:20:22.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:20:22.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:20:22.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:20:22.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:20:22.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:20:22.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:20:22.610 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:20:22.610 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:20:22.610 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:20:22.610 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:20:22.610 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:20:22.610 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:20:22.610 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:20:22.610 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:20:22.620 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:20:22.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:20:22.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:20:22.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:20:22.684 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:20:22.685 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:20:22.685 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:20:22.685 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:20:22.685 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:20:22.685 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:20:22.685 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:20:22.685 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:20:22.685 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:20:22.685 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:20:22.685 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:20:22.685 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:20:22.685 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:20:22.687 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:20:22.687 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:20:22.687 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:20:22.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 77 2026-02-18T08:20:22.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 77 2026-02-18T08:20:22.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:20:22.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=309, tag_tid=21, entry_tid=1], [object_number=308, tag_tid=21, entry_tid=0], [object_number=310, tag_tid=20, entry_tid=46754], [object_number=311, tag_tid=20, entry_tid=46751]]], state=connected] 2026-02-18T08:20:22.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=310, tag_tid=20, entry_tid=46754], [object_number=309, tag_tid=20, entry_tid=46753], [object_number=308, tag_tid=20, entry_tid=46752], [object_number=311, tag_tid=20, entry_tid=46751]]], state=connected] 2026-02-18T08:20:22.719 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:20:22.721 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=309, tag_tid=21, entry_tid=1' 2026-02-18T08:20:22.721 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:20:22.721 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:20:22.721 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:20:22.721 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:20:22.721 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:20:22.721 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:20:22.721 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:20:22.721 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:20:22.722 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:20:22.722 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:20:22.722 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:20:22.722 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:20:22.724 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:20:22.724 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:20:22.724 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:20:22.752 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 77 2026-02-18T08:20:22.752 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 77 2026-02-18T08:20:22.752 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:20:22.752 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=309, tag_tid=21, entry_tid=1], [object_number=308, tag_tid=21, entry_tid=0], [object_number=310, tag_tid=20, entry_tid=46754], [object_number=311, tag_tid=20, entry_tid=46751]]], state=connected] 2026-02-18T08:20:22.752 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=310, tag_tid=20, entry_tid=46754], [object_number=309, tag_tid=20, entry_tid=46753], [object_number=308, tag_tid=20, entry_tid=46752], [object_number=311, tag_tid=20, entry_tid=46751]]], state=connected] 2026-02-18T08:20:22.755 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:20:22.756 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=310, tag_tid=20, entry_tid=46754' 2026-02-18T08:20:22.756 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=309, tag_tid=21, entry_tid=1' -a 'object_number=309, tag_tid=21, entry_tid=1' = 'object_number=310, tag_tid=20, entry_tid=46754' 2026-02-18T08:20:22.756 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=310, tag_tid=20, entry_tid=46754' '!=' 'object_number=310, tag_tid=20, entry_tid=46754' 2026-02-18T08:20:22.756 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:20:22.756 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1.6 2026-02-18T08:20:24.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:20:24.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:20:24.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:20:24.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:20:24.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:20:24.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:20:24.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:20:24.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:20:24.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:20:24.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:20:24.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:20:24.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:20:24.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:20:24.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:20:24.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:20:24.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:20:24.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:20:24.360 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:20:24.360 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:20:24.360 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:20:24.360 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:20:24.360 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:20:24.360 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:20:24.360 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:20:24.360 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:20:24.361 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:20:24.361 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:20:24.361 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:20:24.361 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:20:24.361 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:20:24.361 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:20:24.361 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:20:24.361 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:20:24.361 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:20:24.361 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:20:24.361 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:20:24.361 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:20:24.361 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:20:24.361 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:20:24.361 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:20:24.361 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:20:24.361 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:20:24.361 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:20:24.361 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:20:24.361 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:20:24.362 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:20:24.362 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:20:24.362 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:20:24.362 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:20:24.374 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:20:24.375 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:20:24.375 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:20:24.375 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:20:24.432 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:20:24.432 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:20:24.432 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:20:24.435 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:20:24.435 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:20:24.435 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:20:24.435 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:20:24.436 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:20:24.436 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:20:24.436 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:20:24.436 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:20:24.436 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:20:24.436 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:20:24.436 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:20:24.436 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:20:24.436 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:20:24.436 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:20:24.436 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:20:24.436 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:20:24.437 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:20:24.437 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:20:24.437 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:20:24.437 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:20:24.437 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:20:24.437 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:20:24.437 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:20:24.437 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:20:24.437 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:20:24.437 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:20:24.437 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:20:24.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:20:24.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:20:24.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:20:24.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:20:24.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:20:24.446 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:20:24.446 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:20:24.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:20:24.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:20:24.503 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:20:24.503 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:20:24.503 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:20:24.506 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:20:24.506 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:20:24.506 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:20:24.506 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:20:24.506 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:20:24.506 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:20:24.506 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:20:24.506 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:20:24.506 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:20:24.507 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:20:24.507 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:20:24.507 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:20:24.507 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:20:24.507 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:20:24.507 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:20:24.507 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:20:24.507 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:20:24.507 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:20:24.507 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:20:24.507 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:20:24.507 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:20:24.508 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:20:24.508 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:20:24.508 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:20:24.508 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:20:24.508 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:20:24.508 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:20:24.508 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:20:24.508 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:20:24.508 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:20:24.508 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:20:24.508 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:20:24.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:20:24.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:20:24.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:20:24.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:20:24.574 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:20:24.574 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:20:24.574 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:20:24.576 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:20:24.577 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:20:24.577 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:20:24.577 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:20:24.577 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:20:24.577 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:20:24.577 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:20:24.577 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:20:24.577 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:20:24.577 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:20:24.577 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:20:24.577 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:20:24.578 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:20:24.578 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:20:24.578 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:20:24.578 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:20:24.578 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:20:24.578 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:20:24.578 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:20:24.578 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:20:24.578 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:20:24.578 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:20:24.578 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:20:24.578 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:20:24.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:20:24.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:20:24.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:20:24.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:20:24.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:20:24.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:20:24.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:20:24.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:20:24.586 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:20:24.587 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:20:24.587 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:20:24.587 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:20:24.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:20:24.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:20:24.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:20:24.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:20:24.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:20:24.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:20:24.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:20:24.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:20:24.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:20:24.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:20:24.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:20:24.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:20:24.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:20:24.653 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:20:24.653 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:20:24.653 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:20:24.681 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 77 2026-02-18T08:20:24.681 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 77 2026-02-18T08:20:24.681 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:20:24.681 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=309, tag_tid=21, entry_tid=1], [object_number=308, tag_tid=21, entry_tid=0], [object_number=310, tag_tid=20, entry_tid=46754], [object_number=311, tag_tid=20, entry_tid=46751]]], state=connected] 2026-02-18T08:20:24.682 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=310, tag_tid=20, entry_tid=46754], [object_number=309, tag_tid=20, entry_tid=46753], [object_number=308, tag_tid=20, entry_tid=46752], [object_number=311, tag_tid=20, entry_tid=46751]]], state=connected] 2026-02-18T08:20:24.684 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:20:24.686 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=309, tag_tid=21, entry_tid=1' 2026-02-18T08:20:24.686 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:20:24.686 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:20:24.686 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:20:24.686 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:20:24.686 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:20:24.686 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:20:24.686 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:20:24.686 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:20:24.686 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:20:24.687 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:20:24.687 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:20:24.687 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:20:24.688 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:20:24.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:20:24.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:20:24.722 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 77 2026-02-18T08:20:24.722 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 77 2026-02-18T08:20:24.722 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:20:24.722 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=309, tag_tid=21, entry_tid=1], [object_number=308, tag_tid=21, entry_tid=0], [object_number=310, tag_tid=20, entry_tid=46754], [object_number=311, tag_tid=20, entry_tid=46751]]], state=connected] 2026-02-18T08:20:24.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=310, tag_tid=20, entry_tid=46754], [object_number=309, tag_tid=20, entry_tid=46753], [object_number=308, tag_tid=20, entry_tid=46752], [object_number=311, tag_tid=20, entry_tid=46751]]], state=connected] 2026-02-18T08:20:24.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:20:24.728 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=310, tag_tid=20, entry_tid=46754' 2026-02-18T08:20:24.728 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=309, tag_tid=21, entry_tid=1' -a 'object_number=309, tag_tid=21, entry_tid=1' = 'object_number=310, tag_tid=20, entry_tid=46754' 2026-02-18T08:20:24.728 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=310, tag_tid=20, entry_tid=46754' '!=' 'object_number=310, tag_tid=20, entry_tid=46754' 2026-02-18T08:20:24.728 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:20:24.729 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 2 2026-02-18T08:20:26.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:20:26.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:20:26.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:20:26.731 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:20:26.731 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:20:26.731 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:20:26.731 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:20:26.731 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:20:26.731 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:20:26.731 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:20:26.731 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:20:26.731 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:20:26.731 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:20:26.732 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:20:26.732 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:20:26.732 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:20:26.734 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:20:26.735 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:20:26.735 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:20:26.735 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:20:26.735 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:20:26.735 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:20:26.735 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:20:26.735 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:20:26.735 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:20:26.735 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:20:26.735 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:20:26.735 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:20:26.736 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:20:26.736 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:20:26.736 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:20:26.736 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:20:26.736 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:20:26.736 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:20:26.736 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:20:26.736 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:20:26.736 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:20:26.736 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:20:26.736 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:20:26.736 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:20:26.736 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:20:26.736 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:20:26.737 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:20:26.737 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:20:26.737 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:20:26.737 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:20:26.737 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:20:26.737 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:20:26.737 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:20:26.745 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:20:26.745 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:20:26.745 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:20:26.746 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:20:26.796 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:20:26.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:20:26.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:20:26.799 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:20:26.800 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:20:26.800 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:20:26.800 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:20:26.800 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:20:26.800 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:20:26.800 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:20:26.800 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:20:26.800 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:20:26.800 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:20:26.800 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:20:26.801 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:20:26.801 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:20:26.801 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:20:26.801 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:20:26.801 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:20:26.801 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:20:26.801 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:20:26.801 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:20:26.801 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:20:26.801 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:20:26.801 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:20:26.801 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:20:26.802 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:20:26.802 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:20:26.802 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:20:26.802 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:20:26.802 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:20:26.802 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:20:26.802 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:20:26.802 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:20:26.802 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:20:26.810 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:20:26.810 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:20:26.810 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:20:26.811 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:20:26.868 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:20:26.868 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:20:26.868 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:20:26.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:20:26.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:20:26.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:20:26.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:20:26.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:20:26.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:20:26.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:20:26.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:20:26.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:20:26.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:20:26.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:20:26.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:20:26.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:20:26.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:20:26.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:20:26.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:20:26.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:20:26.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:20:26.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:20:26.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:20:26.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:20:26.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:20:26.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:20:26.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:20:26.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:20:26.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:20:26.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:20:26.874 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:20:26.874 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:20:26.874 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:20:26.874 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:20:26.874 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:20:26.881 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:20:26.882 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:20:26.882 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:20:26.882 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:20:26.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:20:26.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:20:26.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:20:26.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:20:26.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:20:26.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:20:26.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:20:26.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:20:26.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:20:26.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:20:26.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:20:26.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:20:26.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:20:26.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:20:26.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:20:26.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:20:26.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:20:26.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:20:26.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:20:26.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:20:26.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:20:26.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:20:26.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:20:26.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:20:26.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:20:26.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:20:26.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:20:26.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:20:26.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:20:26.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:20:26.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:20:26.944 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:20:26.944 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:20:26.944 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:20:26.944 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:20:26.952 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:20:26.953 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:20:26.953 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:20:26.953 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:20:27.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:20:27.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:20:27.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:20:27.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:20:27.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:20:27.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:20:27.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:20:27.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:20:27.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:20:27.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:20:27.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:20:27.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:20:27.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:20:27.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:20:27.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:20:27.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:20:27.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 77 2026-02-18T08:20:27.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 77 2026-02-18T08:20:27.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:20:27.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=309, tag_tid=21, entry_tid=1], [object_number=308, tag_tid=21, entry_tid=0], [object_number=310, tag_tid=20, entry_tid=46754], [object_number=311, tag_tid=20, entry_tid=46751]]], state=connected] 2026-02-18T08:20:27.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=309, tag_tid=21, entry_tid=1], [object_number=308, tag_tid=21, entry_tid=0], [object_number=310, tag_tid=20, entry_tid=46754], [object_number=311, tag_tid=20, entry_tid=46751]]], state=connected] 2026-02-18T08:20:27.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:20:27.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=309, tag_tid=21, entry_tid=1' 2026-02-18T08:20:27.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:20:27.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:20:27.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:20:27.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:20:27.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:20:27.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:20:27.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:20:27.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:20:27.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:20:27.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:20:27.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:20:27.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:20:27.055 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:20:27.055 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:20:27.055 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:20:27.092 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 77 2026-02-18T08:20:27.092 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 77 2026-02-18T08:20:27.092 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:20:27.092 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=309, tag_tid=21, entry_tid=1], [object_number=308, tag_tid=21, entry_tid=0], [object_number=310, tag_tid=20, entry_tid=46754], [object_number=311, tag_tid=20, entry_tid=46751]]], state=connected] 2026-02-18T08:20:27.092 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=309, tag_tid=21, entry_tid=1], [object_number=308, tag_tid=21, entry_tid=0], [object_number=310, tag_tid=20, entry_tid=46754], [object_number=311, tag_tid=20, entry_tid=46751]]], state=connected] 2026-02-18T08:20:27.095 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:20:27.097 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=309, tag_tid=21, entry_tid=1' 2026-02-18T08:20:27.097 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=309, tag_tid=21, entry_tid=1' -a 'object_number=309, tag_tid=21, entry_tid=1' = 'object_number=309, tag_tid=21, entry_tid=1' 2026-02-18T08:20:27.097 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:20:27.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_snap_present cluster1 mirror test snap7 2026-02-18T08:20:27.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:20:27.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:20:27.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:20:27.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap7 2026-02-18T08:20:27.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-18T08:20:27.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-18T08:20:27.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1 2026-02-18T08:20:28.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster1 info mirror/test@snap7 2026-02-18T08:20:28.129 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:rbd image 'test': 2026-02-18T08:20:28.129 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: size 512 MiB in 128 objects 2026-02-18T08:20:28.129 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: order 22 (4 MiB objects) 2026-02-18T08:20:28.129 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: snapshot_count: 13 2026-02-18T08:20:28.129 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: id: 10a74aa8ebc0 2026-02-18T08:20:28.129 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: block_name_prefix: rbd_data.10a74aa8ebc0 2026-02-18T08:20:28.130 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: format: 2 2026-02-18T08:20:28.130 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: features: layering, exclusive-lock, journaling 2026-02-18T08:20:28.130 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: op_features: 2026-02-18T08:20:28.130 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: flags: 2026-02-18T08:20:28.130 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: create_timestamp: Wed Feb 18 08:10:48 2026 2026-02-18T08:20:28.130 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: access_timestamp: Wed Feb 18 08:18:56 2026 2026-02-18T08:20:28.130 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: modify_timestamp: Wed Feb 18 08:19:19 2026 2026-02-18T08:20:28.130 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: protected: False 2026-02-18T08:20:28.130 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: journal: 10a74aa8ebc0 2026-02-18T08:20:28.130 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring state: enabled 2026-02-18T08:20:28.130 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring mode: journal 2026-02-18T08:20:28.131 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring global id: 0b548823-d751-43df-a076-4efac2da4d3d 2026-02-18T08:20:28.131 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring primary: false 2026-02-18T08:20:28.132 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:20:28.132 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n snap6-clean ']' 2026-02-18T08:20:28.132 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ compare_image_snaps mirror test snap6-clean 2026-02-18T08:20:28.132 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:20:28.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:20:28.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap6-clean 2026-02-18T08:20:28.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local ret=0 2026-02-18T08:20:28.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local rmt_export=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.export 2026-02-18T08:20:28.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loc_export=/tmp/tmp.vybh1m9yJF/cluster1-mirror-test.export 2026-02-18T08:20:28.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.export /tmp/tmp.vybh1m9yJF/cluster1-mirror-test.export 2026-02-18T08:20:28.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 -p mirror export test@snap6-clean /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.export 2026-02-18T08:20:28.607 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-18T08:20:28.623 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster1 -p mirror export test@snap6-clean /tmp/tmp.vybh1m9yJF/cluster1-mirror-test.export 2026-02-18T08:20:29.256 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-18T08:20:29.273 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmp /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.export /tmp/tmp.vybh1m9yJF/cluster1-mirror-test.export 2026-02-18T08:20:29.460 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.export /tmp/tmp.vybh1m9yJF/cluster1-mirror-test.export 2026-02-18T08:20:29.603 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:20:29.603 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ compare_image_snaps mirror test snap7 2026-02-18T08:20:29.603 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:20:29.603 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:20:29.603 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap7 2026-02-18T08:20:29.603 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local ret=0 2026-02-18T08:20:29.603 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local rmt_export=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.export 2026-02-18T08:20:29.603 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loc_export=/tmp/tmp.vybh1m9yJF/cluster1-mirror-test.export 2026-02-18T08:20:29.603 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.export /tmp/tmp.vybh1m9yJF/cluster1-mirror-test.export 2026-02-18T08:20:29.603 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 -p mirror export test@snap7 /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.export 2026-02-18T08:20:30.075 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-18T08:20:30.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster1 -p mirror export test@snap7 /tmp/tmp.vybh1m9yJF/cluster1-mirror-test.export 2026-02-18T08:20:30.717 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-18T08:20:30.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmp /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.export /tmp/tmp.vybh1m9yJF/cluster1-mirror-test.export 2026-02-18T08:20:30.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.export /tmp/tmp.vybh1m9yJF/cluster1-mirror-test.export 2026-02-18T08:20:31.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:20:31.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ clean_snap_name=snap7-clean 2026-02-18T08:20:31.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_snap cluster2 mirror test snap7-clean 2026-02-18T08:20:31.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:20:31.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:20:31.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:20:31.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap7-clean 2026-02-18T08:20:31.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 -p mirror snap create test@snap7-clean --debug-rbd=20 --debug-journaler=20 2026-02-18T08:20:31.915 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 10` 2026-02-18T08:20:31.915 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ stress_write_image cluster2 mirror test 2026-02-18T08:20:31.915 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:20:31.915 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:20:31.915 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:20:31.915 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-18T08:20:31.917 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local duration=7 2026-02-18T08:20:31.917 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set +e 2026-02-18T08:20:31.917 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ timeout 7s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror test --debug-rbd=20 --debug-journaler=20 2026-02-18T08:20:32.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:100 IOs, 30252032 bytes 2026-02-18T08:20:32.936 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:200 IOs, 54881280 bytes 2026-02-18T08:20:33.130 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:300 IOs, 65377280 bytes 2026-02-18T08:20:33.670 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:400 IOs, 105017344 bytes 2026-02-18T08:20:34.037 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:500 IOs, 126876672 bytes 2026-02-18T08:20:35.037 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:600 IOs, 222588928 bytes 2026-02-18T08:20:36.227 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:700 IOs, 251243520 bytes 2026-02-18T08:20:37.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:800 IOs, 332139520 bytes 2026-02-18T08:20:37.573 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:900 IOs, 356043776 bytes 2026-02-18T08:20:38.214 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1000 IOs, 417949696 bytes 2026-02-18T08:20:38.929 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ error_code=124 2026-02-18T08:20:38.929 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set -e 2026-02-18T08:20:38.929 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 124 -eq 124 ']' 2026-02-18T08:20:38.929 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:20:38.929 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+replaying 2026-02-18T08:20:38.929 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:20:38.929 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:20:38.929 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:20:38.929 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local state_pattern=up+replaying 2026-02-18T08:20:38.930 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local description_pattern= 2026-02-18T08:20:38.930 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local service_pattern= 2026-02-18T08:20:38.930 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:20:38.930 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1 2026-02-18T08:20:39.930 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test_status_in_pool_dir cluster1 mirror test up+replaying '' '' 2026-02-18T08:20:39.930 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:20:39.930 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:20:39.930 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:20:39.930 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local state_pattern=up+replaying 2026-02-18T08:20:39.931 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local description_pattern= 2026-02-18T08:20:39.931 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local service_pattern= 2026-02-18T08:20:39.931 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local status 2026-02-18T08:20:39.931 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ CEPH_ARGS= 2026-02-18T08:20:39.931 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-18T08:20:39.971 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ status='test: 2026-02-18T08:20:39.971 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: global_id: 0b548823-d751-43df-a076-4efac2da4d3d 2026-02-18T08:20:39.971 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: state: up+replaying 2026-02-18T08:20:39.971 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: description: replaying, {"bytes_per_second":9396425.07,"entries_behind_primary":2252,"entries_per_second":589.17,"non_primary_position":{"entry_tid":44503,"object_number":307,"tag_tid":20},"primary_position":{"entry_tid":1,"object_number":309,"tag_tid":21},"seconds_until_synced":3} 2026-02-18T08:20:39.971 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: service: mirror.3 on trial163 2026-02-18T08:20:39.971 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: last_update: 2026-02-18 08:20:16 2026-02-18T08:20:39.971 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: peer_sites: 2026-02-18T08:20:39.971 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: name: cluster2 2026-02-18T08:20:39.971 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: state: up+stopped 2026-02-18T08:20:39.971 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: description: local image is primary 2026-02-18T08:20:39.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: last_update: 2026-02-18 08:20:16' 2026-02-18T08:20:39.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ grep '^ state: .*up+replaying' 2026-02-18T08:20:39.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: state: up+replaying 2026-02-18T08:20:39.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ grep '^ description: .*' 2026-02-18T08:20:39.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: description: replaying, {"bytes_per_second":9396425.07,"entries_behind_primary":2252,"entries_per_second":589.17,"non_primary_position":{"entry_tid":44503,"object_number":307,"tag_tid":20},"primary_position":{"entry_tid":1,"object_number":309,"tag_tid":21},"seconds_until_synced":3} 2026-02-18T08:20:39.974 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n '' ']' 2026-02-18T08:20:39.974 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ echo up+replaying 2026-02-18T08:20:39.974 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ grep '^up+' 2026-02-18T08:20:39.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:up+replaying 2026-02-18T08:20:39.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ grep 'service: *mirror..* on ' 2026-02-18T08:20:39.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: service: mirror.3 on trial163 2026-02-18T08:20:39.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local last_update 2026-02-18T08:20:39.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-18T08:20:39.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_update='2026-02-18 08:20:16' 2026-02-18T08:20:39.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+replaying '2026-02-18 08:20:16' 2026-02-18T08:20:39.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:20:39.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:20:39.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:20:39.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local state_pattern=up+replaying 2026-02-18T08:20:39.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'prev_last_update=2026-02-18 08:20:16' 2026-02-18T08:20:39.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local status 2026-02-18T08:20:39.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ CEPH_ARGS= 2026-02-18T08:20:39.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-18T08:20:40.019 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ status='OKOKOK142194268mirror.3trial16320.2.0-670-gadc19233falseOK43304333mirror.2trial16320.2.0-670-gadc19233falseOK43534356mirror.1trial16320.2.0-670-gadc19233trueOK44374440mirror.0trial16320.2.0-670-gadc19233falseOKtest0b548823-d751-43df-a076-4efac2da4d3dup+replayingreplaying, {"bytes_per_second":9396425.07,"entries_behind_primary":2252,"entries_per_second":589.17,"non_primary_position":{"entry_tid":44503,"object_number":307,"tag_tid":20},"primary_position":{"entry_tid":1,"object_number":309,"tag_tid":21},"seconds_until_synced":3}42194268mirror.3trial1632026-02-18 08:20:16cluster2c5e4f85e-ec31-430b-a71d-56e000d0de0bup+stoppedlocal image is primary2026-02-18 08:20:16' 2026-02-18T08:20:40.019 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local last_update state 2026-02-18T08:20:40.020 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-18T08:20:40.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_update='2026-02-18 08:20:16' 2026-02-18T08:20:40.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-18T08:20:40.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ state=up+replaying 2026-02-18T08:20:40.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ echo up+replaying 2026-02-18T08:20:40.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ grep up+replaying 2026-02-18T08:20:40.027 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:up+replaying 2026-02-18T08:20:40.028 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:20:40.028 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:20:40.028 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ snap_name=snap8 2026-02-18T08:20:40.028 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_snap cluster2 mirror test snap8 2026-02-18T08:20:40.028 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:20:40.028 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:20:40.029 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:20:40.029 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap8 2026-02-18T08:20:40.029 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 -p mirror snap create test@snap8 --debug-rbd=20 --debug-journaler=20 2026-02-18T08:21:09.455 INFO:tasks.rbd_mirror_thrash:kill cluster2.client.mirror.0 2026-02-18T08:21:09.456 INFO:tasks.rbd_mirror.cluster2.client.mirror.0:Sent signal 15 2026-02-18T08:21:09.456 INFO:tasks.rbd_mirror_thrash:waiting for 4.706907681420453 secs before reviving daemons 2026-02-18T08:21:09.457 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.trial163.stderr:2026-02-18T08:21:09.449+0000 7fc1bd8be640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.0 (PID: 61171) UID: 1000 2026-02-18T08:21:11.672 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_replay_started cluster1 mirror test 2026-02-18T08:21:11.672 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:21:11.672 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:21:11.672 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:21:11.672 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_replay_state cluster1 mirror test started 2026-02-18T08:21:11.672 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:21:11.672 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:21:11.672 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:21:11.672 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local state=started 2026-02-18T08:21:11.673 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-18T08:21:11.673 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:21:11.673 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1 2026-02-18T08:21:12.673 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test_image_replay_state cluster1 mirror test started 2026-02-18T08:21:12.673 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:21:12.673 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:21:12.673 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:21:12.673 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local test_state=started 2026-02-18T08:21:12.673 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local status_result 2026-02-18T08:21:12.673 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_state=stopped 2026-02-18T08:21:12.674 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-18T08:21:12.674 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -i state 2026-02-18T08:21:12.674 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_instance=cluster1 2026-02-18T08:21:12.674 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-18T08:21:12.674 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:21:12.674 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance=cluster1 2026-02-18T08:21:12.674 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local loop_instance 2026-02-18T08:21:12.674 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:21:12.674 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sleep 0 2026-02-18T08:21:12.675 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:21:12.675 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ seq 0 3 2026-02-18T08:21:12.676 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:21:12.676 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-18T08:21:12.676 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:0 2026-02-18T08:21:12.676 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-18T08:21:12.676 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:21:12.676 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:21:12.676 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:21:12.676 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-18T08:21:12.676 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:21:12.676 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:21:12.677 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:21:12.677 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:21:12.677 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-18T08:21:12.677 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:21:12.677 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:21:12.677 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:21:12.677 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:21:12.677 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:0 2026-02-18T08:21:12.677 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-18T08:21:12.677 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-18T08:21:12.677 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:21:12.677 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:0 2026-02-18T08:21:12.677 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T08:21:12.677 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-18T08:21:12.677 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:21:12.677 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:21:12.677 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-18T08:21:12.677 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-18T08:21:12.677 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-18T08:21:12.677 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=0 2026-02-18T08:21:12.677 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=0 2026-02-18T08:21:12.678 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:21:12.688 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:21:12.688 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:21:12.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:21:12.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-18T08:21:12.740 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:21:12.740 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status : get status for rbd mirror 2026-02-18T08:21:12.740 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:21:12.742 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:21:12.742 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test 2026-02-18T08:21:12.742 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:1 2026-02-18T08:21:12.742 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-18T08:21:12.742 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:21:12.742 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:21:12.742 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:21:12.742 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-18T08:21:12.742 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:21:12.742 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:21:12.742 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:21:12.742 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:21:12.742 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-18T08:21:12.742 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:21:12.742 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:21:12.742 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:21:12.743 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:21:12.743 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:1 2026-02-18T08:21:12.743 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-18T08:21:12.743 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-18T08:21:12.743 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:21:12.743 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:1 2026-02-18T08:21:12.743 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T08:21:12.743 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-18T08:21:12.743 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:21:12.743 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:21:12.743 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-18T08:21:12.743 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-18T08:21:12.743 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-18T08:21:12.743 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=1 2026-02-18T08:21:12.743 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=1 2026-02-18T08:21:12.743 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:21:12.754 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:21:12.754 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:21:12.754 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:21:12.754 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test 2026-02-18T08:21:12.805 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:21:12.805 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status : get status for rbd mirror 2026-02-18T08:21:12.805 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:21:12.809 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:21:12.809 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/test 2026-02-18T08:21:12.809 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:2 2026-02-18T08:21:12.809 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-18T08:21:12.809 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:21:12.809 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:21:12.809 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:21:12.810 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-18T08:21:12.810 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:21:12.810 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:21:12.810 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:21:12.810 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:21:12.810 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-18T08:21:12.810 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:21:12.810 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:21:12.810 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:21:12.810 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:21:12.810 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:2 2026-02-18T08:21:12.811 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-18T08:21:12.811 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-18T08:21:12.811 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:21:12.811 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:2 2026-02-18T08:21:12.811 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T08:21:12.811 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-18T08:21:12.811 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:21:12.811 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:21:12.811 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-18T08:21:12.811 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-18T08:21:12.811 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-18T08:21:12.811 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=2 2026-02-18T08:21:12.812 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=2 2026-02-18T08:21:12.812 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:21:12.819 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:21:12.819 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:21:12.819 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:21:12.819 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/test 2026-02-18T08:21:12.868 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:21:12.869 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status : get status for rbd mirror 2026-02-18T08:21:12.869 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:21:12.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:21:12.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/test 2026-02-18T08:21:12.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:3 2026-02-18T08:21:12.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-18T08:21:12.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:21:12.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:21:12.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:21:12.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-18T08:21:12.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:21:12.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:21:12.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:21:12.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:21:12.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-18T08:21:12.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:21:12.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:21:12.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:21:12.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:21:12.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:3 2026-02-18T08:21:12.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-18T08:21:12.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-18T08:21:12.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:21:12.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:3 2026-02-18T08:21:12.874 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T08:21:12.874 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-18T08:21:12.874 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:21:12.874 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:21:12.874 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-18T08:21:12.874 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-18T08:21:12.874 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-18T08:21:12.874 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=3 2026-02-18T08:21:12.874 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=3 2026-02-18T08:21:12.874 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:21:12.882 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:21:12.883 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:21:12.883 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:21:12.883 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/test 2026-02-18T08:21:12.948 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ return 0 2026-02-18T08:21:12.949 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ status_result=' "state": "Replaying"' 2026-02-18T08:21:12.950 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ echo ' "state": "Replaying"' 2026-02-18T08:21:12.950 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ grep -i Replaying 2026-02-18T08:21:12.950 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: "state": "Replaying" 2026-02-18T08:21:12.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ current_state=started 2026-02-18T08:21:12.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test started = started 2026-02-18T08:21:12.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:21:12.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror test 2026-02-18T08:21:12.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_cluster=cluster1 2026-02-18T08:21:12.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:21:12.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_pool=mirror 2026-02-18T08:21:12.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local remote_pool=mirror 2026-02-18T08:21:12.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:21:12.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' journal = journal ']' 2026-02-18T08:21:12.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror mirror test 2026-02-18T08:21:12.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_cluster=cluster1 2026-02-18T08:21:12.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:21:12.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_pool=mirror 2026-02-18T08:21:12.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local remote_pool=mirror 2026-02-18T08:21:12.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:21:12.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-02-18T08:21:12.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-02-18T08:21:12.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:21:12.952 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:21:12.952 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:21:13.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:21:13.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:21:13.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:21:13.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:21:13.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:21:13.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:21:13.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:21:13.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:21:13.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:21:13.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:21:13.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:21:13.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:21:13.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:21:13.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:21:13.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:21:13.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:21:13.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:21:13.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:21:13.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:21:13.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:21:13.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:21:13.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:21:13.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:21:13.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:21:13.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:21:13.155 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:21:13.155 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:21:13.155 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:21:13.155 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:21:13.155 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:21:13.155 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:21:13.155 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:21:13.155 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:21:13.155 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:21:13.155 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:21:13.155 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:21:13.155 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:21:13.155 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:21:13.155 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:21:13.155 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:21:13.155 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:21:13.155 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:21:13.155 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:21:13.155 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:21:13.155 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:21:13.155 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:21:13.155 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:21:13.155 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:21:13.155 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:21:13.166 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:21:13.166 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:21:13.166 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:21:13.166 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:21:13.222 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:21:13.222 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:21:13.222 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:21:13.223 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:21:13.223 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:21:13.224 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:21:13.224 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:21:13.224 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:21:13.224 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:21:13.224 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:21:13.224 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:21:13.224 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:21:13.224 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:21:13.224 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:21:13.224 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:21:13.224 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:21:13.224 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:21:13.224 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:21:13.224 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:21:13.224 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:21:13.224 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:21:13.224 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:21:13.224 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:21:13.224 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:21:13.224 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:21:13.224 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:21:13.224 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:21:13.224 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:21:13.224 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:21:13.224 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:21:13.224 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:21:13.225 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:21:13.225 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:21:13.225 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:21:13.225 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:21:13.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:21:13.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:21:13.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:21:13.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:21:13.285 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:21:13.285 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:21:13.285 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:21:13.287 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:21:13.287 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:21:13.287 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:21:13.287 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:21:13.287 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:21:13.287 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:21:13.287 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:21:13.287 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:21:13.287 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:21:13.287 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:21:13.287 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:21:13.287 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:21:13.287 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:21:13.287 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:21:13.287 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:21:13.287 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:21:13.288 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:21:13.288 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:21:13.288 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:21:13.288 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:21:13.288 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:21:13.288 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:21:13.288 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:21:13.288 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:21:13.288 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:21:13.288 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:21:13.288 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:21:13.288 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:21:13.288 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:21:13.288 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:21:13.288 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:21:13.288 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:21:13.298 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:21:13.298 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:21:13.298 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:21:13.298 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:21:13.355 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:21:13.355 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:21:13.355 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:21:13.357 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:21:13.357 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:21:13.357 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:21:13.357 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:21:13.357 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:21:13.357 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:21:13.357 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:21:13.357 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:21:13.357 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:21:13.357 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:21:13.357 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:21:13.357 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:21:13.357 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:21:13.357 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:21:13.357 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:21:13.357 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:21:13.357 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:21:13.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:21:13.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:21:13.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:21:13.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:21:13.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:21:13.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:21:13.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:21:13.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:21:13.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:21:13.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:21:13.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:21:13.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:21:13.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:21:13.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:21:13.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:21:13.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:21:13.370 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:21:13.370 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:21:13.370 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:21:13.429 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:21:13.429 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:21:13.430 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:21:13.430 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:21:13.430 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:21:13.430 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:21:13.430 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:21:13.430 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:21:13.430 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:21:13.430 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:21:13.430 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:21:13.430 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:21:13.430 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:21:13.432 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:21:13.432 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:21:13.432 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:21:13.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 82 2026-02-18T08:21:13.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 84 2026-02-18T08:21:13.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:21:13.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=337, tag_tid=24, entry_tid=1], [object_number=336, tag_tid=24, entry_tid=0], [object_number=339, tag_tid=23, entry_tid=29391], [object_number=338, tag_tid=23, entry_tid=29390]]], state=connected] 2026-02-18T08:21:13.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=331, tag_tid=23, entry_tid=23531], [object_number=330, tag_tid=23, entry_tid=23530], [object_number=329, tag_tid=23, entry_tid=23529], [object_number=328, tag_tid=23, entry_tid=23528]]], state=connected] 2026-02-18T08:21:13.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:21:13.465 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=337, tag_tid=24, entry_tid=1' 2026-02-18T08:21:13.465 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:21:13.465 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:21:13.465 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:21:13.465 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:21:13.465 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:21:13.465 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:21:13.465 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:21:13.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:21:13.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:21:13.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:21:13.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:21:13.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:21:13.468 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:21:13.468 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:21:13.468 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:21:13.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 82 2026-02-18T08:21:13.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 84 2026-02-18T08:21:13.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:21:13.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=337, tag_tid=24, entry_tid=1], [object_number=336, tag_tid=24, entry_tid=0], [object_number=339, tag_tid=23, entry_tid=29391], [object_number=338, tag_tid=23, entry_tid=29390]]], state=connected] 2026-02-18T08:21:13.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=331, tag_tid=23, entry_tid=23531], [object_number=330, tag_tid=23, entry_tid=23530], [object_number=329, tag_tid=23, entry_tid=23529], [object_number=328, tag_tid=23, entry_tid=23528]]], state=connected] 2026-02-18T08:21:13.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:21:13.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=331, tag_tid=23, entry_tid=23531' 2026-02-18T08:21:13.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=337, tag_tid=24, entry_tid=1' -a 'object_number=337, tag_tid=24, entry_tid=1' = 'object_number=331, tag_tid=23, entry_tid=23531' 2026-02-18T08:21:13.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=331, tag_tid=23, entry_tid=23531' '!=' '' 2026-02-18T08:21:13.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:21:13.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=331, tag_tid=23, entry_tid=23531' = '' 2026-02-18T08:21:13.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=331, tag_tid=23, entry_tid=23531' 2026-02-18T08:21:13.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=337, tag_tid=24, entry_tid=1' 2026-02-18T08:21:13.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:21:13.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:21:13.501 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=24 2026-02-18T08:21:13.502 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=331, tag_tid=23, entry_tid=23531' 2026-02-18T08:21:13.502 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:21:13.502 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:21:13.503 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=23 2026-02-18T08:21:13.504 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=337, tag_tid=24, entry_tid=1' 2026-02-18T08:21:13.504 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:21:13.504 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:21:13.505 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:21:13.506 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=331, tag_tid=23, entry_tid=23531' 2026-02-18T08:21:13.506 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:21:13.506 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:21:13.508 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=23531 2026-02-18T08:21:13.508 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 24 = 23 -a 1 -le 23531 2026-02-18T08:21:13.508 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:21:13.508 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:21:13.508 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:21:13.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:21:13.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:21:13.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:21:13.711 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:21:13.711 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:21:13.711 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:21:13.711 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:21:13.711 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:21:13.711 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:21:13.711 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:21:13.711 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:21:13.711 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:21:13.711 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:21:13.711 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:21:13.712 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:21:13.712 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:21:13.714 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:21:13.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:21:13.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:21:13.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:21:13.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:21:13.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:21:13.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:21:13.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:21:13.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:21:13.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:21:13.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:21:13.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:21:13.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:21:13.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:21:13.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:21:13.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:21:13.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:21:13.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:21:13.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:21:13.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:21:13.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:21:13.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:21:13.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:21:13.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:21:13.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:21:13.717 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:21:13.717 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:21:13.717 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:21:13.717 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:21:13.717 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:21:13.717 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:21:13.717 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:21:13.717 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:21:13.724 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:21:13.724 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:21:13.724 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:21:13.724 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:21:13.785 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:21:13.786 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:21:13.786 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:21:13.788 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:21:13.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:21:13.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:21:13.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:21:13.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:21:13.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:21:13.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:21:13.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:21:13.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:21:13.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:21:13.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:21:13.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:21:13.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:21:13.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:21:13.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:21:13.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:21:13.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:21:13.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:21:13.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:21:13.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:21:13.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:21:13.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:21:13.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:21:13.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:21:13.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:21:13.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:21:13.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:21:13.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:21:13.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:21:13.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:21:13.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:21:13.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:21:13.799 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:21:13.799 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:21:13.799 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:21:13.799 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:21:13.847 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:21:13.847 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:21:13.847 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:21:13.850 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:21:13.850 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:21:13.850 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:21:13.850 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:21:13.850 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:21:13.850 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:21:13.850 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:21:13.850 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:21:13.850 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:21:13.850 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:21:13.850 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:21:13.851 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:21:13.851 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:21:13.851 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:21:13.851 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:21:13.851 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:21:13.851 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:21:13.851 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:21:13.851 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:21:13.851 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:21:13.851 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:21:13.851 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:21:13.851 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:21:13.852 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:21:13.852 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:21:13.852 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:21:13.852 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:21:13.852 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:21:13.852 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:21:13.852 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:21:13.852 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:21:13.852 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:21:13.861 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:21:13.861 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:21:13.862 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:21:13.862 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:21:13.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:21:13.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:21:13.909 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:21:13.911 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:21:13.911 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:21:13.911 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:21:13.911 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:21:13.912 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:21:13.912 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:21:13.912 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:21:13.912 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:21:13.912 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:21:13.912 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:21:13.912 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:21:13.912 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:21:13.912 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:21:13.912 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:21:13.912 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:21:13.912 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:21:13.913 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:21:13.913 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:21:13.913 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:21:13.913 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:21:13.913 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:21:13.913 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:21:13.913 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:21:13.913 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:21:13.913 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:21:13.913 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:21:13.913 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:21:13.913 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:21:13.914 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:21:13.914 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:21:13.914 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:21:13.914 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:21:13.922 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:21:13.922 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:21:13.922 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:21:13.922 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:21:13.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:21:13.982 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:21:13.982 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:21:13.982 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:21:13.982 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:21:13.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:21:13.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:21:13.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:21:13.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:21:13.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:21:13.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:21:13.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:21:13.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:21:13.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:21:13.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:21:13.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:21:14.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 82 2026-02-18T08:21:14.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 84 2026-02-18T08:21:14.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:21:14.012 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=337, tag_tid=24, entry_tid=1], [object_number=336, tag_tid=24, entry_tid=0], [object_number=339, tag_tid=23, entry_tid=29391], [object_number=338, tag_tid=23, entry_tid=29390]]], state=connected] 2026-02-18T08:21:14.012 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=330, tag_tid=23, entry_tid=23886], [object_number=329, tag_tid=23, entry_tid=23885], [object_number=328, tag_tid=23, entry_tid=23884], [object_number=331, tag_tid=23, entry_tid=23883]]], state=connected] 2026-02-18T08:21:14.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:21:14.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=337, tag_tid=24, entry_tid=1' 2026-02-18T08:21:14.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:21:14.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:21:14.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:21:14.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:21:14.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:21:14.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:21:14.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:21:14.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:21:14.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:21:14.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:21:14.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:21:14.018 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:21:14.018 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:21:14.019 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:21:14.019 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:21:14.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 82 2026-02-18T08:21:14.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 84 2026-02-18T08:21:14.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:21:14.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=337, tag_tid=24, entry_tid=1], [object_number=336, tag_tid=24, entry_tid=0], [object_number=339, tag_tid=23, entry_tid=29391], [object_number=338, tag_tid=23, entry_tid=29390]]], state=connected] 2026-02-18T08:21:14.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=330, tag_tid=23, entry_tid=23886], [object_number=329, tag_tid=23, entry_tid=23885], [object_number=328, tag_tid=23, entry_tid=23884], [object_number=331, tag_tid=23, entry_tid=23883]]], state=connected] 2026-02-18T08:21:14.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:21:14.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=330, tag_tid=23, entry_tid=23886' 2026-02-18T08:21:14.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=337, tag_tid=24, entry_tid=1' -a 'object_number=337, tag_tid=24, entry_tid=1' = 'object_number=330, tag_tid=23, entry_tid=23886' 2026-02-18T08:21:14.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=330, tag_tid=23, entry_tid=23886' '!=' 'object_number=331, tag_tid=23, entry_tid=23531' 2026-02-18T08:21:14.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:21:14.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=330, tag_tid=23, entry_tid=23886' = 'object_number=331, tag_tid=23, entry_tid=23531' 2026-02-18T08:21:14.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=330, tag_tid=23, entry_tid=23886' 2026-02-18T08:21:14.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=337, tag_tid=24, entry_tid=1' 2026-02-18T08:21:14.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:21:14.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:21:14.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=24 2026-02-18T08:21:14.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=330, tag_tid=23, entry_tid=23886' 2026-02-18T08:21:14.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:21:14.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:21:14.055 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=23 2026-02-18T08:21:14.056 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=337, tag_tid=24, entry_tid=1' 2026-02-18T08:21:14.056 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:21:14.056 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:21:14.058 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:21:14.058 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=330, tag_tid=23, entry_tid=23886' 2026-02-18T08:21:14.058 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:21:14.059 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:21:14.060 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=23886 2026-02-18T08:21:14.060 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 24 = 23 -a 1 -le 23886 2026-02-18T08:21:14.060 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:21:14.060 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:21:14.060 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:21:14.164 INFO:tasks.rbd_mirror_thrash:waiting for cluster2.client.mirror.0 2026-02-18T08:21:14.164 INFO:teuthology.orchestra.run:waiting for 600 2026-02-18T08:21:14.165 INFO:tasks.rbd_mirror_thrash:reviving cluster2.client.mirror.0 2026-02-18T08:21:14.165 INFO:tasks.rbd_mirror.cluster2.client.mirror.0:Restarting daemon 2026-02-18T08:21:14.165 DEBUG:teuthology.orchestra.run.trial163:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.0 2026-02-18T08:21:14.167 INFO:tasks.rbd_mirror.cluster2.client.mirror.0:Started 2026-02-18T08:21:14.167 INFO:tasks.rbd_mirror_thrash:waiting for 107.36154182227207 secs before thrashing 2026-02-18T08:21:14.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:21:14.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:21:14.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:21:14.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:21:14.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:21:14.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:21:14.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:21:14.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:21:14.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:21:14.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:21:14.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:21:14.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:21:14.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:21:14.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:21:14.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:21:14.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:21:14.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:21:14.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:21:14.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:21:14.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:21:14.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:21:14.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:21:14.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:21:14.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:21:14.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:21:14.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:21:14.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:21:14.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:21:14.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:21:14.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:21:14.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:21:14.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:21:14.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:21:14.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:21:14.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:21:14.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:21:14.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:21:14.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:21:14.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:21:14.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:21:14.269 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:21:14.269 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:21:14.269 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:21:14.269 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:21:14.269 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:21:14.269 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:21:14.269 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:21:14.269 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:21:14.269 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:21:14.275 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:21:14.275 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:21:14.276 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:21:14.276 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:21:14.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:21:14.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:21:14.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:21:14.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:21:14.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:21:14.344 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:21:14.344 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:21:14.344 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:21:14.344 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:21:14.344 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:21:14.344 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:21:14.344 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:21:14.344 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:21:14.344 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:21:14.344 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:21:14.344 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:21:14.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:21:14.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:21:14.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:21:14.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:21:14.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:21:14.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:21:14.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:21:14.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:21:14.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:21:14.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:21:14.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:21:14.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:21:14.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:21:14.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:21:14.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:21:14.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:21:14.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:21:14.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:21:14.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:21:14.354 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:21:14.355 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:21:14.355 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:21:14.355 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:21:14.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:21:14.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:21:14.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:21:14.412 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:21:14.413 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:21:14.413 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:21:14.413 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:21:14.413 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:21:14.413 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:21:14.413 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:21:14.413 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:21:14.413 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:21:14.413 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:21:14.413 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:21:14.413 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:21:14.413 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:21:14.413 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:21:14.413 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:21:14.413 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:21:14.414 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:21:14.414 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:21:14.414 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:21:14.414 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:21:14.414 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:21:14.414 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:21:14.414 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:21:14.414 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:21:14.414 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:21:14.414 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:21:14.414 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:21:14.414 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:21:14.414 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:21:14.414 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:21:14.414 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:21:14.414 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:21:14.423 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:21:14.423 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:21:14.423 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:21:14.423 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:21:14.478 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:21:14.478 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:21:14.478 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:21:14.481 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:21:14.481 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:21:14.481 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:21:14.482 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:21:14.482 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:21:14.482 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:21:14.482 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:21:14.482 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:21:14.482 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:21:14.482 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:21:14.482 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:21:14.482 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:21:14.482 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:21:14.482 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:21:14.482 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:21:14.483 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:21:14.483 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:21:14.483 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:21:14.483 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:21:14.483 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:21:14.483 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:21:14.483 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:21:14.483 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:21:14.483 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:21:14.483 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:21:14.483 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:21:14.484 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:21:14.484 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:21:14.484 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:21:14.484 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:21:14.484 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:21:14.484 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:21:14.491 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:21:14.491 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:21:14.491 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:21:14.492 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:21:14.551 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:21:14.552 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:21:14.552 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:21:14.552 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:21:14.552 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:21:14.552 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:21:14.552 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:21:14.552 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:21:14.552 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:21:14.553 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:21:14.553 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:21:14.553 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:21:14.553 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:21:14.553 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:21:14.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:21:14.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:21:14.582 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 82 2026-02-18T08:21:14.582 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 84 2026-02-18T08:21:14.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:21:14.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=337, tag_tid=24, entry_tid=1], [object_number=336, tag_tid=24, entry_tid=0], [object_number=339, tag_tid=23, entry_tid=29391], [object_number=338, tag_tid=23, entry_tid=29390]]], state=connected] 2026-02-18T08:21:14.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=330, tag_tid=23, entry_tid=24318], [object_number=329, tag_tid=23, entry_tid=24317], [object_number=328, tag_tid=23, entry_tid=24316], [object_number=331, tag_tid=23, entry_tid=24315]]], state=connected] 2026-02-18T08:21:14.585 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:21:14.587 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=337, tag_tid=24, entry_tid=1' 2026-02-18T08:21:14.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:21:14.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:21:14.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:21:14.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:21:14.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:21:14.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:21:14.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:21:14.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:21:14.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:21:14.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:21:14.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:21:14.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:21:14.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:21:14.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:21:14.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:21:14.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 82 2026-02-18T08:21:14.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 84 2026-02-18T08:21:14.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:21:14.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=337, tag_tid=24, entry_tid=1], [object_number=336, tag_tid=24, entry_tid=0], [object_number=339, tag_tid=23, entry_tid=29391], [object_number=338, tag_tid=23, entry_tid=29390]]], state=connected] 2026-02-18T08:21:14.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=330, tag_tid=23, entry_tid=24318], [object_number=329, tag_tid=23, entry_tid=24317], [object_number=328, tag_tid=23, entry_tid=24316], [object_number=331, tag_tid=23, entry_tid=24315]]], state=connected] 2026-02-18T08:21:14.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:21:14.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=330, tag_tid=23, entry_tid=24318' 2026-02-18T08:21:14.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=337, tag_tid=24, entry_tid=1' -a 'object_number=337, tag_tid=24, entry_tid=1' = 'object_number=330, tag_tid=23, entry_tid=24318' 2026-02-18T08:21:14.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=330, tag_tid=23, entry_tid=24318' '!=' 'object_number=330, tag_tid=23, entry_tid=23886' 2026-02-18T08:21:14.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:21:14.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=330, tag_tid=23, entry_tid=24318' = 'object_number=330, tag_tid=23, entry_tid=23886' 2026-02-18T08:21:14.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=330, tag_tid=23, entry_tid=24318' 2026-02-18T08:21:14.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=337, tag_tid=24, entry_tid=1' 2026-02-18T08:21:14.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:21:14.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:21:14.635 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=24 2026-02-18T08:21:14.635 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=330, tag_tid=23, entry_tid=24318' 2026-02-18T08:21:14.636 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:21:14.636 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:21:14.637 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=23 2026-02-18T08:21:14.638 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=337, tag_tid=24, entry_tid=1' 2026-02-18T08:21:14.638 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:21:14.638 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:21:14.639 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:21:14.640 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=330, tag_tid=23, entry_tid=24318' 2026-02-18T08:21:14.640 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:21:14.640 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:21:14.642 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=24318 2026-02-18T08:21:14.642 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 24 = 23 -a 1 -le 24318 2026-02-18T08:21:14.642 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:21:14.642 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:21:14.642 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:21:14.843 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:21:14.843 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:21:14.844 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:21:14.844 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:21:14.844 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:21:14.844 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:21:14.844 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:21:14.844 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:21:14.844 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:21:14.844 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:21:14.844 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:21:14.844 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:21:14.845 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:21:14.845 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:21:14.845 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:21:14.847 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:21:14.847 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:21:14.847 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:21:14.848 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:21:14.848 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:21:14.848 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:21:14.848 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:21:14.848 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:21:14.848 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:21:14.848 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:21:14.848 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:21:14.848 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:21:14.848 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:21:14.848 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:21:14.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:21:14.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:21:14.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:21:14.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:21:14.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:21:14.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:21:14.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:21:14.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:21:14.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:21:14.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:21:14.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:21:14.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:21:14.850 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:21:14.850 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:21:14.850 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:21:14.850 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:21:14.850 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:21:14.850 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:21:14.850 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:21:14.850 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:21:14.858 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:21:14.858 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:21:14.858 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:21:14.858 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:21:14.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:21:14.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:21:14.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:21:14.910 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:21:14.910 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:21:14.910 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:21:14.910 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:21:14.910 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:21:14.910 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:21:14.910 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:21:14.911 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:21:14.911 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:21:14.911 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:21:14.911 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:21:14.911 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:21:14.911 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:21:14.911 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:21:14.911 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:21:14.911 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:21:14.911 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:21:14.912 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:21:14.912 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:21:14.912 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:21:14.912 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:21:14.912 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:21:14.912 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:21:14.912 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:21:14.912 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:21:14.912 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:21:14.912 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:21:14.913 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:21:14.913 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:21:14.913 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:21:14.913 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:21:14.913 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:21:14.922 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:21:14.922 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:21:14.922 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:21:14.922 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:21:14.970 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:21:14.970 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:21:14.971 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:21:14.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:21:14.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:21:14.974 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:21:14.974 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:21:14.974 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:21:14.974 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:21:14.974 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:21:14.974 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:21:14.974 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:21:14.974 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:21:14.974 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:21:14.974 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:21:14.974 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:21:14.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:21:14.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:21:14.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:21:14.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:21:14.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:21:14.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:21:14.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:21:14.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:21:14.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:21:14.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:21:14.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:21:14.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:21:14.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:21:14.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:21:14.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:21:14.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:21:14.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:21:14.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:21:14.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:21:14.985 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:21:14.985 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:21:14.986 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:21:14.986 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:21:15.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:21:15.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:21:15.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:21:15.036 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:21:15.036 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:21:15.036 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:21:15.036 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:21:15.036 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:21:15.036 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:21:15.036 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:21:15.036 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:21:15.037 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:21:15.037 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:21:15.037 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:21:15.037 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:21:15.037 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:21:15.037 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:21:15.037 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:21:15.037 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:21:15.037 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:21:15.037 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:21:15.037 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:21:15.037 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:21:15.038 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:21:15.038 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:21:15.038 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:21:15.038 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:21:15.038 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:21:15.038 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:21:15.038 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:21:15.038 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:21:15.038 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:21:15.038 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:21:15.038 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:21:15.038 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:21:15.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:21:15.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:21:15.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:21:15.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:21:15.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:21:15.106 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:21:15.106 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:21:15.106 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:21:15.106 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:21:15.106 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:21:15.106 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:21:15.106 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:21:15.106 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:21:15.106 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:21:15.106 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:21:15.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:21:15.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:21:15.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:21:15.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:21:15.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:21:15.136 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 82 2026-02-18T08:21:15.136 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 84 2026-02-18T08:21:15.136 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:21:15.136 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=337, tag_tid=24, entry_tid=1], [object_number=336, tag_tid=24, entry_tid=0], [object_number=339, tag_tid=23, entry_tid=29391], [object_number=338, tag_tid=23, entry_tid=29390]]], state=connected] 2026-02-18T08:21:15.137 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=331, tag_tid=23, entry_tid=24687], [object_number=330, tag_tid=23, entry_tid=24686], [object_number=329, tag_tid=23, entry_tid=24685], [object_number=328, tag_tid=23, entry_tid=24684]]], state=connected] 2026-02-18T08:21:15.139 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:21:15.141 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=337, tag_tid=24, entry_tid=1' 2026-02-18T08:21:15.142 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:21:15.142 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:21:15.142 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:21:15.142 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:21:15.142 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:21:15.142 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:21:15.142 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:21:15.142 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:21:15.142 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:21:15.143 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:21:15.143 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:21:15.143 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:21:15.144 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:21:15.144 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:21:15.144 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:21:15.173 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 82 2026-02-18T08:21:15.174 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 84 2026-02-18T08:21:15.174 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:21:15.174 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=337, tag_tid=24, entry_tid=1], [object_number=336, tag_tid=24, entry_tid=0], [object_number=339, tag_tid=23, entry_tid=29391], [object_number=338, tag_tid=23, entry_tid=29390]]], state=connected] 2026-02-18T08:21:15.174 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=331, tag_tid=23, entry_tid=24687], [object_number=330, tag_tid=23, entry_tid=24686], [object_number=329, tag_tid=23, entry_tid=24685], [object_number=328, tag_tid=23, entry_tid=24684]]], state=connected] 2026-02-18T08:21:15.176 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:21:15.178 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=331, tag_tid=23, entry_tid=24687' 2026-02-18T08:21:15.179 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=337, tag_tid=24, entry_tid=1' -a 'object_number=337, tag_tid=24, entry_tid=1' = 'object_number=331, tag_tid=23, entry_tid=24687' 2026-02-18T08:21:15.179 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=331, tag_tid=23, entry_tid=24687' '!=' 'object_number=330, tag_tid=23, entry_tid=24318' 2026-02-18T08:21:15.179 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:21:15.179 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=331, tag_tid=23, entry_tid=24687' = 'object_number=330, tag_tid=23, entry_tid=24318' 2026-02-18T08:21:15.179 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=331, tag_tid=23, entry_tid=24687' 2026-02-18T08:21:15.180 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=337, tag_tid=24, entry_tid=1' 2026-02-18T08:21:15.180 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:21:15.180 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:21:15.180 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=24 2026-02-18T08:21:15.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=331, tag_tid=23, entry_tid=24687' 2026-02-18T08:21:15.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:21:15.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:21:15.183 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=23 2026-02-18T08:21:15.183 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=337, tag_tid=24, entry_tid=1' 2026-02-18T08:21:15.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:21:15.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:21:15.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:21:15.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=331, tag_tid=23, entry_tid=24687' 2026-02-18T08:21:15.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:21:15.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:21:15.187 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=24687 2026-02-18T08:21:15.188 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 24 = 23 -a 1 -le 24687 2026-02-18T08:21:15.188 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:21:15.188 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:21:15.188 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:21:15.388 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:21:15.388 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:21:15.388 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:21:15.389 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:21:15.389 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:21:15.389 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:21:15.389 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:21:15.389 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:21:15.389 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:21:15.389 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:21:15.389 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:21:15.389 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:21:15.389 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:21:15.389 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:21:15.389 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:21:15.389 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:21:15.390 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:21:15.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:21:15.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:21:15.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:21:15.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:21:15.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:21:15.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:21:15.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:21:15.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:21:15.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:21:15.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:21:15.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:21:15.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:21:15.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:21:15.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:21:15.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:21:15.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:21:15.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:21:15.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:21:15.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:21:15.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:21:15.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:21:15.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:21:15.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:21:15.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:21:15.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:21:15.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:21:15.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:21:15.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:21:15.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:21:15.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:21:15.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:21:15.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:21:15.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:21:15.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:21:15.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:21:15.405 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:21:15.454 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:21:15.454 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:21:15.454 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:21:15.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:21:15.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:21:15.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:21:15.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:21:15.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:21:15.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:21:15.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:21:15.458 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:21:15.458 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:21:15.458 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:21:15.458 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:21:15.458 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:21:15.458 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:21:15.458 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:21:15.458 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:21:15.458 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:21:15.458 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:21:15.458 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:21:15.458 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:21:15.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:21:15.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:21:15.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:21:15.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:21:15.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:21:15.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:21:15.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:21:15.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:21:15.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:21:15.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:21:15.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:21:15.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:21:15.460 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:21:15.468 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:21:15.468 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:21:15.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:21:15.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:21:15.526 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:21:15.527 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:21:15.527 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:21:15.529 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:21:15.529 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:21:15.529 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:21:15.529 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:21:15.529 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:21:15.530 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:21:15.530 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:21:15.530 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:21:15.530 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:21:15.530 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:21:15.530 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:21:15.530 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:21:15.530 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:21:15.530 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:21:15.530 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:21:15.530 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:21:15.530 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:21:15.531 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:21:15.531 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:21:15.531 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:21:15.531 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:21:15.531 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:21:15.531 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:21:15.531 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:21:15.531 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:21:15.531 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:21:15.531 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:21:15.531 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:21:15.531 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:21:15.532 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:21:15.532 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:21:15.532 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:21:15.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:21:15.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:21:15.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:21:15.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:21:15.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:21:15.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:21:15.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:21:15.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:21:15.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:21:15.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:21:15.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:21:15.594 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:21:15.594 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:21:15.594 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:21:15.594 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:21:15.594 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:21:15.594 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:21:15.594 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:21:15.594 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:21:15.594 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:21:15.594 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:21:15.594 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:21:15.594 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:21:15.595 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:21:15.595 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:21:15.595 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:21:15.595 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:21:15.595 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:21:15.595 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:21:15.595 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:21:15.595 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:21:15.595 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:21:15.595 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:21:15.595 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:21:15.595 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:21:15.596 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:21:15.596 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:21:15.596 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:21:15.596 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:21:15.605 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:21:15.606 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:21:15.606 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:21:15.606 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:21:15.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:21:15.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:21:15.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:21:15.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:21:15.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:21:15.667 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:21:15.667 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:21:15.667 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:21:15.667 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:21:15.667 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:21:15.667 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:21:15.667 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:21:15.667 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:21:15.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:21:15.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:21:15.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:21:15.697 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 83 2026-02-18T08:21:15.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 84 2026-02-18T08:21:15.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:21:15.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=337, tag_tid=24, entry_tid=1], [object_number=336, tag_tid=24, entry_tid=0], [object_number=339, tag_tid=23, entry_tid=29391], [object_number=338, tag_tid=23, entry_tid=29390]]], state=connected] 2026-02-18T08:21:15.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=334, tag_tid=23, entry_tid=24958], [object_number=333, tag_tid=23, entry_tid=24957], [object_number=332, tag_tid=23, entry_tid=24956], [object_number=335, tag_tid=23, entry_tid=24955]]], state=connected] 2026-02-18T08:21:15.700 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:21:15.702 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=337, tag_tid=24, entry_tid=1' 2026-02-18T08:21:15.703 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:21:15.704 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:21:15.704 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:21:15.704 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:21:15.704 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:21:15.704 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:21:15.704 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:21:15.704 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:21:15.704 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:21:15.704 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:21:15.704 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:21:15.704 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:21:15.705 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:21:15.705 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:21:15.705 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:21:15.735 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 83 2026-02-18T08:21:15.735 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 84 2026-02-18T08:21:15.735 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:21:15.735 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=337, tag_tid=24, entry_tid=1], [object_number=336, tag_tid=24, entry_tid=0], [object_number=339, tag_tid=23, entry_tid=29391], [object_number=338, tag_tid=23, entry_tid=29390]]], state=connected] 2026-02-18T08:21:15.735 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=334, tag_tid=23, entry_tid=24958], [object_number=333, tag_tid=23, entry_tid=24957], [object_number=332, tag_tid=23, entry_tid=24956], [object_number=335, tag_tid=23, entry_tid=24955]]], state=connected] 2026-02-18T08:21:15.737 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:21:15.739 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=334, tag_tid=23, entry_tid=24958' 2026-02-18T08:21:15.740 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=337, tag_tid=24, entry_tid=1' -a 'object_number=337, tag_tid=24, entry_tid=1' = 'object_number=334, tag_tid=23, entry_tid=24958' 2026-02-18T08:21:15.740 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=334, tag_tid=23, entry_tid=24958' '!=' 'object_number=331, tag_tid=23, entry_tid=24687' 2026-02-18T08:21:15.740 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:21:15.740 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=334, tag_tid=23, entry_tid=24958' = 'object_number=331, tag_tid=23, entry_tid=24687' 2026-02-18T08:21:15.740 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=334, tag_tid=23, entry_tid=24958' 2026-02-18T08:21:15.741 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=337, tag_tid=24, entry_tid=1' 2026-02-18T08:21:15.741 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:21:15.741 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:21:15.741 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=24 2026-02-18T08:21:15.742 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=334, tag_tid=23, entry_tid=24958' 2026-02-18T08:21:15.742 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:21:15.742 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:21:15.743 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=23 2026-02-18T08:21:15.744 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=337, tag_tid=24, entry_tid=1' 2026-02-18T08:21:15.744 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:21:15.745 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:21:15.746 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:21:15.746 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=334, tag_tid=23, entry_tid=24958' 2026-02-18T08:21:15.747 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:21:15.747 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:21:15.748 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=24958 2026-02-18T08:21:15.748 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 24 = 23 -a 1 -le 24958 2026-02-18T08:21:15.748 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:21:15.749 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:21:15.749 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:21:15.949 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:21:15.949 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:21:15.950 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:21:15.950 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:21:15.950 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:21:15.950 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:21:15.950 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:21:15.950 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:21:15.950 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:21:15.950 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:21:15.950 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:21:15.950 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:21:15.950 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:21:15.950 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:21:15.950 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:21:15.950 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:21:15.950 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:21:15.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:21:15.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:21:15.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:21:15.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:21:15.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:21:15.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:21:15.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:21:15.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:21:15.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:21:15.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:21:15.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:21:15.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:21:15.952 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:21:15.952 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:21:15.952 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:21:15.952 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:21:15.952 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:21:15.952 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:21:15.952 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:21:15.952 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:21:15.952 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:21:15.952 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:21:15.952 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:21:15.952 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:21:15.952 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:21:15.952 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:21:15.952 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:21:15.952 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:21:15.952 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:21:15.952 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:21:15.952 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:21:15.952 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:21:15.963 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:21:15.963 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:21:15.963 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:21:15.963 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:21:16.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:21:16.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:21:16.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:21:16.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:21:16.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:21:16.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:21:16.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:21:16.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:21:16.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:21:16.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:21:16.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:21:16.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:21:16.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:21:16.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:21:16.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:21:16.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:21:16.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:21:16.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:21:16.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:21:16.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:21:16.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:21:16.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:21:16.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:21:16.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:21:16.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:21:16.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:21:16.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:21:16.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:21:16.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:21:16.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:21:16.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:21:16.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:21:16.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:21:16.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:21:16.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:21:16.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:21:16.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:21:16.027 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:21:16.027 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:21:16.075 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:21:16.075 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:21:16.076 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:21:16.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:21:16.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:21:16.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:21:16.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:21:16.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:21:16.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:21:16.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:21:16.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:21:16.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:21:16.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:21:16.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:21:16.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:21:16.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:21:16.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:21:16.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:21:16.080 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:21:16.080 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:21:16.080 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:21:16.080 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:21:16.080 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:21:16.080 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:21:16.080 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:21:16.080 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:21:16.080 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:21:16.080 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:21:16.080 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:21:16.080 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:21:16.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:21:16.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:21:16.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:21:16.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:21:16.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:21:16.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:21:16.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:21:16.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:21:16.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:21:16.147 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:21:16.147 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:21:16.147 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:21:16.150 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:21:16.150 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:21:16.150 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:21:16.150 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:21:16.150 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:21:16.150 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:21:16.150 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:21:16.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:21:16.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:21:16.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:21:16.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:21:16.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:21:16.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:21:16.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:21:16.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:21:16.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:21:16.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:21:16.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:21:16.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:21:16.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:21:16.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:21:16.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:21:16.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:21:16.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:21:16.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:21:16.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:21:16.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:21:16.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:21:16.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:21:16.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:21:16.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:21:16.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:21:16.162 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:21:16.162 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:21:16.163 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:21:16.163 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:21:16.222 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:21:16.223 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:21:16.223 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:21:16.223 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:21:16.223 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:21:16.223 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:21:16.223 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:21:16.223 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:21:16.223 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:21:16.223 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:21:16.223 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:21:16.224 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:21:16.224 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:21:16.225 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:21:16.225 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:21:16.225 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:21:16.255 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 83 2026-02-18T08:21:16.255 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 84 2026-02-18T08:21:16.255 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:21:16.255 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=337, tag_tid=24, entry_tid=1], [object_number=336, tag_tid=24, entry_tid=0], [object_number=339, tag_tid=23, entry_tid=29391], [object_number=338, tag_tid=23, entry_tid=29390]]], state=connected] 2026-02-18T08:21:16.255 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=335, tag_tid=23, entry_tid=25315], [object_number=334, tag_tid=23, entry_tid=25314], [object_number=333, tag_tid=23, entry_tid=25313], [object_number=332, tag_tid=23, entry_tid=25312]]], state=connected] 2026-02-18T08:21:16.257 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:21:16.259 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=337, tag_tid=24, entry_tid=1' 2026-02-18T08:21:16.260 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:21:16.260 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:21:16.260 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:21:16.260 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:21:16.260 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:21:16.261 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:21:16.261 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:21:16.261 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:21:16.261 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:21:16.261 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:21:16.261 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:21:16.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:21:16.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:21:16.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:21:16.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:21:16.291 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 83 2026-02-18T08:21:16.292 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 84 2026-02-18T08:21:16.292 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:21:16.292 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=337, tag_tid=24, entry_tid=1], [object_number=336, tag_tid=24, entry_tid=0], [object_number=339, tag_tid=23, entry_tid=29391], [object_number=338, tag_tid=23, entry_tid=29390]]], state=connected] 2026-02-18T08:21:16.292 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=335, tag_tid=23, entry_tid=25315], [object_number=334, tag_tid=23, entry_tid=25314], [object_number=333, tag_tid=23, entry_tid=25313], [object_number=332, tag_tid=23, entry_tid=25312]]], state=connected] 2026-02-18T08:21:16.294 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:21:16.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=335, tag_tid=23, entry_tid=25315' 2026-02-18T08:21:16.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=337, tag_tid=24, entry_tid=1' -a 'object_number=337, tag_tid=24, entry_tid=1' = 'object_number=335, tag_tid=23, entry_tid=25315' 2026-02-18T08:21:16.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=335, tag_tid=23, entry_tid=25315' '!=' 'object_number=334, tag_tid=23, entry_tid=24958' 2026-02-18T08:21:16.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:21:16.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=335, tag_tid=23, entry_tid=25315' = 'object_number=334, tag_tid=23, entry_tid=24958' 2026-02-18T08:21:16.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=335, tag_tid=23, entry_tid=25315' 2026-02-18T08:21:16.297 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=337, tag_tid=24, entry_tid=1' 2026-02-18T08:21:16.297 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:21:16.297 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:21:16.297 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=24 2026-02-18T08:21:16.298 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=335, tag_tid=23, entry_tid=25315' 2026-02-18T08:21:16.298 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:21:16.298 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:21:16.300 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=23 2026-02-18T08:21:16.301 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=337, tag_tid=24, entry_tid=1' 2026-02-18T08:21:16.301 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:21:16.301 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:21:16.302 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:21:16.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=335, tag_tid=23, entry_tid=25315' 2026-02-18T08:21:16.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:21:16.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:21:16.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=25315 2026-02-18T08:21:16.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 24 = 23 -a 1 -le 25315 2026-02-18T08:21:16.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:21:16.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:21:16.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:21:16.507 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:21:16.507 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:21:16.507 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:21:16.507 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:21:16.507 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:21:16.507 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:21:16.507 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:21:16.507 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:21:16.507 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:21:16.508 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:21:16.508 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:21:16.508 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:21:16.508 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:21:16.508 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:21:16.508 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:21:16.508 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:21:16.511 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:21:16.511 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:21:16.511 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:21:16.511 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:21:16.511 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:21:16.511 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:21:16.511 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:21:16.511 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:21:16.511 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:21:16.512 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:21:16.512 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:21:16.512 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:21:16.512 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:21:16.512 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:21:16.512 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:21:16.512 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:21:16.512 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:21:16.512 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:21:16.512 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:21:16.512 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:21:16.512 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:21:16.513 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:21:16.513 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:21:16.513 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:21:16.513 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:21:16.513 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:21:16.513 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:21:16.513 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:21:16.513 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:21:16.513 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:21:16.513 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:21:16.513 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:21:16.513 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:21:16.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:21:16.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:21:16.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:21:16.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:21:16.584 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:21:16.584 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:21:16.584 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:21:16.587 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:21:16.587 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:21:16.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:21:16.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:21:16.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:21:16.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:21:16.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:21:16.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:21:16.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:21:16.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:21:16.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:21:16.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:21:16.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:21:16.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:21:16.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:21:16.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:21:16.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:21:16.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:21:16.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:21:16.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:21:16.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:21:16.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:21:16.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:21:16.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:21:16.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:21:16.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:21:16.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:21:16.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:21:16.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:21:16.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:21:16.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:21:16.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:21:16.598 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:21:16.599 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:21:16.599 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:21:16.599 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:21:16.649 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:21:16.649 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:21:16.649 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:21:16.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:21:16.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:21:16.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:21:16.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:21:16.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:21:16.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:21:16.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:21:16.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:21:16.653 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:21:16.653 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:21:16.653 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:21:16.653 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:21:16.653 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:21:16.653 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:21:16.653 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:21:16.653 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:21:16.653 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:21:16.653 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:21:16.653 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:21:16.653 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:21:16.654 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:21:16.654 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:21:16.654 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:21:16.654 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:21:16.654 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:21:16.654 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:21:16.654 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:21:16.654 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:21:16.654 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:21:16.654 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:21:16.654 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:21:16.654 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:21:16.663 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:21:16.663 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:21:16.663 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:21:16.663 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:21:16.712 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:21:16.712 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:21:16.712 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:21:16.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:21:16.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:21:16.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:21:16.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:21:16.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:21:16.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:21:16.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:21:16.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:21:16.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:21:16.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:21:16.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:21:16.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:21:16.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:21:16.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:21:16.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:21:16.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:21:16.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:21:16.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:21:16.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:21:16.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:21:16.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:21:16.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:21:16.717 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:21:16.717 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:21:16.717 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:21:16.717 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:21:16.717 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:21:16.717 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:21:16.717 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:21:16.717 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:21:16.717 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:21:16.717 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:21:16.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:21:16.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:21:16.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:21:16.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:21:16.793 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:21:16.794 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:21:16.794 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:21:16.794 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:21:16.794 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:21:16.794 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:21:16.794 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:21:16.794 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:21:16.794 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:21:16.795 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:21:16.795 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:21:16.795 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:21:16.795 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:21:16.796 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:21:16.796 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:21:16.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:21:16.832 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 83 2026-02-18T08:21:16.832 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 84 2026-02-18T08:21:16.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:21:16.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=337, tag_tid=24, entry_tid=1], [object_number=336, tag_tid=24, entry_tid=0], [object_number=339, tag_tid=23, entry_tid=29391], [object_number=338, tag_tid=23, entry_tid=29390]]], state=connected] 2026-02-18T08:21:16.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=334, tag_tid=23, entry_tid=25702], [object_number=333, tag_tid=23, entry_tid=25701], [object_number=332, tag_tid=23, entry_tid=25700], [object_number=335, tag_tid=23, entry_tid=25699]]], state=connected] 2026-02-18T08:21:16.836 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:21:16.838 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=337, tag_tid=24, entry_tid=1' 2026-02-18T08:21:16.839 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:21:16.839 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:21:16.839 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:21:16.840 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:21:16.840 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:21:16.840 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:21:16.840 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:21:16.840 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:21:16.840 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:21:16.840 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:21:16.841 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:21:16.841 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:21:16.841 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:21:16.841 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:21:16.842 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:21:16.876 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 83 2026-02-18T08:21:16.876 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 84 2026-02-18T08:21:16.876 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:21:16.877 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=337, tag_tid=24, entry_tid=1], [object_number=336, tag_tid=24, entry_tid=0], [object_number=339, tag_tid=23, entry_tid=29391], [object_number=338, tag_tid=23, entry_tid=29390]]], state=connected] 2026-02-18T08:21:16.877 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=334, tag_tid=23, entry_tid=25702], [object_number=333, tag_tid=23, entry_tid=25701], [object_number=332, tag_tid=23, entry_tid=25700], [object_number=335, tag_tid=23, entry_tid=25699]]], state=connected] 2026-02-18T08:21:16.880 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:21:16.882 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=334, tag_tid=23, entry_tid=25702' 2026-02-18T08:21:16.882 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=337, tag_tid=24, entry_tid=1' -a 'object_number=337, tag_tid=24, entry_tid=1' = 'object_number=334, tag_tid=23, entry_tid=25702' 2026-02-18T08:21:16.883 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=334, tag_tid=23, entry_tid=25702' '!=' 'object_number=335, tag_tid=23, entry_tid=25315' 2026-02-18T08:21:16.883 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:21:16.883 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=334, tag_tid=23, entry_tid=25702' = 'object_number=335, tag_tid=23, entry_tid=25315' 2026-02-18T08:21:16.883 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=334, tag_tid=23, entry_tid=25702' 2026-02-18T08:21:16.884 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=337, tag_tid=24, entry_tid=1' 2026-02-18T08:21:16.884 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:21:16.884 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:21:16.884 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=24 2026-02-18T08:21:16.885 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=334, tag_tid=23, entry_tid=25702' 2026-02-18T08:21:16.885 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:21:16.886 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:21:16.887 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=23 2026-02-18T08:21:16.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=337, tag_tid=24, entry_tid=1' 2026-02-18T08:21:16.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:21:16.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:21:16.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:21:16.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=334, tag_tid=23, entry_tid=25702' 2026-02-18T08:21:16.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:21:16.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:21:16.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=25702 2026-02-18T08:21:16.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 24 = 23 -a 1 -le 25702 2026-02-18T08:21:16.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:21:16.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:21:16.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:21:17.094 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:21:17.094 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:21:17.094 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:21:17.094 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:21:17.094 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:21:17.094 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:21:17.094 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:21:17.094 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:21:17.095 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:21:17.095 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:21:17.095 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:21:17.095 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:21:17.095 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:21:17.095 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:21:17.095 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:21:17.095 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:21:17.095 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:21:17.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:21:17.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:21:17.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:21:17.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:21:17.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:21:17.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:21:17.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:21:17.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:21:17.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:21:17.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:21:17.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:21:17.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:21:17.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:21:17.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:21:17.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:21:17.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:21:17.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:21:17.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:21:17.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:21:17.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:21:17.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:21:17.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:21:17.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:21:17.100 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:21:17.100 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:21:17.100 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:21:17.100 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:21:17.100 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:21:17.100 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:21:17.100 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:21:17.100 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:21:17.100 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:21:17.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:21:17.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:21:17.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:21:17.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:21:17.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:21:17.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:21:17.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:21:17.171 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:21:17.171 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:21:17.171 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:21:17.171 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:21:17.171 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:21:17.171 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:21:17.171 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:21:17.171 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:21:17.171 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:21:17.171 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:21:17.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:21:17.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:21:17.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:21:17.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:21:17.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:21:17.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:21:17.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:21:17.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:21:17.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:21:17.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:21:17.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:21:17.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:21:17.173 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:21:17.173 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:21:17.173 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:21:17.173 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:21:17.173 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:21:17.173 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:21:17.173 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:21:17.173 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:21:17.173 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:21:17.173 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:21:17.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:21:17.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:21:17.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:21:17.183 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:21:17.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:21:17.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:21:17.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:21:17.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:21:17.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:21:17.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:21:17.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:21:17.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:21:17.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:21:17.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:21:17.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:21:17.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:21:17.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:21:17.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:21:17.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:21:17.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:21:17.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:21:17.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:21:17.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:21:17.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:21:17.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:21:17.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:21:17.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:21:17.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:21:17.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:21:17.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:21:17.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:21:17.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:21:17.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:21:17.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:21:17.238 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:21:17.238 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:21:17.238 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:21:17.238 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:21:17.238 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:21:17.247 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:21:17.247 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:21:17.247 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:21:17.248 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:21:17.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:21:17.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:21:17.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:21:17.298 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:21:17.299 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:21:17.299 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:21:17.299 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:21:17.299 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:21:17.299 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:21:17.299 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:21:17.299 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:21:17.299 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:21:17.299 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:21:17.299 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:21:17.300 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:21:17.300 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:21:17.300 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:21:17.300 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:21:17.300 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:21:17.300 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:21:17.300 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:21:17.300 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:21:17.300 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:21:17.300 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:21:17.300 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:21:17.300 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:21:17.301 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:21:17.301 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:21:17.301 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:21:17.301 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:21:17.301 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:21:17.301 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:21:17.301 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:21:17.301 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:21:17.301 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:21:17.310 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:21:17.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:21:17.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:21:17.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:21:17.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:21:17.370 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:21:17.370 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:21:17.370 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:21:17.371 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:21:17.371 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:21:17.371 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:21:17.371 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:21:17.371 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:21:17.371 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:21:17.371 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:21:17.371 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:21:17.372 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:21:17.372 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:21:17.373 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:21:17.373 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:21:17.399 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 83 2026-02-18T08:21:17.399 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 84 2026-02-18T08:21:17.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:21:17.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=337, tag_tid=24, entry_tid=1], [object_number=336, tag_tid=24, entry_tid=0], [object_number=339, tag_tid=23, entry_tid=29391], [object_number=338, tag_tid=23, entry_tid=29390]]], state=connected] 2026-02-18T08:21:17.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=334, tag_tid=23, entry_tid=26110], [object_number=333, tag_tid=23, entry_tid=26109], [object_number=332, tag_tid=23, entry_tid=26108], [object_number=335, tag_tid=23, entry_tid=26107]]], state=connected] 2026-02-18T08:21:17.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:21:17.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=337, tag_tid=24, entry_tid=1' 2026-02-18T08:21:17.405 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:21:17.405 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:21:17.405 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:21:17.405 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:21:17.405 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:21:17.405 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:21:17.405 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:21:17.406 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:21:17.406 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:21:17.406 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:21:17.406 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:21:17.406 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:21:17.407 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:21:17.407 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:21:17.407 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:21:17.434 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 83 2026-02-18T08:21:17.434 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 84 2026-02-18T08:21:17.434 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:21:17.434 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=337, tag_tid=24, entry_tid=1], [object_number=336, tag_tid=24, entry_tid=0], [object_number=339, tag_tid=23, entry_tid=29391], [object_number=338, tag_tid=23, entry_tid=29390]]], state=connected] 2026-02-18T08:21:17.434 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=334, tag_tid=23, entry_tid=26110], [object_number=333, tag_tid=23, entry_tid=26109], [object_number=332, tag_tid=23, entry_tid=26108], [object_number=335, tag_tid=23, entry_tid=26107]]], state=connected] 2026-02-18T08:21:17.437 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:21:17.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=334, tag_tid=23, entry_tid=26110' 2026-02-18T08:21:17.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=337, tag_tid=24, entry_tid=1' -a 'object_number=337, tag_tid=24, entry_tid=1' = 'object_number=334, tag_tid=23, entry_tid=26110' 2026-02-18T08:21:17.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=334, tag_tid=23, entry_tid=26110' '!=' 'object_number=334, tag_tid=23, entry_tid=25702' 2026-02-18T08:21:17.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:21:17.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=334, tag_tid=23, entry_tid=26110' = 'object_number=334, tag_tid=23, entry_tid=25702' 2026-02-18T08:21:17.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=334, tag_tid=23, entry_tid=26110' 2026-02-18T08:21:17.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=337, tag_tid=24, entry_tid=1' 2026-02-18T08:21:17.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:21:17.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:21:17.440 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=24 2026-02-18T08:21:17.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=334, tag_tid=23, entry_tid=26110' 2026-02-18T08:21:17.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:21:17.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:21:17.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=23 2026-02-18T08:21:17.443 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=337, tag_tid=24, entry_tid=1' 2026-02-18T08:21:17.443 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:21:17.443 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:21:17.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:21:17.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=334, tag_tid=23, entry_tid=26110' 2026-02-18T08:21:17.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:21:17.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:21:17.446 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=26110 2026-02-18T08:21:17.446 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 24 = 23 -a 1 -le 26110 2026-02-18T08:21:17.446 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:21:17.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:21:17.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:21:17.649 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:21:17.649 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:21:17.649 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:21:17.649 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:21:17.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:21:17.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:21:17.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:21:17.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:21:17.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:21:17.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:21:17.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:21:17.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:21:17.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:21:17.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:21:17.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:21:17.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:21:17.653 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:21:17.653 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:21:17.653 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:21:17.653 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:21:17.654 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:21:17.654 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:21:17.654 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:21:17.654 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:21:17.654 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:21:17.654 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:21:17.654 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:21:17.654 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:21:17.654 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:21:17.654 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:21:17.654 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:21:17.654 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:21:17.655 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:21:17.655 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:21:17.655 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:21:17.655 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:21:17.655 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:21:17.655 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:21:17.655 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:21:17.655 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:21:17.655 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:21:17.655 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:21:17.655 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:21:17.655 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:21:17.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:21:17.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:21:17.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:21:17.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:21:17.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:21:17.664 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:21:17.664 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:21:17.664 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:21:17.664 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:21:17.725 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:21:17.725 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:21:17.725 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:21:17.728 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:21:17.728 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:21:17.728 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:21:17.728 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:21:17.728 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:21:17.728 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:21:17.728 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:21:17.728 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:21:17.729 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:21:17.729 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:21:17.729 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:21:17.729 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:21:17.729 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:21:17.729 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:21:17.729 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:21:17.729 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:21:17.729 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:21:17.729 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:21:17.729 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:21:17.729 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:21:17.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:21:17.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:21:17.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:21:17.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:21:17.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:21:17.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:21:17.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:21:17.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:21:17.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:21:17.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:21:17.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:21:17.731 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:21:17.739 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:21:17.739 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:21:17.739 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:21:17.739 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:21:17.787 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:21:17.787 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:21:17.787 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:21:17.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:21:17.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:21:17.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:21:17.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:21:17.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:21:17.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:21:17.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:21:17.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:21:17.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:21:17.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:21:17.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:21:17.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:21:17.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:21:17.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:21:17.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:21:17.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:21:17.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:21:17.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:21:17.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:21:17.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:21:17.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:21:17.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:21:17.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:21:17.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:21:17.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:21:17.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:21:17.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:21:17.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:21:17.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:21:17.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:21:17.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:21:17.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:21:17.801 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:21:17.802 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:21:17.802 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:21:17.802 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:21:17.860 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:21:17.860 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:21:17.860 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:21:17.863 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:21:17.863 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:21:17.863 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:21:17.863 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:21:17.863 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:21:17.863 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:21:17.864 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:21:17.864 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:21:17.864 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:21:17.864 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:21:17.864 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:21:17.864 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:21:17.864 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:21:17.864 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:21:17.864 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:21:17.864 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:21:17.864 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:21:17.864 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:21:17.865 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:21:17.865 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:21:17.865 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:21:17.865 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:21:17.865 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:21:17.865 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:21:17.865 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:21:17.865 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:21:17.865 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:21:17.865 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:21:17.865 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:21:17.865 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:21:17.866 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:21:17.866 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:21:17.875 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:21:17.875 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:21:17.875 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:21:17.875 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:21:17.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:21:17.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:21:17.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:21:17.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:21:17.936 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:21:17.936 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:21:17.936 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:21:17.936 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:21:17.936 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:21:17.936 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:21:17.936 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:21:17.936 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:21:17.936 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:21:17.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:21:17.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:21:17.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:21:17.965 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 83 2026-02-18T08:21:17.965 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 84 2026-02-18T08:21:17.965 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:21:17.966 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=337, tag_tid=24, entry_tid=1], [object_number=336, tag_tid=24, entry_tid=0], [object_number=339, tag_tid=23, entry_tid=29391], [object_number=338, tag_tid=23, entry_tid=29390]]], state=connected] 2026-02-18T08:21:17.966 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=334, tag_tid=23, entry_tid=26478], [object_number=333, tag_tid=23, entry_tid=26477], [object_number=332, tag_tid=23, entry_tid=26476], [object_number=335, tag_tid=23, entry_tid=26475]]], state=connected] 2026-02-18T08:21:17.968 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:21:17.969 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=337, tag_tid=24, entry_tid=1' 2026-02-18T08:21:17.971 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:21:17.971 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:21:17.971 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:21:17.971 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:21:17.971 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:21:17.971 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:21:17.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:21:17.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:21:17.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:21:17.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:21:17.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:21:17.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:21:17.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:21:17.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:21:17.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:21:18.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 83 2026-02-18T08:21:18.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 84 2026-02-18T08:21:18.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:21:18.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=337, tag_tid=24, entry_tid=1], [object_number=336, tag_tid=24, entry_tid=0], [object_number=339, tag_tid=23, entry_tid=29391], [object_number=338, tag_tid=23, entry_tid=29390]]], state=connected] 2026-02-18T08:21:18.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=334, tag_tid=23, entry_tid=26478], [object_number=333, tag_tid=23, entry_tid=26477], [object_number=332, tag_tid=23, entry_tid=26476], [object_number=335, tag_tid=23, entry_tid=26475]]], state=connected] 2026-02-18T08:21:18.004 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:21:18.007 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=334, tag_tid=23, entry_tid=26478' 2026-02-18T08:21:18.007 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=337, tag_tid=24, entry_tid=1' -a 'object_number=337, tag_tid=24, entry_tid=1' = 'object_number=334, tag_tid=23, entry_tid=26478' 2026-02-18T08:21:18.007 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=334, tag_tid=23, entry_tid=26478' '!=' 'object_number=334, tag_tid=23, entry_tid=26110' 2026-02-18T08:21:18.007 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:21:18.007 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=334, tag_tid=23, entry_tid=26478' = 'object_number=334, tag_tid=23, entry_tid=26110' 2026-02-18T08:21:18.007 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=334, tag_tid=23, entry_tid=26478' 2026-02-18T08:21:18.007 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=337, tag_tid=24, entry_tid=1' 2026-02-18T08:21:18.008 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:21:18.008 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:21:18.008 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=24 2026-02-18T08:21:18.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=334, tag_tid=23, entry_tid=26478' 2026-02-18T08:21:18.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:21:18.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:21:18.010 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=23 2026-02-18T08:21:18.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=337, tag_tid=24, entry_tid=1' 2026-02-18T08:21:18.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:21:18.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:21:18.012 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:21:18.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=334, tag_tid=23, entry_tid=26478' 2026-02-18T08:21:18.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:21:18.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:21:18.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=26478 2026-02-18T08:21:18.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 24 = 23 -a 1 -le 26478 2026-02-18T08:21:18.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:21:18.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:21:18.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:21:18.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:21:18.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:21:18.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:21:18.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:21:18.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:21:18.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:21:18.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:21:18.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:21:18.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:21:18.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:21:18.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:21:18.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:21:18.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:21:18.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:21:18.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:21:18.217 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:21:18.217 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:21:18.218 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:21:18.218 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:21:18.218 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:21:18.219 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:21:18.219 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:21:18.219 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:21:18.219 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:21:18.219 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:21:18.219 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:21:18.219 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:21:18.219 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:21:18.219 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:21:18.219 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:21:18.219 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:21:18.219 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:21:18.219 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:21:18.219 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:21:18.219 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:21:18.219 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:21:18.219 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:21:18.219 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:21:18.219 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:21:18.219 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:21:18.220 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:21:18.220 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:21:18.220 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:21:18.220 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:21:18.220 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:21:18.220 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:21:18.220 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:21:18.220 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:21:18.220 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:21:18.230 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:21:18.231 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:21:18.231 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:21:18.231 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:21:18.280 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:21:18.280 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:21:18.280 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:21:18.283 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:21:18.283 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:21:18.283 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:21:18.283 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:21:18.284 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:21:18.284 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:21:18.284 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:21:18.284 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:21:18.284 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:21:18.284 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:21:18.284 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:21:18.284 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:21:18.284 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:21:18.284 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:21:18.284 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:21:18.284 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:21:18.284 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:21:18.285 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:21:18.285 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:21:18.285 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:21:18.285 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:21:18.285 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:21:18.285 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:21:18.285 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:21:18.285 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:21:18.285 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:21:18.285 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:21:18.285 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:21:18.285 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:21:18.286 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:21:18.286 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:21:18.286 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:21:18.294 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:21:18.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:21:18.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:21:18.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:21:18.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:21:18.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:21:18.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:21:18.361 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:21:18.361 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:21:18.361 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:21:18.361 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:21:18.361 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:21:18.362 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:21:18.362 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:21:18.362 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:21:18.362 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:21:18.362 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:21:18.362 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:21:18.362 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:21:18.362 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:21:18.362 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:21:18.362 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:21:18.362 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:21:18.362 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:21:18.363 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:21:18.363 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:21:18.363 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:21:18.363 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:21:18.363 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:21:18.363 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:21:18.363 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:21:18.363 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:21:18.363 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:21:18.363 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:21:18.363 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:21:18.363 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:21:18.364 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:21:18.364 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:21:18.364 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:21:18.371 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:21:18.371 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:21:18.371 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:21:18.372 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:21:18.420 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:21:18.421 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:21:18.421 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:21:18.423 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:21:18.423 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:21:18.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:21:18.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:21:18.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:21:18.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:21:18.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:21:18.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:21:18.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:21:18.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:21:18.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:21:18.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:21:18.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:21:18.425 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:21:18.425 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:21:18.425 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:21:18.425 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:21:18.425 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:21:18.425 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:21:18.425 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:21:18.425 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:21:18.425 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:21:18.425 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:21:18.425 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:21:18.425 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:21:18.425 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:21:18.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:21:18.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:21:18.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:21:18.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:21:18.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:21:18.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:21:18.434 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:21:18.434 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:21:18.434 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:21:18.434 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:21:18.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:21:18.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:21:18.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:21:18.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:21:18.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:21:18.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:21:18.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:21:18.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:21:18.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:21:18.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:21:18.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:21:18.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:21:18.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:21:18.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:21:18.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:21:18.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:21:18.534 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 83 2026-02-18T08:21:18.534 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 84 2026-02-18T08:21:18.534 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:21:18.534 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=337, tag_tid=24, entry_tid=1], [object_number=336, tag_tid=24, entry_tid=0], [object_number=339, tag_tid=23, entry_tid=29391], [object_number=338, tag_tid=23, entry_tid=29390]]], state=connected] 2026-02-18T08:21:18.534 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=334, tag_tid=23, entry_tid=26870], [object_number=333, tag_tid=23, entry_tid=26869], [object_number=332, tag_tid=23, entry_tid=26868], [object_number=335, tag_tid=23, entry_tid=26867]]], state=connected] 2026-02-18T08:21:18.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:21:18.539 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=337, tag_tid=24, entry_tid=1' 2026-02-18T08:21:18.540 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:21:18.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:21:18.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:21:18.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:21:18.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:21:18.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:21:18.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:21:18.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:21:18.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:21:18.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:21:18.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:21:18.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:21:18.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:21:18.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:21:18.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:21:18.570 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 83 2026-02-18T08:21:18.570 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 84 2026-02-18T08:21:18.571 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:21:18.571 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=337, tag_tid=24, entry_tid=1], [object_number=336, tag_tid=24, entry_tid=0], [object_number=339, tag_tid=23, entry_tid=29391], [object_number=338, tag_tid=23, entry_tid=29390]]], state=connected] 2026-02-18T08:21:18.571 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=334, tag_tid=23, entry_tid=26870], [object_number=333, tag_tid=23, entry_tid=26869], [object_number=332, tag_tid=23, entry_tid=26868], [object_number=335, tag_tid=23, entry_tid=26867]]], state=connected] 2026-02-18T08:21:18.573 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:21:18.575 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=334, tag_tid=23, entry_tid=26870' 2026-02-18T08:21:18.575 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=337, tag_tid=24, entry_tid=1' -a 'object_number=337, tag_tid=24, entry_tid=1' = 'object_number=334, tag_tid=23, entry_tid=26870' 2026-02-18T08:21:18.575 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=334, tag_tid=23, entry_tid=26870' '!=' 'object_number=334, tag_tid=23, entry_tid=26478' 2026-02-18T08:21:18.576 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:21:18.576 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=334, tag_tid=23, entry_tid=26870' = 'object_number=334, tag_tid=23, entry_tid=26478' 2026-02-18T08:21:18.576 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=334, tag_tid=23, entry_tid=26870' 2026-02-18T08:21:18.576 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=337, tag_tid=24, entry_tid=1' 2026-02-18T08:21:18.577 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:21:18.577 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:21:18.577 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=24 2026-02-18T08:21:18.578 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=334, tag_tid=23, entry_tid=26870' 2026-02-18T08:21:18.578 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:21:18.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:21:18.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=23 2026-02-18T08:21:18.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=337, tag_tid=24, entry_tid=1' 2026-02-18T08:21:18.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:21:18.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:21:18.582 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:21:18.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=334, tag_tid=23, entry_tid=26870' 2026-02-18T08:21:18.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:21:18.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:21:18.585 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=26870 2026-02-18T08:21:18.585 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 24 = 23 -a 1 -le 26870 2026-02-18T08:21:18.585 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:21:18.585 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:21:18.585 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:21:18.787 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:21:18.787 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:21:18.787 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:21:18.787 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:21:18.787 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:21:18.787 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:21:18.787 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:21:18.788 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:21:18.788 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:21:18.788 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:21:18.788 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:21:18.788 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:21:18.788 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:21:18.788 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:21:18.788 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:21:18.788 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:21:18.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:21:18.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:21:18.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:21:18.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:21:18.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:21:18.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:21:18.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:21:18.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:21:18.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:21:18.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:21:18.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:21:18.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:21:18.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:21:18.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:21:18.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:21:18.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:21:18.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:21:18.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:21:18.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:21:18.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:21:18.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:21:18.793 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:21:18.793 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:21:18.793 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:21:18.793 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:21:18.793 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:21:18.793 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:21:18.793 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:21:18.793 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:21:18.793 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:21:18.793 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:21:18.793 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:21:18.793 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:21:18.802 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:21:18.802 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:21:18.802 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:21:18.802 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:21:18.852 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:21:18.852 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:21:18.853 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:21:18.855 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:21:18.856 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:21:18.856 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:21:18.856 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:21:18.856 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:21:18.856 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:21:18.856 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:21:18.856 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:21:18.856 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:21:18.856 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:21:18.856 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:21:18.857 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:21:18.857 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:21:18.857 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:21:18.857 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:21:18.857 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:21:18.857 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:21:18.857 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:21:18.857 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:21:18.857 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:21:18.857 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:21:18.857 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:21:18.857 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:21:18.858 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:21:18.858 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:21:18.858 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:21:18.858 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:21:18.858 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:21:18.858 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:21:18.858 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:21:18.858 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:21:18.858 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:21:18.867 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:21:18.867 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:21:18.868 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:21:18.868 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:21:18.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:21:18.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:21:18.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:21:18.930 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:21:18.930 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:21:18.930 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:21:18.930 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:21:18.930 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:21:18.930 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:21:18.930 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:21:18.930 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:21:18.931 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:21:18.931 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:21:18.931 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:21:18.931 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:21:18.931 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:21:18.931 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:21:18.931 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:21:18.931 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:21:18.931 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:21:18.931 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:21:18.931 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:21:18.931 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:21:18.931 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:21:18.932 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:21:18.932 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:21:18.932 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:21:18.932 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:21:18.932 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:21:18.932 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:21:18.932 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:21:18.932 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:21:18.932 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:21:18.932 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:21:18.932 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:21:18.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:21:18.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:21:18.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:21:18.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:21:18.992 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:21:18.992 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:21:18.992 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:21:18.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:21:18.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:21:18.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:21:18.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:21:18.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:21:18.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:21:18.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:21:18.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:21:18.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:21:18.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:21:18.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:21:18.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:21:18.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:21:18.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:21:18.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:21:18.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:21:18.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:21:18.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:21:18.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:21:18.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:21:18.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:21:18.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:21:18.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:21:18.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:21:18.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:21:18.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:21:18.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:21:18.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:21:18.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:21:18.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:21:18.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:21:18.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:21:19.005 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:21:19.006 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:21:19.006 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:21:19.006 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:21:19.075 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:21:19.076 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:21:19.076 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:21:19.076 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:21:19.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:21:19.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:21:19.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:21:19.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:21:19.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:21:19.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:21:19.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:21:19.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:21:19.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:21:19.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:21:19.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:21:19.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:21:19.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 83 2026-02-18T08:21:19.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 84 2026-02-18T08:21:19.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:21:19.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=337, tag_tid=24, entry_tid=1], [object_number=336, tag_tid=24, entry_tid=0], [object_number=339, tag_tid=23, entry_tid=29391], [object_number=338, tag_tid=23, entry_tid=29390]]], state=connected] 2026-02-18T08:21:19.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=335, tag_tid=23, entry_tid=27223], [object_number=334, tag_tid=23, entry_tid=27222], [object_number=333, tag_tid=23, entry_tid=27221], [object_number=332, tag_tid=23, entry_tid=27220]]], state=connected] 2026-02-18T08:21:19.111 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:21:19.112 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=337, tag_tid=24, entry_tid=1' 2026-02-18T08:21:19.114 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:21:19.114 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:21:19.114 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:21:19.114 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:21:19.114 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:21:19.114 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:21:19.114 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:21:19.114 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:21:19.114 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:21:19.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:21:19.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:21:19.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:21:19.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:21:19.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:21:19.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:21:19.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 83 2026-02-18T08:21:19.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 84 2026-02-18T08:21:19.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:21:19.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=337, tag_tid=24, entry_tid=1], [object_number=336, tag_tid=24, entry_tid=0], [object_number=339, tag_tid=23, entry_tid=29391], [object_number=338, tag_tid=23, entry_tid=29390]]], state=connected] 2026-02-18T08:21:19.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=335, tag_tid=23, entry_tid=27223], [object_number=334, tag_tid=23, entry_tid=27222], [object_number=333, tag_tid=23, entry_tid=27221], [object_number=332, tag_tid=23, entry_tid=27220]]], state=connected] 2026-02-18T08:21:19.158 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:21:19.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=335, tag_tid=23, entry_tid=27223' 2026-02-18T08:21:19.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=337, tag_tid=24, entry_tid=1' -a 'object_number=337, tag_tid=24, entry_tid=1' = 'object_number=335, tag_tid=23, entry_tid=27223' 2026-02-18T08:21:19.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=335, tag_tid=23, entry_tid=27223' '!=' 'object_number=334, tag_tid=23, entry_tid=26870' 2026-02-18T08:21:19.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:21:19.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=335, tag_tid=23, entry_tid=27223' = 'object_number=334, tag_tid=23, entry_tid=26870' 2026-02-18T08:21:19.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=335, tag_tid=23, entry_tid=27223' 2026-02-18T08:21:19.161 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=337, tag_tid=24, entry_tid=1' 2026-02-18T08:21:19.161 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:21:19.161 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:21:19.162 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=24 2026-02-18T08:21:19.162 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=335, tag_tid=23, entry_tid=27223' 2026-02-18T08:21:19.163 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:21:19.163 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:21:19.164 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=23 2026-02-18T08:21:19.165 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=337, tag_tid=24, entry_tid=1' 2026-02-18T08:21:19.165 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:21:19.165 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:21:19.166 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:21:19.167 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=335, tag_tid=23, entry_tid=27223' 2026-02-18T08:21:19.167 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:21:19.167 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:21:19.169 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=27223 2026-02-18T08:21:19.169 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 24 = 23 -a 1 -le 27223 2026-02-18T08:21:19.169 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:21:19.169 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:21:19.169 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:21:19.370 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:21:19.371 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:21:19.371 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:21:19.371 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:21:19.371 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:21:19.371 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:21:19.371 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:21:19.371 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:21:19.371 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:21:19.371 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:21:19.371 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:21:19.372 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:21:19.372 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:21:19.372 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:21:19.372 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:21:19.374 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:21:19.374 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:21:19.375 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:21:19.375 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:21:19.375 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:21:19.375 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:21:19.375 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:21:19.375 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:21:19.375 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:21:19.375 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:21:19.375 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:21:19.375 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:21:19.375 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:21:19.375 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:21:19.376 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:21:19.376 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:21:19.376 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:21:19.376 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:21:19.376 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:21:19.376 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:21:19.376 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:21:19.376 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:21:19.376 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:21:19.376 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:21:19.376 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:21:19.376 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:21:19.376 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:21:19.377 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:21:19.377 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:21:19.377 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:21:19.377 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:21:19.377 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:21:19.377 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:21:19.377 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:21:19.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:21:19.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:21:19.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:21:19.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:21:19.437 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:21:19.437 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:21:19.437 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:21:19.440 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:21:19.440 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:21:19.440 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:21:19.440 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:21:19.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:21:19.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:21:19.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:21:19.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:21:19.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:21:19.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:21:19.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:21:19.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:21:19.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:21:19.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:21:19.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:21:19.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:21:19.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:21:19.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:21:19.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:21:19.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:21:19.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:21:19.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:21:19.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:21:19.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:21:19.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:21:19.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:21:19.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:21:19.443 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:21:19.443 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:21:19.443 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:21:19.443 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:21:19.443 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:21:19.451 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:21:19.452 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:21:19.452 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:21:19.452 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:21:19.512 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:21:19.512 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:21:19.512 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:21:19.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:21:19.516 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:21:19.516 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:21:19.516 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:21:19.516 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:21:19.516 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:21:19.516 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:21:19.516 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:21:19.516 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:21:19.516 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:21:19.516 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:21:19.516 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:21:19.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:21:19.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:21:19.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:21:19.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:21:19.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:21:19.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:21:19.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:21:19.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:21:19.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:21:19.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:21:19.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:21:19.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:21:19.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:21:19.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:21:19.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:21:19.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:21:19.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:21:19.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:21:19.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:21:19.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:21:19.526 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:21:19.526 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:21:19.526 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:21:19.526 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:21:19.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:21:19.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:21:19.584 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:21:19.586 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:21:19.586 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:21:19.586 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:21:19.586 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:21:19.586 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:21:19.587 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:21:19.587 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:21:19.587 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:21:19.587 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:21:19.587 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:21:19.587 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:21:19.587 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:21:19.587 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:21:19.587 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:21:19.587 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:21:19.587 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:21:19.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:21:19.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:21:19.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:21:19.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:21:19.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:21:19.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:21:19.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:21:19.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:21:19.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:21:19.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:21:19.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:21:19.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:21:19.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:21:19.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:21:19.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:21:19.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:21:19.597 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:21:19.598 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:21:19.598 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:21:19.598 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:21:19.659 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:21:19.660 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:21:19.660 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:21:19.660 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:21:19.660 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:21:19.660 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:21:19.661 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:21:19.661 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:21:19.661 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:21:19.661 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:21:19.661 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:21:19.661 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:21:19.661 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:21:19.661 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:21:19.662 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:21:19.662 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:21:19.691 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 83 2026-02-18T08:21:19.691 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 84 2026-02-18T08:21:19.691 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:21:19.691 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=337, tag_tid=24, entry_tid=1], [object_number=336, tag_tid=24, entry_tid=0], [object_number=339, tag_tid=23, entry_tid=29391], [object_number=338, tag_tid=23, entry_tid=29390]]], state=connected] 2026-02-18T08:21:19.691 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=333, tag_tid=23, entry_tid=27573], [object_number=332, tag_tid=23, entry_tid=27572], [object_number=335, tag_tid=23, entry_tid=27571], [object_number=334, tag_tid=23, entry_tid=27570]]], state=connected] 2026-02-18T08:21:19.694 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:21:19.696 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=337, tag_tid=24, entry_tid=1' 2026-02-18T08:21:19.697 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:21:19.697 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:21:19.697 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:21:19.697 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:21:19.697 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:21:19.697 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:21:19.697 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:21:19.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:21:19.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:21:19.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:21:19.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:21:19.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:21:19.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:21:19.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:21:19.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:21:19.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 83 2026-02-18T08:21:19.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 84 2026-02-18T08:21:19.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:21:19.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=337, tag_tid=24, entry_tid=1], [object_number=336, tag_tid=24, entry_tid=0], [object_number=339, tag_tid=23, entry_tid=29391], [object_number=338, tag_tid=23, entry_tid=29390]]], state=connected] 2026-02-18T08:21:19.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=333, tag_tid=23, entry_tid=27573], [object_number=332, tag_tid=23, entry_tid=27572], [object_number=335, tag_tid=23, entry_tid=27571], [object_number=334, tag_tid=23, entry_tid=27570]]], state=connected] 2026-02-18T08:21:19.729 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:21:19.731 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=333, tag_tid=23, entry_tid=27573' 2026-02-18T08:21:19.731 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=337, tag_tid=24, entry_tid=1' -a 'object_number=337, tag_tid=24, entry_tid=1' = 'object_number=333, tag_tid=23, entry_tid=27573' 2026-02-18T08:21:19.731 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=333, tag_tid=23, entry_tid=27573' '!=' 'object_number=335, tag_tid=23, entry_tid=27223' 2026-02-18T08:21:19.731 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:21:19.732 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=333, tag_tid=23, entry_tid=27573' = 'object_number=335, tag_tid=23, entry_tid=27223' 2026-02-18T08:21:19.732 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=333, tag_tid=23, entry_tid=27573' 2026-02-18T08:21:19.732 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=337, tag_tid=24, entry_tid=1' 2026-02-18T08:21:19.732 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:21:19.732 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:21:19.733 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=24 2026-02-18T08:21:19.734 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=333, tag_tid=23, entry_tid=27573' 2026-02-18T08:21:19.734 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:21:19.734 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:21:19.735 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=23 2026-02-18T08:21:19.736 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=337, tag_tid=24, entry_tid=1' 2026-02-18T08:21:19.736 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:21:19.736 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:21:19.737 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:21:19.738 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=333, tag_tid=23, entry_tid=27573' 2026-02-18T08:21:19.738 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:21:19.738 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:21:19.740 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=27573 2026-02-18T08:21:19.740 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 24 = 23 -a 1 -le 27573 2026-02-18T08:21:19.740 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:21:19.740 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:21:19.741 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:21:19.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:21:19.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:21:19.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:21:19.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:21:19.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:21:19.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:21:19.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:21:19.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:21:19.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:21:19.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:21:19.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:21:19.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:21:19.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:21:19.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:21:19.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:21:19.946 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:21:19.946 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:21:19.946 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:21:19.946 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:21:19.946 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:21:19.946 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:21:19.946 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:21:19.946 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:21:19.946 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:21:19.947 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:21:19.947 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:21:19.947 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:21:19.947 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:21:19.947 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:21:19.947 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:21:19.947 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:21:19.947 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:21:19.947 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:21:19.947 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:21:19.947 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:21:19.947 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:21:19.947 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:21:19.948 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:21:19.948 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:21:19.948 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:21:19.948 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:21:19.948 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:21:19.948 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:21:19.948 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:21:19.948 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:21:19.948 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:21:19.948 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:21:19.948 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:21:19.948 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:21:19.956 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:21:19.956 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:21:19.956 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:21:19.956 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:21:20.005 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:21:20.005 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:21:20.005 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:21:20.008 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:21:20.008 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:21:20.008 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:21:20.008 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:21:20.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:21:20.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:21:20.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:21:20.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:21:20.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:21:20.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:21:20.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:21:20.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:21:20.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:21:20.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:21:20.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:21:20.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:21:20.010 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:21:20.010 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:21:20.010 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:21:20.010 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:21:20.010 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:21:20.010 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:21:20.010 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:21:20.010 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:21:20.010 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:21:20.010 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:21:20.010 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:21:20.010 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:21:20.010 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:21:20.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:21:20.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:21:20.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:21:20.019 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:21:20.019 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:21:20.019 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:21:20.019 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:21:20.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:21:20.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:21:20.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:21:20.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:21:20.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:21:20.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:21:20.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:21:20.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:21:20.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:21:20.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:21:20.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:21:20.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:21:20.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:21:20.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:21:20.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:21:20.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:21:20.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:21:20.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:21:20.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:21:20.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:21:20.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:21:20.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:21:20.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:21:20.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:21:20.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:21:20.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:21:20.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:21:20.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:21:20.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:21:20.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:21:20.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:21:20.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:21:20.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:21:20.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:21:20.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:21:20.092 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:21:20.092 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:21:20.092 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:21:20.093 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:21:20.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:21:20.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:21:20.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:21:20.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:21:20.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:21:20.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:21:20.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:21:20.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:21:20.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:21:20.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:21:20.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:21:20.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:21:20.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:21:20.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:21:20.155 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:21:20.155 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:21:20.155 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:21:20.155 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:21:20.155 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:21:20.155 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:21:20.155 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:21:20.155 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:21:20.155 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:21:20.155 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:21:20.155 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:21:20.155 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:21:20.156 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:21:20.156 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:21:20.156 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:21:20.156 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:21:20.156 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:21:20.156 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:21:20.156 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:21:20.156 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:21:20.156 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:21:20.166 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:21:20.166 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:21:20.166 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:21:20.166 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:21:20.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:21:20.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:21:20.229 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:21:20.229 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:21:20.229 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:21:20.229 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:21:20.229 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:21:20.229 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:21:20.229 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:21:20.229 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:21:20.229 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:21:20.229 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:21:20.229 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:21:20.230 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:21:20.230 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:21:20.230 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:21:20.259 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 83 2026-02-18T08:21:20.259 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 84 2026-02-18T08:21:20.259 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:21:20.259 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=337, tag_tid=24, entry_tid=1], [object_number=336, tag_tid=24, entry_tid=0], [object_number=339, tag_tid=23, entry_tid=29391], [object_number=338, tag_tid=23, entry_tid=29390]]], state=connected] 2026-02-18T08:21:20.259 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=334, tag_tid=23, entry_tid=27918], [object_number=333, tag_tid=23, entry_tid=27917], [object_number=332, tag_tid=23, entry_tid=27916], [object_number=335, tag_tid=23, entry_tid=27915]]], state=connected] 2026-02-18T08:21:20.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:21:20.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=337, tag_tid=24, entry_tid=1' 2026-02-18T08:21:20.266 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:21:20.266 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:21:20.266 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:21:20.266 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:21:20.266 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:21:20.266 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:21:20.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:21:20.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:21:20.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:21:20.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:21:20.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:21:20.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:21:20.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:21:20.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:21:20.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:21:20.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 83 2026-02-18T08:21:20.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 84 2026-02-18T08:21:20.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:21:20.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=337, tag_tid=24, entry_tid=1], [object_number=336, tag_tid=24, entry_tid=0], [object_number=339, tag_tid=23, entry_tid=29391], [object_number=338, tag_tid=23, entry_tid=29390]]], state=connected] 2026-02-18T08:21:20.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=334, tag_tid=23, entry_tid=27918], [object_number=333, tag_tid=23, entry_tid=27917], [object_number=332, tag_tid=23, entry_tid=27916], [object_number=335, tag_tid=23, entry_tid=27915]]], state=connected] 2026-02-18T08:21:20.298 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:21:20.299 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=334, tag_tid=23, entry_tid=27918' 2026-02-18T08:21:20.300 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=337, tag_tid=24, entry_tid=1' -a 'object_number=337, tag_tid=24, entry_tid=1' = 'object_number=334, tag_tid=23, entry_tid=27918' 2026-02-18T08:21:20.300 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=334, tag_tid=23, entry_tid=27918' '!=' 'object_number=333, tag_tid=23, entry_tid=27573' 2026-02-18T08:21:20.300 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:21:20.300 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=334, tag_tid=23, entry_tid=27918' = 'object_number=333, tag_tid=23, entry_tid=27573' 2026-02-18T08:21:20.300 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=334, tag_tid=23, entry_tid=27918' 2026-02-18T08:21:20.301 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=337, tag_tid=24, entry_tid=1' 2026-02-18T08:21:20.301 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:21:20.301 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:21:20.301 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=24 2026-02-18T08:21:20.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=334, tag_tid=23, entry_tid=27918' 2026-02-18T08:21:20.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:21:20.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:21:20.304 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=23 2026-02-18T08:21:20.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=337, tag_tid=24, entry_tid=1' 2026-02-18T08:21:20.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:21:20.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:21:20.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:21:20.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=334, tag_tid=23, entry_tid=27918' 2026-02-18T08:21:20.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:21:20.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:21:20.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=27918 2026-02-18T08:21:20.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 24 = 23 -a 1 -le 27918 2026-02-18T08:21:20.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:21:20.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:21:20.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:21:20.510 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:21:20.510 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:21:20.510 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:21:20.510 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:21:20.510 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:21:20.510 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:21:20.510 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:21:20.510 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:21:20.510 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:21:20.510 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:21:20.511 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:21:20.511 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:21:20.511 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:21:20.511 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:21:20.511 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:21:20.511 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:21:20.511 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:21:20.511 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:21:20.511 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:21:20.511 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:21:20.511 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:21:20.511 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:21:20.512 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:21:20.512 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:21:20.512 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:21:20.512 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:21:20.512 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:21:20.512 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:21:20.512 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:21:20.512 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:21:20.513 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:21:20.513 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:21:20.513 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:21:20.513 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:21:20.513 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:21:20.513 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:21:20.513 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:21:20.513 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:21:20.513 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:21:20.513 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:21:20.513 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:21:20.513 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:21:20.513 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:21:20.513 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:21:20.513 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:21:20.513 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:21:20.513 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:21:20.513 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:21:20.513 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:21:20.524 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:21:20.524 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:21:20.524 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:21:20.524 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:21:20.573 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:21:20.573 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:21:20.573 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:21:20.576 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:21:20.576 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:21:20.576 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:21:20.576 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:21:20.577 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:21:20.577 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:21:20.577 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:21:20.577 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:21:20.577 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:21:20.577 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:21:20.577 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:21:20.577 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:21:20.577 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:21:20.577 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:21:20.578 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:21:20.578 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:21:20.578 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:21:20.578 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:21:20.578 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:21:20.578 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:21:20.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:21:20.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:21:20.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:21:20.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:21:20.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:21:20.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:21:20.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:21:20.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:21:20.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:21:20.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:21:20.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:21:20.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:21:20.587 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:21:20.587 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:21:20.587 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:21:20.587 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:21:20.648 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:21:20.648 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:21:20.648 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:21:20.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:21:20.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:21:20.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:21:20.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:21:20.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:21:20.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:21:20.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:21:20.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:21:20.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:21:20.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:21:20.653 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:21:20.653 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:21:20.653 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:21:20.653 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:21:20.653 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:21:20.653 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:21:20.654 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:21:20.654 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:21:20.654 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:21:20.654 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:21:20.654 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:21:20.654 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:21:20.654 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:21:20.655 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:21:20.655 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:21:20.655 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:21:20.655 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:21:20.655 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:21:20.655 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:21:20.655 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:21:20.655 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:21:20.655 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:21:20.662 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:21:20.662 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:21:20.662 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:21:20.663 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:21:20.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:21:20.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:21:20.711 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:21:20.714 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:21:20.714 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:21:20.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:21:20.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:21:20.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:21:20.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:21:20.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:21:20.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:21:20.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:21:20.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:21:20.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:21:20.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:21:20.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:21:20.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:21:20.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:21:20.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:21:20.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:21:20.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:21:20.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:21:20.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:21:20.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:21:20.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:21:20.717 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:21:20.717 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:21:20.717 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:21:20.717 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:21:20.717 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:21:20.717 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:21:20.717 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:21:20.717 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:21:20.717 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:21:20.718 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:21:20.725 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:21:20.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:21:20.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:21:20.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:21:20.788 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:21:20.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:21:20.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:21:20.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:21:20.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:21:20.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:21:20.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:21:20.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:21:20.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:21:20.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:21:20.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:21:20.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:21:20.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:21:20.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:21:20.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:21:20.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:21:20.820 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 83 2026-02-18T08:21:20.820 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 84 2026-02-18T08:21:20.820 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:21:20.821 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=337, tag_tid=24, entry_tid=1], [object_number=336, tag_tid=24, entry_tid=0], [object_number=339, tag_tid=23, entry_tid=29391], [object_number=338, tag_tid=23, entry_tid=29390]]], state=connected] 2026-02-18T08:21:20.821 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=332, tag_tid=23, entry_tid=28264], [object_number=335, tag_tid=23, entry_tid=28263], [object_number=334, tag_tid=23, entry_tid=28262], [object_number=333, tag_tid=23, entry_tid=28261]]], state=connected] 2026-02-18T08:21:20.823 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:21:20.824 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=337, tag_tid=24, entry_tid=1' 2026-02-18T08:21:20.827 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:21:20.827 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:21:20.827 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:21:20.827 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:21:20.827 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:21:20.828 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:21:20.828 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:21:20.828 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:21:20.828 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:21:20.828 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:21:20.828 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:21:20.829 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:21:20.830 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:21:20.830 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:21:20.830 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:21:20.866 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 83 2026-02-18T08:21:20.866 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 84 2026-02-18T08:21:20.866 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:21:20.866 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=337, tag_tid=24, entry_tid=1], [object_number=336, tag_tid=24, entry_tid=0], [object_number=339, tag_tid=23, entry_tid=29391], [object_number=338, tag_tid=23, entry_tid=29390]]], state=connected] 2026-02-18T08:21:20.867 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=332, tag_tid=23, entry_tid=28264], [object_number=335, tag_tid=23, entry_tid=28263], [object_number=334, tag_tid=23, entry_tid=28262], [object_number=333, tag_tid=23, entry_tid=28261]]], state=connected] 2026-02-18T08:21:20.870 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:21:20.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=332, tag_tid=23, entry_tid=28264' 2026-02-18T08:21:20.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=337, tag_tid=24, entry_tid=1' -a 'object_number=337, tag_tid=24, entry_tid=1' = 'object_number=332, tag_tid=23, entry_tid=28264' 2026-02-18T08:21:20.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=332, tag_tid=23, entry_tid=28264' '!=' 'object_number=334, tag_tid=23, entry_tid=27918' 2026-02-18T08:21:20.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:21:20.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=332, tag_tid=23, entry_tid=28264' = 'object_number=334, tag_tid=23, entry_tid=27918' 2026-02-18T08:21:20.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=332, tag_tid=23, entry_tid=28264' 2026-02-18T08:21:20.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=337, tag_tid=24, entry_tid=1' 2026-02-18T08:21:20.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:21:20.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:21:20.874 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=24 2026-02-18T08:21:20.874 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=332, tag_tid=23, entry_tid=28264' 2026-02-18T08:21:20.875 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:21:20.875 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:21:20.876 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=23 2026-02-18T08:21:20.877 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=337, tag_tid=24, entry_tid=1' 2026-02-18T08:21:20.877 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:21:20.877 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:21:20.878 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:21:20.879 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=332, tag_tid=23, entry_tid=28264' 2026-02-18T08:21:20.879 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:21:20.879 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:21:20.881 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=28264 2026-02-18T08:21:20.881 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 24 = 23 -a 1 -le 28264 2026-02-18T08:21:20.881 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:21:20.881 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:21:20.881 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:21:21.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:21:21.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:21:21.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:21:21.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:21:21.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:21:21.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:21:21.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:21:21.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:21:21.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:21:21.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:21:21.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:21:21.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:21:21.086 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:21:21.086 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:21:21.086 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:21:21.086 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:21:21.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:21:21.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:21:21.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:21:21.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:21:21.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:21:21.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:21:21.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:21:21.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:21:21.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:21:21.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:21:21.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:21:21.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:21:21.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:21:21.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:21:21.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:21:21.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:21:21.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:21:21.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:21:21.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:21:21.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:21:21.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:21:21.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:21:21.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:21:21.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:21:21.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:21:21.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:21:21.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:21:21.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:21:21.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:21:21.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:21:21.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:21:21.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:21:21.092 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:21:21.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:21:21.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:21:21.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:21:21.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:21:21.148 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:21:21.148 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:21:21.148 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:21:21.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:21:21.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:21:21.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:21:21.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:21:21.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:21:21.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:21:21.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:21:21.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:21:21.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:21:21.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:21:21.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:21:21.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:21:21.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:21:21.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:21:21.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:21:21.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:21:21.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:21:21.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:21:21.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:21:21.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:21:21.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:21:21.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:21:21.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:21:21.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:21:21.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:21:21.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:21:21.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:21:21.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:21:21.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:21:21.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:21:21.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:21:21.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:21:21.162 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:21:21.163 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:21:21.163 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:21:21.164 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:21:21.212 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:21:21.212 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:21:21.212 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:21:21.217 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:21:21.217 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:21:21.217 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:21:21.217 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:21:21.218 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:21:21.218 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:21:21.218 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:21:21.218 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:21:21.218 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:21:21.218 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:21:21.219 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:21:21.219 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:21:21.219 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:21:21.219 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:21:21.219 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:21:21.220 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:21:21.220 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:21:21.220 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:21:21.220 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:21:21.220 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:21:21.220 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:21:21.221 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:21:21.221 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:21:21.221 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:21:21.221 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:21:21.221 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:21:21.222 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:21:21.222 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:21:21.222 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:21:21.222 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:21:21.222 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:21:21.222 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:21:21.227 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:21:21.227 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:21:21.227 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:21:21.227 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:21:21.275 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:21:21.275 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:21:21.275 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:21:21.277 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:21:21.277 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:21:21.277 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:21:21.277 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:21:21.277 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:21:21.277 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:21:21.278 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:21:21.278 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:21:21.278 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:21:21.278 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:21:21.278 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:21:21.278 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:21:21.278 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:21:21.278 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:21:21.278 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:21:21.278 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:21:21.278 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:21:21.278 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:21:21.278 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:21:21.278 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:21:21.278 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:21:21.278 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:21:21.278 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:21:21.279 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:21:21.279 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:21:21.279 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:21:21.279 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:21:21.279 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:21:21.279 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:21:21.279 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:21:21.279 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:21:21.279 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:21:21.289 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:21:21.289 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:21:21.289 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:21:21.289 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:21:21.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:21:21.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:21:21.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:21:21.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:21:21.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:21:21.360 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:21:21.360 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:21:21.360 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:21:21.360 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:21:21.360 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:21:21.360 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:21:21.361 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:21:21.361 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:21:21.361 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:21:21.362 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:21:21.362 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:21:21.393 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 83 2026-02-18T08:21:21.393 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 84 2026-02-18T08:21:21.393 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:21:21.393 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=337, tag_tid=24, entry_tid=1], [object_number=336, tag_tid=24, entry_tid=0], [object_number=339, tag_tid=23, entry_tid=29391], [object_number=338, tag_tid=23, entry_tid=29390]]], state=connected] 2026-02-18T08:21:21.393 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=333, tag_tid=23, entry_tid=28625], [object_number=332, tag_tid=23, entry_tid=28624], [object_number=335, tag_tid=23, entry_tid=28623], [object_number=334, tag_tid=23, entry_tid=28622]]], state=connected] 2026-02-18T08:21:21.395 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:21:21.397 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=337, tag_tid=24, entry_tid=1' 2026-02-18T08:21:21.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:21:21.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:21:21.399 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:21:21.399 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:21:21.399 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:21:21.399 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:21:21.399 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:21:21.399 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:21:21.399 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:21:21.399 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:21:21.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:21:21.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:21:21.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:21:21.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:21:21.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:21:21.430 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 83 2026-02-18T08:21:21.430 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 84 2026-02-18T08:21:21.430 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:21:21.430 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=337, tag_tid=24, entry_tid=1], [object_number=336, tag_tid=24, entry_tid=0], [object_number=339, tag_tid=23, entry_tid=29391], [object_number=338, tag_tid=23, entry_tid=29390]]], state=connected] 2026-02-18T08:21:21.430 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=333, tag_tid=23, entry_tid=28625], [object_number=332, tag_tid=23, entry_tid=28624], [object_number=335, tag_tid=23, entry_tid=28623], [object_number=334, tag_tid=23, entry_tid=28622]]], state=connected] 2026-02-18T08:21:21.433 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:21:21.435 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=333, tag_tid=23, entry_tid=28625' 2026-02-18T08:21:21.436 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=337, tag_tid=24, entry_tid=1' -a 'object_number=337, tag_tid=24, entry_tid=1' = 'object_number=333, tag_tid=23, entry_tid=28625' 2026-02-18T08:21:21.436 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=333, tag_tid=23, entry_tid=28625' '!=' 'object_number=332, tag_tid=23, entry_tid=28264' 2026-02-18T08:21:21.436 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:21:21.436 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=333, tag_tid=23, entry_tid=28625' = 'object_number=332, tag_tid=23, entry_tid=28264' 2026-02-18T08:21:21.436 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=333, tag_tid=23, entry_tid=28625' 2026-02-18T08:21:21.436 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=337, tag_tid=24, entry_tid=1' 2026-02-18T08:21:21.436 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:21:21.436 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:21:21.437 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=24 2026-02-18T08:21:21.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=333, tag_tid=23, entry_tid=28625' 2026-02-18T08:21:21.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:21:21.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:21:21.439 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=23 2026-02-18T08:21:21.440 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=337, tag_tid=24, entry_tid=1' 2026-02-18T08:21:21.440 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:21:21.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:21:21.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:21:21.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=333, tag_tid=23, entry_tid=28625' 2026-02-18T08:21:21.443 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:21:21.443 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:21:21.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=28625 2026-02-18T08:21:21.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 24 = 23 -a 1 -le 28625 2026-02-18T08:21:21.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:21:21.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:21:21.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:21:21.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:21:21.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:21:21.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:21:21.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:21:21.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:21:21.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:21:21.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:21:21.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:21:21.646 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:21:21.646 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:21:21.646 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:21:21.646 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:21:21.646 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:21:21.646 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:21:21.646 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:21:21.646 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:21:21.646 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:21:21.647 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:21:21.647 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:21:21.647 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:21:21.647 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:21:21.647 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:21:21.647 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:21:21.647 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:21:21.647 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:21:21.647 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:21:21.647 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:21:21.647 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:21:21.648 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:21:21.648 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:21:21.648 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:21:21.648 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:21:21.648 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:21:21.648 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:21:21.648 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:21:21.648 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:21:21.648 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:21:21.648 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:21:21.648 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:21:21.648 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:21:21.648 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:21:21.648 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:21:21.648 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:21:21.648 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:21:21.648 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:21:21.648 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:21:21.648 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:21:21.649 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:21:21.649 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:21:21.660 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:21:21.660 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:21:21.660 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:21:21.660 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:21:21.711 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:21:21.711 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:21:21.711 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:21:21.714 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:21:21.714 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:21:21.714 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:21:21.714 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:21:21.714 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:21:21.714 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:21:21.714 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:21:21.714 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:21:21.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:21:21.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:21:21.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:21:21.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:21:21.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:21:21.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:21:21.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:21:21.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:21:21.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:21:21.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:21:21.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:21:21.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:21:21.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:21:21.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:21:21.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:21:21.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:21:21.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:21:21.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:21:21.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:21:21.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:21:21.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:21:21.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:21:21.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:21:21.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:21:21.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:21:21.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:21:21.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:21:21.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:21:21.774 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:21:21.774 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:21:21.774 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:21:21.776 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:21:21.777 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:21:21.777 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:21:21.777 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:21:21.777 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:21:21.777 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:21:21.777 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:21:21.777 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:21:21.777 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:21:21.777 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:21:21.777 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:21:21.777 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:21:21.777 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:21:21.778 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:21:21.778 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:21:21.778 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:21:21.778 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:21:21.778 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:21:21.778 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:21:21.778 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:21:21.778 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:21:21.778 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:21:21.778 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:21:21.778 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:21:21.778 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:21:21.778 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:21:21.778 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:21:21.779 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:21:21.779 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:21:21.779 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:21:21.779 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:21:21.779 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:21:21.786 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:21:21.787 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:21:21.787 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:21:21.787 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:21:21.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:21:21.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:21:21.835 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:21:21.837 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:21:21.838 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:21:21.838 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:21:21.838 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:21:21.838 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:21:21.838 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:21:21.838 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:21:21.838 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:21:21.838 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:21:21.838 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:21:21.838 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:21:21.839 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:21:21.839 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:21:21.839 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:21:21.839 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:21:21.839 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:21:21.839 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:21:21.839 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:21:21.839 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:21:21.839 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:21:21.839 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:21:21.839 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:21:21.839 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:21:21.840 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:21:21.840 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:21:21.840 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:21:21.840 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:21:21.840 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:21:21.840 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:21:21.840 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:21:21.840 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:21:21.840 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:21:21.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:21:21.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:21:21.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:21:21.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:21:21.909 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:21:21.910 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:21:21.911 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:21:21.911 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:21:21.911 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:21:21.911 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:21:21.911 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:21:21.911 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:21:21.911 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:21:21.911 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:21:21.911 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:21:21.912 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:21:21.912 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:21:21.913 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:21:21.913 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:21:21.913 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:21:21.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 84 2026-02-18T08:21:21.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 84 2026-02-18T08:21:21.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:21:21.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=337, tag_tid=24, entry_tid=1], [object_number=336, tag_tid=24, entry_tid=0], [object_number=339, tag_tid=23, entry_tid=29391], [object_number=338, tag_tid=23, entry_tid=29390]]], state=connected] 2026-02-18T08:21:21.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=337, tag_tid=23, entry_tid=28949], [object_number=336, tag_tid=23, entry_tid=28948], [object_number=339, tag_tid=23, entry_tid=28947], [object_number=338, tag_tid=23, entry_tid=28946]]], state=connected] 2026-02-18T08:21:21.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:21:21.947 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=337, tag_tid=24, entry_tid=1' 2026-02-18T08:21:21.950 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:21:21.950 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:21:21.950 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:21:21.950 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:21:21.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:21:21.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:21:21.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:21:21.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:21:21.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:21:21.952 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:21:21.952 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:21:21.952 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:21:21.952 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:21:21.952 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:21:21.952 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:21:21.985 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 84 2026-02-18T08:21:21.985 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 84 2026-02-18T08:21:21.985 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:21:21.986 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=337, tag_tid=24, entry_tid=1], [object_number=336, tag_tid=24, entry_tid=0], [object_number=339, tag_tid=23, entry_tid=29391], [object_number=338, tag_tid=23, entry_tid=29390]]], state=connected] 2026-02-18T08:21:21.986 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=337, tag_tid=23, entry_tid=28949], [object_number=336, tag_tid=23, entry_tid=28948], [object_number=339, tag_tid=23, entry_tid=28947], [object_number=338, tag_tid=23, entry_tid=28946]]], state=connected] 2026-02-18T08:21:21.989 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:21:21.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=337, tag_tid=23, entry_tid=28949' 2026-02-18T08:21:21.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=337, tag_tid=24, entry_tid=1' -a 'object_number=337, tag_tid=24, entry_tid=1' = 'object_number=337, tag_tid=23, entry_tid=28949' 2026-02-18T08:21:21.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=337, tag_tid=23, entry_tid=28949' '!=' 'object_number=333, tag_tid=23, entry_tid=28625' 2026-02-18T08:21:21.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:21:21.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=337, tag_tid=23, entry_tid=28949' = 'object_number=333, tag_tid=23, entry_tid=28625' 2026-02-18T08:21:21.992 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=337, tag_tid=23, entry_tid=28949' 2026-02-18T08:21:21.992 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=337, tag_tid=24, entry_tid=1' 2026-02-18T08:21:21.992 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:21:21.992 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:21:21.993 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=24 2026-02-18T08:21:21.994 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=337, tag_tid=23, entry_tid=28949' 2026-02-18T08:21:21.994 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:21:21.994 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:21:21.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=23 2026-02-18T08:21:21.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=337, tag_tid=24, entry_tid=1' 2026-02-18T08:21:21.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:21:21.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:21:21.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:21:21.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=337, tag_tid=23, entry_tid=28949' 2026-02-18T08:21:21.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:21:21.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:21:22.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=28949 2026-02-18T08:21:22.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 24 = 23 -a 1 -le 28949 2026-02-18T08:21:22.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:21:22.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:21:22.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:21:22.202 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:21:22.202 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:21:22.202 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:21:22.202 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:21:22.203 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:21:22.203 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:21:22.203 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:21:22.203 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:21:22.203 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:21:22.203 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:21:22.203 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:21:22.203 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:21:22.203 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:21:22.203 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:21:22.204 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:21:22.204 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:21:22.206 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:21:22.206 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:21:22.207 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:21:22.207 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:21:22.207 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:21:22.207 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:21:22.207 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:21:22.207 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:21:22.207 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:21:22.207 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:21:22.207 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:21:22.207 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:21:22.207 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:21:22.208 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:21:22.208 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:21:22.208 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:21:22.208 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:21:22.208 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:21:22.208 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:21:22.208 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:21:22.208 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:21:22.208 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:21:22.208 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:21:22.208 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:21:22.208 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:21:22.208 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:21:22.209 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:21:22.209 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:21:22.209 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:21:22.209 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:21:22.209 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:21:22.209 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:21:22.209 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:21:22.215 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:21:22.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:21:22.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:21:22.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:21:22.266 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:21:22.266 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:21:22.266 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:21:22.269 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:21:22.269 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:21:22.269 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:21:22.269 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:21:22.269 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:21:22.270 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:21:22.270 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:21:22.270 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:21:22.270 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:21:22.270 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:21:22.270 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:21:22.270 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:21:22.270 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:21:22.270 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:21:22.270 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:21:22.270 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:21:22.270 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:21:22.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:21:22.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:21:22.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:21:22.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:21:22.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:21:22.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:21:22.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:21:22.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:21:22.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:21:22.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:21:22.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:21:22.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:21:22.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:21:22.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:21:22.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:21:22.279 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:21:22.280 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:21:22.280 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:21:22.280 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:21:22.329 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:21:22.329 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:21:22.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:21:22.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:21:22.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:21:22.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:21:22.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:21:22.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:21:22.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:21:22.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:21:22.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:21:22.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:21:22.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:21:22.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:21:22.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:21:22.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:21:22.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:21:22.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:21:22.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:21:22.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:21:22.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:21:22.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:21:22.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:21:22.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:21:22.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:21:22.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:21:22.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:21:22.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:21:22.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:21:22.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:21:22.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:21:22.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:21:22.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:21:22.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:21:22.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:21:22.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:21:22.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:21:22.344 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:21:22.344 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:21:22.406 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:21:22.407 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:21:22.407 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:21:22.408 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:21:22.408 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:21:22.408 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:21:22.408 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:21:22.409 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:21:22.409 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:21:22.409 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:21:22.409 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:21:22.409 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:21:22.409 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:21:22.409 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:21:22.409 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:21:22.409 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:21:22.409 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:21:22.409 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:21:22.409 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:21:22.409 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:21:22.409 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:21:22.409 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:21:22.409 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:21:22.409 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:21:22.409 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:21:22.409 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:21:22.409 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:21:22.409 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:21:22.409 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:21:22.409 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:21:22.409 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:21:22.409 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:21:22.409 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:21:22.409 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:21:22.409 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:21:22.421 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:21:22.422 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:21:22.422 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:21:22.422 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:21:22.484 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:21:22.485 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:21:22.485 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:21:22.485 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:21:22.485 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:21:22.485 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:21:22.486 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:21:22.486 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:21:22.486 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:21:22.486 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:21:22.486 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:21:22.486 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:21:22.486 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:21:22.487 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:21:22.487 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:21:22.487 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:21:22.516 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 84 2026-02-18T08:21:22.516 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 84 2026-02-18T08:21:22.516 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:21:22.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=337, tag_tid=24, entry_tid=1], [object_number=336, tag_tid=24, entry_tid=0], [object_number=339, tag_tid=23, entry_tid=29391], [object_number=338, tag_tid=23, entry_tid=29390]]], state=connected] 2026-02-18T08:21:22.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=338, tag_tid=23, entry_tid=29298], [object_number=337, tag_tid=23, entry_tid=29297], [object_number=336, tag_tid=23, entry_tid=29296], [object_number=339, tag_tid=23, entry_tid=29295]]], state=connected] 2026-02-18T08:21:22.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:21:22.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=337, tag_tid=24, entry_tid=1' 2026-02-18T08:21:22.524 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:21:22.524 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:21:22.524 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:21:22.524 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:21:22.524 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:21:22.525 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:21:22.525 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:21:22.525 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:21:22.525 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:21:22.525 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:21:22.526 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:21:22.526 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:21:22.526 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:21:22.527 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:21:22.527 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:21:22.552 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 84 2026-02-18T08:21:22.552 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 84 2026-02-18T08:21:22.552 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:21:22.553 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=337, tag_tid=24, entry_tid=1], [object_number=336, tag_tid=24, entry_tid=0], [object_number=339, tag_tid=23, entry_tid=29391], [object_number=338, tag_tid=23, entry_tid=29390]]], state=connected] 2026-02-18T08:21:22.553 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=338, tag_tid=23, entry_tid=29298], [object_number=337, tag_tid=23, entry_tid=29297], [object_number=336, tag_tid=23, entry_tid=29296], [object_number=339, tag_tid=23, entry_tid=29295]]], state=connected] 2026-02-18T08:21:22.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:21:22.557 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=338, tag_tid=23, entry_tid=29298' 2026-02-18T08:21:22.557 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=337, tag_tid=24, entry_tid=1' -a 'object_number=337, tag_tid=24, entry_tid=1' = 'object_number=338, tag_tid=23, entry_tid=29298' 2026-02-18T08:21:22.557 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=338, tag_tid=23, entry_tid=29298' '!=' 'object_number=337, tag_tid=23, entry_tid=28949' 2026-02-18T08:21:22.558 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:21:22.558 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=338, tag_tid=23, entry_tid=29298' = 'object_number=337, tag_tid=23, entry_tid=28949' 2026-02-18T08:21:22.558 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=338, tag_tid=23, entry_tid=29298' 2026-02-18T08:21:22.558 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=337, tag_tid=24, entry_tid=1' 2026-02-18T08:21:22.559 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:21:22.559 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:21:22.559 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=24 2026-02-18T08:21:22.560 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=338, tag_tid=23, entry_tid=29298' 2026-02-18T08:21:22.560 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:21:22.560 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:21:22.561 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=23 2026-02-18T08:21:22.562 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=337, tag_tid=24, entry_tid=1' 2026-02-18T08:21:22.562 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:21:22.562 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:21:22.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:21:22.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=338, tag_tid=23, entry_tid=29298' 2026-02-18T08:21:22.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:21:22.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:21:22.566 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=29298 2026-02-18T08:21:22.566 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 24 = 23 -a 1 -le 29298 2026-02-18T08:21:22.566 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:21:22.566 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:21:22.566 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:21:22.768 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:21:22.768 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:21:22.769 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:21:22.769 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:21:22.769 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:21:22.769 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:21:22.769 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:21:22.769 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:21:22.769 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:21:22.769 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:21:22.769 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:21:22.770 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:21:22.770 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:21:22.770 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:21:22.770 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:21:22.770 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:21:22.773 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:21:22.773 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:21:22.773 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:21:22.773 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:21:22.773 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:21:22.773 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:21:22.773 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:21:22.774 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:21:22.774 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:21:22.774 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:21:22.774 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:21:22.774 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:21:22.774 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:21:22.774 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:21:22.774 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:21:22.774 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:21:22.774 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:21:22.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:21:22.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:21:22.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:21:22.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:21:22.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:21:22.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:21:22.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:21:22.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:21:22.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:21:22.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:21:22.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:21:22.776 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:21:22.776 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:21:22.776 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:21:22.776 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:21:22.776 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:21:22.784 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:21:22.784 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:21:22.785 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:21:22.785 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:21:22.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:21:22.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:21:22.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:21:22.837 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:21:22.837 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:21:22.837 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:21:22.837 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:21:22.837 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:21:22.837 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:21:22.837 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:21:22.837 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:21:22.837 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:21:22.838 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:21:22.838 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:21:22.838 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:21:22.838 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:21:22.838 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:21:22.838 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:21:22.838 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:21:22.838 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:21:22.838 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:21:22.838 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:21:22.839 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:21:22.839 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:21:22.839 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:21:22.839 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:21:22.839 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:21:22.839 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:21:22.839 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:21:22.839 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:21:22.839 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:21:22.839 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:21:22.840 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:21:22.840 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:21:22.840 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:21:22.848 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:21:22.848 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:21:22.848 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:21:22.848 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:21:22.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:21:22.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:21:22.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:21:22.910 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:21:22.910 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:21:22.910 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:21:22.910 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:21:22.910 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:21:22.910 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:21:22.910 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:21:22.910 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:21:22.911 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:21:22.911 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:21:22.911 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:21:22.911 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:21:22.911 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:21:22.911 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:21:22.911 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:21:22.911 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:21:22.911 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:21:22.911 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:21:22.911 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:21:22.911 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:21:22.912 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:21:22.912 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:21:22.912 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:21:22.912 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:21:22.912 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:21:22.912 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:21:22.912 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:21:22.912 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:21:22.912 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:21:22.912 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:21:22.912 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:21:22.913 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:21:22.922 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:21:22.923 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:21:22.923 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:21:22.923 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:21:22.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:21:22.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:21:22.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:21:22.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:21:22.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:21:22.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:21:22.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:21:22.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:21:22.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:21:22.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:21:22.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:21:22.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:21:22.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:21:22.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:21:22.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:21:22.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:21:22.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:21:22.985 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:21:22.985 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:21:22.985 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:21:22.985 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:21:22.985 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:21:22.985 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:21:22.985 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:21:22.985 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:21:22.985 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:21:22.985 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:21:22.985 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:21:22.985 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:21:22.986 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:21:22.986 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:21:22.986 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:21:22.986 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:21:22.986 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:21:22.986 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:21:22.994 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:21:22.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:21:22.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:21:22.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:21:23.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:21:23.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:21:23.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:21:23.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:21:23.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:21:23.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:21:23.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:21:23.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:21:23.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:21:23.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:21:23.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:21:23.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:21:23.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:21:23.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:21:23.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:21:23.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:21:23.102 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 84 2026-02-18T08:21:23.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 84 2026-02-18T08:21:23.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:21:23.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=337, tag_tid=24, entry_tid=1], [object_number=336, tag_tid=24, entry_tid=0], [object_number=339, tag_tid=23, entry_tid=29391], [object_number=338, tag_tid=23, entry_tid=29390]]], state=connected] 2026-02-18T08:21:23.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=336, tag_tid=23, entry_tid=29392], [object_number=339, tag_tid=23, entry_tid=29391], [object_number=338, tag_tid=23, entry_tid=29390], [object_number=337, tag_tid=23, entry_tid=29389]]], state=connected] 2026-02-18T08:21:23.106 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:21:23.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=337, tag_tid=24, entry_tid=1' 2026-02-18T08:21:23.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:21:23.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:21:23.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:21:23.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:21:23.110 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:21:23.110 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:21:23.110 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:21:23.110 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:21:23.110 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:21:23.110 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:21:23.110 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:21:23.110 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:21:23.111 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:21:23.111 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:21:23.112 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:21:23.146 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 84 2026-02-18T08:21:23.147 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 84 2026-02-18T08:21:23.147 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:21:23.147 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=337, tag_tid=24, entry_tid=1], [object_number=336, tag_tid=24, entry_tid=0], [object_number=339, tag_tid=23, entry_tid=29391], [object_number=338, tag_tid=23, entry_tid=29390]]], state=connected] 2026-02-18T08:21:23.147 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=336, tag_tid=23, entry_tid=29392], [object_number=339, tag_tid=23, entry_tid=29391], [object_number=338, tag_tid=23, entry_tid=29390], [object_number=337, tag_tid=23, entry_tid=29389]]], state=connected] 2026-02-18T08:21:23.150 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:21:23.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=336, tag_tid=23, entry_tid=29392' 2026-02-18T08:21:23.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=337, tag_tid=24, entry_tid=1' -a 'object_number=337, tag_tid=24, entry_tid=1' = 'object_number=336, tag_tid=23, entry_tid=29392' 2026-02-18T08:21:23.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=336, tag_tid=23, entry_tid=29392' '!=' 'object_number=338, tag_tid=23, entry_tid=29298' 2026-02-18T08:21:23.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:21:23.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=336, tag_tid=23, entry_tid=29392' = 'object_number=338, tag_tid=23, entry_tid=29298' 2026-02-18T08:21:23.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=336, tag_tid=23, entry_tid=29392' 2026-02-18T08:21:23.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=337, tag_tid=24, entry_tid=1' 2026-02-18T08:21:23.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:21:23.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:21:23.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=24 2026-02-18T08:21:23.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=336, tag_tid=23, entry_tid=29392' 2026-02-18T08:21:23.155 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:21:23.155 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:21:23.156 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=23 2026-02-18T08:21:23.157 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=337, tag_tid=24, entry_tid=1' 2026-02-18T08:21:23.157 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:21:23.157 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:21:23.158 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:21:23.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=336, tag_tid=23, entry_tid=29392' 2026-02-18T08:21:23.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:21:23.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:21:23.161 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=29392 2026-02-18T08:21:23.161 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 24 = 23 -a 1 -le 29392 2026-02-18T08:21:23.161 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:21:23.161 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:21:23.161 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:21:23.367 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:21:23.368 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:21:23.368 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:21:23.368 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:21:23.368 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:21:23.368 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:21:23.368 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:21:23.368 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:21:23.368 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:21:23.368 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:21:23.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:21:23.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:21:23.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:21:23.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:21:23.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:21:23.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:21:23.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:21:23.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:21:23.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:21:23.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:21:23.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:21:23.370 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:21:23.370 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:21:23.370 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:21:23.370 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:21:23.370 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:21:23.370 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:21:23.370 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:21:23.370 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:21:23.370 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:21:23.370 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:21:23.370 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:21:23.370 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:21:23.371 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:21:23.371 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:21:23.371 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:21:23.371 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:21:23.371 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:21:23.371 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:21:23.371 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:21:23.371 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:21:23.371 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:21:23.371 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:21:23.371 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:21:23.371 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:21:23.372 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:21:23.372 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:21:23.372 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:21:23.372 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:21:23.379 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:21:23.380 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:21:23.380 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:21:23.380 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:21:23.440 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:21:23.440 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:21:23.440 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:21:23.443 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:21:23.443 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:21:23.443 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:21:23.443 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:21:23.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:21:23.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:21:23.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:21:23.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:21:23.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:21:23.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:21:23.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:21:23.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:21:23.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:21:23.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:21:23.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:21:23.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:21:23.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:21:23.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:21:23.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:21:23.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:21:23.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:21:23.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:21:23.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:21:23.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:21:23.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:21:23.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:21:23.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:21:23.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:21:23.446 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:21:23.446 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:21:23.446 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:21:23.446 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:21:23.454 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:21:23.454 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:21:23.455 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:21:23.455 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:21:23.511 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:21:23.511 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:21:23.511 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:21:23.514 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:21:23.514 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:21:23.514 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:21:23.514 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:21:23.514 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:21:23.514 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:21:23.514 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:21:23.514 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:21:23.514 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:21:23.514 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:21:23.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:21:23.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:21:23.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:21:23.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:21:23.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:21:23.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:21:23.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:21:23.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:21:23.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:21:23.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:21:23.516 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:21:23.516 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:21:23.516 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:21:23.516 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:21:23.516 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:21:23.516 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:21:23.516 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:21:23.516 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:21:23.516 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:21:23.516 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:21:23.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:21:23.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:21:23.525 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:21:23.525 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:21:23.525 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:21:23.526 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:21:23.573 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:21:23.573 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:21:23.573 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:21:23.576 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:21:23.576 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:21:23.577 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:21:23.577 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:21:23.577 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:21:23.577 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:21:23.577 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:21:23.577 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:21:23.577 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:21:23.578 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:21:23.578 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:21:23.578 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:21:23.578 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:21:23.578 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:21:23.578 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:21:23.578 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:21:23.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:21:23.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:21:23.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:21:23.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:21:23.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:21:23.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:21:23.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:21:23.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:21:23.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:21:23.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:21:23.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:21:23.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:21:23.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:21:23.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:21:23.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:21:23.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:21:23.587 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:21:23.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:21:23.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:21:23.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:21:23.646 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:21:23.647 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:21:23.647 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:21:23.647 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:21:23.647 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:21:23.647 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:21:23.647 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:21:23.647 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:21:23.647 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:21:23.648 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:21:23.648 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:21:23.648 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:21:23.648 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:21:23.649 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:21:23.649 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:21:23.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:21:23.677 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 84 2026-02-18T08:21:23.677 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 84 2026-02-18T08:21:23.677 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:21:23.677 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=337, tag_tid=24, entry_tid=1], [object_number=336, tag_tid=24, entry_tid=0], [object_number=339, tag_tid=23, entry_tid=29391], [object_number=338, tag_tid=23, entry_tid=29390]]], state=connected] 2026-02-18T08:21:23.677 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=337, tag_tid=24, entry_tid=1], [object_number=336, tag_tid=24, entry_tid=0], [object_number=339, tag_tid=23, entry_tid=29391], [object_number=338, tag_tid=23, entry_tid=29390]]], state=connected] 2026-02-18T08:21:23.680 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:21:23.681 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=337, tag_tid=24, entry_tid=1' 2026-02-18T08:21:23.683 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:21:23.683 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:21:23.683 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:21:23.683 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:21:23.683 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:21:23.683 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:21:23.683 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:21:23.683 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:21:23.684 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:21:23.684 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:21:23.684 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:21:23.684 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:21:23.685 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:21:23.685 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:21:23.685 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:21:23.711 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 84 2026-02-18T08:21:23.711 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 84 2026-02-18T08:21:23.711 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:21:23.711 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=337, tag_tid=24, entry_tid=1], [object_number=336, tag_tid=24, entry_tid=0], [object_number=339, tag_tid=23, entry_tid=29391], [object_number=338, tag_tid=23, entry_tid=29390]]], state=connected] 2026-02-18T08:21:23.711 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=337, tag_tid=24, entry_tid=1], [object_number=336, tag_tid=24, entry_tid=0], [object_number=339, tag_tid=23, entry_tid=29391], [object_number=338, tag_tid=23, entry_tid=29390]]], state=connected] 2026-02-18T08:21:23.713 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:21:23.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=337, tag_tid=24, entry_tid=1' 2026-02-18T08:21:23.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=337, tag_tid=24, entry_tid=1' -a 'object_number=337, tag_tid=24, entry_tid=1' = 'object_number=337, tag_tid=24, entry_tid=1' 2026-02-18T08:21:23.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:21:23.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_snap_present cluster1 mirror test snap8 2026-02-18T08:21:23.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:21:23.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:21:23.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:21:23.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap8 2026-02-18T08:21:23.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-18T08:21:23.717 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-18T08:21:23.717 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1 2026-02-18T08:21:24.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster1 info mirror/test@snap8 2026-02-18T08:21:24.746 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:rbd image 'test': 2026-02-18T08:21:24.746 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: size 512 MiB in 128 objects 2026-02-18T08:21:24.746 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: order 22 (4 MiB objects) 2026-02-18T08:21:24.746 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: snapshot_count: 15 2026-02-18T08:21:24.746 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: id: 10a74aa8ebc0 2026-02-18T08:21:24.746 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: block_name_prefix: rbd_data.10a74aa8ebc0 2026-02-18T08:21:24.746 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: format: 2 2026-02-18T08:21:24.746 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: features: layering, exclusive-lock, journaling 2026-02-18T08:21:24.746 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: op_features: 2026-02-18T08:21:24.747 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: flags: 2026-02-18T08:21:24.747 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: create_timestamp: Wed Feb 18 08:10:48 2026 2026-02-18T08:21:24.747 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: access_timestamp: Wed Feb 18 08:20:28 2026 2026-02-18T08:21:24.747 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: modify_timestamp: Wed Feb 18 08:20:35 2026 2026-02-18T08:21:24.747 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: protected: False 2026-02-18T08:21:24.747 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: journal: 10a74aa8ebc0 2026-02-18T08:21:24.747 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring state: enabled 2026-02-18T08:21:24.747 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring mode: journal 2026-02-18T08:21:24.747 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring global id: 0b548823-d751-43df-a076-4efac2da4d3d 2026-02-18T08:21:24.747 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring primary: false 2026-02-18T08:21:24.749 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:21:24.749 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n snap7-clean ']' 2026-02-18T08:21:24.749 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ compare_image_snaps mirror test snap7-clean 2026-02-18T08:21:24.749 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:21:24.749 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:21:24.749 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap7-clean 2026-02-18T08:21:24.749 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local ret=0 2026-02-18T08:21:24.749 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local rmt_export=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.export 2026-02-18T08:21:24.750 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loc_export=/tmp/tmp.vybh1m9yJF/cluster1-mirror-test.export 2026-02-18T08:21:24.750 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.export /tmp/tmp.vybh1m9yJF/cluster1-mirror-test.export 2026-02-18T08:21:24.750 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 -p mirror export test@snap7-clean /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.export 2026-02-18T08:21:25.249 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-18T08:21:25.261 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster1 -p mirror export test@snap7-clean /tmp/tmp.vybh1m9yJF/cluster1-mirror-test.export 2026-02-18T08:21:26.147 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-18T08:21:26.162 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmp /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.export /tmp/tmp.vybh1m9yJF/cluster1-mirror-test.export 2026-02-18T08:21:26.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.export /tmp/tmp.vybh1m9yJF/cluster1-mirror-test.export 2026-02-18T08:21:26.488 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:21:26.488 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ compare_image_snaps mirror test snap8 2026-02-18T08:21:26.488 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:21:26.488 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:21:26.488 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap8 2026-02-18T08:21:26.488 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local ret=0 2026-02-18T08:21:26.488 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local rmt_export=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.export 2026-02-18T08:21:26.489 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loc_export=/tmp/tmp.vybh1m9yJF/cluster1-mirror-test.export 2026-02-18T08:21:26.489 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.export /tmp/tmp.vybh1m9yJF/cluster1-mirror-test.export 2026-02-18T08:21:26.489 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 -p mirror export test@snap8 /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.export 2026-02-18T08:21:26.962 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-18T08:21:26.974 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster1 -p mirror export test@snap8 /tmp/tmp.vybh1m9yJF/cluster1-mirror-test.export 2026-02-18T08:21:27.008 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.0 with skip (0.6803300069900193) > weight (0.25) 2026-02-18T08:21:27.009 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.1 with skip (0.6219279757367565) > weight (0.25) 2026-02-18T08:21:27.009 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.2 with skip (0.606313719544043) > weight (0.25) 2026-02-18T08:21:27.009 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.3 with skip (0.250032715080181) > weight (0.25) 2026-02-18T08:21:27.009 INFO:tasks.rbd_mirror_thrash:waiting for 104.09332045285427 secs before thrashing 2026-02-18T08:21:27.619 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-18T08:21:27.637 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmp /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.export /tmp/tmp.vybh1m9yJF/cluster1-mirror-test.export 2026-02-18T08:21:27.824 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.export /tmp/tmp.vybh1m9yJF/cluster1-mirror-test.export 2026-02-18T08:21:27.966 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:21:27.966 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ clean_snap_name=snap8-clean 2026-02-18T08:21:27.967 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_snap cluster2 mirror test snap8-clean 2026-02-18T08:21:27.967 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:21:27.967 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:21:27.967 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:21:27.967 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap8-clean 2026-02-18T08:21:27.967 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 -p mirror snap create test@snap8-clean --debug-rbd=20 --debug-journaler=20 2026-02-18T08:21:28.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 10` 2026-02-18T08:21:28.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ stress_write_image cluster2 mirror test 2026-02-18T08:21:28.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:21:28.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:21:28.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:21:28.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-18T08:21:28.691 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local duration=12 2026-02-18T08:21:28.691 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set +e 2026-02-18T08:21:28.691 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ timeout 12s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror test --debug-rbd=20 --debug-journaler=20 2026-02-18T08:21:29.202 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:100 IOs, 28099584 bytes 2026-02-18T08:21:29.788 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:200 IOs, 86806528 bytes 2026-02-18T08:21:30.806 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:300 IOs, 174813184 bytes 2026-02-18T08:21:31.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:400 IOs, 224893952 bytes 2026-02-18T08:21:32.248 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:500 IOs, 248474624 bytes 2026-02-18T08:21:32.677 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:600 IOs, 291444736 bytes 2026-02-18T08:21:33.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:700 IOs, 333109248 bytes 2026-02-18T08:21:34.214 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:800 IOs, 382453760 bytes 2026-02-18T08:21:34.886 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:900 IOs, 423648256 bytes 2026-02-18T08:21:35.390 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1000 IOs, 467287040 bytes 2026-02-18T08:21:36.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1100 IOs, 508419072 bytes 2026-02-18T08:21:37.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1200 IOs, 571138048 bytes 2026-02-18T08:21:37.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1300 IOs, 599348224 bytes 2026-02-18T08:21:38.253 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1400 IOs, 650556416 bytes 2026-02-18T08:21:39.864 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1500 IOs, 698641408 bytes 2026-02-18T08:21:40.700 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ error_code=124 2026-02-18T08:21:40.700 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set -e 2026-02-18T08:21:40.701 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 124 -eq 124 ']' 2026-02-18T08:21:40.701 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:21:40.701 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+replaying 2026-02-18T08:21:40.701 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:21:40.701 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:21:40.701 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:21:40.701 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local state_pattern=up+replaying 2026-02-18T08:21:40.701 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local description_pattern= 2026-02-18T08:21:40.701 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local service_pattern= 2026-02-18T08:21:40.701 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:21:40.701 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1 2026-02-18T08:21:41.701 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test_status_in_pool_dir cluster1 mirror test up+replaying '' '' 2026-02-18T08:21:41.701 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:21:41.701 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:21:41.701 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:21:41.701 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local state_pattern=up+replaying 2026-02-18T08:21:41.701 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local description_pattern= 2026-02-18T08:21:41.701 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local service_pattern= 2026-02-18T08:21:41.701 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local status 2026-02-18T08:21:41.702 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ CEPH_ARGS= 2026-02-18T08:21:41.702 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-18T08:21:41.742 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ status='test: 2026-02-18T08:21:41.742 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: global_id: 0b548823-d751-43df-a076-4efac2da4d3d 2026-02-18T08:21:41.742 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: state: up+replaying 2026-02-18T08:21:41.742 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: description: replaying, {"bytes_per_second":10215013.73,"entries_behind_primary":4078,"entries_per_second":637.0700000000001,"non_primary_position":{"entry_tid":25315,"object_number":335,"tag_tid":23},"primary_position":{"entry_tid":1,"object_number":337,"tag_tid":24},"seconds_until_synced":6} 2026-02-18T08:21:41.742 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: service: mirror.3 on trial163 2026-02-18T08:21:41.743 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: last_update: 2026-02-18 08:21:16 2026-02-18T08:21:41.743 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: peer_sites: 2026-02-18T08:21:41.743 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: name: cluster2 2026-02-18T08:21:41.743 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: state: up+stopped 2026-02-18T08:21:41.743 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: description: local image is primary 2026-02-18T08:21:41.743 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: last_update: 2026-02-18 08:21:16' 2026-02-18T08:21:41.743 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ grep '^ state: .*up+replaying' 2026-02-18T08:21:41.743 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: state: up+replaying 2026-02-18T08:21:41.744 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ grep '^ description: .*' 2026-02-18T08:21:41.744 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: description: replaying, {"bytes_per_second":10215013.73,"entries_behind_primary":4078,"entries_per_second":637.0700000000001,"non_primary_position":{"entry_tid":25315,"object_number":335,"tag_tid":23},"primary_position":{"entry_tid":1,"object_number":337,"tag_tid":24},"seconds_until_synced":6} 2026-02-18T08:21:41.745 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n '' ']' 2026-02-18T08:21:41.745 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ echo up+replaying 2026-02-18T08:21:41.745 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ grep '^up+' 2026-02-18T08:21:41.746 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:up+replaying 2026-02-18T08:21:41.746 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ grep 'service: *mirror..* on ' 2026-02-18T08:21:41.748 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: service: mirror.3 on trial163 2026-02-18T08:21:41.748 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local last_update 2026-02-18T08:21:41.748 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-18T08:21:41.750 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_update='2026-02-18 08:21:16' 2026-02-18T08:21:41.750 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+replaying '2026-02-18 08:21:16' 2026-02-18T08:21:41.750 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:21:41.750 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:21:41.751 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:21:41.751 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local state_pattern=up+replaying 2026-02-18T08:21:41.751 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'prev_last_update=2026-02-18 08:21:16' 2026-02-18T08:21:41.751 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local status 2026-02-18T08:21:41.751 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ CEPH_ARGS= 2026-02-18T08:21:41.751 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-18T08:21:41.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ status='OKOKOK142194268mirror.3trial16320.2.0-670-gadc19233falseOK43304333mirror.2trial16320.2.0-670-gadc19233falseOK43534356mirror.1trial16320.2.0-670-gadc19233trueOK44374440mirror.0trial16320.2.0-670-gadc19233falseOKtest0b548823-d751-43df-a076-4efac2da4d3dup+replayingreplaying, {"bytes_per_second":10215013.73,"entries_behind_primary":4078,"entries_per_second":637.0700000000001,"non_primary_position":{"entry_tid":25315,"object_number":335,"tag_tid":23},"primary_position":{"entry_tid":1,"object_number":337,"tag_tid":24},"seconds_until_synced":6}42194268mirror.3trial1632026-02-18 08:21:16cluster2c5e4f85e-ec31-430b-a71d-56e000d0de0bup+stoppedlocal image is primary2026-02-18 08:21:16' 2026-02-18T08:21:41.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local last_update state 2026-02-18T08:21:41.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-18T08:21:41.794 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_update='2026-02-18 08:21:16' 2026-02-18T08:21:41.794 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-18T08:21:41.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ state=up+replaying 2026-02-18T08:21:41.798 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ echo up+replaying 2026-02-18T08:21:41.798 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ grep up+replaying 2026-02-18T08:21:41.799 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:up+replaying 2026-02-18T08:21:41.800 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:21:41.800 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:21:41.800 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ snap_name=snap9 2026-02-18T08:21:41.800 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_snap cluster2 mirror test snap9 2026-02-18T08:21:41.800 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:21:41.800 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:21:41.801 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:21:41.801 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap9 2026-02-18T08:21:41.801 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 -p mirror snap create test@snap9 --debug-rbd=20 --debug-journaler=20 2026-02-18T08:22:12.718 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_replay_started cluster1 mirror test 2026-02-18T08:22:12.719 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:22:12.719 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:22:12.719 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:22:12.719 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_replay_state cluster1 mirror test started 2026-02-18T08:22:12.719 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:22:12.719 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:22:12.719 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:22:12.720 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local state=started 2026-02-18T08:22:12.720 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-18T08:22:12.720 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:22:12.720 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1 2026-02-18T08:22:13.720 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test_image_replay_state cluster1 mirror test started 2026-02-18T08:22:13.721 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:22:13.721 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:22:13.721 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:22:13.721 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local test_state=started 2026-02-18T08:22:13.721 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local status_result 2026-02-18T08:22:13.721 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_state=stopped 2026-02-18T08:22:13.721 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-18T08:22:13.721 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -i state 2026-02-18T08:22:13.722 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_instance=cluster1 2026-02-18T08:22:13.722 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-18T08:22:13.722 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:22:13.722 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance=cluster1 2026-02-18T08:22:13.722 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local loop_instance 2026-02-18T08:22:13.722 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:22:13.722 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sleep 0 2026-02-18T08:22:13.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:22:13.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ seq 0 3 2026-02-18T08:22:13.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:22:13.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-18T08:22:13.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:0 2026-02-18T08:22:13.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-18T08:22:13.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:22:13.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:22:13.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:22:13.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-18T08:22:13.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:22:13.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:13.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:22:13.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:22:13.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-18T08:22:13.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:22:13.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:22:13.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:22:13.728 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:22:13.728 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:0 2026-02-18T08:22:13.728 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-18T08:22:13.728 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-18T08:22:13.728 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:22:13.728 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:0 2026-02-18T08:22:13.728 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T08:22:13.728 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-18T08:22:13.728 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:22:13.728 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:22:13.728 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-18T08:22:13.728 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-18T08:22:13.729 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-18T08:22:13.729 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=0 2026-02-18T08:22:13.729 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=0 2026-02-18T08:22:13.729 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:22:13.735 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:22:13.736 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:22:13.736 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:22:13.736 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-18T08:22:13.787 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:22:13.787 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status : get status for rbd mirror 2026-02-18T08:22:13.787 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:22:13.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:22:13.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test 2026-02-18T08:22:13.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:1 2026-02-18T08:22:13.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-18T08:22:13.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:22:13.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:22:13.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:22:13.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-18T08:22:13.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:22:13.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:13.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:22:13.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:22:13.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-18T08:22:13.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:22:13.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:22:13.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:22:13.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:22:13.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:1 2026-02-18T08:22:13.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-18T08:22:13.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-18T08:22:13.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:22:13.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:1 2026-02-18T08:22:13.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T08:22:13.793 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-18T08:22:13.793 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:22:13.793 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:22:13.793 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-18T08:22:13.793 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-18T08:22:13.793 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-18T08:22:13.793 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=1 2026-02-18T08:22:13.793 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=1 2026-02-18T08:22:13.793 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:22:13.802 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:22:13.803 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:22:13.803 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:22:13.803 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test 2026-02-18T08:22:13.861 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:22:13.861 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status : get status for rbd mirror 2026-02-18T08:22:13.861 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:22:13.864 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:22:13.864 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/test 2026-02-18T08:22:13.864 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:2 2026-02-18T08:22:13.864 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-18T08:22:13.864 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:22:13.864 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:22:13.865 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:22:13.865 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-18T08:22:13.865 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:22:13.865 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:13.865 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:22:13.865 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:22:13.865 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-18T08:22:13.865 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:22:13.865 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:22:13.865 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:22:13.866 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:22:13.866 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:2 2026-02-18T08:22:13.866 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-18T08:22:13.866 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-18T08:22:13.866 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:22:13.866 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:2 2026-02-18T08:22:13.866 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T08:22:13.866 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-18T08:22:13.866 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:22:13.866 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:22:13.866 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-18T08:22:13.866 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-18T08:22:13.867 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-18T08:22:13.867 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=2 2026-02-18T08:22:13.867 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=2 2026-02-18T08:22:13.867 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:22:13.875 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:22:13.876 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:22:13.876 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:22:13.876 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/test 2026-02-18T08:22:13.933 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:22:13.933 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status : get status for rbd mirror 2026-02-18T08:22:13.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:22:13.936 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:22:13.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/test 2026-02-18T08:22:13.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:3 2026-02-18T08:22:13.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-18T08:22:13.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:22:13.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:22:13.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:22:13.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-18T08:22:13.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:22:13.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:13.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:22:13.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:22:13.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-18T08:22:13.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:22:13.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:22:13.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:22:13.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:22:13.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:3 2026-02-18T08:22:13.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-18T08:22:13.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-18T08:22:13.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:22:13.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:3 2026-02-18T08:22:13.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T08:22:13.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-18T08:22:13.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:22:13.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:22:13.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-18T08:22:13.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-18T08:22:13.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-18T08:22:13.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=3 2026-02-18T08:22:13.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=3 2026-02-18T08:22:13.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:22:13.946 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:22:13.947 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:22:13.947 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:22:13.948 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/test 2026-02-18T08:22:14.012 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ return 0 2026-02-18T08:22:14.012 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ status_result=' "state": "Replaying"' 2026-02-18T08:22:14.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ echo ' "state": "Replaying"' 2026-02-18T08:22:14.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ grep -i Replaying 2026-02-18T08:22:14.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: "state": "Replaying" 2026-02-18T08:22:14.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ current_state=started 2026-02-18T08:22:14.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test started = started 2026-02-18T08:22:14.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:22:14.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror test 2026-02-18T08:22:14.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_cluster=cluster1 2026-02-18T08:22:14.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:22:14.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_pool=mirror 2026-02-18T08:22:14.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local remote_pool=mirror 2026-02-18T08:22:14.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:22:14.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' journal = journal ']' 2026-02-18T08:22:14.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror mirror test 2026-02-18T08:22:14.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_cluster=cluster1 2026-02-18T08:22:14.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:22:14.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_pool=mirror 2026-02-18T08:22:14.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local remote_pool=mirror 2026-02-18T08:22:14.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:22:14.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-02-18T08:22:14.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-02-18T08:22:14.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:22:14.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:22:14.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:22:14.217 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:22:14.217 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:22:14.217 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:22:14.217 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:22:14.217 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:22:14.217 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:22:14.217 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:22:14.218 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:22:14.218 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:22:14.218 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:14.218 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:22:14.218 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:22:14.218 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:22:14.218 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:22:14.218 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:22:14.218 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:22:14.221 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:22:14.221 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:22:14.221 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:22:14.221 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:22:14.221 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:14.222 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:22:14.222 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:22:14.222 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:22:14.222 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:22:14.222 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:22:14.222 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:22:14.222 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:22:14.222 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:22:14.222 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:22:14.222 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:14.222 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:22:14.223 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:22:14.223 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:22:14.223 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:22:14.223 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:22:14.223 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:22:14.223 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:22:14.223 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:22:14.223 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:22:14.223 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:22:14.223 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:14.223 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:22:14.223 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:22:14.223 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:22:14.224 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:22:14.224 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:22:14.224 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:22:14.224 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:22:14.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:22:14.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:22:14.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:22:14.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:22:14.283 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:22:14.283 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:22:14.283 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:22:14.286 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:22:14.286 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:22:14.286 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:22:14.286 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:14.286 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:22:14.286 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:22:14.286 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:22:14.286 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:22:14.287 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:22:14.287 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:22:14.287 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:22:14.287 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:22:14.287 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:22:14.287 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:14.287 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:22:14.287 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:22:14.287 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:22:14.287 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:22:14.287 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:22:14.288 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:22:14.288 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:22:14.288 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:22:14.288 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:22:14.288 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:22:14.288 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:14.288 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:22:14.288 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:22:14.288 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:22:14.289 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:22:14.289 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:22:14.289 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:22:14.289 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:22:14.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:22:14.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:22:14.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:22:14.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:22:14.355 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:22:14.355 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:22:14.355 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:22:14.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:22:14.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:22:14.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:22:14.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:14.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:22:14.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:22:14.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:22:14.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:22:14.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:22:14.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:22:14.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:22:14.360 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:22:14.360 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:22:14.360 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:14.360 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:22:14.360 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:22:14.360 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:22:14.360 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:22:14.360 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:22:14.360 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:22:14.360 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:22:14.360 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:22:14.360 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:22:14.361 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:22:14.361 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:14.361 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:22:14.361 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:22:14.361 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:22:14.361 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:22:14.361 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:22:14.361 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:22:14.361 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:22:14.371 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:22:14.371 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:22:14.372 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:22:14.372 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:22:14.421 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:22:14.421 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:22:14.421 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:22:14.423 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:22:14.423 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:22:14.423 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:22:14.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:14.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:22:14.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:22:14.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:22:14.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:22:14.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:22:14.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:22:14.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:22:14.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:22:14.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:22:14.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:14.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:22:14.425 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:22:14.425 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:22:14.425 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:22:14.425 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:22:14.425 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:22:14.425 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:22:14.425 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:22:14.425 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:22:14.425 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:22:14.425 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:14.425 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:22:14.425 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:22:14.425 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:22:14.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:22:14.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:22:14.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:22:14.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:22:14.433 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:22:14.434 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:22:14.434 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:22:14.434 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:22:14.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:22:14.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:22:14.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:22:14.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:22:14.500 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:22:14.500 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:22:14.500 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:22:14.500 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:22:14.500 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:22:14.500 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:22:14.500 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:22:14.500 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:22:14.501 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:22:14.501 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:14.502 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:22:14.502 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:14.539 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 90 2026-02-18T08:22:14.539 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 95 2026-02-18T08:22:14.539 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:22:14.539 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=381, tag_tid=27, entry_tid=1], [object_number=380, tag_tid=27, entry_tid=0], [object_number=382, tag_tid=26, entry_tid=46994], [object_number=383, tag_tid=26, entry_tid=46991]]], state=connected] 2026-02-18T08:22:14.539 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=360, tag_tid=26, entry_tid=24520], [object_number=363, tag_tid=26, entry_tid=24519], [object_number=362, tag_tid=26, entry_tid=24518], [object_number=361, tag_tid=26, entry_tid=24517]]], state=connected] 2026-02-18T08:22:14.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:14.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=381, tag_tid=27, entry_tid=1' 2026-02-18T08:22:14.545 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:22:14.545 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:22:14.545 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:22:14.545 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:22:14.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:22:14.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:22:14.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:22:14.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:22:14.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:22:14.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:22:14.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:22:14.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:22:14.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:14.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:22:14.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:14.575 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 90 2026-02-18T08:22:14.575 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 95 2026-02-18T08:22:14.575 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:22:14.575 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=381, tag_tid=27, entry_tid=1], [object_number=380, tag_tid=27, entry_tid=0], [object_number=382, tag_tid=26, entry_tid=46994], [object_number=383, tag_tid=26, entry_tid=46991]]], state=connected] 2026-02-18T08:22:14.575 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=360, tag_tid=26, entry_tid=24520], [object_number=363, tag_tid=26, entry_tid=24519], [object_number=362, tag_tid=26, entry_tid=24518], [object_number=361, tag_tid=26, entry_tid=24517]]], state=connected] 2026-02-18T08:22:14.578 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:14.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=360, tag_tid=26, entry_tid=24520' 2026-02-18T08:22:14.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=381, tag_tid=27, entry_tid=1' -a 'object_number=381, tag_tid=27, entry_tid=1' = 'object_number=360, tag_tid=26, entry_tid=24520' 2026-02-18T08:22:14.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=360, tag_tid=26, entry_tid=24520' '!=' '' 2026-02-18T08:22:14.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:22:14.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=360, tag_tid=26, entry_tid=24520' = '' 2026-02-18T08:22:14.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=360, tag_tid=26, entry_tid=24520' 2026-02-18T08:22:14.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=381, tag_tid=27, entry_tid=1' 2026-02-18T08:22:14.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:22:14.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:22:14.582 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=27 2026-02-18T08:22:14.582 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=360, tag_tid=26, entry_tid=24520' 2026-02-18T08:22:14.582 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:22:14.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:22:14.584 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=26 2026-02-18T08:22:14.585 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=381, tag_tid=27, entry_tid=1' 2026-02-18T08:22:14.585 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:22:14.585 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:22:14.586 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:22:14.587 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=360, tag_tid=26, entry_tid=24520' 2026-02-18T08:22:14.587 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:22:14.587 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:22:14.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=24520 2026-02-18T08:22:14.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 27 = 26 -a 1 -le 24520 2026-02-18T08:22:14.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:22:14.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:22:14.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:22:14.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:22:14.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:22:14.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:22:14.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:22:14.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:22:14.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:22:14.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:22:14.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:22:14.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:22:14.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:14.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:22:14.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:22:14.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:22:14.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:22:14.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:22:14.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:22:14.795 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:22:14.795 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:22:14.795 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:22:14.795 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:22:14.795 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:14.795 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:22:14.795 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:22:14.795 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:22:14.795 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:22:14.796 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:22:14.796 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:22:14.796 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:22:14.796 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:22:14.796 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:22:14.796 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:14.796 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:22:14.796 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:22:14.796 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:22:14.796 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:22:14.796 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:22:14.796 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:22:14.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:22:14.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:22:14.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:22:14.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:22:14.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:14.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:22:14.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:22:14.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:22:14.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:22:14.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:22:14.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:22:14.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:22:14.804 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:22:14.805 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:22:14.805 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:22:14.805 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:22:14.853 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:22:14.853 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:22:14.853 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:22:14.856 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:22:14.856 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:22:14.856 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:22:14.856 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:14.856 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:22:14.856 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:22:14.856 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:22:14.856 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:22:14.856 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:22:14.857 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:22:14.857 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:22:14.857 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:22:14.857 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:22:14.857 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:14.857 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:22:14.857 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:22:14.857 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:22:14.857 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:22:14.857 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:22:14.857 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:22:14.857 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:22:14.858 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:22:14.858 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:22:14.858 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:22:14.858 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:14.858 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:22:14.858 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:22:14.858 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:22:14.858 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:22:14.858 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:22:14.858 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:22:14.858 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:22:14.866 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:22:14.867 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:22:14.867 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:22:14.867 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:22:14.914 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:22:14.914 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:22:14.914 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:22:14.917 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:22:14.917 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:22:14.917 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:22:14.918 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:14.918 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:22:14.918 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:22:14.918 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:22:14.918 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:22:14.918 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:22:14.918 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:22:14.918 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:22:14.918 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:22:14.918 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:22:14.918 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:14.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:22:14.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:22:14.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:22:14.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:22:14.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:22:14.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:22:14.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:22:14.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:22:14.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:22:14.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:22:14.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:14.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:22:14.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:22:14.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:22:14.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:22:14.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:22:14.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:22:14.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:22:14.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:22:14.928 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:22:14.928 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:22:14.928 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:22:14.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:22:14.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:22:14.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:22:14.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:22:14.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:22:14.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:22:14.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:14.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:22:14.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:22:14.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:22:14.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:22:14.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:22:14.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:22:14.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:22:14.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:22:14.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:22:14.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:14.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:22:14.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:22:14.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:22:14.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:22:14.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:22:14.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:22:14.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:22:14.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:22:14.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:22:14.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:22:14.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:14.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:22:14.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:22:14.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:22:14.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:22:14.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:22:14.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:22:14.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:22:14.989 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:22:14.990 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:22:14.990 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:22:14.990 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:22:15.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:22:15.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:22:15.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:22:15.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:22:15.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:22:15.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:22:15.055 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:22:15.055 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:22:15.055 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:22:15.055 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:22:15.055 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:22:15.055 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:22:15.055 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:22:15.056 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:15.056 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:22:15.056 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:15.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 90 2026-02-18T08:22:15.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 95 2026-02-18T08:22:15.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:22:15.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=381, tag_tid=27, entry_tid=1], [object_number=380, tag_tid=27, entry_tid=0], [object_number=382, tag_tid=26, entry_tid=46994], [object_number=383, tag_tid=26, entry_tid=46991]]], state=connected] 2026-02-18T08:22:15.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=360, tag_tid=26, entry_tid=24872], [object_number=363, tag_tid=26, entry_tid=24871], [object_number=362, tag_tid=26, entry_tid=24870], [object_number=361, tag_tid=26, entry_tid=24869]]], state=connected] 2026-02-18T08:22:15.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:15.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=381, tag_tid=27, entry_tid=1' 2026-02-18T08:22:15.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:22:15.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:22:15.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:22:15.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:22:15.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:22:15.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:22:15.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:22:15.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:22:15.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:22:15.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:22:15.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:22:15.092 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:22:15.092 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:15.092 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:22:15.092 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:15.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 90 2026-02-18T08:22:15.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 95 2026-02-18T08:22:15.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:22:15.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=381, tag_tid=27, entry_tid=1], [object_number=380, tag_tid=27, entry_tid=0], [object_number=382, tag_tid=26, entry_tid=46994], [object_number=383, tag_tid=26, entry_tid=46991]]], state=connected] 2026-02-18T08:22:15.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=360, tag_tid=26, entry_tid=24872], [object_number=363, tag_tid=26, entry_tid=24871], [object_number=362, tag_tid=26, entry_tid=24870], [object_number=361, tag_tid=26, entry_tid=24869]]], state=connected] 2026-02-18T08:22:15.131 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:15.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=360, tag_tid=26, entry_tid=24872' 2026-02-18T08:22:15.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=381, tag_tid=27, entry_tid=1' -a 'object_number=381, tag_tid=27, entry_tid=1' = 'object_number=360, tag_tid=26, entry_tid=24872' 2026-02-18T08:22:15.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=360, tag_tid=26, entry_tid=24872' '!=' 'object_number=360, tag_tid=26, entry_tid=24520' 2026-02-18T08:22:15.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:22:15.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=360, tag_tid=26, entry_tid=24872' = 'object_number=360, tag_tid=26, entry_tid=24520' 2026-02-18T08:22:15.134 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=360, tag_tid=26, entry_tid=24872' 2026-02-18T08:22:15.134 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=381, tag_tid=27, entry_tid=1' 2026-02-18T08:22:15.134 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:22:15.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:22:15.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=27 2026-02-18T08:22:15.136 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=360, tag_tid=26, entry_tid=24872' 2026-02-18T08:22:15.136 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:22:15.136 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:22:15.138 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=26 2026-02-18T08:22:15.138 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=381, tag_tid=27, entry_tid=1' 2026-02-18T08:22:15.139 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:22:15.139 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:22:15.140 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:22:15.141 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=360, tag_tid=26, entry_tid=24872' 2026-02-18T08:22:15.141 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:22:15.141 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:22:15.142 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=24872 2026-02-18T08:22:15.143 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 27 = 26 -a 1 -le 24872 2026-02-18T08:22:15.143 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:22:15.143 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:22:15.143 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:22:15.344 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:22:15.344 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:22:15.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:22:15.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:22:15.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:22:15.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:22:15.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:22:15.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:22:15.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:22:15.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:15.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:22:15.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:22:15.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:22:15.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:22:15.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:22:15.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:22:15.348 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:22:15.348 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:22:15.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:22:15.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:22:15.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:15.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:22:15.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:22:15.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:22:15.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:22:15.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:22:15.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:22:15.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:22:15.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:22:15.350 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:22:15.350 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:15.350 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:22:15.350 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:22:15.350 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:22:15.350 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:22:15.350 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:22:15.350 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:22:15.350 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:22:15.350 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:22:15.350 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:22:15.350 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:22:15.351 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:15.351 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:22:15.351 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:22:15.351 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:22:15.351 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:22:15.351 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:22:15.351 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:22:15.351 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:22:15.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:22:15.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:22:15.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:22:15.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:22:15.416 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:22:15.416 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:22:15.416 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:22:15.419 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:22:15.419 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:22:15.419 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:22:15.419 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:15.419 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:22:15.419 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:22:15.419 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:22:15.420 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:22:15.420 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:22:15.420 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:22:15.420 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:22:15.420 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:22:15.420 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:22:15.420 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:15.420 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:22:15.420 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:22:15.420 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:22:15.420 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:22:15.420 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:22:15.421 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:22:15.421 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:22:15.421 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:22:15.421 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:22:15.421 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:22:15.421 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:15.421 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:22:15.421 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:22:15.421 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:22:15.421 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:22:15.421 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:22:15.421 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:22:15.422 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:22:15.431 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:22:15.431 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:22:15.431 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:22:15.431 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:22:15.478 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:22:15.479 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:22:15.479 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:22:15.481 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:22:15.482 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:22:15.482 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:22:15.482 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:15.482 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:22:15.482 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:22:15.482 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:22:15.482 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:22:15.482 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:22:15.482 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:22:15.482 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:22:15.482 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:22:15.483 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:22:15.483 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:15.483 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:22:15.483 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:22:15.483 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:22:15.483 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:22:15.483 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:22:15.483 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:22:15.483 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:22:15.483 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:22:15.483 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:22:15.483 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:22:15.484 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:15.484 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:22:15.484 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:22:15.484 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:22:15.484 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:22:15.484 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:22:15.484 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:22:15.484 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:22:15.492 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:22:15.492 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:22:15.493 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:22:15.493 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:22:15.540 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:22:15.540 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:22:15.540 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:22:15.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:22:15.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:22:15.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:22:15.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:15.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:22:15.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:22:15.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:22:15.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:22:15.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:22:15.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:22:15.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:22:15.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:22:15.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:22:15.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:15.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:22:15.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:22:15.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:22:15.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:22:15.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:22:15.545 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:22:15.545 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:22:15.545 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:22:15.545 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:22:15.545 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:22:15.545 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:15.545 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:22:15.545 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:22:15.545 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:22:15.545 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:22:15.545 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:22:15.545 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:22:15.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:22:15.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:22:15.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:22:15.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:22:15.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:22:15.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:22:15.623 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:22:15.623 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:22:15.623 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:22:15.623 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:22:15.623 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:22:15.623 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:22:15.623 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:22:15.623 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:22:15.623 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:22:15.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:22:15.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:22:15.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:22:15.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:15.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:22:15.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:15.664 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 90 2026-02-18T08:22:15.664 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 95 2026-02-18T08:22:15.664 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:22:15.664 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=381, tag_tid=27, entry_tid=1], [object_number=380, tag_tid=27, entry_tid=0], [object_number=382, tag_tid=26, entry_tid=46994], [object_number=383, tag_tid=26, entry_tid=46991]]], state=connected] 2026-02-18T08:22:15.664 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=361, tag_tid=26, entry_tid=25241], [object_number=360, tag_tid=26, entry_tid=25240], [object_number=363, tag_tid=26, entry_tid=25239], [object_number=362, tag_tid=26, entry_tid=25238]]], state=connected] 2026-02-18T08:22:15.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:15.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=381, tag_tid=27, entry_tid=1' 2026-02-18T08:22:15.670 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:22:15.670 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:22:15.670 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:22:15.670 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:22:15.670 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:22:15.671 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:22:15.671 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:22:15.671 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:22:15.671 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:22:15.671 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:22:15.671 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:22:15.672 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:22:15.672 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:15.672 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:22:15.672 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:15.712 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 90 2026-02-18T08:22:15.712 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 95 2026-02-18T08:22:15.712 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:22:15.712 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=381, tag_tid=27, entry_tid=1], [object_number=380, tag_tid=27, entry_tid=0], [object_number=382, tag_tid=26, entry_tid=46994], [object_number=383, tag_tid=26, entry_tid=46991]]], state=connected] 2026-02-18T08:22:15.712 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=361, tag_tid=26, entry_tid=25241], [object_number=360, tag_tid=26, entry_tid=25240], [object_number=363, tag_tid=26, entry_tid=25239], [object_number=362, tag_tid=26, entry_tid=25238]]], state=connected] 2026-02-18T08:22:15.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:15.717 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=361, tag_tid=26, entry_tid=25241' 2026-02-18T08:22:15.717 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=381, tag_tid=27, entry_tid=1' -a 'object_number=381, tag_tid=27, entry_tid=1' = 'object_number=361, tag_tid=26, entry_tid=25241' 2026-02-18T08:22:15.717 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=361, tag_tid=26, entry_tid=25241' '!=' 'object_number=360, tag_tid=26, entry_tid=24872' 2026-02-18T08:22:15.718 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:22:15.718 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=361, tag_tid=26, entry_tid=25241' = 'object_number=360, tag_tid=26, entry_tid=24872' 2026-02-18T08:22:15.718 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=361, tag_tid=26, entry_tid=25241' 2026-02-18T08:22:15.718 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=381, tag_tid=27, entry_tid=1' 2026-02-18T08:22:15.718 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:22:15.719 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:22:15.720 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=27 2026-02-18T08:22:15.720 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=361, tag_tid=26, entry_tid=25241' 2026-02-18T08:22:15.720 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:22:15.721 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:22:15.722 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=26 2026-02-18T08:22:15.722 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=381, tag_tid=27, entry_tid=1' 2026-02-18T08:22:15.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:22:15.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:22:15.724 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:22:15.725 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=361, tag_tid=26, entry_tid=25241' 2026-02-18T08:22:15.725 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:22:15.725 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:22:15.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=25241 2026-02-18T08:22:15.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 27 = 26 -a 1 -le 25241 2026-02-18T08:22:15.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:22:15.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:22:15.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:22:15.928 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:22:15.928 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:22:15.929 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:22:15.929 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:22:15.929 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:22:15.929 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:22:15.929 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:22:15.929 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:22:15.929 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:22:15.929 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:15.929 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:22:15.929 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:22:15.929 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:22:15.930 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:22:15.930 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:22:15.930 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:22:15.932 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:22:15.933 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:22:15.933 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:22:15.933 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:22:15.933 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:15.933 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:22:15.933 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:22:15.933 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:22:15.933 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:22:15.933 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:22:15.933 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:22:15.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:22:15.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:22:15.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:22:15.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:15.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:22:15.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:22:15.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:22:15.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:22:15.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:22:15.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:22:15.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:22:15.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:22:15.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:22:15.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:22:15.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:15.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:22:15.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:22:15.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:22:15.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:22:15.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:22:15.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:22:15.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:22:15.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:22:15.944 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:22:15.944 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:22:15.944 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:22:15.993 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:22:15.993 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:22:15.994 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:22:15.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:22:15.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:22:15.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:22:15.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:15.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:22:15.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:22:15.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:22:15.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:22:15.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:22:15.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:22:15.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:22:15.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:22:15.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:22:15.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:15.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:22:15.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:22:15.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:22:15.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:22:15.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:22:15.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:22:15.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:22:15.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:22:15.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:22:15.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:22:15.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:15.999 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:22:15.999 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:22:15.999 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:22:15.999 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:22:15.999 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:22:15.999 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:22:15.999 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:22:16.007 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:22:16.008 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:22:16.008 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:22:16.008 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:22:16.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:22:16.067 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:22:16.067 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:22:16.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:22:16.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:22:16.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:22:16.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:16.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:22:16.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:22:16.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:22:16.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:22:16.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:22:16.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:22:16.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:22:16.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:22:16.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:22:16.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:16.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:22:16.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:22:16.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:22:16.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:22:16.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:22:16.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:22:16.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:22:16.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:22:16.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:22:16.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:22:16.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:16.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:22:16.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:22:16.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:22:16.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:22:16.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:22:16.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:22:16.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:22:16.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:22:16.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:22:16.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:22:16.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:22:16.130 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:22:16.130 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:22:16.130 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:22:16.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:22:16.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:22:16.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:22:16.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:16.134 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:22:16.134 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:22:16.134 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:22:16.134 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:22:16.134 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:22:16.134 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:22:16.134 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:22:16.134 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:22:16.134 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:22:16.134 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:16.134 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:22:16.134 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:22:16.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:22:16.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:22:16.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:22:16.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:22:16.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:22:16.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:22:16.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:22:16.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:22:16.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:16.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:22:16.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:22:16.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:22:16.136 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:22:16.136 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:22:16.136 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:22:16.136 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:22:16.143 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:22:16.144 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:22:16.144 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:22:16.144 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:22:16.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:22:16.206 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:22:16.206 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:22:16.206 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:22:16.207 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:22:16.207 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:22:16.207 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:22:16.207 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:22:16.207 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:22:16.207 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:22:16.207 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:22:16.208 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:22:16.208 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:22:16.208 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:16.209 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:22:16.209 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:16.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 90 2026-02-18T08:22:16.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 95 2026-02-18T08:22:16.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:22:16.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=381, tag_tid=27, entry_tid=1], [object_number=380, tag_tid=27, entry_tid=0], [object_number=382, tag_tid=26, entry_tid=46994], [object_number=383, tag_tid=26, entry_tid=46991]]], state=connected] 2026-02-18T08:22:16.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=362, tag_tid=26, entry_tid=25598], [object_number=361, tag_tid=26, entry_tid=25597], [object_number=360, tag_tid=26, entry_tid=25596], [object_number=363, tag_tid=26, entry_tid=25595]]], state=connected] 2026-02-18T08:22:16.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:16.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=381, tag_tid=27, entry_tid=1' 2026-02-18T08:22:16.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:22:16.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:22:16.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:22:16.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:22:16.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:22:16.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:22:16.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:22:16.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:22:16.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:22:16.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:22:16.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:22:16.244 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:22:16.244 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:16.245 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:22:16.245 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:16.273 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 90 2026-02-18T08:22:16.273 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 95 2026-02-18T08:22:16.273 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:22:16.273 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=381, tag_tid=27, entry_tid=1], [object_number=380, tag_tid=27, entry_tid=0], [object_number=382, tag_tid=26, entry_tid=46994], [object_number=383, tag_tid=26, entry_tid=46991]]], state=connected] 2026-02-18T08:22:16.273 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=362, tag_tid=26, entry_tid=25598], [object_number=361, tag_tid=26, entry_tid=25597], [object_number=360, tag_tid=26, entry_tid=25596], [object_number=363, tag_tid=26, entry_tid=25595]]], state=connected] 2026-02-18T08:22:16.275 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:16.277 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=362, tag_tid=26, entry_tid=25598' 2026-02-18T08:22:16.277 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=381, tag_tid=27, entry_tid=1' -a 'object_number=381, tag_tid=27, entry_tid=1' = 'object_number=362, tag_tid=26, entry_tid=25598' 2026-02-18T08:22:16.277 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=362, tag_tid=26, entry_tid=25598' '!=' 'object_number=361, tag_tid=26, entry_tid=25241' 2026-02-18T08:22:16.278 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:22:16.278 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=362, tag_tid=26, entry_tid=25598' = 'object_number=361, tag_tid=26, entry_tid=25241' 2026-02-18T08:22:16.278 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=362, tag_tid=26, entry_tid=25598' 2026-02-18T08:22:16.278 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=381, tag_tid=27, entry_tid=1' 2026-02-18T08:22:16.278 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:22:16.279 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:22:16.279 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=27 2026-02-18T08:22:16.280 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=362, tag_tid=26, entry_tid=25598' 2026-02-18T08:22:16.280 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:22:16.280 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:22:16.281 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=26 2026-02-18T08:22:16.282 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=381, tag_tid=27, entry_tid=1' 2026-02-18T08:22:16.283 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:22:16.283 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:22:16.284 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:22:16.285 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=362, tag_tid=26, entry_tid=25598' 2026-02-18T08:22:16.285 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:22:16.285 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:22:16.286 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=25598 2026-02-18T08:22:16.286 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 27 = 26 -a 1 -le 25598 2026-02-18T08:22:16.286 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:22:16.287 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:22:16.287 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:22:16.488 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:22:16.488 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:22:16.488 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:22:16.488 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:22:16.488 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:22:16.488 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:22:16.488 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:22:16.488 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:22:16.489 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:22:16.489 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:16.489 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:22:16.489 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:22:16.489 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:22:16.489 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:22:16.489 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:22:16.492 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:22:16.492 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:22:16.492 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:22:16.492 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:22:16.492 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:22:16.492 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:16.492 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:22:16.492 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:22:16.493 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:22:16.493 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:22:16.493 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:22:16.493 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:22:16.493 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:22:16.493 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:22:16.493 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:22:16.493 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:16.493 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:22:16.493 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:22:16.493 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:22:16.493 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:22:16.493 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:22:16.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:22:16.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:22:16.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:22:16.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:22:16.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:22:16.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:16.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:22:16.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:22:16.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:22:16.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:22:16.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:22:16.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:22:16.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:22:16.503 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:22:16.503 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:22:16.503 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:22:16.503 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:22:16.565 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:22:16.565 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:22:16.565 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:22:16.568 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:22:16.568 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:22:16.568 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:22:16.568 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:16.568 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:22:16.568 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:22:16.568 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:22:16.568 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:22:16.568 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:22:16.568 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:22:16.569 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:22:16.569 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:22:16.569 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:22:16.569 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:16.569 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:22:16.569 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:22:16.569 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:22:16.569 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:22:16.569 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:22:16.569 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:22:16.569 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:22:16.569 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:22:16.569 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:22:16.570 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:22:16.570 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:16.570 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:22:16.570 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:22:16.570 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:22:16.570 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:22:16.570 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:22:16.570 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:22:16.570 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:22:16.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:22:16.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:22:16.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:22:16.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:22:16.637 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:22:16.637 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:22:16.637 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:22:16.640 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:22:16.640 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:22:16.640 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:22:16.640 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:16.640 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:22:16.641 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:22:16.641 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:22:16.641 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:22:16.641 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:22:16.641 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:22:16.641 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:22:16.641 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:22:16.641 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:22:16.641 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:16.641 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:22:16.641 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:22:16.641 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:22:16.642 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:22:16.642 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:22:16.642 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:22:16.642 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:22:16.642 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:22:16.642 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:22:16.642 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:22:16.642 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:16.642 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:22:16.642 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:22:16.642 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:22:16.642 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:22:16.643 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:22:16.643 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:22:16.643 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:22:16.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:22:16.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:22:16.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:22:16.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:22:16.709 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:22:16.709 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:22:16.709 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:22:16.712 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:22:16.712 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:22:16.712 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:22:16.712 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:16.713 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:22:16.713 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:22:16.713 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:22:16.713 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:22:16.713 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:22:16.713 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:22:16.713 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:22:16.713 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:22:16.713 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:22:16.713 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:16.713 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:22:16.713 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:22:16.714 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:22:16.714 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:22:16.714 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:22:16.714 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:22:16.714 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:22:16.714 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:22:16.714 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:22:16.714 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:22:16.714 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:16.714 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:22:16.714 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:22:16.714 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:22:16.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:22:16.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:22:16.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:22:16.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:22:16.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:22:16.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:22:16.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:22:16.724 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:22:16.793 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:22:16.794 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:22:16.794 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:22:16.794 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:22:16.794 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:22:16.794 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:22:16.794 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:22:16.794 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:22:16.795 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:22:16.795 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:22:16.795 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:22:16.795 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:22:16.795 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:22:16.796 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:16.796 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:22:16.796 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:16.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 90 2026-02-18T08:22:16.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 95 2026-02-18T08:22:16.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:22:16.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=381, tag_tid=27, entry_tid=1], [object_number=380, tag_tid=27, entry_tid=0], [object_number=382, tag_tid=26, entry_tid=46994], [object_number=383, tag_tid=26, entry_tid=46991]]], state=connected] 2026-02-18T08:22:16.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=360, tag_tid=26, entry_tid=25956], [object_number=363, tag_tid=26, entry_tid=25955], [object_number=362, tag_tid=26, entry_tid=25954], [object_number=361, tag_tid=26, entry_tid=25953]]], state=connected] 2026-02-18T08:22:16.838 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:16.839 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=381, tag_tid=27, entry_tid=1' 2026-02-18T08:22:16.840 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:22:16.840 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:22:16.840 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:22:16.840 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:22:16.840 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:22:16.841 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:22:16.841 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:22:16.841 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:22:16.841 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:22:16.841 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:22:16.841 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:22:16.841 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:22:16.842 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:16.842 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:22:16.842 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:16.870 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 90 2026-02-18T08:22:16.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 95 2026-02-18T08:22:16.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:22:16.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=381, tag_tid=27, entry_tid=1], [object_number=380, tag_tid=27, entry_tid=0], [object_number=382, tag_tid=26, entry_tid=46994], [object_number=383, tag_tid=26, entry_tid=46991]]], state=connected] 2026-02-18T08:22:16.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=360, tag_tid=26, entry_tid=25956], [object_number=363, tag_tid=26, entry_tid=25955], [object_number=362, tag_tid=26, entry_tid=25954], [object_number=361, tag_tid=26, entry_tid=25953]]], state=connected] 2026-02-18T08:22:16.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:16.875 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=360, tag_tid=26, entry_tid=25956' 2026-02-18T08:22:16.875 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=381, tag_tid=27, entry_tid=1' -a 'object_number=381, tag_tid=27, entry_tid=1' = 'object_number=360, tag_tid=26, entry_tid=25956' 2026-02-18T08:22:16.876 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=360, tag_tid=26, entry_tid=25956' '!=' 'object_number=362, tag_tid=26, entry_tid=25598' 2026-02-18T08:22:16.876 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:22:16.876 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=360, tag_tid=26, entry_tid=25956' = 'object_number=362, tag_tid=26, entry_tid=25598' 2026-02-18T08:22:16.876 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=360, tag_tid=26, entry_tid=25956' 2026-02-18T08:22:16.876 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=381, tag_tid=27, entry_tid=1' 2026-02-18T08:22:16.876 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:22:16.876 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:22:16.877 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=27 2026-02-18T08:22:16.878 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=360, tag_tid=26, entry_tid=25956' 2026-02-18T08:22:16.878 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:22:16.878 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:22:16.879 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=26 2026-02-18T08:22:16.880 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=381, tag_tid=27, entry_tid=1' 2026-02-18T08:22:16.880 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:22:16.880 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:22:16.881 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:22:16.882 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=360, tag_tid=26, entry_tid=25956' 2026-02-18T08:22:16.882 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:22:16.882 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:22:16.884 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=25956 2026-02-18T08:22:16.884 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 27 = 26 -a 1 -le 25956 2026-02-18T08:22:16.884 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:22:16.884 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:22:16.885 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:22:17.086 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:22:17.086 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:22:17.086 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:22:17.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:22:17.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:22:17.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:22:17.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:22:17.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:22:17.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:22:17.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:17.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:22:17.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:22:17.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:22:17.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:22:17.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:22:17.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:22:17.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:22:17.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:22:17.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:22:17.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:22:17.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:17.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:22:17.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:22:17.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:22:17.092 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:22:17.092 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:22:17.092 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:22:17.092 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:22:17.092 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:22:17.092 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:22:17.092 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:17.092 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:22:17.092 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:22:17.092 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:22:17.092 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:22:17.092 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:22:17.093 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:22:17.093 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:22:17.093 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:22:17.093 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:22:17.093 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:22:17.093 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:17.093 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:22:17.093 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:22:17.093 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:22:17.093 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:22:17.093 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:22:17.093 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:22:17.094 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:22:17.100 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:22:17.100 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:22:17.101 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:22:17.101 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:22:17.149 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:22:17.149 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:22:17.149 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:22:17.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:22:17.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:22:17.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:22:17.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:17.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:22:17.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:22:17.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:22:17.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:22:17.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:22:17.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:22:17.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:22:17.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:22:17.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:22:17.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:17.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:22:17.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:22:17.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:22:17.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:22:17.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:22:17.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:22:17.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:22:17.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:22:17.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:22:17.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:22:17.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:17.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:22:17.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:22:17.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:22:17.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:22:17.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:22:17.155 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:22:17.155 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:22:17.162 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:22:17.163 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:22:17.163 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:22:17.163 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:22:17.212 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:22:17.212 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:22:17.212 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:22:17.214 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:22:17.215 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:22:17.215 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:22:17.215 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:17.215 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:22:17.215 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:22:17.215 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:22:17.215 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:22:17.215 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:22:17.215 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:22:17.215 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:22:17.215 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:22:17.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:22:17.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:17.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:22:17.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:22:17.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:22:17.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:22:17.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:22:17.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:22:17.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:22:17.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:22:17.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:22:17.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:22:17.217 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:17.217 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:22:17.217 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:22:17.217 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:22:17.217 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:22:17.217 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:22:17.217 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:22:17.217 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:22:17.225 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:22:17.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:22:17.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:22:17.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:22:17.274 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:22:17.275 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:22:17.275 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:22:17.277 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:22:17.277 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:22:17.277 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:22:17.278 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:17.278 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:22:17.278 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:22:17.278 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:22:17.278 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:22:17.278 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:22:17.278 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:22:17.278 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:22:17.278 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:22:17.278 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:22:17.278 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:17.278 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:22:17.279 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:22:17.279 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:22:17.279 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:22:17.279 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:22:17.279 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:22:17.279 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:22:17.279 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:22:17.279 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:22:17.279 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:22:17.279 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:17.279 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:22:17.279 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:22:17.280 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:22:17.280 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:22:17.280 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:22:17.280 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:22:17.280 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:22:17.287 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:22:17.288 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:22:17.288 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:22:17.288 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:22:17.356 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:22:17.357 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:22:17.357 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:22:17.357 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:22:17.357 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:22:17.357 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:22:17.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:22:17.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:22:17.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:22:17.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:22:17.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:22:17.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:22:17.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:22:17.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:17.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:22:17.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:17.397 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 90 2026-02-18T08:22:17.397 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 95 2026-02-18T08:22:17.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:22:17.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=381, tag_tid=27, entry_tid=1], [object_number=380, tag_tid=27, entry_tid=0], [object_number=382, tag_tid=26, entry_tid=46994], [object_number=383, tag_tid=26, entry_tid=46991]]], state=connected] 2026-02-18T08:22:17.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=363, tag_tid=26, entry_tid=26323], [object_number=362, tag_tid=26, entry_tid=26322], [object_number=361, tag_tid=26, entry_tid=26321], [object_number=360, tag_tid=26, entry_tid=26320]]], state=connected] 2026-02-18T08:22:17.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:17.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=381, tag_tid=27, entry_tid=1' 2026-02-18T08:22:17.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:22:17.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:22:17.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:22:17.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:22:17.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:22:17.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:22:17.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:22:17.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:22:17.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:22:17.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:22:17.405 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:22:17.405 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:22:17.405 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:17.406 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:22:17.406 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:17.435 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 90 2026-02-18T08:22:17.435 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 95 2026-02-18T08:22:17.435 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:22:17.436 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=381, tag_tid=27, entry_tid=1], [object_number=380, tag_tid=27, entry_tid=0], [object_number=382, tag_tid=26, entry_tid=46994], [object_number=383, tag_tid=26, entry_tid=46991]]], state=connected] 2026-02-18T08:22:17.436 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=363, tag_tid=26, entry_tid=26323], [object_number=362, tag_tid=26, entry_tid=26322], [object_number=361, tag_tid=26, entry_tid=26321], [object_number=360, tag_tid=26, entry_tid=26320]]], state=connected] 2026-02-18T08:22:17.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:17.440 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=363, tag_tid=26, entry_tid=26323' 2026-02-18T08:22:17.440 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=381, tag_tid=27, entry_tid=1' -a 'object_number=381, tag_tid=27, entry_tid=1' = 'object_number=363, tag_tid=26, entry_tid=26323' 2026-02-18T08:22:17.440 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=363, tag_tid=26, entry_tid=26323' '!=' 'object_number=360, tag_tid=26, entry_tid=25956' 2026-02-18T08:22:17.440 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:22:17.440 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=363, tag_tid=26, entry_tid=26323' = 'object_number=360, tag_tid=26, entry_tid=25956' 2026-02-18T08:22:17.440 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=363, tag_tid=26, entry_tid=26323' 2026-02-18T08:22:17.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=381, tag_tid=27, entry_tid=1' 2026-02-18T08:22:17.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:22:17.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:22:17.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=27 2026-02-18T08:22:17.443 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=363, tag_tid=26, entry_tid=26323' 2026-02-18T08:22:17.443 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:22:17.443 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:22:17.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=26 2026-02-18T08:22:17.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=381, tag_tid=27, entry_tid=1' 2026-02-18T08:22:17.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:22:17.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:22:17.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:22:17.448 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=363, tag_tid=26, entry_tid=26323' 2026-02-18T08:22:17.448 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:22:17.448 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:22:17.449 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=26323 2026-02-18T08:22:17.449 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 27 = 26 -a 1 -le 26323 2026-02-18T08:22:17.449 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:22:17.449 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:22:17.449 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:22:17.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:22:17.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:22:17.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:22:17.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:22:17.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:22:17.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:22:17.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:22:17.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:22:17.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:22:17.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:17.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:22:17.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:22:17.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:22:17.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:22:17.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:22:17.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:22:17.655 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:22:17.655 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:22:17.655 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:22:17.655 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:22:17.655 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:17.655 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:22:17.655 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:22:17.655 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:22:17.655 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:22:17.655 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:22:17.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:22:17.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:22:17.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:22:17.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:22:17.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:17.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:22:17.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:22:17.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:22:17.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:22:17.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:22:17.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:22:17.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:22:17.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:22:17.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:22:17.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:22:17.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:17.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:22:17.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:22:17.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:22:17.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:22:17.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:22:17.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:22:17.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:22:17.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:22:17.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:22:17.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:22:17.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:22:17.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:22:17.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:22:17.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:22:17.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:22:17.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:22:17.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:22:17.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:17.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:22:17.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:22:17.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:22:17.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:22:17.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:22:17.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:22:17.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:22:17.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:22:17.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:22:17.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:17.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:22:17.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:22:17.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:22:17.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:22:17.728 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:22:17.728 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:22:17.728 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:22:17.728 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:22:17.728 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:22:17.728 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:22:17.728 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:17.728 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:22:17.728 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:22:17.728 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:22:17.729 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:22:17.729 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:22:17.729 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:22:17.729 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:22:17.738 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:22:17.738 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:22:17.738 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:22:17.739 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:22:17.795 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:22:17.795 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:22:17.795 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:22:17.798 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:22:17.798 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:22:17.798 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:22:17.798 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:17.798 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:22:17.798 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:22:17.798 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:22:17.799 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:22:17.799 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:22:17.799 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:22:17.799 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:22:17.799 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:22:17.799 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:22:17.799 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:17.799 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:22:17.799 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:22:17.799 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:22:17.799 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:22:17.799 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:22:17.800 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:22:17.800 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:22:17.800 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:22:17.800 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:22:17.800 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:22:17.800 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:17.800 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:22:17.800 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:22:17.800 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:22:17.800 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:22:17.800 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:22:17.800 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:22:17.801 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:22:17.810 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:22:17.810 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:22:17.811 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:22:17.811 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:22:17.867 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:22:17.867 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:22:17.867 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:22:17.870 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:22:17.870 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:22:17.870 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:22:17.870 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:17.870 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:22:17.870 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:22:17.870 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:22:17.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:22:17.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:22:17.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:22:17.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:22:17.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:22:17.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:22:17.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:17.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:22:17.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:22:17.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:22:17.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:22:17.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:22:17.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:22:17.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:22:17.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:22:17.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:22:17.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:22:17.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:17.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:22:17.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:22:17.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:22:17.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:22:17.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:22:17.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:22:17.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:22:17.881 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:22:17.882 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:22:17.882 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:22:17.882 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:22:17.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:22:17.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:22:17.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:22:17.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:22:17.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:22:17.944 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:22:17.944 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:22:17.944 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:22:17.944 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:22:17.944 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:22:17.944 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:22:17.944 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:22:17.944 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:22:17.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:17.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:22:17.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:17.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 90 2026-02-18T08:22:17.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 95 2026-02-18T08:22:17.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:22:17.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=381, tag_tid=27, entry_tid=1], [object_number=380, tag_tid=27, entry_tid=0], [object_number=382, tag_tid=26, entry_tid=46994], [object_number=383, tag_tid=26, entry_tid=46991]]], state=connected] 2026-02-18T08:22:17.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=362, tag_tid=26, entry_tid=26690], [object_number=361, tag_tid=26, entry_tid=26689], [object_number=360, tag_tid=26, entry_tid=26688], [object_number=363, tag_tid=26, entry_tid=26687]]], state=connected] 2026-02-18T08:22:17.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:17.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=381, tag_tid=27, entry_tid=1' 2026-02-18T08:22:17.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:22:17.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:22:17.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:22:17.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:22:17.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:22:17.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:22:17.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:22:17.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:22:17.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:22:17.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:22:17.982 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:22:17.982 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:22:17.982 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:17.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:22:17.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:18.010 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 90 2026-02-18T08:22:18.010 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 95 2026-02-18T08:22:18.010 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:22:18.010 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=381, tag_tid=27, entry_tid=1], [object_number=380, tag_tid=27, entry_tid=0], [object_number=382, tag_tid=26, entry_tid=46994], [object_number=383, tag_tid=26, entry_tid=46991]]], state=connected] 2026-02-18T08:22:18.010 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=362, tag_tid=26, entry_tid=26690], [object_number=361, tag_tid=26, entry_tid=26689], [object_number=360, tag_tid=26, entry_tid=26688], [object_number=363, tag_tid=26, entry_tid=26687]]], state=connected] 2026-02-18T08:22:18.012 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:18.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=362, tag_tid=26, entry_tid=26690' 2026-02-18T08:22:18.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=381, tag_tid=27, entry_tid=1' -a 'object_number=381, tag_tid=27, entry_tid=1' = 'object_number=362, tag_tid=26, entry_tid=26690' 2026-02-18T08:22:18.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=362, tag_tid=26, entry_tid=26690' '!=' 'object_number=363, tag_tid=26, entry_tid=26323' 2026-02-18T08:22:18.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:22:18.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=362, tag_tid=26, entry_tid=26690' = 'object_number=363, tag_tid=26, entry_tid=26323' 2026-02-18T08:22:18.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=362, tag_tid=26, entry_tid=26690' 2026-02-18T08:22:18.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=381, tag_tid=27, entry_tid=1' 2026-02-18T08:22:18.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:22:18.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:22:18.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=27 2026-02-18T08:22:18.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=362, tag_tid=26, entry_tid=26690' 2026-02-18T08:22:18.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:22:18.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:22:18.018 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=26 2026-02-18T08:22:18.019 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=381, tag_tid=27, entry_tid=1' 2026-02-18T08:22:18.019 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:22:18.019 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:22:18.021 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:22:18.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=362, tag_tid=26, entry_tid=26690' 2026-02-18T08:22:18.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:22:18.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:22:18.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=26690 2026-02-18T08:22:18.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 27 = 26 -a 1 -le 26690 2026-02-18T08:22:18.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:22:18.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:22:18.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:22:18.225 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:22:18.225 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:22:18.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:22:18.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:22:18.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:22:18.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:22:18.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:22:18.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:22:18.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:22:18.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:18.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:22:18.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:22:18.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:22:18.227 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:22:18.227 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:22:18.227 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:22:18.229 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:22:18.230 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:22:18.230 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:22:18.230 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:22:18.230 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:18.230 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:22:18.230 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:22:18.230 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:22:18.230 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:22:18.230 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:22:18.230 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:22:18.231 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:22:18.231 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:22:18.231 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:22:18.231 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:18.231 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:22:18.231 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:22:18.231 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:22:18.231 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:22:18.231 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:22:18.231 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:22:18.231 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:22:18.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:22:18.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:22:18.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:22:18.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:18.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:22:18.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:22:18.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:22:18.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:22:18.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:22:18.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:22:18.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:22:18.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:22:18.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:22:18.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:22:18.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:22:18.290 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:22:18.290 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:22:18.290 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:22:18.293 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:22:18.293 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:22:18.293 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:22:18.293 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:18.293 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:22:18.293 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:22:18.293 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:22:18.293 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:22:18.294 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:22:18.294 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:22:18.294 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:22:18.294 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:22:18.294 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:22:18.294 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:18.294 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:22:18.294 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:22:18.294 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:22:18.294 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:22:18.294 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:22:18.294 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:22:18.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:22:18.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:22:18.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:22:18.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:22:18.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:18.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:22:18.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:22:18.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:22:18.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:22:18.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:22:18.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:22:18.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:22:18.304 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:22:18.304 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:22:18.304 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:22:18.304 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:22:18.354 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:22:18.354 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:22:18.354 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:22:18.357 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:22:18.357 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:22:18.357 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:22:18.357 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:18.357 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:22:18.357 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:22:18.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:22:18.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:22:18.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:22:18.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:22:18.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:22:18.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:22:18.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:22:18.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:18.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:22:18.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:22:18.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:22:18.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:22:18.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:22:18.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:22:18.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:22:18.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:22:18.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:22:18.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:22:18.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:18.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:22:18.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:22:18.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:22:18.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:22:18.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:22:18.360 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:22:18.360 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:22:18.367 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:22:18.367 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:22:18.368 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:22:18.368 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:22:18.417 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:22:18.417 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:22:18.417 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:22:18.421 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:22:18.421 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:22:18.421 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:22:18.421 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:18.421 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:22:18.421 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:22:18.421 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:22:18.421 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:22:18.421 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:22:18.421 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:22:18.422 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:22:18.422 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:22:18.422 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:22:18.422 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:18.422 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:22:18.422 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:22:18.422 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:22:18.422 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:22:18.422 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:22:18.422 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:22:18.422 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:22:18.422 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:22:18.423 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:22:18.423 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:22:18.423 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:18.423 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:22:18.423 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:22:18.423 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:22:18.423 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:22:18.423 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:22:18.423 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:22:18.423 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:22:18.431 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:22:18.431 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:22:18.431 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:22:18.431 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:22:18.501 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:22:18.503 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:22:18.503 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:22:18.503 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:22:18.503 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:22:18.503 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:22:18.503 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:22:18.503 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:22:18.503 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:22:18.503 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:22:18.503 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:22:18.504 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:22:18.504 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:22:18.504 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:18.505 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:22:18.505 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:18.535 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 90 2026-02-18T08:22:18.535 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 95 2026-02-18T08:22:18.535 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:22:18.535 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=381, tag_tid=27, entry_tid=1], [object_number=380, tag_tid=27, entry_tid=0], [object_number=382, tag_tid=26, entry_tid=46994], [object_number=383, tag_tid=26, entry_tid=46991]]], state=connected] 2026-02-18T08:22:18.535 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=361, tag_tid=26, entry_tid=27045], [object_number=360, tag_tid=26, entry_tid=27044], [object_number=363, tag_tid=26, entry_tid=27043], [object_number=362, tag_tid=26, entry_tid=27042]]], state=connected] 2026-02-18T08:22:18.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:18.539 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=381, tag_tid=27, entry_tid=1' 2026-02-18T08:22:18.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:22:18.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:22:18.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:22:18.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:22:18.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:22:18.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:22:18.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:22:18.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:22:18.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:22:18.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:22:18.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:22:18.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:22:18.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:18.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:22:18.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:18.578 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 90 2026-02-18T08:22:18.578 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 95 2026-02-18T08:22:18.578 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:22:18.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=381, tag_tid=27, entry_tid=1], [object_number=380, tag_tid=27, entry_tid=0], [object_number=382, tag_tid=26, entry_tid=46994], [object_number=383, tag_tid=26, entry_tid=46991]]], state=connected] 2026-02-18T08:22:18.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=361, tag_tid=26, entry_tid=27045], [object_number=360, tag_tid=26, entry_tid=27044], [object_number=363, tag_tid=26, entry_tid=27043], [object_number=362, tag_tid=26, entry_tid=27042]]], state=connected] 2026-02-18T08:22:18.582 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:18.584 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=361, tag_tid=26, entry_tid=27045' 2026-02-18T08:22:18.584 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=381, tag_tid=27, entry_tid=1' -a 'object_number=381, tag_tid=27, entry_tid=1' = 'object_number=361, tag_tid=26, entry_tid=27045' 2026-02-18T08:22:18.584 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=361, tag_tid=26, entry_tid=27045' '!=' 'object_number=362, tag_tid=26, entry_tid=26690' 2026-02-18T08:22:18.584 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:22:18.584 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=361, tag_tid=26, entry_tid=27045' = 'object_number=362, tag_tid=26, entry_tid=26690' 2026-02-18T08:22:18.584 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=361, tag_tid=26, entry_tid=27045' 2026-02-18T08:22:18.585 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=381, tag_tid=27, entry_tid=1' 2026-02-18T08:22:18.585 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:22:18.586 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:22:18.586 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=27 2026-02-18T08:22:18.587 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=361, tag_tid=26, entry_tid=27045' 2026-02-18T08:22:18.587 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:22:18.587 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:22:18.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=26 2026-02-18T08:22:18.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=381, tag_tid=27, entry_tid=1' 2026-02-18T08:22:18.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:22:18.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:22:18.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:22:18.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=361, tag_tid=26, entry_tid=27045' 2026-02-18T08:22:18.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:22:18.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:22:18.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=27045 2026-02-18T08:22:18.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 27 = 26 -a 1 -le 27045 2026-02-18T08:22:18.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:22:18.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:22:18.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:22:18.795 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:22:18.795 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:22:18.795 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:22:18.795 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:22:18.795 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:22:18.795 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:22:18.796 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:22:18.796 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:22:18.796 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:22:18.796 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:18.796 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:22:18.796 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:22:18.796 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:22:18.796 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:22:18.796 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:22:18.796 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:22:18.799 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:22:18.799 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:22:18.799 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:22:18.799 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:22:18.799 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:18.800 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:22:18.800 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:22:18.800 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:22:18.800 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:22:18.800 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:22:18.800 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:22:18.800 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:22:18.800 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:22:18.800 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:22:18.800 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:18.800 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:22:18.800 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:22:18.801 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:22:18.801 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:22:18.801 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:22:18.801 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:22:18.801 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:22:18.801 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:22:18.801 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:22:18.801 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:22:18.801 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:18.801 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:22:18.801 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:22:18.802 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:22:18.802 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:22:18.802 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:22:18.802 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:22:18.802 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:22:18.808 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:22:18.809 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:22:18.809 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:22:18.809 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:22:18.859 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:22:18.859 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:22:18.859 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:22:18.862 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:22:18.862 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:22:18.862 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:22:18.862 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:18.862 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:22:18.862 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:22:18.862 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:22:18.863 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:22:18.863 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:22:18.863 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:22:18.863 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:22:18.863 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:22:18.863 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:22:18.863 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:18.863 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:22:18.863 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:22:18.863 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:22:18.863 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:22:18.864 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:22:18.864 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:22:18.864 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:22:18.864 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:22:18.864 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:22:18.864 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:22:18.864 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:18.864 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:22:18.864 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:22:18.864 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:22:18.864 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:22:18.864 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:22:18.865 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:22:18.865 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:22:18.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:22:18.874 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:22:18.874 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:22:18.874 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:22:18.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:22:18.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:22:18.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:22:18.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:22:18.928 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:22:18.928 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:22:18.928 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:18.928 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:22:18.928 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:22:18.928 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:22:18.928 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:22:18.928 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:22:18.928 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:22:18.929 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:22:18.929 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:22:18.929 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:22:18.929 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:18.929 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:22:18.929 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:22:18.929 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:22:18.929 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:22:18.929 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:22:18.929 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:22:18.929 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:22:18.929 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:22:18.930 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:22:18.930 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:22:18.930 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:18.930 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:22:18.930 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:22:18.930 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:22:18.930 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:22:18.930 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:22:18.930 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:22:18.930 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:22:18.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:22:18.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:22:18.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:22:18.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:22:18.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:22:18.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:22:18.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:22:19.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:22:19.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:22:19.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:22:19.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:19.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:22:19.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:22:19.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:22:19.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:22:19.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:22:19.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:22:19.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:22:19.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:22:19.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:22:19.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:19.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:22:19.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:22:19.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:22:19.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:22:19.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:22:19.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:22:19.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:22:19.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:22:19.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:22:19.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:22:19.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:19.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:22:19.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:22:19.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:22:19.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:22:19.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:22:19.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:22:19.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:22:19.010 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:22:19.010 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:22:19.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:22:19.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:22:19.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:22:19.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:22:19.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:22:19.074 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:22:19.074 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:22:19.074 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:22:19.074 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:22:19.074 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:22:19.074 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:22:19.074 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:22:19.074 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:22:19.074 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:22:19.074 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:22:19.075 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:19.075 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:22:19.075 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:19.105 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 90 2026-02-18T08:22:19.105 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 95 2026-02-18T08:22:19.105 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:22:19.105 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=381, tag_tid=27, entry_tid=1], [object_number=380, tag_tid=27, entry_tid=0], [object_number=382, tag_tid=26, entry_tid=46994], [object_number=383, tag_tid=26, entry_tid=46991]]], state=connected] 2026-02-18T08:22:19.105 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=361, tag_tid=26, entry_tid=27385], [object_number=360, tag_tid=26, entry_tid=27384], [object_number=363, tag_tid=26, entry_tid=27383], [object_number=362, tag_tid=26, entry_tid=27382]]], state=connected] 2026-02-18T08:22:19.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:19.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=381, tag_tid=27, entry_tid=1' 2026-02-18T08:22:19.110 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:22:19.110 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:22:19.111 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:22:19.111 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:22:19.111 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:22:19.111 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:22:19.111 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:22:19.111 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:22:19.111 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:22:19.111 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:22:19.112 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:22:19.112 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:22:19.113 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:19.113 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:22:19.113 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:19.141 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 90 2026-02-18T08:22:19.141 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 95 2026-02-18T08:22:19.141 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:22:19.141 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=381, tag_tid=27, entry_tid=1], [object_number=380, tag_tid=27, entry_tid=0], [object_number=382, tag_tid=26, entry_tid=46994], [object_number=383, tag_tid=26, entry_tid=46991]]], state=connected] 2026-02-18T08:22:19.141 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=361, tag_tid=26, entry_tid=27385], [object_number=360, tag_tid=26, entry_tid=27384], [object_number=363, tag_tid=26, entry_tid=27383], [object_number=362, tag_tid=26, entry_tid=27382]]], state=connected] 2026-02-18T08:22:19.144 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:19.145 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=361, tag_tid=26, entry_tid=27385' 2026-02-18T08:22:19.145 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=381, tag_tid=27, entry_tid=1' -a 'object_number=381, tag_tid=27, entry_tid=1' = 'object_number=361, tag_tid=26, entry_tid=27385' 2026-02-18T08:22:19.145 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=361, tag_tid=26, entry_tid=27385' '!=' 'object_number=361, tag_tid=26, entry_tid=27045' 2026-02-18T08:22:19.145 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:22:19.145 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=361, tag_tid=26, entry_tid=27385' = 'object_number=361, tag_tid=26, entry_tid=27045' 2026-02-18T08:22:19.145 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=361, tag_tid=26, entry_tid=27385' 2026-02-18T08:22:19.146 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=381, tag_tid=27, entry_tid=1' 2026-02-18T08:22:19.146 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:22:19.146 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:22:19.147 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=27 2026-02-18T08:22:19.148 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=361, tag_tid=26, entry_tid=27385' 2026-02-18T08:22:19.148 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:22:19.148 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:22:19.149 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=26 2026-02-18T08:22:19.150 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=381, tag_tid=27, entry_tid=1' 2026-02-18T08:22:19.150 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:22:19.150 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:22:19.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:22:19.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=361, tag_tid=26, entry_tid=27385' 2026-02-18T08:22:19.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:22:19.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:22:19.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=27385 2026-02-18T08:22:19.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 27 = 26 -a 1 -le 27385 2026-02-18T08:22:19.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:22:19.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:22:19.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:22:19.355 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:22:19.355 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:22:19.355 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:22:19.356 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:22:19.356 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:22:19.356 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:22:19.356 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:22:19.356 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:22:19.356 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:22:19.356 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:19.356 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:22:19.356 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:22:19.356 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:22:19.356 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:22:19.356 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:22:19.356 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:22:19.356 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:22:19.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:22:19.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:22:19.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:22:19.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:19.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:22:19.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:22:19.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:22:19.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:22:19.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:22:19.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:22:19.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:22:19.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:22:19.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:22:19.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:19.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:22:19.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:22:19.360 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:22:19.360 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:22:19.360 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:22:19.360 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:22:19.360 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:22:19.360 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:22:19.360 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:22:19.360 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:22:19.360 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:19.360 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:22:19.360 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:22:19.360 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:22:19.360 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:22:19.360 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:22:19.360 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:22:19.360 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:22:19.371 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:22:19.371 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:22:19.371 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:22:19.371 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:22:19.421 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:22:19.421 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:22:19.421 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:22:19.423 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:22:19.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:22:19.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:22:19.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:19.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:22:19.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:22:19.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:22:19.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:22:19.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:22:19.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:22:19.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:22:19.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:22:19.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:22:19.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:19.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:22:19.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:22:19.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:22:19.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:22:19.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:22:19.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:22:19.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:22:19.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:22:19.425 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:22:19.425 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:22:19.425 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:19.425 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:22:19.425 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:22:19.425 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:22:19.425 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:22:19.425 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:22:19.425 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:22:19.425 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:22:19.435 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:22:19.435 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:22:19.435 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:22:19.435 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:22:19.492 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:22:19.492 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:22:19.492 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:22:19.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:22:19.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:22:19.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:22:19.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:19.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:22:19.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:22:19.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:22:19.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:22:19.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:22:19.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:22:19.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:22:19.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:22:19.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:22:19.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:19.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:22:19.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:22:19.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:22:19.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:22:19.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:22:19.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:22:19.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:22:19.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:22:19.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:22:19.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:22:19.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:19.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:22:19.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:22:19.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:22:19.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:22:19.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:22:19.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:22:19.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:22:19.506 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:22:19.506 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:22:19.506 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:22:19.506 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:22:19.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:22:19.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:22:19.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:22:19.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:22:19.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:22:19.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:22:19.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:19.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:22:19.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:22:19.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:22:19.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:22:19.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:22:19.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:22:19.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:22:19.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:22:19.557 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:22:19.557 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:19.557 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:22:19.557 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:22:19.557 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:22:19.557 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:22:19.557 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:22:19.557 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:22:19.557 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:22:19.557 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:22:19.557 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:22:19.557 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:22:19.557 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:19.557 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:22:19.557 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:22:19.557 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:22:19.557 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:22:19.557 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:22:19.557 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:22:19.557 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:22:19.567 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:22:19.567 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:22:19.567 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:22:19.567 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:22:19.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:22:19.635 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:22:19.635 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:22:19.635 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:22:19.635 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:22:19.635 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:22:19.635 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:22:19.635 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:22:19.635 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:22:19.635 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:22:19.635 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:22:19.635 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:22:19.635 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:22:19.637 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:19.637 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:22:19.637 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:19.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 90 2026-02-18T08:22:19.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 95 2026-02-18T08:22:19.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:22:19.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=381, tag_tid=27, entry_tid=1], [object_number=380, tag_tid=27, entry_tid=0], [object_number=382, tag_tid=26, entry_tid=46994], [object_number=383, tag_tid=26, entry_tid=46991]]], state=connected] 2026-02-18T08:22:19.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=362, tag_tid=26, entry_tid=27730], [object_number=361, tag_tid=26, entry_tid=27729], [object_number=360, tag_tid=26, entry_tid=27728], [object_number=363, tag_tid=26, entry_tid=27727]]], state=connected] 2026-02-18T08:22:19.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:19.670 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=381, tag_tid=27, entry_tid=1' 2026-02-18T08:22:19.671 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:22:19.671 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:22:19.671 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:22:19.671 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:22:19.671 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:22:19.671 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:22:19.671 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:22:19.671 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:22:19.671 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:22:19.671 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:22:19.672 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:22:19.672 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:22:19.673 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:19.673 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:22:19.674 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:19.709 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 90 2026-02-18T08:22:19.709 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 95 2026-02-18T08:22:19.709 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:22:19.709 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=381, tag_tid=27, entry_tid=1], [object_number=380, tag_tid=27, entry_tid=0], [object_number=382, tag_tid=26, entry_tid=46994], [object_number=383, tag_tid=26, entry_tid=46991]]], state=connected] 2026-02-18T08:22:19.709 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=362, tag_tid=26, entry_tid=27730], [object_number=361, tag_tid=26, entry_tid=27729], [object_number=360, tag_tid=26, entry_tid=27728], [object_number=363, tag_tid=26, entry_tid=27727]]], state=connected] 2026-02-18T08:22:19.712 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:19.714 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=362, tag_tid=26, entry_tid=27730' 2026-02-18T08:22:19.714 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=381, tag_tid=27, entry_tid=1' -a 'object_number=381, tag_tid=27, entry_tid=1' = 'object_number=362, tag_tid=26, entry_tid=27730' 2026-02-18T08:22:19.714 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=362, tag_tid=26, entry_tid=27730' '!=' 'object_number=361, tag_tid=26, entry_tid=27385' 2026-02-18T08:22:19.714 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:22:19.714 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=362, tag_tid=26, entry_tid=27730' = 'object_number=361, tag_tid=26, entry_tid=27385' 2026-02-18T08:22:19.714 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=362, tag_tid=26, entry_tid=27730' 2026-02-18T08:22:19.714 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=381, tag_tid=27, entry_tid=1' 2026-02-18T08:22:19.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:22:19.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:22:19.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=27 2026-02-18T08:22:19.717 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=362, tag_tid=26, entry_tid=27730' 2026-02-18T08:22:19.717 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:22:19.717 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:22:19.718 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=26 2026-02-18T08:22:19.719 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=381, tag_tid=27, entry_tid=1' 2026-02-18T08:22:19.719 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:22:19.719 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:22:19.721 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:22:19.721 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=362, tag_tid=26, entry_tid=27730' 2026-02-18T08:22:19.721 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:22:19.722 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:22:19.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=27730 2026-02-18T08:22:19.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 27 = 26 -a 1 -le 27730 2026-02-18T08:22:19.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:22:19.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:22:19.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:22:19.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:22:19.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:22:19.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:22:19.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:22:19.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:22:19.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:22:19.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:22:19.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:22:19.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:22:19.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:19.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:22:19.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:22:19.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:22:19.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:22:19.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:22:19.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:22:19.930 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:22:19.930 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:22:19.930 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:22:19.930 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:22:19.930 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:19.930 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:22:19.930 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:22:19.930 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:22:19.931 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:22:19.931 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:22:19.931 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:22:19.931 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:22:19.931 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:22:19.931 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:22:19.931 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:19.931 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:22:19.931 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:22:19.932 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:22:19.932 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:22:19.932 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:22:19.932 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:22:19.932 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:22:19.932 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:22:19.932 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:22:19.932 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:22:19.932 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:19.933 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:22:19.933 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:22:19.933 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:22:19.933 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:22:19.933 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:22:19.933 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:22:19.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:22:19.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:22:19.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:22:19.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:22:19.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:22:19.990 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:22:19.990 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:22:19.990 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:22:19.993 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:22:19.993 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:22:19.993 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:22:19.994 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:19.994 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:22:19.994 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:22:19.994 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:22:19.994 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:22:19.994 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:22:19.994 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:22:19.994 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:22:19.994 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:22:19.994 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:22:19.994 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:19.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:22:19.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:22:19.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:22:19.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:22:19.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:22:19.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:22:19.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:22:19.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:22:19.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:22:19.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:22:19.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:19.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:22:19.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:22:19.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:22:19.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:22:19.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:22:19.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:22:19.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:22:20.004 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:22:20.004 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:22:20.004 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:22:20.004 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:22:20.062 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:22:20.062 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:22:20.062 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:22:20.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:22:20.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:22:20.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:22:20.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:20.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:22:20.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:22:20.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:22:20.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:22:20.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:22:20.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:22:20.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:22:20.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:22:20.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:22:20.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:20.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:22:20.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:22:20.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:22:20.067 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:22:20.067 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:22:20.067 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:22:20.067 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:22:20.067 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:22:20.067 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:22:20.067 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:22:20.067 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:20.067 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:22:20.067 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:22:20.067 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:22:20.068 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:22:20.068 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:22:20.068 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:22:20.068 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:22:20.076 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:22:20.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:22:20.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:22:20.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:22:20.134 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:22:20.134 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:22:20.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:22:20.137 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:22:20.137 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:22:20.138 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:22:20.138 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:20.138 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:22:20.138 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:22:20.138 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:22:20.138 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:22:20.138 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:22:20.138 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:22:20.138 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:22:20.138 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:22:20.138 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:22:20.139 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:20.139 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:22:20.139 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:22:20.139 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:22:20.139 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:22:20.139 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:22:20.139 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:22:20.139 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:22:20.139 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:22:20.139 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:22:20.139 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:22:20.139 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:20.140 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:22:20.140 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:22:20.140 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:22:20.140 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:22:20.140 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:22:20.140 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:22:20.140 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:22:20.148 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:22:20.148 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:22:20.148 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:22:20.149 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:22:20.217 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:22:20.217 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:22:20.218 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:22:20.218 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:22:20.218 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:22:20.218 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:22:20.218 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:22:20.218 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:22:20.218 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:22:20.218 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:22:20.218 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:22:20.219 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:22:20.219 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:22:20.219 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:20.220 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:22:20.220 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:20.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 90 2026-02-18T08:22:20.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 95 2026-02-18T08:22:20.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:22:20.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=381, tag_tid=27, entry_tid=1], [object_number=380, tag_tid=27, entry_tid=0], [object_number=382, tag_tid=26, entry_tid=46994], [object_number=383, tag_tid=26, entry_tid=46991]]], state=connected] 2026-02-18T08:22:20.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=362, tag_tid=26, entry_tid=28098], [object_number=361, tag_tid=26, entry_tid=28097], [object_number=360, tag_tid=26, entry_tid=28096], [object_number=363, tag_tid=26, entry_tid=28095]]], state=connected] 2026-02-18T08:22:20.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:20.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=381, tag_tid=27, entry_tid=1' 2026-02-18T08:22:20.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:22:20.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:22:20.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:22:20.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:22:20.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:22:20.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:22:20.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:22:20.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:22:20.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:22:20.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:22:20.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:22:20.266 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:22:20.266 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:20.266 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:22:20.266 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:20.304 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 90 2026-02-18T08:22:20.304 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 95 2026-02-18T08:22:20.304 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:22:20.304 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=381, tag_tid=27, entry_tid=1], [object_number=380, tag_tid=27, entry_tid=0], [object_number=382, tag_tid=26, entry_tid=46994], [object_number=383, tag_tid=26, entry_tid=46991]]], state=connected] 2026-02-18T08:22:20.304 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=362, tag_tid=26, entry_tid=28098], [object_number=361, tag_tid=26, entry_tid=28097], [object_number=360, tag_tid=26, entry_tid=28096], [object_number=363, tag_tid=26, entry_tid=28095]]], state=connected] 2026-02-18T08:22:20.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:20.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=362, tag_tid=26, entry_tid=28098' 2026-02-18T08:22:20.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=381, tag_tid=27, entry_tid=1' -a 'object_number=381, tag_tid=27, entry_tid=1' = 'object_number=362, tag_tid=26, entry_tid=28098' 2026-02-18T08:22:20.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=362, tag_tid=26, entry_tid=28098' '!=' 'object_number=362, tag_tid=26, entry_tid=27730' 2026-02-18T08:22:20.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:22:20.310 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=362, tag_tid=26, entry_tid=28098' = 'object_number=362, tag_tid=26, entry_tid=27730' 2026-02-18T08:22:20.310 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=362, tag_tid=26, entry_tid=28098' 2026-02-18T08:22:20.310 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=381, tag_tid=27, entry_tid=1' 2026-02-18T08:22:20.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:22:20.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:22:20.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=27 2026-02-18T08:22:20.312 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=362, tag_tid=26, entry_tid=28098' 2026-02-18T08:22:20.312 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:22:20.312 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:22:20.314 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=26 2026-02-18T08:22:20.314 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=381, tag_tid=27, entry_tid=1' 2026-02-18T08:22:20.315 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:22:20.315 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:22:20.316 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:22:20.317 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=362, tag_tid=26, entry_tid=28098' 2026-02-18T08:22:20.317 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:22:20.317 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:22:20.318 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=28098 2026-02-18T08:22:20.318 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 27 = 26 -a 1 -le 28098 2026-02-18T08:22:20.318 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:22:20.319 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:22:20.319 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:22:20.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:22:20.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:22:20.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:22:20.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:22:20.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:22:20.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:22:20.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:22:20.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:22:20.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:22:20.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:20.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:22:20.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:22:20.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:22:20.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:22:20.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:22:20.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:22:20.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:22:20.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:22:20.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:22:20.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:22:20.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:20.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:22:20.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:22:20.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:22:20.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:22:20.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:22:20.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:22:20.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:22:20.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:22:20.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:22:20.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:20.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:22:20.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:22:20.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:22:20.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:22:20.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:22:20.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:22:20.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:22:20.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:22:20.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:22:20.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:22:20.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:20.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:22:20.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:22:20.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:22:20.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:22:20.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:22:20.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:22:20.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:22:20.535 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:22:20.535 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:22:20.535 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:22:20.535 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:22:20.584 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:22:20.584 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:22:20.584 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:22:20.587 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:22:20.587 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:22:20.587 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:22:20.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:20.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:22:20.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:22:20.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:22:20.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:22:20.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:22:20.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:22:20.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:22:20.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:22:20.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:22:20.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:20.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:22:20.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:22:20.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:22:20.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:22:20.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:22:20.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:22:20.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:22:20.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:22:20.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:22:20.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:22:20.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:20.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:22:20.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:22:20.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:22:20.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:22:20.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:22:20.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:22:20.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:22:20.598 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:22:20.599 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:22:20.599 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:22:20.599 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:22:20.648 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:22:20.648 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:22:20.648 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:22:20.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:22:20.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:22:20.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:22:20.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:20.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:22:20.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:22:20.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:22:20.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:22:20.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:22:20.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:22:20.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:22:20.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:22:20.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:22:20.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:20.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:22:20.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:22:20.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:22:20.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:22:20.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:22:20.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:22:20.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:22:20.653 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:22:20.653 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:22:20.653 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:22:20.653 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:20.653 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:22:20.653 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:22:20.653 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:22:20.653 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:22:20.653 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:22:20.653 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:22:20.654 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:22:20.663 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:22:20.664 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:22:20.664 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:22:20.664 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:22:20.712 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:22:20.712 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:22:20.712 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:22:20.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:22:20.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:22:20.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:22:20.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:20.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:22:20.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:22:20.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:22:20.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:22:20.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:22:20.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:22:20.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:22:20.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:22:20.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:22:20.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:20.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:22:20.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:22:20.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:22:20.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:22:20.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:22:20.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:22:20.717 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:22:20.717 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:22:20.717 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:22:20.717 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:22:20.717 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:20.717 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:22:20.717 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:22:20.717 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:22:20.717 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:22:20.717 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:22:20.717 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:22:20.718 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:22:20.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:22:20.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:22:20.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:22:20.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:22:20.785 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:22:20.787 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:22:20.787 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:22:20.787 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:22:20.787 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:22:20.787 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:22:20.787 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:22:20.787 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:22:20.787 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:22:20.787 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:22:20.787 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:22:20.788 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:22:20.788 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:22:20.788 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:20.788 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:22:20.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:20.821 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 90 2026-02-18T08:22:20.821 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 95 2026-02-18T08:22:20.821 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:22:20.821 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=381, tag_tid=27, entry_tid=1], [object_number=380, tag_tid=27, entry_tid=0], [object_number=382, tag_tid=26, entry_tid=46994], [object_number=383, tag_tid=26, entry_tid=46991]]], state=connected] 2026-02-18T08:22:20.821 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=362, tag_tid=26, entry_tid=28462], [object_number=361, tag_tid=26, entry_tid=28461], [object_number=360, tag_tid=26, entry_tid=28460], [object_number=363, tag_tid=26, entry_tid=28459]]], state=connected] 2026-02-18T08:22:20.825 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:20.826 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=381, tag_tid=27, entry_tid=1' 2026-02-18T08:22:20.827 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:22:20.827 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:22:20.828 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:22:20.828 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:22:20.828 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:22:20.828 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:22:20.828 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:22:20.828 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:22:20.828 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:22:20.828 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:22:20.829 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:22:20.829 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:22:20.829 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:20.830 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:22:20.830 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:20.860 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 90 2026-02-18T08:22:20.860 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 95 2026-02-18T08:22:20.860 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:22:20.860 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=381, tag_tid=27, entry_tid=1], [object_number=380, tag_tid=27, entry_tid=0], [object_number=382, tag_tid=26, entry_tid=46994], [object_number=383, tag_tid=26, entry_tid=46991]]], state=connected] 2026-02-18T08:22:20.860 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=362, tag_tid=26, entry_tid=28462], [object_number=361, tag_tid=26, entry_tid=28461], [object_number=360, tag_tid=26, entry_tid=28460], [object_number=363, tag_tid=26, entry_tid=28459]]], state=connected] 2026-02-18T08:22:20.863 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:20.864 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=362, tag_tid=26, entry_tid=28462' 2026-02-18T08:22:20.864 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=381, tag_tid=27, entry_tid=1' -a 'object_number=381, tag_tid=27, entry_tid=1' = 'object_number=362, tag_tid=26, entry_tid=28462' 2026-02-18T08:22:20.864 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=362, tag_tid=26, entry_tid=28462' '!=' 'object_number=362, tag_tid=26, entry_tid=28098' 2026-02-18T08:22:20.865 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:22:20.865 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=362, tag_tid=26, entry_tid=28462' = 'object_number=362, tag_tid=26, entry_tid=28098' 2026-02-18T08:22:20.865 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=362, tag_tid=26, entry_tid=28462' 2026-02-18T08:22:20.865 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=381, tag_tid=27, entry_tid=1' 2026-02-18T08:22:20.866 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:22:20.866 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:22:20.866 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=27 2026-02-18T08:22:20.867 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=362, tag_tid=26, entry_tid=28462' 2026-02-18T08:22:20.867 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:22:20.867 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:22:20.868 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=26 2026-02-18T08:22:20.869 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=381, tag_tid=27, entry_tid=1' 2026-02-18T08:22:20.869 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:22:20.869 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:22:20.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:22:20.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=362, tag_tid=26, entry_tid=28462' 2026-02-18T08:22:20.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:22:20.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:22:20.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=28462 2026-02-18T08:22:20.874 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 27 = 26 -a 1 -le 28462 2026-02-18T08:22:20.874 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:22:20.874 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:22:20.874 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:22:21.075 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:22:21.075 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:22:21.075 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:22:21.075 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:22:21.075 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:22:21.076 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:22:21.076 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:22:21.076 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:22:21.076 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:22:21.076 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:21.076 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:22:21.076 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:22:21.076 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:22:21.076 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:22:21.076 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:22:21.076 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:22:21.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:22:21.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:22:21.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:22:21.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:22:21.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:21.080 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:22:21.080 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:22:21.080 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:22:21.080 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:22:21.080 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:22:21.080 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:22:21.080 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:22:21.080 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:22:21.080 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:22:21.080 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:21.080 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:22:21.080 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:22:21.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:22:21.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:22:21.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:22:21.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:22:21.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:22:21.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:22:21.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:22:21.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:22:21.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:21.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:22:21.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:22:21.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:22:21.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:22:21.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:22:21.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:22:21.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:22:21.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:22:21.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:22:21.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:22:21.092 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:22:21.149 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:22:21.149 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:22:21.149 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:22:21.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:22:21.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:22:21.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:22:21.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:21.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:22:21.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:22:21.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:22:21.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:22:21.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:22:21.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:22:21.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:22:21.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:22:21.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:22:21.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:21.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:22:21.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:22:21.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:22:21.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:22:21.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:22:21.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:22:21.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:22:21.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:22:21.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:22:21.155 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:22:21.155 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:21.155 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:22:21.155 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:22:21.155 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:22:21.155 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:22:21.155 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:22:21.155 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:22:21.155 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:22:21.165 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:22:21.165 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:22:21.165 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:22:21.166 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:22:21.213 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:22:21.213 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:22:21.213 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:22:21.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:22:21.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:22:21.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:22:21.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:21.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:22:21.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:22:21.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:22:21.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:22:21.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:22:21.217 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:22:21.217 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:22:21.217 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:22:21.217 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:22:21.217 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:21.217 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:22:21.217 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:22:21.217 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:22:21.217 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:22:21.217 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:22:21.217 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:22:21.217 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:22:21.218 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:22:21.218 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:22:21.218 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:22:21.218 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:21.218 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:22:21.218 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:22:21.218 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:22:21.218 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:22:21.218 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:22:21.218 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:22:21.218 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:22:21.227 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:22:21.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:22:21.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:22:21.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:22:21.275 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:22:21.275 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:22:21.275 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:22:21.277 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:22:21.278 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:22:21.278 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:22:21.278 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:21.278 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:22:21.278 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:22:21.278 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:22:21.278 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:22:21.278 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:22:21.278 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:22:21.278 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:22:21.278 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:22:21.279 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:22:21.279 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:21.279 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:22:21.279 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:22:21.279 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:22:21.279 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:22:21.279 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:22:21.279 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:22:21.279 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:22:21.279 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:22:21.279 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:22:21.279 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:22:21.280 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:21.280 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:22:21.280 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:22:21.280 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:22:21.280 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:22:21.280 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:22:21.280 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:22:21.280 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:22:21.287 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:22:21.288 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:22:21.288 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:22:21.288 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:22:21.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:22:21.350 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:22:21.350 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:22:21.350 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:22:21.350 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:22:21.350 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:22:21.350 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:22:21.351 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:22:21.351 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:22:21.351 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:22:21.351 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:22:21.351 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:22:21.351 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:22:21.352 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:21.353 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:22:21.353 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:21.381 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 91 2026-02-18T08:22:21.381 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 95 2026-02-18T08:22:21.381 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:22:21.381 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=381, tag_tid=27, entry_tid=1], [object_number=380, tag_tid=27, entry_tid=0], [object_number=382, tag_tid=26, entry_tid=46994], [object_number=383, tag_tid=26, entry_tid=46991]]], state=connected] 2026-02-18T08:22:21.381 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=366, tag_tid=26, entry_tid=28726], [object_number=365, tag_tid=26, entry_tid=28725], [object_number=364, tag_tid=26, entry_tid=28724], [object_number=367, tag_tid=26, entry_tid=28723]]], state=connected] 2026-02-18T08:22:21.384 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:21.385 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=381, tag_tid=27, entry_tid=1' 2026-02-18T08:22:21.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:22:21.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:22:21.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:22:21.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:22:21.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:22:21.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:22:21.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:22:21.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:22:21.387 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:22:21.387 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:22:21.387 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:22:21.387 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:22:21.388 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:21.388 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:22:21.388 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:21.416 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 91 2026-02-18T08:22:21.416 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 95 2026-02-18T08:22:21.416 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:22:21.416 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=381, tag_tid=27, entry_tid=1], [object_number=380, tag_tid=27, entry_tid=0], [object_number=382, tag_tid=26, entry_tid=46994], [object_number=383, tag_tid=26, entry_tid=46991]]], state=connected] 2026-02-18T08:22:21.416 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=366, tag_tid=26, entry_tid=28726], [object_number=365, tag_tid=26, entry_tid=28725], [object_number=364, tag_tid=26, entry_tid=28724], [object_number=367, tag_tid=26, entry_tid=28723]]], state=connected] 2026-02-18T08:22:21.418 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:21.420 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=366, tag_tid=26, entry_tid=28726' 2026-02-18T08:22:21.420 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=381, tag_tid=27, entry_tid=1' -a 'object_number=381, tag_tid=27, entry_tid=1' = 'object_number=366, tag_tid=26, entry_tid=28726' 2026-02-18T08:22:21.420 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=366, tag_tid=26, entry_tid=28726' '!=' 'object_number=362, tag_tid=26, entry_tid=28462' 2026-02-18T08:22:21.420 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:22:21.421 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=366, tag_tid=26, entry_tid=28726' = 'object_number=362, tag_tid=26, entry_tid=28462' 2026-02-18T08:22:21.421 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=366, tag_tid=26, entry_tid=28726' 2026-02-18T08:22:21.421 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=381, tag_tid=27, entry_tid=1' 2026-02-18T08:22:21.421 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:22:21.421 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:22:21.422 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=27 2026-02-18T08:22:21.423 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=366, tag_tid=26, entry_tid=28726' 2026-02-18T08:22:21.423 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:22:21.423 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:22:21.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=26 2026-02-18T08:22:21.425 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=381, tag_tid=27, entry_tid=1' 2026-02-18T08:22:21.425 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:22:21.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:22:21.427 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:22:21.428 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=366, tag_tid=26, entry_tid=28726' 2026-02-18T08:22:21.428 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:22:21.428 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:22:21.430 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=28726 2026-02-18T08:22:21.430 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 27 = 26 -a 1 -le 28726 2026-02-18T08:22:21.430 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:22:21.430 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:22:21.430 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:22:21.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:22:21.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:22:21.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:22:21.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:22:21.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:22:21.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:22:21.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:22:21.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:22:21.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:22:21.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:21.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:22:21.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:22:21.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:22:21.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:22:21.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:22:21.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:22:21.635 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:22:21.635 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:22:21.635 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:22:21.635 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:22:21.635 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:21.635 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:22:21.636 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:22:21.636 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:22:21.636 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:22:21.636 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:22:21.636 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:22:21.636 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:22:21.636 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:22:21.636 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:22:21.636 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:21.636 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:22:21.636 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:22:21.636 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:22:21.637 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:22:21.637 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:22:21.637 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:22:21.637 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:22:21.637 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:22:21.637 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:22:21.637 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:22:21.637 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:21.637 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:22:21.637 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:22:21.637 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:22:21.637 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:22:21.638 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:22:21.638 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:22:21.638 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:22:21.647 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:22:21.648 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:22:21.648 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:22:21.648 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:22:21.697 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:22:21.697 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:22:21.697 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:22:21.700 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:22:21.700 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:22:21.700 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:22:21.700 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:21.700 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:22:21.700 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:22:21.700 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:22:21.700 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:22:21.700 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:22:21.700 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:22:21.700 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:22:21.701 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:22:21.701 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:22:21.701 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:21.701 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:22:21.701 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:22:21.701 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:22:21.701 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:22:21.701 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:22:21.701 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:22:21.701 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:22:21.701 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:22:21.701 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:22:21.702 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:22:21.702 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:21.702 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:22:21.702 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:22:21.702 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:22:21.702 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:22:21.702 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:22:21.702 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:22:21.702 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:22:21.712 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:22:21.712 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:22:21.712 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:22:21.712 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:22:21.760 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:22:21.760 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:22:21.760 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:22:21.763 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:22:21.763 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:22:21.763 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:22:21.763 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:21.763 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:22:21.763 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:22:21.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:22:21.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:22:21.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:22:21.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:22:21.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:22:21.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:22:21.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:22:21.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:21.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:22:21.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:22:21.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:22:21.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:22:21.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:22:21.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:22:21.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:22:21.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:22:21.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:22:21.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:22:21.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:21.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:22:21.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:22:21.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:22:21.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:22:21.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:22:21.766 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:22:21.766 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:22:21.774 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:22:21.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:22:21.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:22:21.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:22:21.824 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:22:21.824 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:22:21.824 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:22:21.827 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:22:21.827 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:22:21.827 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:22:21.827 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:21.828 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:22:21.828 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:22:21.828 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:22:21.828 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:22:21.828 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:22:21.828 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:22:21.828 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:22:21.828 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:22:21.828 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:22:21.828 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:21.829 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:22:21.829 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:22:21.829 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:22:21.829 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:22:21.829 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:22:21.829 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:22:21.829 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:22:21.829 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:22:21.829 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:22:21.829 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:22:21.830 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:21.830 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:22:21.830 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:22:21.830 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:22:21.830 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:22:21.830 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:22:21.830 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:22:21.830 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:22:21.839 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:22:21.840 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:22:21.840 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:22:21.840 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:22:21.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:22:21.904 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:22:21.904 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:22:21.904 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:22:21.904 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:22:21.904 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:22:21.904 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:22:21.904 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:22:21.904 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:22:21.904 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:22:21.905 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:22:21.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:22:21.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:22:21.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:21.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:22:21.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:21.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 91 2026-02-18T08:22:21.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 95 2026-02-18T08:22:21.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:22:21.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=381, tag_tid=27, entry_tid=1], [object_number=380, tag_tid=27, entry_tid=0], [object_number=382, tag_tid=26, entry_tid=46994], [object_number=383, tag_tid=26, entry_tid=46991]]], state=connected] 2026-02-18T08:22:21.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=365, tag_tid=26, entry_tid=29089], [object_number=364, tag_tid=26, entry_tid=29088], [object_number=367, tag_tid=26, entry_tid=29087], [object_number=366, tag_tid=26, entry_tid=29086]]], state=connected] 2026-02-18T08:22:21.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:21.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=381, tag_tid=27, entry_tid=1' 2026-02-18T08:22:21.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:22:21.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:22:21.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:22:21.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:22:21.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:22:21.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:22:21.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:22:21.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:22:21.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:22:21.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:22:21.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:22:21.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:22:21.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:21.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:22:21.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:21.971 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 91 2026-02-18T08:22:21.971 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 95 2026-02-18T08:22:21.971 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:22:21.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=381, tag_tid=27, entry_tid=1], [object_number=380, tag_tid=27, entry_tid=0], [object_number=382, tag_tid=26, entry_tid=46994], [object_number=383, tag_tid=26, entry_tid=46991]]], state=connected] 2026-02-18T08:22:21.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=365, tag_tid=26, entry_tid=29089], [object_number=364, tag_tid=26, entry_tid=29088], [object_number=367, tag_tid=26, entry_tid=29087], [object_number=366, tag_tid=26, entry_tid=29086]]], state=connected] 2026-02-18T08:22:21.974 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:21.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=365, tag_tid=26, entry_tid=29089' 2026-02-18T08:22:21.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=381, tag_tid=27, entry_tid=1' -a 'object_number=381, tag_tid=27, entry_tid=1' = 'object_number=365, tag_tid=26, entry_tid=29089' 2026-02-18T08:22:21.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=365, tag_tid=26, entry_tid=29089' '!=' 'object_number=366, tag_tid=26, entry_tid=28726' 2026-02-18T08:22:21.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:22:21.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=365, tag_tid=26, entry_tid=29089' = 'object_number=366, tag_tid=26, entry_tid=28726' 2026-02-18T08:22:21.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=365, tag_tid=26, entry_tid=29089' 2026-02-18T08:22:21.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=381, tag_tid=27, entry_tid=1' 2026-02-18T08:22:21.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:22:21.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:22:21.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=27 2026-02-18T08:22:21.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=365, tag_tid=26, entry_tid=29089' 2026-02-18T08:22:21.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:22:21.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:22:21.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=26 2026-02-18T08:22:21.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=381, tag_tid=27, entry_tid=1' 2026-02-18T08:22:21.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:22:21.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:22:21.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:22:21.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=365, tag_tid=26, entry_tid=29089' 2026-02-18T08:22:21.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:22:21.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:22:21.985 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=29089 2026-02-18T08:22:21.985 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 27 = 26 -a 1 -le 29089 2026-02-18T08:22:21.986 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:22:21.986 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:22:21.986 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:22:22.188 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:22:22.188 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:22:22.188 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:22:22.188 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:22:22.188 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:22:22.188 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:22:22.188 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:22:22.189 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:22:22.189 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:22:22.189 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:22.189 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:22:22.189 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:22:22.189 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:22:22.189 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:22:22.189 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:22:22.189 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:22:22.189 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:22:22.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:22:22.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:22:22.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:22:22.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:22.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:22:22.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:22:22.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:22:22.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:22:22.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:22:22.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:22:22.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:22:22.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:22:22.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:22:22.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:22.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:22:22.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:22:22.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:22:22.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:22:22.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:22:22.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:22:22.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:22:22.194 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:22:22.194 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:22:22.194 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:22:22.194 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:22.194 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:22:22.194 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:22:22.194 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:22:22.194 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:22:22.194 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:22:22.194 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:22:22.194 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:22:22.202 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:22:22.202 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:22:22.202 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:22:22.202 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:22:22.260 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:22:22.260 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:22:22.260 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:22:22.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:22:22.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:22:22.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:22:22.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:22.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:22:22.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:22:22.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:22:22.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:22:22.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:22:22.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:22:22.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:22:22.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:22:22.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:22:22.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:22.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:22:22.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:22:22.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:22:22.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:22:22.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:22:22.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:22:22.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:22:22.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:22:22.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:22:22.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:22:22.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:22.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:22:22.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:22:22.266 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:22:22.266 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:22:22.266 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:22:22.266 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:22:22.266 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:22:22.274 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:22:22.275 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:22:22.275 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:22:22.275 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:22:22.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:22:22.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:22:22.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:22:22.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:22:22.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:22:22.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:22:22.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:22.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:22:22.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:22:22.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:22:22.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:22:22.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:22:22.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:22:22.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:22:22.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:22:22.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:22:22.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:22.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:22:22.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:22:22.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:22:22.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:22:22.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:22:22.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:22:22.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:22:22.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:22:22.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:22:22.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:22:22.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:22.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:22:22.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:22:22.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:22:22.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:22:22.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:22:22.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:22:22.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:22:22.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:22:22.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:22:22.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:22:22.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:22:22.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:22:22.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:22:22.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:22:22.407 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:22:22.408 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:22:22.408 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:22:22.408 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:22.408 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:22:22.408 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:22:22.408 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:22:22.408 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:22:22.408 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:22:22.408 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:22:22.408 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:22:22.408 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:22:22.409 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:22:22.409 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:22.409 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:22:22.409 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:22:22.409 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:22:22.409 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:22:22.409 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:22:22.409 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:22:22.409 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:22:22.409 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:22:22.409 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:22:22.409 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:22:22.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:22.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:22:22.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:22:22.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:22:22.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:22:22.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:22:22.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:22:22.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:22:22.418 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:22:22.419 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:22:22.419 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:22:22.419 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:22:22.488 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:22:22.489 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:22:22.489 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:22:22.489 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:22:22.489 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:22:22.489 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:22:22.489 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:22:22.490 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:22:22.490 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:22:22.490 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:22:22.490 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:22:22.490 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:22:22.490 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:22:22.490 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:22.491 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:22:22.491 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:22.528 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 91 2026-02-18T08:22:22.528 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 95 2026-02-18T08:22:22.528 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:22:22.528 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=381, tag_tid=27, entry_tid=1], [object_number=380, tag_tid=27, entry_tid=0], [object_number=382, tag_tid=26, entry_tid=46994], [object_number=383, tag_tid=26, entry_tid=46991]]], state=connected] 2026-02-18T08:22:22.528 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=366, tag_tid=26, entry_tid=29426], [object_number=365, tag_tid=26, entry_tid=29425], [object_number=364, tag_tid=26, entry_tid=29424], [object_number=367, tag_tid=26, entry_tid=29423]]], state=connected] 2026-02-18T08:22:22.532 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:22.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=381, tag_tid=27, entry_tid=1' 2026-02-18T08:22:22.535 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:22:22.535 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:22:22.535 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:22:22.535 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:22:22.535 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:22:22.535 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:22:22.536 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:22:22.536 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:22:22.536 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:22:22.536 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:22:22.536 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:22:22.536 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:22:22.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:22.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:22:22.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:22.568 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 91 2026-02-18T08:22:22.568 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 95 2026-02-18T08:22:22.568 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:22:22.568 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=381, tag_tid=27, entry_tid=1], [object_number=380, tag_tid=27, entry_tid=0], [object_number=382, tag_tid=26, entry_tid=46994], [object_number=383, tag_tid=26, entry_tid=46991]]], state=connected] 2026-02-18T08:22:22.568 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=366, tag_tid=26, entry_tid=29426], [object_number=365, tag_tid=26, entry_tid=29425], [object_number=364, tag_tid=26, entry_tid=29424], [object_number=367, tag_tid=26, entry_tid=29423]]], state=connected] 2026-02-18T08:22:22.570 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:22.572 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=366, tag_tid=26, entry_tid=29426' 2026-02-18T08:22:22.572 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=381, tag_tid=27, entry_tid=1' -a 'object_number=381, tag_tid=27, entry_tid=1' = 'object_number=366, tag_tid=26, entry_tid=29426' 2026-02-18T08:22:22.572 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=366, tag_tid=26, entry_tid=29426' '!=' 'object_number=365, tag_tid=26, entry_tid=29089' 2026-02-18T08:22:22.573 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:22:22.573 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=366, tag_tid=26, entry_tid=29426' = 'object_number=365, tag_tid=26, entry_tid=29089' 2026-02-18T08:22:22.573 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=366, tag_tid=26, entry_tid=29426' 2026-02-18T08:22:22.574 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=381, tag_tid=27, entry_tid=1' 2026-02-18T08:22:22.574 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:22:22.574 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:22:22.574 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=27 2026-02-18T08:22:22.575 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=366, tag_tid=26, entry_tid=29426' 2026-02-18T08:22:22.575 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:22:22.576 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:22:22.577 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=26 2026-02-18T08:22:22.578 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=381, tag_tid=27, entry_tid=1' 2026-02-18T08:22:22.578 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:22:22.578 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:22:22.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:22:22.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=366, tag_tid=26, entry_tid=29426' 2026-02-18T08:22:22.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:22:22.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:22:22.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=29426 2026-02-18T08:22:22.582 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 27 = 26 -a 1 -le 29426 2026-02-18T08:22:22.582 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:22:22.582 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:22:22.582 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:22:22.783 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:22:22.784 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:22:22.784 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:22:22.784 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:22:22.784 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:22:22.784 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:22:22.784 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:22:22.784 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:22:22.784 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:22:22.785 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:22.785 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:22:22.785 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:22:22.785 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:22:22.785 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:22:22.785 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:22:22.785 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:22:22.788 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:22:22.788 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:22:22.788 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:22:22.788 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:22:22.788 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:22.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:22:22.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:22:22.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:22:22.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:22:22.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:22:22.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:22:22.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:22:22.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:22:22.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:22:22.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:22.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:22:22.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:22:22.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:22:22.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:22:22.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:22:22.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:22:22.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:22:22.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:22:22.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:22:22.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:22:22.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:22.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:22:22.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:22:22.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:22:22.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:22:22.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:22:22.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:22:22.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:22:22.798 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:22:22.798 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:22:22.799 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:22:22.799 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:22:22.850 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:22:22.851 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:22:22.851 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:22:22.854 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:22:22.854 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:22:22.854 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:22:22.854 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:22.854 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:22:22.854 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:22:22.854 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:22:22.855 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:22:22.855 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:22:22.855 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:22:22.855 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:22:22.855 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:22:22.855 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:22:22.855 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:22.855 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:22:22.855 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:22:22.855 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:22:22.856 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:22:22.856 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:22:22.856 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:22:22.856 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:22:22.856 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:22:22.856 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:22:22.856 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:22:22.856 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:22.856 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:22:22.856 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:22:22.856 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:22:22.857 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:22:22.857 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:22:22.857 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:22:22.857 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:22:22.864 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:22:22.865 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:22:22.865 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:22:22.865 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:22:22.923 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:22:22.923 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:22:22.923 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:22:22.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:22:22.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:22:22.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:22:22.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:22.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:22:22.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:22:22.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:22:22.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:22:22.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:22:22.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:22:22.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:22:22.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:22:22.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:22:22.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:22.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:22:22.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:22:22.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:22:22.928 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:22:22.928 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:22:22.928 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:22:22.928 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:22:22.928 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:22:22.928 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:22:22.928 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:22:22.928 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:22.928 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:22:22.928 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:22:22.929 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:22:22.929 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:22:22.929 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:22:22.929 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:22:22.929 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:22:22.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:22:22.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:22:22.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:22:22.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:22:22.987 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:22:22.987 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:22:22.987 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:22:22.990 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:22:22.990 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:22:22.990 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:22:22.990 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:22.990 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:22:22.990 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:22:22.990 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:22:22.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:22:22.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:22:22.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:22:22.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:22:22.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:22:22.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:22:22.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:22.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:22:22.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:22:22.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:22:22.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:22:22.992 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:22:22.992 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:22:22.992 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:22:22.992 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:22:22.992 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:22:22.992 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:22:22.992 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:22.992 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:22:22.992 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:22:22.992 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:22:22.992 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:22:22.993 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:22:22.993 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:22:22.993 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:22:23.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:22:23.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:22:23.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:22:23.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:22:23.069 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:22:23.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:22:23.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:22:23.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:22:23.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:22:23.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:22:23.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:22:23.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:22:23.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:22:23.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:22:23.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:22:23.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:22:23.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:22:23.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:23.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:22:23.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:23.112 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 91 2026-02-18T08:22:23.112 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 95 2026-02-18T08:22:23.112 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:22:23.112 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=381, tag_tid=27, entry_tid=1], [object_number=380, tag_tid=27, entry_tid=0], [object_number=382, tag_tid=26, entry_tid=46994], [object_number=383, tag_tid=26, entry_tid=46991]]], state=connected] 2026-02-18T08:22:23.112 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=365, tag_tid=26, entry_tid=29777], [object_number=364, tag_tid=26, entry_tid=29776], [object_number=367, tag_tid=26, entry_tid=29775], [object_number=366, tag_tid=26, entry_tid=29774]]], state=connected] 2026-02-18T08:22:23.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:23.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=381, tag_tid=27, entry_tid=1' 2026-02-18T08:22:23.119 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:22:23.119 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:22:23.119 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:22:23.119 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:22:23.119 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:22:23.119 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:22:23.119 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:22:23.119 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:22:23.120 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:22:23.120 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:22:23.120 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:22:23.120 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:22:23.121 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:23.121 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:22:23.121 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:23.148 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 91 2026-02-18T08:22:23.148 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 95 2026-02-18T08:22:23.148 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:22:23.148 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=381, tag_tid=27, entry_tid=1], [object_number=380, tag_tid=27, entry_tid=0], [object_number=382, tag_tid=26, entry_tid=46994], [object_number=383, tag_tid=26, entry_tid=46991]]], state=connected] 2026-02-18T08:22:23.148 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=365, tag_tid=26, entry_tid=29777], [object_number=364, tag_tid=26, entry_tid=29776], [object_number=367, tag_tid=26, entry_tid=29775], [object_number=366, tag_tid=26, entry_tid=29774]]], state=connected] 2026-02-18T08:22:23.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:23.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=365, tag_tid=26, entry_tid=29777' 2026-02-18T08:22:23.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=381, tag_tid=27, entry_tid=1' -a 'object_number=381, tag_tid=27, entry_tid=1' = 'object_number=365, tag_tid=26, entry_tid=29777' 2026-02-18T08:22:23.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=365, tag_tid=26, entry_tid=29777' '!=' 'object_number=366, tag_tid=26, entry_tid=29426' 2026-02-18T08:22:23.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:22:23.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=365, tag_tid=26, entry_tid=29777' = 'object_number=366, tag_tid=26, entry_tid=29426' 2026-02-18T08:22:23.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=365, tag_tid=26, entry_tid=29777' 2026-02-18T08:22:23.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=381, tag_tid=27, entry_tid=1' 2026-02-18T08:22:23.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:22:23.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:22:23.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=27 2026-02-18T08:22:23.155 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=365, tag_tid=26, entry_tid=29777' 2026-02-18T08:22:23.156 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:22:23.156 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:22:23.157 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=26 2026-02-18T08:22:23.158 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=381, tag_tid=27, entry_tid=1' 2026-02-18T08:22:23.158 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:22:23.158 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:22:23.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:22:23.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=365, tag_tid=26, entry_tid=29777' 2026-02-18T08:22:23.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:22:23.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:22:23.162 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=29777 2026-02-18T08:22:23.162 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 27 = 26 -a 1 -le 29777 2026-02-18T08:22:23.162 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:22:23.162 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:22:23.162 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:22:23.364 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:22:23.364 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:22:23.364 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:22:23.364 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:22:23.364 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:22:23.364 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:22:23.364 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:22:23.365 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:22:23.365 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:22:23.365 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:23.365 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:22:23.365 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:22:23.365 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:22:23.365 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:22:23.365 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:22:23.365 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:22:23.368 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:22:23.368 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:22:23.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:22:23.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:22:23.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:23.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:22:23.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:22:23.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:22:23.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:22:23.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:22:23.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:22:23.370 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:22:23.370 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:22:23.370 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:22:23.370 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:23.370 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:22:23.370 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:22:23.370 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:22:23.370 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:22:23.370 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:22:23.370 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:22:23.370 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:22:23.370 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:22:23.371 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:22:23.371 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:22:23.371 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:23.371 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:22:23.371 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:22:23.371 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:22:23.371 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:22:23.371 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:22:23.371 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:22:23.371 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:22:23.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:22:23.379 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:22:23.379 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:22:23.379 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:22:23.430 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:22:23.430 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:22:23.430 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:22:23.433 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:22:23.433 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:22:23.433 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:22:23.434 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:23.434 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:22:23.434 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:22:23.434 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:22:23.434 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:22:23.434 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:22:23.434 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:22:23.434 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:22:23.434 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:22:23.434 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:22:23.434 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:23.434 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:22:23.435 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:22:23.435 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:22:23.435 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:22:23.435 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:22:23.435 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:22:23.435 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:22:23.435 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:22:23.435 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:22:23.435 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:22:23.435 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:23.435 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:22:23.436 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:22:23.436 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:22:23.436 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:22:23.436 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:22:23.436 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:22:23.436 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:22:23.443 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:22:23.443 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:22:23.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:22:23.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:22:23.493 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:22:23.493 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:22:23.493 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:22:23.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:22:23.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:22:23.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:22:23.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:23.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:22:23.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:22:23.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:22:23.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:22:23.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:22:23.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:22:23.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:22:23.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:22:23.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:22:23.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:23.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:22:23.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:22:23.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:22:23.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:22:23.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:22:23.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:22:23.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:22:23.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:22:23.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:22:23.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:22:23.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:23.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:22:23.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:22:23.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:22:23.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:22:23.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:22:23.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:22:23.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:22:23.506 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:22:23.506 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:22:23.507 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:22:23.507 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:22:23.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:22:23.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:22:23.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:22:23.559 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:22:23.559 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:22:23.559 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:22:23.559 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:23.559 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:22:23.559 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:22:23.559 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:22:23.559 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:22:23.559 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:22:23.560 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:22:23.560 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:22:23.560 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:22:23.560 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:22:23.560 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:23.560 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:22:23.560 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:22:23.560 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:22:23.560 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:22:23.560 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:22:23.561 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:22:23.561 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:22:23.561 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:22:23.561 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:22:23.561 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:22:23.561 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:23.561 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:22:23.561 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:22:23.561 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:22:23.561 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:22:23.561 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:22:23.562 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:22:23.562 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:22:23.569 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:22:23.570 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:22:23.570 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:22:23.570 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:22:23.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:22:23.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:22:23.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:22:23.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:22:23.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:22:23.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:22:23.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:22:23.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:22:23.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:22:23.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:22:23.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:22:23.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:22:23.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:22:23.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:23.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:22:23.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:23.663 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 91 2026-02-18T08:22:23.663 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 95 2026-02-18T08:22:23.663 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:22:23.663 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=381, tag_tid=27, entry_tid=1], [object_number=380, tag_tid=27, entry_tid=0], [object_number=382, tag_tid=26, entry_tid=46994], [object_number=383, tag_tid=26, entry_tid=46991]]], state=connected] 2026-02-18T08:22:23.663 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=365, tag_tid=26, entry_tid=30105], [object_number=364, tag_tid=26, entry_tid=30104], [object_number=367, tag_tid=26, entry_tid=30103], [object_number=366, tag_tid=26, entry_tid=30102]]], state=connected] 2026-02-18T08:22:23.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:23.667 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=381, tag_tid=27, entry_tid=1' 2026-02-18T08:22:23.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:22:23.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:22:23.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:22:23.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:22:23.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:22:23.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:22:23.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:22:23.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:22:23.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:22:23.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:22:23.670 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:22:23.670 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:22:23.670 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:23.671 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:22:23.671 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:23.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 91 2026-02-18T08:22:23.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 95 2026-02-18T08:22:23.700 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:22:23.700 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=381, tag_tid=27, entry_tid=1], [object_number=380, tag_tid=27, entry_tid=0], [object_number=382, tag_tid=26, entry_tid=46994], [object_number=383, tag_tid=26, entry_tid=46991]]], state=connected] 2026-02-18T08:22:23.700 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=365, tag_tid=26, entry_tid=30105], [object_number=364, tag_tid=26, entry_tid=30104], [object_number=367, tag_tid=26, entry_tid=30103], [object_number=366, tag_tid=26, entry_tid=30102]]], state=connected] 2026-02-18T08:22:23.702 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:23.704 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=365, tag_tid=26, entry_tid=30105' 2026-02-18T08:22:23.704 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=381, tag_tid=27, entry_tid=1' -a 'object_number=381, tag_tid=27, entry_tid=1' = 'object_number=365, tag_tid=26, entry_tid=30105' 2026-02-18T08:22:23.704 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=365, tag_tid=26, entry_tid=30105' '!=' 'object_number=365, tag_tid=26, entry_tid=29777' 2026-02-18T08:22:23.704 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:22:23.704 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=365, tag_tid=26, entry_tid=30105' = 'object_number=365, tag_tid=26, entry_tid=29777' 2026-02-18T08:22:23.704 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=365, tag_tid=26, entry_tid=30105' 2026-02-18T08:22:23.705 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=381, tag_tid=27, entry_tid=1' 2026-02-18T08:22:23.705 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:22:23.705 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:22:23.706 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=27 2026-02-18T08:22:23.707 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=365, tag_tid=26, entry_tid=30105' 2026-02-18T08:22:23.707 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:22:23.707 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:22:23.708 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=26 2026-02-18T08:22:23.709 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=381, tag_tid=27, entry_tid=1' 2026-02-18T08:22:23.709 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:22:23.709 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:22:23.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:22:23.711 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=365, tag_tid=26, entry_tid=30105' 2026-02-18T08:22:23.711 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:22:23.711 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:22:23.713 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=30105 2026-02-18T08:22:23.713 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 27 = 26 -a 1 -le 30105 2026-02-18T08:22:23.713 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:22:23.713 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:22:23.713 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:22:23.915 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:22:23.915 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:22:23.915 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:22:23.915 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:22:23.915 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:22:23.915 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:22:23.915 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:22:23.915 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:22:23.916 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:22:23.916 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:23.916 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:22:23.916 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:22:23.916 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:22:23.916 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:22:23.916 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:22:23.916 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:22:23.916 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:22:23.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:22:23.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:22:23.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:22:23.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:23.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:22:23.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:22:23.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:22:23.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:22:23.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:22:23.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:22:23.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:22:23.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:22:23.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:22:23.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:23.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:22:23.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:22:23.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:22:23.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:22:23.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:22:23.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:22:23.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:22:23.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:22:23.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:22:23.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:22:23.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:23.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:22:23.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:22:23.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:22:23.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:22:23.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:22:23.922 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:22:23.922 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:22:23.928 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:22:23.929 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:22:23.929 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:22:23.929 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:22:23.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:22:23.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:22:23.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:22:23.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:22:23.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:22:23.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:22:23.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:23.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:22:23.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:22:23.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:22:23.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:22:23.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:22:23.982 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:22:23.982 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:22:23.982 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:22:23.982 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:22:23.982 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:23.982 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:22:23.982 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:22:23.982 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:22:23.982 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:22:23.982 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:22:23.982 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:22:23.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:22:23.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:22:23.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:22:23.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:22:23.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:23.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:22:23.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:22:23.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:22:23.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:22:23.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:22:23.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:22:23.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:22:23.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:22:23.992 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:22:23.992 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:22:23.992 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:22:24.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:22:24.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:22:24.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:22:24.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:22:24.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:22:24.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:22:24.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:24.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:22:24.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:22:24.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:22:24.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:22:24.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:22:24.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:22:24.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:22:24.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:22:24.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:22:24.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:24.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:22:24.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:22:24.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:22:24.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:22:24.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:22:24.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:22:24.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:22:24.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:22:24.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:22:24.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:22:24.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:24.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:22:24.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:22:24.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:22:24.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:22:24.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:22:24.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:22:24.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:22:24.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:22:24.062 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:22:24.062 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:22:24.062 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:22:24.111 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:22:24.111 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:22:24.112 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:22:24.114 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:22:24.114 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:22:24.114 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:22:24.114 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:24.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:22:24.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:22:24.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:22:24.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:22:24.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:22:24.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:22:24.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:22:24.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:22:24.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:22:24.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:24.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:22:24.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:22:24.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:22:24.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:22:24.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:22:24.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:22:24.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:22:24.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:22:24.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:22:24.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:22:24.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:24.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:22:24.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:22:24.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:22:24.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:22:24.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:22:24.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:22:24.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:22:24.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:22:24.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:22:24.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:22:24.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:22:24.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:22:24.187 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:22:24.187 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:22:24.187 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:22:24.188 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:22:24.188 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:22:24.188 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:22:24.188 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:22:24.188 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:22:24.188 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:22:24.188 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:22:24.188 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:22:24.188 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:22:24.189 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:24.189 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:22:24.189 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:24.422 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 91 2026-02-18T08:22:24.423 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 95 2026-02-18T08:22:24.423 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:22:24.423 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=381, tag_tid=27, entry_tid=1], [object_number=380, tag_tid=27, entry_tid=0], [object_number=382, tag_tid=26, entry_tid=46994], [object_number=383, tag_tid=26, entry_tid=46991]]], state=connected] 2026-02-18T08:22:24.423 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=364, tag_tid=26, entry_tid=30448], [object_number=367, tag_tid=26, entry_tid=30447], [object_number=366, tag_tid=26, entry_tid=30446], [object_number=365, tag_tid=26, entry_tid=30445]]], state=connected] 2026-02-18T08:22:24.427 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:24.428 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=381, tag_tid=27, entry_tid=1' 2026-02-18T08:22:24.429 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:22:24.429 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:22:24.429 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:22:24.429 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:22:24.430 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:22:24.430 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:22:24.430 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:22:24.430 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:22:24.430 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:22:24.430 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:22:24.431 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:22:24.431 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:22:24.431 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:24.432 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:22:24.432 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:24.468 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 91 2026-02-18T08:22:24.468 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 95 2026-02-18T08:22:24.468 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:22:24.468 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=381, tag_tid=27, entry_tid=1], [object_number=380, tag_tid=27, entry_tid=0], [object_number=382, tag_tid=26, entry_tid=46994], [object_number=383, tag_tid=26, entry_tid=46991]]], state=connected] 2026-02-18T08:22:24.468 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=364, tag_tid=26, entry_tid=30448], [object_number=367, tag_tid=26, entry_tid=30447], [object_number=366, tag_tid=26, entry_tid=30446], [object_number=365, tag_tid=26, entry_tid=30445]]], state=connected] 2026-02-18T08:22:24.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:24.473 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=364, tag_tid=26, entry_tid=30448' 2026-02-18T08:22:24.473 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=381, tag_tid=27, entry_tid=1' -a 'object_number=381, tag_tid=27, entry_tid=1' = 'object_number=364, tag_tid=26, entry_tid=30448' 2026-02-18T08:22:24.473 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=364, tag_tid=26, entry_tid=30448' '!=' 'object_number=365, tag_tid=26, entry_tid=30105' 2026-02-18T08:22:24.473 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:22:24.473 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=364, tag_tid=26, entry_tid=30448' = 'object_number=365, tag_tid=26, entry_tid=30105' 2026-02-18T08:22:24.473 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=364, tag_tid=26, entry_tid=30448' 2026-02-18T08:22:24.474 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=381, tag_tid=27, entry_tid=1' 2026-02-18T08:22:24.474 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:22:24.474 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:22:24.475 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=27 2026-02-18T08:22:24.476 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=364, tag_tid=26, entry_tid=30448' 2026-02-18T08:22:24.476 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:22:24.476 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:22:24.477 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=26 2026-02-18T08:22:24.478 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=381, tag_tid=27, entry_tid=1' 2026-02-18T08:22:24.478 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:22:24.478 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:22:24.480 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:22:24.480 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=364, tag_tid=26, entry_tid=30448' 2026-02-18T08:22:24.481 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:22:24.481 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:22:24.482 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=30448 2026-02-18T08:22:24.482 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 27 = 26 -a 1 -le 30448 2026-02-18T08:22:24.483 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:22:24.483 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:22:24.483 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:22:24.684 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:22:24.684 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:22:24.684 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:22:24.684 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:22:24.684 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:22:24.685 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:22:24.685 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:22:24.685 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:22:24.685 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:22:24.685 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:24.685 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:22:24.685 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:22:24.685 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:22:24.685 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:22:24.686 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:22:24.688 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:22:24.688 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:22:24.688 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:22:24.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:22:24.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:22:24.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:24.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:22:24.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:22:24.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:22:24.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:22:24.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:22:24.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:22:24.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:22:24.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:22:24.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:22:24.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:24.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:22:24.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:22:24.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:22:24.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:22:24.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:22:24.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:22:24.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:22:24.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:22:24.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:22:24.691 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:22:24.691 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:24.691 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:22:24.691 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:22:24.691 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:22:24.691 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:22:24.691 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:22:24.691 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:22:24.691 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:22:24.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:22:24.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:22:24.700 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:22:24.700 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:22:24.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:22:24.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:22:24.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:22:24.760 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:22:24.760 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:22:24.760 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:22:24.760 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:24.760 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:22:24.760 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:22:24.760 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:22:24.761 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:22:24.761 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:22:24.761 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:22:24.761 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:22:24.761 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:22:24.761 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:22:24.761 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:24.761 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:22:24.761 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:22:24.761 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:22:24.762 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:22:24.762 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:22:24.762 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:22:24.762 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:22:24.762 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:22:24.762 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:22:24.762 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:22:24.762 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:24.762 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:22:24.762 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:22:24.762 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:22:24.763 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:22:24.763 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:22:24.763 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:22:24.763 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:22:24.771 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:22:24.772 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:22:24.772 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:22:24.772 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:22:24.828 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:22:24.828 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:22:24.828 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:22:24.831 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:22:24.831 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:22:24.831 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:22:24.831 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:24.831 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:22:24.831 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:22:24.832 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:22:24.832 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:22:24.832 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:22:24.832 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:22:24.832 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:22:24.832 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:22:24.832 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:22:24.832 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:24.832 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:22:24.832 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:22:24.832 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:22:24.832 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:22:24.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:22:24.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:22:24.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:22:24.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:22:24.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:22:24.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:22:24.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:24.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:22:24.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:22:24.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:22:24.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:22:24.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:22:24.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:22:24.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:22:24.841 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:22:24.841 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:22:24.841 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:22:24.842 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:22:24.894 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:22:24.894 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:22:24.894 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:22:24.897 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:22:24.897 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:22:24.897 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:22:24.897 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:24.897 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:22:24.897 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:22:24.897 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:22:24.897 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:22:24.897 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:22:24.897 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:22:24.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:22:24.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:22:24.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:22:24.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:24.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:22:24.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:22:24.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:22:24.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:22:24.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:22:24.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:22:24.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:22:24.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:22:24.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:22:24.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:22:24.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:24.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:22:24.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:22:24.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:22:24.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:22:24.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:22:24.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:22:24.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:22:24.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:22:24.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:22:24.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:22:24.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:22:24.969 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:22:24.971 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:22:24.971 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:22:24.971 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:22:24.971 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:22:24.971 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:22:24.971 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:22:24.971 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:22:24.971 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:22:24.971 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:22:24.971 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:22:24.971 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:22:24.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:22:24.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:24.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:22:24.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:25.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 91 2026-02-18T08:22:25.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 95 2026-02-18T08:22:25.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:22:25.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=381, tag_tid=27, entry_tid=1], [object_number=380, tag_tid=27, entry_tid=0], [object_number=382, tag_tid=26, entry_tid=46994], [object_number=383, tag_tid=26, entry_tid=46991]]], state=connected] 2026-02-18T08:22:25.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=365, tag_tid=26, entry_tid=30933], [object_number=364, tag_tid=26, entry_tid=30932], [object_number=367, tag_tid=26, entry_tid=30931], [object_number=366, tag_tid=26, entry_tid=30930]]], state=connected] 2026-02-18T08:22:25.004 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:25.005 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=381, tag_tid=27, entry_tid=1' 2026-02-18T08:22:25.006 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:22:25.006 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:22:25.006 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:22:25.007 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:22:25.007 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:22:25.007 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:22:25.007 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:22:25.007 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:22:25.007 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:22:25.007 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:22:25.008 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:22:25.008 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:22:25.008 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:25.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:22:25.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:25.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 91 2026-02-18T08:22:25.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 95 2026-02-18T08:22:25.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:22:25.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=381, tag_tid=27, entry_tid=1], [object_number=380, tag_tid=27, entry_tid=0], [object_number=382, tag_tid=26, entry_tid=46994], [object_number=383, tag_tid=26, entry_tid=46991]]], state=connected] 2026-02-18T08:22:25.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=365, tag_tid=26, entry_tid=30933], [object_number=364, tag_tid=26, entry_tid=30932], [object_number=367, tag_tid=26, entry_tid=30931], [object_number=366, tag_tid=26, entry_tid=30930]]], state=connected] 2026-02-18T08:22:25.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:25.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=365, tag_tid=26, entry_tid=30933' 2026-02-18T08:22:25.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=381, tag_tid=27, entry_tid=1' -a 'object_number=381, tag_tid=27, entry_tid=1' = 'object_number=365, tag_tid=26, entry_tid=30933' 2026-02-18T08:22:25.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=365, tag_tid=26, entry_tid=30933' '!=' 'object_number=364, tag_tid=26, entry_tid=30448' 2026-02-18T08:22:25.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:22:25.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=365, tag_tid=26, entry_tid=30933' = 'object_number=364, tag_tid=26, entry_tid=30448' 2026-02-18T08:22:25.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=365, tag_tid=26, entry_tid=30933' 2026-02-18T08:22:25.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=381, tag_tid=27, entry_tid=1' 2026-02-18T08:22:25.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:22:25.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:22:25.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=27 2026-02-18T08:22:25.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=365, tag_tid=26, entry_tid=30933' 2026-02-18T08:22:25.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:22:25.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:22:25.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=26 2026-02-18T08:22:25.055 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=381, tag_tid=27, entry_tid=1' 2026-02-18T08:22:25.055 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:22:25.055 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:22:25.056 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:22:25.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=365, tag_tid=26, entry_tid=30933' 2026-02-18T08:22:25.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:22:25.058 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:22:25.059 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=30933 2026-02-18T08:22:25.059 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 27 = 26 -a 1 -le 30933 2026-02-18T08:22:25.059 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:22:25.059 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:22:25.059 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:22:25.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:22:25.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:22:25.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:22:25.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:22:25.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:22:25.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:22:25.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:22:25.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:22:25.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:22:25.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:25.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:22:25.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:22:25.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:22:25.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:22:25.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:22:25.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:22:25.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:22:25.266 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:22:25.266 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:22:25.266 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:22:25.266 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:25.266 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:22:25.266 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:22:25.266 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:22:25.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:22:25.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:22:25.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:22:25.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:22:25.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:22:25.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:22:25.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:25.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:22:25.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:22:25.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:22:25.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:22:25.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:22:25.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:22:25.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:22:25.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:22:25.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:22:25.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:22:25.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:25.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:22:25.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:22:25.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:22:25.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:22:25.269 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:22:25.269 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:22:25.269 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:22:25.276 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:22:25.276 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:22:25.276 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:22:25.276 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:22:25.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:22:25.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:22:25.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:22:25.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:22:25.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:22:25.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:22:25.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:25.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:22:25.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:22:25.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:22:25.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:22:25.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:22:25.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:22:25.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:22:25.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:22:25.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:22:25.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:25.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:22:25.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:22:25.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:22:25.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:22:25.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:22:25.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:22:25.344 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:22:25.344 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:22:25.344 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:22:25.344 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:22:25.344 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:25.344 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:22:25.344 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:22:25.344 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:22:25.344 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:22:25.344 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:22:25.344 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:22:25.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:22:25.353 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:22:25.353 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:22:25.353 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:22:25.353 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:22:25.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:22:25.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:22:25.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:22:25.403 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:22:25.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:22:25.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:22:25.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:25.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:22:25.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:22:25.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:22:25.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:22:25.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:22:25.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:22:25.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:22:25.405 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:22:25.405 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:22:25.405 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:25.405 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:22:25.405 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:22:25.405 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:22:25.405 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:22:25.405 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:22:25.405 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:22:25.405 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:22:25.405 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:22:25.405 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:22:25.406 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:22:25.406 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:25.406 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:22:25.406 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:22:25.406 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:22:25.406 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:22:25.406 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:22:25.406 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:22:25.406 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:22:25.415 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:22:25.415 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:22:25.415 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:22:25.415 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:22:25.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:22:25.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:22:25.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:22:25.467 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:22:25.467 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:22:25.467 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:22:25.467 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:25.467 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:22:25.467 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:22:25.467 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:22:25.467 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:22:25.467 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:22:25.468 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:22:25.468 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:22:25.468 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:22:25.468 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:22:25.468 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:25.468 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:22:25.468 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:22:25.468 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:22:25.468 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:22:25.468 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:22:25.468 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:22:25.468 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:22:25.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:22:25.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:22:25.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:22:25.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:25.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:22:25.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:22:25.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:22:25.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:22:25.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:22:25.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:22:25.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:22:25.478 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:22:25.478 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:22:25.478 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:22:25.479 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:22:25.566 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:22:25.566 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:22:25.566 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:22:25.566 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:22:25.567 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:22:25.567 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:22:25.567 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:22:25.567 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:22:25.567 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:22:25.567 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:22:25.567 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:22:25.567 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:22:25.568 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:22:25.568 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:25.568 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:22:25.569 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:25.606 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 91 2026-02-18T08:22:25.606 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 95 2026-02-18T08:22:25.606 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:22:25.607 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=381, tag_tid=27, entry_tid=1], [object_number=380, tag_tid=27, entry_tid=0], [object_number=382, tag_tid=26, entry_tid=46994], [object_number=383, tag_tid=26, entry_tid=46991]]], state=connected] 2026-02-18T08:22:25.607 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=365, tag_tid=26, entry_tid=31185], [object_number=364, tag_tid=26, entry_tid=31184], [object_number=367, tag_tid=26, entry_tid=31183], [object_number=366, tag_tid=26, entry_tid=31182]]], state=connected] 2026-02-18T08:22:25.610 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:25.612 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=381, tag_tid=27, entry_tid=1' 2026-02-18T08:22:25.613 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:22:25.613 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:22:25.613 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:22:25.613 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:22:25.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:22:25.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:22:25.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:22:25.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:22:25.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:22:25.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:22:25.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:22:25.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:22:25.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:25.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:22:25.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:25.642 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 91 2026-02-18T08:22:25.642 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 95 2026-02-18T08:22:25.642 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:22:25.642 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=381, tag_tid=27, entry_tid=1], [object_number=380, tag_tid=27, entry_tid=0], [object_number=382, tag_tid=26, entry_tid=46994], [object_number=383, tag_tid=26, entry_tid=46991]]], state=connected] 2026-02-18T08:22:25.643 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=365, tag_tid=26, entry_tid=31185], [object_number=364, tag_tid=26, entry_tid=31184], [object_number=367, tag_tid=26, entry_tid=31183], [object_number=366, tag_tid=26, entry_tid=31182]]], state=connected] 2026-02-18T08:22:25.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:25.647 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=365, tag_tid=26, entry_tid=31185' 2026-02-18T08:22:25.647 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=381, tag_tid=27, entry_tid=1' -a 'object_number=381, tag_tid=27, entry_tid=1' = 'object_number=365, tag_tid=26, entry_tid=31185' 2026-02-18T08:22:25.647 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=365, tag_tid=26, entry_tid=31185' '!=' 'object_number=365, tag_tid=26, entry_tid=30933' 2026-02-18T08:22:25.647 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:22:25.647 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=365, tag_tid=26, entry_tid=31185' = 'object_number=365, tag_tid=26, entry_tid=30933' 2026-02-18T08:22:25.648 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=365, tag_tid=26, entry_tid=31185' 2026-02-18T08:22:25.648 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=381, tag_tid=27, entry_tid=1' 2026-02-18T08:22:25.648 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:22:25.648 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:22:25.649 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=27 2026-02-18T08:22:25.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=365, tag_tid=26, entry_tid=31185' 2026-02-18T08:22:25.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:22:25.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:22:25.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=26 2026-02-18T08:22:25.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=381, tag_tid=27, entry_tid=1' 2026-02-18T08:22:25.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:22:25.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:22:25.654 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:22:25.654 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=365, tag_tid=26, entry_tid=31185' 2026-02-18T08:22:25.655 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:22:25.655 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:22:25.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=31185 2026-02-18T08:22:25.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 27 = 26 -a 1 -le 31185 2026-02-18T08:22:25.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:22:25.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:22:25.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:22:25.858 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:22:25.859 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:22:25.859 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:22:25.859 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:22:25.859 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:22:25.859 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:22:25.859 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:22:25.859 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:22:25.859 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:22:25.860 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:25.860 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:22:25.860 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:22:25.860 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:22:25.860 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:22:25.860 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:22:25.860 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:22:25.863 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:22:25.863 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:22:25.863 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:22:25.863 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:22:25.863 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:25.863 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:22:25.864 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:22:25.864 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:22:25.864 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:22:25.864 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:22:25.864 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:22:25.864 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:22:25.864 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:22:25.864 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:22:25.864 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:25.864 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:22:25.864 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:22:25.864 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:22:25.865 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:22:25.865 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:22:25.865 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:22:25.865 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:22:25.865 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:22:25.865 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:22:25.865 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:22:25.865 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:25.865 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:22:25.865 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:22:25.865 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:22:25.866 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:22:25.866 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:22:25.866 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:22:25.866 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:22:25.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:22:25.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:22:25.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:22:25.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:22:25.930 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:22:25.930 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:22:25.930 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:22:25.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:22:25.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:22:25.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:22:25.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:25.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:22:25.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:22:25.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:22:25.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:22:25.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:22:25.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:22:25.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:22:25.936 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:22:25.936 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:22:25.936 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:25.936 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:22:25.936 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:22:25.936 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:22:25.936 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:22:25.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:22:25.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:22:25.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:22:25.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:22:25.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:22:25.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:22:25.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:25.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:22:25.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:22:25.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:22:25.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:22:25.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:22:25.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:22:25.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:22:25.944 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:22:25.944 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:22:25.944 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:22:25.944 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:22:25.994 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:22:25.994 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:22:25.994 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:22:25.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:22:25.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:22:25.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:22:25.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:25.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:22:25.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:22:25.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:22:25.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:22:25.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:22:25.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:22:25.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:22:25.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:22:25.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:22:25.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:25.999 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:22:25.999 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:22:25.999 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:22:25.999 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:22:25.999 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:22:25.999 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:22:25.999 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:22:25.999 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:22:26.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:22:26.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:22:26.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:26.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:22:26.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:22:26.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:22:26.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:22:26.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:22:26.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:22:26.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:22:26.007 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:22:26.008 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:22:26.008 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:22:26.008 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:22:26.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:22:26.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:22:26.058 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:22:26.060 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:22:26.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:22:26.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:22:26.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:26.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:22:26.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:22:26.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:22:26.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:22:26.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:22:26.062 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:22:26.062 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:22:26.062 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:22:26.062 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:22:26.062 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:26.062 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:22:26.062 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:22:26.062 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:22:26.062 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:22:26.062 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:22:26.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:22:26.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:22:26.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:22:26.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:22:26.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:22:26.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:26.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:22:26.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:22:26.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:22:26.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:22:26.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:22:26.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:22:26.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:22:26.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:22:26.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:22:26.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:22:26.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:22:26.142 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:22:26.142 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:22:26.143 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:22:26.143 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:22:26.143 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:22:26.143 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:22:26.143 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:22:26.143 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:22:26.144 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:22:26.144 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:22:26.144 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:22:26.144 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:22:26.144 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:22:26.145 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:26.146 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:22:26.146 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:26.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 91 2026-02-18T08:22:26.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 95 2026-02-18T08:22:26.173 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:22:26.173 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=381, tag_tid=27, entry_tid=1], [object_number=380, tag_tid=27, entry_tid=0], [object_number=382, tag_tid=26, entry_tid=46994], [object_number=383, tag_tid=26, entry_tid=46991]]], state=connected] 2026-02-18T08:22:26.173 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=365, tag_tid=26, entry_tid=31537], [object_number=364, tag_tid=26, entry_tid=31536], [object_number=367, tag_tid=26, entry_tid=31535], [object_number=366, tag_tid=26, entry_tid=31534]]], state=connected] 2026-02-18T08:22:26.175 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:26.176 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=381, tag_tid=27, entry_tid=1' 2026-02-18T08:22:26.178 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:22:26.178 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:22:26.178 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:22:26.178 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:22:26.178 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:22:26.178 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:22:26.178 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:22:26.179 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:22:26.179 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:22:26.179 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:22:26.179 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:22:26.179 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:22:26.180 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:26.180 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:22:26.180 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:26.209 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 91 2026-02-18T08:22:26.209 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 95 2026-02-18T08:22:26.210 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:22:26.210 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=381, tag_tid=27, entry_tid=1], [object_number=380, tag_tid=27, entry_tid=0], [object_number=382, tag_tid=26, entry_tid=46994], [object_number=383, tag_tid=26, entry_tid=46991]]], state=connected] 2026-02-18T08:22:26.210 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=365, tag_tid=26, entry_tid=31537], [object_number=364, tag_tid=26, entry_tid=31536], [object_number=367, tag_tid=26, entry_tid=31535], [object_number=366, tag_tid=26, entry_tid=31534]]], state=connected] 2026-02-18T08:22:26.212 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:26.213 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=365, tag_tid=26, entry_tid=31537' 2026-02-18T08:22:26.214 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=381, tag_tid=27, entry_tid=1' -a 'object_number=381, tag_tid=27, entry_tid=1' = 'object_number=365, tag_tid=26, entry_tid=31537' 2026-02-18T08:22:26.214 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=365, tag_tid=26, entry_tid=31537' '!=' 'object_number=365, tag_tid=26, entry_tid=31185' 2026-02-18T08:22:26.214 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:22:26.214 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=365, tag_tid=26, entry_tid=31537' = 'object_number=365, tag_tid=26, entry_tid=31185' 2026-02-18T08:22:26.214 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=365, tag_tid=26, entry_tid=31537' 2026-02-18T08:22:26.215 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=381, tag_tid=27, entry_tid=1' 2026-02-18T08:22:26.215 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:22:26.215 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:22:26.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=27 2026-02-18T08:22:26.217 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=365, tag_tid=26, entry_tid=31537' 2026-02-18T08:22:26.217 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:22:26.217 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:22:26.218 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=26 2026-02-18T08:22:26.219 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=381, tag_tid=27, entry_tid=1' 2026-02-18T08:22:26.219 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:22:26.219 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:22:26.220 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:22:26.221 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=365, tag_tid=26, entry_tid=31537' 2026-02-18T08:22:26.221 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:22:26.221 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:22:26.223 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=31537 2026-02-18T08:22:26.223 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 27 = 26 -a 1 -le 31537 2026-02-18T08:22:26.223 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:22:26.223 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:22:26.223 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:22:26.425 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:22:26.425 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:22:26.425 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:22:26.425 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:22:26.425 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:22:26.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:22:26.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:22:26.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:22:26.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:22:26.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:26.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:22:26.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:22:26.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:22:26.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:22:26.427 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:22:26.429 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:22:26.429 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:22:26.430 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:22:26.430 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:22:26.430 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:22:26.430 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:26.430 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:22:26.430 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:22:26.430 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:22:26.430 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:22:26.430 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:22:26.430 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:22:26.431 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:22:26.431 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:22:26.431 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:22:26.431 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:26.431 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:22:26.431 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:22:26.431 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:22:26.431 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:22:26.431 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:22:26.431 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:22:26.432 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:22:26.432 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:22:26.432 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:22:26.432 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:22:26.432 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:26.432 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:22:26.432 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:22:26.432 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:22:26.432 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:22:26.432 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:22:26.432 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:22:26.432 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:22:26.440 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:22:26.440 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:22:26.440 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:22:26.440 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:22:26.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:22:26.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:22:26.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:22:26.501 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:22:26.501 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:22:26.501 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:22:26.501 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:26.501 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:22:26.501 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:22:26.501 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:22:26.501 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:22:26.501 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:22:26.501 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:22:26.502 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:22:26.502 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:22:26.502 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:22:26.502 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:26.502 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:22:26.502 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:22:26.502 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:22:26.502 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:22:26.502 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:22:26.502 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:22:26.502 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:22:26.503 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:22:26.503 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:22:26.503 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:22:26.503 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:26.503 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:22:26.503 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:22:26.503 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:22:26.503 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:22:26.503 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:22:26.503 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:22:26.503 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:22:26.512 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:22:26.513 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:22:26.513 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:22:26.513 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:22:26.562 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:22:26.562 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:22:26.562 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:22:26.565 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:22:26.565 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:22:26.565 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:22:26.565 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:26.565 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:22:26.565 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:22:26.565 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:22:26.566 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:22:26.566 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:22:26.566 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:22:26.566 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:22:26.566 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:22:26.566 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:22:26.566 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:26.566 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:22:26.566 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:22:26.566 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:22:26.566 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:22:26.566 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:22:26.567 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:22:26.567 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:22:26.567 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:22:26.567 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:22:26.567 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:22:26.567 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:26.567 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:22:26.567 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:22:26.567 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:22:26.567 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:22:26.567 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:22:26.568 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:22:26.568 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:22:26.576 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:22:26.576 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:22:26.577 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:22:26.577 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:22:26.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:22:26.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:22:26.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:22:26.636 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:22:26.636 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:22:26.636 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:22:26.636 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:26.636 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:22:26.636 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:22:26.636 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:22:26.637 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:22:26.637 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:22:26.637 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:22:26.637 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:22:26.637 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:22:26.637 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:22:26.637 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:26.637 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:22:26.637 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:22:26.637 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:22:26.637 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:22:26.637 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:22:26.638 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:22:26.638 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:22:26.638 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:22:26.638 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:22:26.638 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:22:26.638 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:26.638 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:22:26.638 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:22:26.638 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:22:26.638 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:22:26.638 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:22:26.638 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:22:26.639 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:22:26.647 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:22:26.648 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:22:26.648 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:22:26.648 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:22:26.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:22:26.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:22:26.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:22:26.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:22:26.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:22:26.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:22:26.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:22:26.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:22:26.711 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:22:26.711 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:22:26.711 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:22:26.711 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:22:26.711 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:22:26.712 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:26.712 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:22:26.712 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:26.750 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 91 2026-02-18T08:22:26.750 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 95 2026-02-18T08:22:26.750 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:22:26.750 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=381, tag_tid=27, entry_tid=1], [object_number=380, tag_tid=27, entry_tid=0], [object_number=382, tag_tid=26, entry_tid=46994], [object_number=383, tag_tid=26, entry_tid=46991]]], state=connected] 2026-02-18T08:22:26.750 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=365, tag_tid=26, entry_tid=31889], [object_number=364, tag_tid=26, entry_tid=31888], [object_number=367, tag_tid=26, entry_tid=31887], [object_number=366, tag_tid=26, entry_tid=31886]]], state=connected] 2026-02-18T08:22:26.754 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:26.755 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=381, tag_tid=27, entry_tid=1' 2026-02-18T08:22:26.756 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:22:26.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:22:26.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:22:26.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:22:26.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:22:26.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:22:26.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:22:26.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:22:26.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:22:26.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:22:26.758 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:22:26.758 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:22:26.758 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:26.759 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:22:26.759 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:26.796 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 91 2026-02-18T08:22:26.796 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 95 2026-02-18T08:22:26.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:22:26.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=381, tag_tid=27, entry_tid=1], [object_number=380, tag_tid=27, entry_tid=0], [object_number=382, tag_tid=26, entry_tid=46994], [object_number=383, tag_tid=26, entry_tid=46991]]], state=connected] 2026-02-18T08:22:26.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=365, tag_tid=26, entry_tid=31889], [object_number=364, tag_tid=26, entry_tid=31888], [object_number=367, tag_tid=26, entry_tid=31887], [object_number=366, tag_tid=26, entry_tid=31886]]], state=connected] 2026-02-18T08:22:26.800 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:26.802 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=365, tag_tid=26, entry_tid=31889' 2026-02-18T08:22:26.802 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=381, tag_tid=27, entry_tid=1' -a 'object_number=381, tag_tid=27, entry_tid=1' = 'object_number=365, tag_tid=26, entry_tid=31889' 2026-02-18T08:22:26.802 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=365, tag_tid=26, entry_tid=31889' '!=' 'object_number=365, tag_tid=26, entry_tid=31537' 2026-02-18T08:22:26.802 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:22:26.803 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=365, tag_tid=26, entry_tid=31889' = 'object_number=365, tag_tid=26, entry_tid=31537' 2026-02-18T08:22:26.803 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=365, tag_tid=26, entry_tid=31889' 2026-02-18T08:22:26.803 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=381, tag_tid=27, entry_tid=1' 2026-02-18T08:22:26.804 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:22:26.804 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:22:26.804 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=27 2026-02-18T08:22:26.805 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=365, tag_tid=26, entry_tid=31889' 2026-02-18T08:22:26.805 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:22:26.805 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:22:26.807 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=26 2026-02-18T08:22:26.808 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=381, tag_tid=27, entry_tid=1' 2026-02-18T08:22:26.808 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:22:26.808 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:22:26.809 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:22:26.809 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=365, tag_tid=26, entry_tid=31889' 2026-02-18T08:22:26.810 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:22:26.810 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:22:26.812 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=31889 2026-02-18T08:22:26.812 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 27 = 26 -a 1 -le 31889 2026-02-18T08:22:26.812 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:22:26.812 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:22:26.812 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:22:27.012 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:22:27.012 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:22:27.012 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:22:27.012 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:22:27.012 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:22:27.012 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:22:27.012 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:22:27.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:22:27.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:22:27.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:27.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:22:27.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:22:27.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:22:27.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:22:27.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:22:27.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:22:27.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:22:27.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:22:27.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:22:27.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:22:27.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:27.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:22:27.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:22:27.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:22:27.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:22:27.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:22:27.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:22:27.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:22:27.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:22:27.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:22:27.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:27.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:22:27.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:22:27.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:22:27.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:22:27.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:22:27.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:22:27.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:22:27.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:22:27.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:22:27.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:22:27.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:27.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:22:27.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:22:27.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:22:27.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:22:27.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:22:27.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:22:27.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:22:27.028 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:22:27.029 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:22:27.029 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:22:27.029 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:22:27.086 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:22:27.086 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:22:27.086 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:22:27.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:22:27.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:22:27.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:22:27.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:27.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:22:27.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:22:27.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:22:27.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:22:27.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:22:27.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:22:27.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:22:27.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:22:27.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:22:27.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:27.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:22:27.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:22:27.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:22:27.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:22:27.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:22:27.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:22:27.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:22:27.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:22:27.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:22:27.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:22:27.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:27.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:22:27.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:22:27.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:22:27.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:22:27.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:22:27.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:22:27.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:22:27.100 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:22:27.100 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:22:27.101 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:22:27.101 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:22:27.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:22:27.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:22:27.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:22:27.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:22:27.155 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:22:27.155 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:22:27.155 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:27.155 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:22:27.155 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:22:27.155 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:22:27.155 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:22:27.155 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:22:27.155 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:22:27.155 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:22:27.156 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:22:27.156 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:22:27.156 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:27.156 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:22:27.156 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:22:27.156 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:22:27.156 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:22:27.156 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:22:27.156 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:22:27.156 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:22:27.156 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:22:27.156 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:22:27.157 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:22:27.157 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:27.157 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:22:27.157 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:22:27.157 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:22:27.157 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:22:27.157 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:22:27.157 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:22:27.157 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:22:27.167 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:22:27.167 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:22:27.167 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:22:27.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:22:27.224 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:22:27.224 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:22:27.224 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:22:27.227 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:22:27.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:22:27.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:22:27.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:27.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:22:27.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:22:27.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:22:27.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:22:27.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:22:27.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:22:27.229 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:22:27.229 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:22:27.229 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:22:27.229 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:27.229 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:22:27.229 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:22:27.229 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:22:27.229 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:22:27.229 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:22:27.229 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:22:27.229 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:22:27.229 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:22:27.230 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:22:27.230 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:22:27.230 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:27.230 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:22:27.230 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:22:27.230 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:22:27.230 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:22:27.230 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:22:27.230 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:22:27.230 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:22:27.238 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:22:27.238 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:22:27.238 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:22:27.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:22:27.298 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:22:27.299 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:22:27.299 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:22:27.300 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:22:27.300 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:22:27.300 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:22:27.300 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:22:27.300 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:22:27.300 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:22:27.300 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:22:27.300 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:22:27.300 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:22:27.300 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:22:27.301 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:27.301 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:22:27.302 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:27.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 91 2026-02-18T08:22:27.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 95 2026-02-18T08:22:27.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:22:27.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=381, tag_tid=27, entry_tid=1], [object_number=380, tag_tid=27, entry_tid=0], [object_number=382, tag_tid=26, entry_tid=46994], [object_number=383, tag_tid=26, entry_tid=46991]]], state=connected] 2026-02-18T08:22:27.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=365, tag_tid=26, entry_tid=32277], [object_number=364, tag_tid=26, entry_tid=32276], [object_number=367, tag_tid=26, entry_tid=32275], [object_number=366, tag_tid=26, entry_tid=32274]]], state=connected] 2026-02-18T08:22:27.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:27.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=381, tag_tid=27, entry_tid=1' 2026-02-18T08:22:27.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:22:27.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:22:27.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:22:27.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:22:27.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:22:27.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:22:27.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:22:27.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:22:27.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:22:27.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:22:27.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:22:27.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:22:27.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:27.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:22:27.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:27.376 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 91 2026-02-18T08:22:27.376 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 95 2026-02-18T08:22:27.377 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:22:27.377 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=381, tag_tid=27, entry_tid=1], [object_number=380, tag_tid=27, entry_tid=0], [object_number=382, tag_tid=26, entry_tid=46994], [object_number=383, tag_tid=26, entry_tid=46991]]], state=connected] 2026-02-18T08:22:27.377 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=365, tag_tid=26, entry_tid=32277], [object_number=364, tag_tid=26, entry_tid=32276], [object_number=367, tag_tid=26, entry_tid=32275], [object_number=366, tag_tid=26, entry_tid=32274]]], state=connected] 2026-02-18T08:22:27.380 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:27.382 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=365, tag_tid=26, entry_tid=32277' 2026-02-18T08:22:27.382 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=381, tag_tid=27, entry_tid=1' -a 'object_number=381, tag_tid=27, entry_tid=1' = 'object_number=365, tag_tid=26, entry_tid=32277' 2026-02-18T08:22:27.382 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=365, tag_tid=26, entry_tid=32277' '!=' 'object_number=365, tag_tid=26, entry_tid=31889' 2026-02-18T08:22:27.382 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:22:27.382 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=365, tag_tid=26, entry_tid=32277' = 'object_number=365, tag_tid=26, entry_tid=31889' 2026-02-18T08:22:27.382 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=365, tag_tid=26, entry_tid=32277' 2026-02-18T08:22:27.383 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=381, tag_tid=27, entry_tid=1' 2026-02-18T08:22:27.383 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:22:27.383 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:22:27.384 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=27 2026-02-18T08:22:27.385 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=365, tag_tid=26, entry_tid=32277' 2026-02-18T08:22:27.385 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:22:27.385 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:22:27.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=26 2026-02-18T08:22:27.387 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=381, tag_tid=27, entry_tid=1' 2026-02-18T08:22:27.387 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:22:27.387 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:22:27.388 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:22:27.389 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=365, tag_tid=26, entry_tid=32277' 2026-02-18T08:22:27.389 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:22:27.389 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:22:27.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=32277 2026-02-18T08:22:27.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 27 = 26 -a 1 -le 32277 2026-02-18T08:22:27.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:22:27.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:22:27.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:22:27.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:22:27.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:22:27.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:22:27.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:22:27.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:22:27.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:22:27.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:22:27.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:22:27.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:22:27.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:27.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:22:27.594 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:22:27.594 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:22:27.594 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:22:27.594 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:22:27.594 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:22:27.597 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:22:27.597 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:22:27.597 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:22:27.597 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:22:27.597 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:27.597 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:22:27.597 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:22:27.597 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:22:27.597 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:22:27.597 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:22:27.598 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:22:27.598 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:22:27.598 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:22:27.598 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:22:27.598 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:27.598 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:22:27.598 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:22:27.598 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:22:27.598 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:22:27.598 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:22:27.598 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:22:27.598 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:22:27.599 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:22:27.599 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:22:27.599 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:22:27.599 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:27.599 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:22:27.599 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:22:27.599 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:22:27.599 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:22:27.599 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:22:27.599 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:22:27.599 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:22:27.607 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:22:27.607 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:22:27.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:22:27.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:22:27.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:22:27.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:22:27.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:22:27.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:22:27.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:22:27.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:22:27.670 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:27.670 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:22:27.670 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:22:27.670 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:22:27.670 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:22:27.670 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:22:27.670 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:22:27.670 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:22:27.670 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:22:27.670 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:22:27.670 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:27.670 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:22:27.671 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:22:27.671 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:22:27.671 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:22:27.671 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:22:27.671 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:22:27.671 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:22:27.671 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:22:27.671 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:22:27.671 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:22:27.671 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:27.671 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:22:27.671 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:22:27.671 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:22:27.672 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:22:27.672 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:22:27.672 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:22:27.672 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:22:27.680 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:22:27.681 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:22:27.681 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:22:27.681 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:22:27.740 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:22:27.740 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:22:27.740 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:22:27.743 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:22:27.743 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:22:27.743 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:22:27.743 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:27.743 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:22:27.744 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:22:27.744 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:22:27.744 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:22:27.744 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:22:27.744 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:22:27.744 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:22:27.744 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:22:27.744 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:22:27.744 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:27.744 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:22:27.744 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:22:27.745 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:22:27.745 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:22:27.745 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:22:27.745 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:22:27.745 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:22:27.745 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:22:27.745 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:22:27.745 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:22:27.745 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:27.745 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:22:27.745 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:22:27.745 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:22:27.746 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:22:27.746 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:22:27.746 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:22:27.746 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:22:27.753 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:22:27.754 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:22:27.754 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:22:27.754 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:22:27.804 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:22:27.804 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:22:27.805 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:22:27.807 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:22:27.808 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:22:27.808 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:22:27.808 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:27.808 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:22:27.808 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:22:27.808 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:22:27.808 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:22:27.808 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:22:27.808 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:22:27.808 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:22:27.809 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:22:27.809 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:22:27.809 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:27.809 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:22:27.809 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:22:27.809 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:22:27.809 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:22:27.809 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:22:27.809 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:22:27.809 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:22:27.810 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:22:27.810 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:22:27.810 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:22:27.810 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:27.810 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:22:27.810 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:22:27.810 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:22:27.810 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:22:27.810 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:22:27.810 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:22:27.810 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:22:27.818 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:22:27.818 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:22:27.818 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:22:27.818 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:22:27.879 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:22:27.881 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:22:27.881 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:22:27.881 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:22:27.881 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:22:27.881 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:22:27.881 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:22:27.881 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:22:27.881 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:22:27.881 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:22:27.881 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:22:27.882 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:22:27.882 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:22:27.882 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:27.882 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:22:27.883 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:27.910 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 91 2026-02-18T08:22:27.911 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 95 2026-02-18T08:22:27.911 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:22:27.911 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=381, tag_tid=27, entry_tid=1], [object_number=380, tag_tid=27, entry_tid=0], [object_number=382, tag_tid=26, entry_tid=46994], [object_number=383, tag_tid=26, entry_tid=46991]]], state=connected] 2026-02-18T08:22:27.911 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=364, tag_tid=26, entry_tid=32620], [object_number=367, tag_tid=26, entry_tid=32619], [object_number=366, tag_tid=26, entry_tid=32618], [object_number=365, tag_tid=26, entry_tid=32617]]], state=connected] 2026-02-18T08:22:27.913 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:27.915 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=381, tag_tid=27, entry_tid=1' 2026-02-18T08:22:27.916 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:22:27.916 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:22:27.916 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:22:27.917 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:22:27.917 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:22:27.917 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:22:27.917 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:22:27.917 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:22:27.917 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:22:27.917 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:22:27.917 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:22:27.917 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:22:27.918 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:27.918 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:22:27.918 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:27.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 91 2026-02-18T08:22:27.952 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 95 2026-02-18T08:22:27.952 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:22:27.952 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=381, tag_tid=27, entry_tid=1], [object_number=380, tag_tid=27, entry_tid=0], [object_number=382, tag_tid=26, entry_tid=46994], [object_number=383, tag_tid=26, entry_tid=46991]]], state=connected] 2026-02-18T08:22:27.952 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=364, tag_tid=26, entry_tid=32620], [object_number=367, tag_tid=26, entry_tid=32619], [object_number=366, tag_tid=26, entry_tid=32618], [object_number=365, tag_tid=26, entry_tid=32617]]], state=connected] 2026-02-18T08:22:27.955 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:27.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=364, tag_tid=26, entry_tid=32620' 2026-02-18T08:22:27.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=381, tag_tid=27, entry_tid=1' -a 'object_number=381, tag_tid=27, entry_tid=1' = 'object_number=364, tag_tid=26, entry_tid=32620' 2026-02-18T08:22:27.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=364, tag_tid=26, entry_tid=32620' '!=' 'object_number=365, tag_tid=26, entry_tid=32277' 2026-02-18T08:22:27.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:22:27.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=364, tag_tid=26, entry_tid=32620' = 'object_number=365, tag_tid=26, entry_tid=32277' 2026-02-18T08:22:27.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=364, tag_tid=26, entry_tid=32620' 2026-02-18T08:22:27.958 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=381, tag_tid=27, entry_tid=1' 2026-02-18T08:22:27.958 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:22:27.958 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:22:27.959 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=27 2026-02-18T08:22:27.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=364, tag_tid=26, entry_tid=32620' 2026-02-18T08:22:27.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:22:27.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:22:27.961 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=26 2026-02-18T08:22:27.962 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=381, tag_tid=27, entry_tid=1' 2026-02-18T08:22:27.962 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:22:27.962 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:22:27.964 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:22:27.964 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=364, tag_tid=26, entry_tid=32620' 2026-02-18T08:22:27.964 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:22:27.965 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:22:27.966 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=32620 2026-02-18T08:22:27.966 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 27 = 26 -a 1 -le 32620 2026-02-18T08:22:27.966 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:22:27.967 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:22:27.967 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:22:28.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:22:28.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:22:28.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:22:28.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:22:28.169 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:22:28.169 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:22:28.169 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:22:28.169 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:22:28.169 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:22:28.169 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:28.169 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:22:28.169 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:22:28.169 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:22:28.170 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:22:28.170 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:22:28.173 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:22:28.173 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:22:28.173 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:22:28.173 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:22:28.173 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:22:28.173 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:28.173 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:22:28.173 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:22:28.173 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:22:28.173 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:22:28.174 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:22:28.174 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:22:28.174 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:22:28.174 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:22:28.174 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:22:28.174 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:28.174 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:22:28.174 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:22:28.174 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:22:28.174 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:22:28.174 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:22:28.175 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:22:28.175 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:22:28.175 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:22:28.175 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:22:28.175 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:22:28.175 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:28.175 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:22:28.175 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:22:28.175 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:22:28.175 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:22:28.175 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:22:28.176 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:22:28.176 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:22:28.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:22:28.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:22:28.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:22:28.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:22:28.234 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:22:28.234 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:22:28.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:22:28.238 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:22:28.238 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:22:28.238 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:22:28.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:28.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:22:28.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:22:28.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:22:28.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:22:28.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:22:28.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:22:28.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:22:28.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:22:28.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:22:28.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:28.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:22:28.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:22:28.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:22:28.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:22:28.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:22:28.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:22:28.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:22:28.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:22:28.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:22:28.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:22:28.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:28.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:22:28.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:22:28.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:22:28.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:22:28.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:22:28.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:22:28.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:22:28.248 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:22:28.249 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:22:28.249 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:22:28.249 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:22:28.301 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:22:28.301 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:22:28.301 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:22:28.304 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:22:28.304 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:22:28.304 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:22:28.304 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:28.304 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:22:28.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:22:28.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:22:28.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:22:28.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:22:28.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:22:28.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:22:28.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:22:28.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:22:28.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:28.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:22:28.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:22:28.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:22:28.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:22:28.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:22:28.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:22:28.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:22:28.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:22:28.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:22:28.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:22:28.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:28.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:22:28.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:22:28.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:22:28.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:22:28.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:22:28.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:22:28.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:22:28.315 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:22:28.316 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:22:28.316 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:22:28.316 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:22:28.365 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:22:28.365 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:22:28.366 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:22:28.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:22:28.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:22:28.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:22:28.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:28.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:22:28.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:22:28.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:22:28.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:22:28.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:22:28.370 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:22:28.370 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:22:28.370 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:22:28.370 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:22:28.370 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:28.370 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:22:28.370 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:22:28.370 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:22:28.370 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:22:28.370 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:22:28.370 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:22:28.370 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:22:28.371 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:22:28.371 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:22:28.371 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:22:28.371 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:28.371 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:22:28.371 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:22:28.371 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:22:28.371 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:22:28.371 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:22:28.371 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:22:28.371 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:22:28.380 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:22:28.380 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:22:28.380 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:22:28.380 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:22:28.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:22:28.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:22:28.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:22:28.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:22:28.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:22:28.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:22:28.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:22:28.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:22:28.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:22:28.467 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:22:28.467 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:22:28.467 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:22:28.467 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:22:28.467 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:28.468 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:22:28.468 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:28.506 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 92 2026-02-18T08:22:28.506 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 95 2026-02-18T08:22:28.506 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:22:28.506 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=381, tag_tid=27, entry_tid=1], [object_number=380, tag_tid=27, entry_tid=0], [object_number=382, tag_tid=26, entry_tid=46994], [object_number=383, tag_tid=26, entry_tid=46991]]], state=connected] 2026-02-18T08:22:28.506 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=369, tag_tid=26, entry_tid=32893], [object_number=368, tag_tid=26, entry_tid=32892], [object_number=371, tag_tid=26, entry_tid=32891], [object_number=370, tag_tid=26, entry_tid=32890]]], state=connected] 2026-02-18T08:22:28.510 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:28.511 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=381, tag_tid=27, entry_tid=1' 2026-02-18T08:22:28.512 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:22:28.512 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:22:28.512 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:22:28.512 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:22:28.513 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:22:28.513 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:22:28.513 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:22:28.513 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:22:28.513 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:22:28.513 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:22:28.513 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:22:28.514 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:22:28.514 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:28.514 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:22:28.514 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:28.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 92 2026-02-18T08:22:28.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 95 2026-02-18T08:22:28.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:22:28.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=381, tag_tid=27, entry_tid=1], [object_number=380, tag_tid=27, entry_tid=0], [object_number=382, tag_tid=26, entry_tid=46994], [object_number=383, tag_tid=26, entry_tid=46991]]], state=connected] 2026-02-18T08:22:28.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=369, tag_tid=26, entry_tid=32893], [object_number=368, tag_tid=26, entry_tid=32892], [object_number=371, tag_tid=26, entry_tid=32891], [object_number=370, tag_tid=26, entry_tid=32890]]], state=connected] 2026-02-18T08:22:28.552 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:28.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=369, tag_tid=26, entry_tid=32893' 2026-02-18T08:22:28.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=381, tag_tid=27, entry_tid=1' -a 'object_number=381, tag_tid=27, entry_tid=1' = 'object_number=369, tag_tid=26, entry_tid=32893' 2026-02-18T08:22:28.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=369, tag_tid=26, entry_tid=32893' '!=' 'object_number=364, tag_tid=26, entry_tid=32620' 2026-02-18T08:22:28.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:22:28.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=369, tag_tid=26, entry_tid=32893' = 'object_number=364, tag_tid=26, entry_tid=32620' 2026-02-18T08:22:28.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=369, tag_tid=26, entry_tid=32893' 2026-02-18T08:22:28.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=381, tag_tid=27, entry_tid=1' 2026-02-18T08:22:28.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:22:28.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:22:28.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=27 2026-02-18T08:22:28.557 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=369, tag_tid=26, entry_tid=32893' 2026-02-18T08:22:28.557 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:22:28.557 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:22:28.558 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=26 2026-02-18T08:22:28.559 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=381, tag_tid=27, entry_tid=1' 2026-02-18T08:22:28.559 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:22:28.559 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:22:28.560 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:22:28.561 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=369, tag_tid=26, entry_tid=32893' 2026-02-18T08:22:28.561 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:22:28.561 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:22:28.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=32893 2026-02-18T08:22:28.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 27 = 26 -a 1 -le 32893 2026-02-18T08:22:28.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:22:28.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:22:28.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:22:28.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:22:28.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:22:28.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:22:28.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:22:28.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:22:28.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:22:28.766 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:22:28.766 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:22:28.766 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:22:28.766 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:28.766 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:22:28.766 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:22:28.766 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:22:28.766 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:22:28.766 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:22:28.766 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:22:28.769 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:22:28.769 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:22:28.769 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:22:28.769 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:22:28.770 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:28.770 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:22:28.770 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:22:28.770 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:22:28.770 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:22:28.770 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:22:28.770 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:22:28.770 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:22:28.770 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:22:28.770 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:22:28.770 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:28.770 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:22:28.771 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:22:28.771 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:22:28.771 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:22:28.771 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:22:28.771 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:22:28.771 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:22:28.771 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:22:28.771 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:22:28.771 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:22:28.771 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:28.771 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:22:28.771 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:22:28.771 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:22:28.772 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:22:28.772 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:22:28.772 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:22:28.772 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:22:28.778 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:22:28.779 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:22:28.779 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:22:28.779 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:22:28.827 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:22:28.827 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:22:28.827 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:22:28.830 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:22:28.830 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:22:28.830 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:22:28.831 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:28.831 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:22:28.831 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:22:28.831 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:22:28.831 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:22:28.831 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:22:28.831 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:22:28.831 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:22:28.831 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:22:28.831 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:22:28.831 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:28.832 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:22:28.832 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:22:28.832 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:22:28.832 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:22:28.832 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:22:28.832 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:22:28.832 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:22:28.832 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:22:28.832 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:22:28.832 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:22:28.832 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:28.832 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:22:28.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:22:28.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:22:28.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:22:28.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:22:28.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:22:28.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:22:28.841 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:22:28.842 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:22:28.842 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:22:28.842 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:22:28.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:22:28.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:22:28.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:22:28.896 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:22:28.896 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:22:28.897 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:22:28.897 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:28.897 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:22:28.897 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:22:28.897 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:22:28.897 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:22:28.897 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:22:28.897 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:22:28.897 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:22:28.897 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:22:28.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:22:28.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:28.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:22:28.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:22:28.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:22:28.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:22:28.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:22:28.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:22:28.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:22:28.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:22:28.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:22:28.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:22:28.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:28.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:22:28.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:22:28.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:22:28.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:22:28.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:22:28.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:22:28.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:22:28.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:22:28.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:22:28.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:22:28.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:22:28.956 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:22:28.956 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:22:28.956 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:22:28.959 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:22:28.959 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:22:28.959 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:22:28.959 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:28.959 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:22:28.959 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:22:28.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:22:28.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:22:28.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:22:28.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:22:28.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:22:28.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:22:28.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:22:28.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:28.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:22:28.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:22:28.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:22:28.961 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:22:28.961 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:22:28.961 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:22:28.961 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:22:28.961 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:22:28.961 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:22:28.961 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:22:28.961 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:28.961 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:22:28.961 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:22:28.961 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:22:28.961 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:22:28.962 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:22:28.962 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:22:28.962 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:22:28.969 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:22:28.970 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:22:28.970 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:22:28.970 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:22:29.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:22:29.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:22:29.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:22:29.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:22:29.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:22:29.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:22:29.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:22:29.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:22:29.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:22:29.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:22:29.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:22:29.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:22:29.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:22:29.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:29.035 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:22:29.035 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:29.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 92 2026-02-18T08:22:29.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 95 2026-02-18T08:22:29.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:22:29.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=381, tag_tid=27, entry_tid=1], [object_number=380, tag_tid=27, entry_tid=0], [object_number=382, tag_tid=26, entry_tid=46994], [object_number=383, tag_tid=26, entry_tid=46991]]], state=connected] 2026-02-18T08:22:29.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=370, tag_tid=26, entry_tid=33238], [object_number=369, tag_tid=26, entry_tid=33237], [object_number=368, tag_tid=26, entry_tid=33236], [object_number=371, tag_tid=26, entry_tid=33235]]], state=connected] 2026-02-18T08:22:29.075 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:29.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=381, tag_tid=27, entry_tid=1' 2026-02-18T08:22:29.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:22:29.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:22:29.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:22:29.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:22:29.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:22:29.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:22:29.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:22:29.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:22:29.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:22:29.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:22:29.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:22:29.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:22:29.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:29.080 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:22:29.080 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:29.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 92 2026-02-18T08:22:29.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 95 2026-02-18T08:22:29.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:22:29.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=381, tag_tid=27, entry_tid=1], [object_number=380, tag_tid=27, entry_tid=0], [object_number=382, tag_tid=26, entry_tid=46994], [object_number=383, tag_tid=26, entry_tid=46991]]], state=connected] 2026-02-18T08:22:29.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=370, tag_tid=26, entry_tid=33238], [object_number=369, tag_tid=26, entry_tid=33237], [object_number=368, tag_tid=26, entry_tid=33236], [object_number=371, tag_tid=26, entry_tid=33235]]], state=connected] 2026-02-18T08:22:29.119 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:29.121 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=370, tag_tid=26, entry_tid=33238' 2026-02-18T08:22:29.121 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=381, tag_tid=27, entry_tid=1' -a 'object_number=381, tag_tid=27, entry_tid=1' = 'object_number=370, tag_tid=26, entry_tid=33238' 2026-02-18T08:22:29.121 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=370, tag_tid=26, entry_tid=33238' '!=' 'object_number=369, tag_tid=26, entry_tid=32893' 2026-02-18T08:22:29.122 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:22:29.122 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=370, tag_tid=26, entry_tid=33238' = 'object_number=369, tag_tid=26, entry_tid=32893' 2026-02-18T08:22:29.122 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=370, tag_tid=26, entry_tid=33238' 2026-02-18T08:22:29.122 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=381, tag_tid=27, entry_tid=1' 2026-02-18T08:22:29.122 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:22:29.122 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:22:29.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=27 2026-02-18T08:22:29.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=370, tag_tid=26, entry_tid=33238' 2026-02-18T08:22:29.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:22:29.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:22:29.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=26 2026-02-18T08:22:29.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=381, tag_tid=27, entry_tid=1' 2026-02-18T08:22:29.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:22:29.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:22:29.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:22:29.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=370, tag_tid=26, entry_tid=33238' 2026-02-18T08:22:29.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:22:29.129 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:22:29.130 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=33238 2026-02-18T08:22:29.130 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 27 = 26 -a 1 -le 33238 2026-02-18T08:22:29.130 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:22:29.130 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:22:29.131 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:22:29.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:22:29.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:22:29.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:22:29.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:22:29.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:22:29.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:22:29.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:22:29.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:22:29.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:22:29.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:29.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:22:29.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:22:29.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:22:29.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:22:29.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:22:29.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:22:29.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:22:29.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:22:29.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:22:29.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:22:29.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:29.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:22:29.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:22:29.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:22:29.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:22:29.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:22:29.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:22:29.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:22:29.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:22:29.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:22:29.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:29.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:22:29.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:22:29.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:22:29.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:22:29.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:22:29.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:22:29.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:22:29.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:22:29.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:22:29.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:22:29.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:29.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:22:29.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:22:29.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:22:29.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:22:29.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:22:29.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:22:29.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:22:29.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:22:29.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:22:29.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:22:29.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:22:29.397 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:22:29.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:22:29.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:22:29.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:22:29.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:22:29.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:22:29.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:29.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:22:29.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:22:29.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:22:29.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:22:29.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:22:29.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:22:29.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:22:29.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:22:29.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:22:29.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:29.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:22:29.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:22:29.403 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:22:29.403 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:22:29.403 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:22:29.403 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:22:29.403 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:22:29.403 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:22:29.403 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:22:29.403 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:22:29.403 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:29.403 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:22:29.403 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:22:29.403 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:22:29.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:22:29.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:22:29.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:22:29.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:22:29.412 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:22:29.412 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:22:29.412 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:22:29.413 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:22:29.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:22:29.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:22:29.462 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:22:29.465 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:22:29.465 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:22:29.465 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:22:29.465 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:29.465 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:22:29.465 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:22:29.465 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:22:29.465 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:22:29.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:22:29.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:22:29.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:22:29.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:22:29.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:22:29.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:29.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:22:29.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:22:29.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:22:29.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:22:29.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:22:29.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:22:29.467 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:22:29.467 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:22:29.467 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:22:29.467 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:22:29.467 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:29.467 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:22:29.467 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:22:29.467 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:22:29.467 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:22:29.467 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:22:29.467 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:22:29.468 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:22:29.475 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:22:29.476 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:22:29.476 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:22:29.476 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:22:29.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:22:29.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:22:29.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:22:29.536 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:22:29.536 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:22:29.536 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:22:29.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:29.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:22:29.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:22:29.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:22:29.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:22:29.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:22:29.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:22:29.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:22:29.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:22:29.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:22:29.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:29.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:22:29.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:22:29.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:22:29.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:22:29.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:22:29.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:22:29.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:22:29.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:22:29.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:22:29.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:22:29.539 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:29.539 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:22:29.539 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:22:29.539 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:22:29.539 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:22:29.539 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:22:29.539 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:22:29.539 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:22:29.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:22:29.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:22:29.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:22:29.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:22:29.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:22:29.610 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:22:29.610 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:22:29.610 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:22:29.610 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:22:29.611 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:22:29.611 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:22:29.611 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:22:29.611 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:22:29.611 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:22:29.611 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:22:29.611 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:22:29.611 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:22:29.612 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:29.612 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:22:29.612 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:29.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 92 2026-02-18T08:22:29.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 95 2026-02-18T08:22:29.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:22:29.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=381, tag_tid=27, entry_tid=1], [object_number=380, tag_tid=27, entry_tid=0], [object_number=382, tag_tid=26, entry_tid=46994], [object_number=383, tag_tid=26, entry_tid=46991]]], state=connected] 2026-02-18T08:22:29.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=371, tag_tid=26, entry_tid=33567], [object_number=370, tag_tid=26, entry_tid=33566], [object_number=369, tag_tid=26, entry_tid=33565], [object_number=368, tag_tid=26, entry_tid=33564]]], state=connected] 2026-02-18T08:22:29.647 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:29.648 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=381, tag_tid=27, entry_tid=1' 2026-02-18T08:22:29.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:22:29.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:22:29.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:22:29.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:22:29.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:22:29.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:22:29.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:22:29.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:22:29.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:22:29.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:22:29.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:22:29.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:22:29.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:29.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:22:29.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:29.681 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 92 2026-02-18T08:22:29.681 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 95 2026-02-18T08:22:29.682 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:22:29.682 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=381, tag_tid=27, entry_tid=1], [object_number=380, tag_tid=27, entry_tid=0], [object_number=382, tag_tid=26, entry_tid=46994], [object_number=383, tag_tid=26, entry_tid=46991]]], state=connected] 2026-02-18T08:22:29.682 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=371, tag_tid=26, entry_tid=33567], [object_number=370, tag_tid=26, entry_tid=33566], [object_number=369, tag_tid=26, entry_tid=33565], [object_number=368, tag_tid=26, entry_tid=33564]]], state=connected] 2026-02-18T08:22:29.684 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:29.686 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=371, tag_tid=26, entry_tid=33567' 2026-02-18T08:22:29.686 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=381, tag_tid=27, entry_tid=1' -a 'object_number=381, tag_tid=27, entry_tid=1' = 'object_number=371, tag_tid=26, entry_tid=33567' 2026-02-18T08:22:29.686 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=371, tag_tid=26, entry_tid=33567' '!=' 'object_number=370, tag_tid=26, entry_tid=33238' 2026-02-18T08:22:29.686 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:22:29.686 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=371, tag_tid=26, entry_tid=33567' = 'object_number=370, tag_tid=26, entry_tid=33238' 2026-02-18T08:22:29.686 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=371, tag_tid=26, entry_tid=33567' 2026-02-18T08:22:29.687 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=381, tag_tid=27, entry_tid=1' 2026-02-18T08:22:29.687 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:22:29.687 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:22:29.688 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=27 2026-02-18T08:22:29.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=371, tag_tid=26, entry_tid=33567' 2026-02-18T08:22:29.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:22:29.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:22:29.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=26 2026-02-18T08:22:29.691 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=381, tag_tid=27, entry_tid=1' 2026-02-18T08:22:29.691 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:22:29.691 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:22:29.692 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:22:29.693 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=371, tag_tid=26, entry_tid=33567' 2026-02-18T08:22:29.693 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:22:29.694 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:22:29.695 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=33567 2026-02-18T08:22:29.695 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 27 = 26 -a 1 -le 33567 2026-02-18T08:22:29.695 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:22:29.695 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:22:29.695 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:22:29.897 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:22:29.897 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:22:29.897 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:22:29.897 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:22:29.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:22:29.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:22:29.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:22:29.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:22:29.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:22:29.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:29.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:22:29.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:22:29.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:22:29.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:22:29.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:22:29.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:22:29.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:22:29.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:22:29.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:22:29.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:22:29.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:29.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:22:29.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:22:29.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:22:29.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:22:29.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:22:29.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:22:29.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:22:29.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:22:29.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:22:29.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:29.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:22:29.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:22:29.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:22:29.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:22:29.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:22:29.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:22:29.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:22:29.904 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:22:29.904 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:22:29.904 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:22:29.904 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:29.904 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:22:29.904 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:22:29.904 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:22:29.904 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:22:29.904 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:22:29.904 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:22:29.904 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:22:29.911 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:22:29.912 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:22:29.912 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:22:29.912 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:22:29.968 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:22:29.969 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:22:29.969 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:22:29.971 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:22:29.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:22:29.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:22:29.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:29.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:22:29.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:22:29.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:22:29.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:22:29.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:22:29.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:22:29.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:22:29.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:22:29.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:22:29.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:29.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:22:29.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:22:29.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:22:29.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:22:29.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:22:29.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:22:29.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:22:29.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:22:29.974 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:22:29.974 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:22:29.974 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:29.974 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:22:29.974 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:22:29.974 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:22:29.974 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:22:29.974 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:22:29.974 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:22:29.974 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:22:29.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:22:29.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:22:29.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:22:29.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:22:30.040 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:22:30.040 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:22:30.041 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:22:30.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:22:30.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:22:30.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:22:30.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:30.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:22:30.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:22:30.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:22:30.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:22:30.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:22:30.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:22:30.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:22:30.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:22:30.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:22:30.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:30.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:22:30.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:22:30.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:22:30.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:22:30.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:22:30.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:22:30.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:22:30.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:22:30.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:22:30.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:22:30.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:30.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:22:30.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:22:30.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:22:30.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:22:30.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:22:30.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:22:30.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:22:30.055 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:22:30.055 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:22:30.055 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:22:30.056 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:22:30.111 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:22:30.111 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:22:30.111 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:22:30.114 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:22:30.114 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:22:30.114 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:22:30.114 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:30.114 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:22:30.114 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:22:30.114 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:22:30.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:22:30.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:22:30.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:22:30.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:22:30.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:22:30.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:22:30.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:30.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:22:30.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:22:30.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:22:30.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:22:30.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:22:30.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:22:30.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:22:30.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:22:30.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:22:30.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:22:30.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:30.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:22:30.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:22:30.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:22:30.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:22:30.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:22:30.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:22:30.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:22:30.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:22:30.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:22:30.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:22:30.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:22:30.183 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:22:30.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:22:30.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:22:30.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:22:30.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:22:30.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:22:30.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:22:30.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:22:30.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:22:30.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:22:30.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:22:30.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:22:30.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:22:30.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:30.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:22:30.187 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:30.214 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 92 2026-02-18T08:22:30.215 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 95 2026-02-18T08:22:30.215 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:22:30.215 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=381, tag_tid=27, entry_tid=1], [object_number=380, tag_tid=27, entry_tid=0], [object_number=382, tag_tid=26, entry_tid=46994], [object_number=383, tag_tid=26, entry_tid=46991]]], state=connected] 2026-02-18T08:22:30.215 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=369, tag_tid=26, entry_tid=33941], [object_number=368, tag_tid=26, entry_tid=33940], [object_number=371, tag_tid=26, entry_tid=33939], [object_number=370, tag_tid=26, entry_tid=33938]]], state=connected] 2026-02-18T08:22:30.217 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:30.219 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=381, tag_tid=27, entry_tid=1' 2026-02-18T08:22:30.220 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:22:30.220 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:22:30.220 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:22:30.220 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:22:30.221 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:22:30.221 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:22:30.221 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:22:30.221 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:22:30.221 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:22:30.221 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:22:30.222 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:22:30.222 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:22:30.222 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:30.223 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:22:30.223 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:30.250 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 92 2026-02-18T08:22:30.250 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 95 2026-02-18T08:22:30.250 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:22:30.250 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=381, tag_tid=27, entry_tid=1], [object_number=380, tag_tid=27, entry_tid=0], [object_number=382, tag_tid=26, entry_tid=46994], [object_number=383, tag_tid=26, entry_tid=46991]]], state=connected] 2026-02-18T08:22:30.250 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=369, tag_tid=26, entry_tid=33941], [object_number=368, tag_tid=26, entry_tid=33940], [object_number=371, tag_tid=26, entry_tid=33939], [object_number=370, tag_tid=26, entry_tid=33938]]], state=connected] 2026-02-18T08:22:30.252 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:30.254 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=369, tag_tid=26, entry_tid=33941' 2026-02-18T08:22:30.255 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=381, tag_tid=27, entry_tid=1' -a 'object_number=381, tag_tid=27, entry_tid=1' = 'object_number=369, tag_tid=26, entry_tid=33941' 2026-02-18T08:22:30.255 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=369, tag_tid=26, entry_tid=33941' '!=' 'object_number=371, tag_tid=26, entry_tid=33567' 2026-02-18T08:22:30.255 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:22:30.255 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=369, tag_tid=26, entry_tid=33941' = 'object_number=371, tag_tid=26, entry_tid=33567' 2026-02-18T08:22:30.255 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=369, tag_tid=26, entry_tid=33941' 2026-02-18T08:22:30.256 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=381, tag_tid=27, entry_tid=1' 2026-02-18T08:22:30.256 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:22:30.256 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:22:30.257 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=27 2026-02-18T08:22:30.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=369, tag_tid=26, entry_tid=33941' 2026-02-18T08:22:30.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:22:30.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:22:30.259 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=26 2026-02-18T08:22:30.260 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=381, tag_tid=27, entry_tid=1' 2026-02-18T08:22:30.260 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:22:30.260 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:22:30.261 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:22:30.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=369, tag_tid=26, entry_tid=33941' 2026-02-18T08:22:30.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:22:30.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:22:30.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=33941 2026-02-18T08:22:30.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 27 = 26 -a 1 -le 33941 2026-02-18T08:22:30.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:22:30.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:22:30.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:22:30.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:22:30.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:22:30.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:22:30.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:22:30.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:22:30.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:22:30.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:22:30.467 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:22:30.467 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:22:30.467 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:30.467 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:22:30.467 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:22:30.467 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:22:30.467 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:22:30.467 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:22:30.470 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:22:30.470 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:22:30.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:22:30.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:22:30.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:22:30.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:30.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:22:30.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:22:30.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:22:30.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:22:30.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:22:30.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:22:30.472 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:22:30.472 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:22:30.472 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:22:30.472 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:30.472 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:22:30.472 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:22:30.472 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:22:30.472 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:22:30.472 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:22:30.472 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:22:30.472 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:22:30.473 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:22:30.473 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:22:30.473 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:22:30.473 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:30.473 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:22:30.473 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:22:30.473 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:22:30.473 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:22:30.473 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:22:30.473 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:22:30.473 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:22:30.480 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:22:30.480 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:22:30.480 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:22:30.481 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:22:30.539 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:22:30.539 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:22:30.539 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:22:30.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:22:30.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:22:30.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:22:30.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:30.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:22:30.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:22:30.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:22:30.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:22:30.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:22:30.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:22:30.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:22:30.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:22:30.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:22:30.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:30.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:22:30.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:22:30.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:22:30.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:22:30.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:22:30.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:22:30.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:22:30.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:22:30.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:22:30.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:22:30.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:30.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:22:30.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:22:30.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:22:30.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:22:30.545 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:22:30.545 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:22:30.545 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:22:30.552 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:22:30.552 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:22:30.553 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:22:30.553 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:22:30.603 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:22:30.604 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:22:30.604 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:22:30.607 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:22:30.607 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:22:30.607 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:22:30.607 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:30.607 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:22:30.607 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:22:30.607 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:22:30.607 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:22:30.607 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:22:30.607 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:22:30.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:22:30.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:22:30.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:22:30.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:30.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:22:30.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:22:30.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:22:30.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:22:30.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:22:30.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:22:30.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:22:30.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:22:30.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:22:30.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:22:30.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:30.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:22:30.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:22:30.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:22:30.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:22:30.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:22:30.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:22:30.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:22:30.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:22:30.618 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:22:30.618 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:22:30.618 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:22:30.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:22:30.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:22:30.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:22:30.673 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:22:30.673 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:22:30.673 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:22:30.673 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:30.673 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:22:30.673 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:22:30.673 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:22:30.673 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:22:30.674 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:22:30.674 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:22:30.674 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:22:30.674 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:22:30.674 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:22:30.674 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:30.674 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:22:30.674 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:22:30.674 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:22:30.674 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:22:30.674 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:22:30.674 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:22:30.675 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:22:30.675 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:22:30.675 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:22:30.675 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:22:30.675 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:30.675 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:22:30.675 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:22:30.675 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:22:30.675 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:22:30.675 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:22:30.675 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:22:30.676 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:22:30.682 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:22:30.683 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:22:30.683 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:22:30.683 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:22:30.745 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:22:30.746 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:22:30.746 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:22:30.746 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:22:30.746 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:22:30.746 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:22:30.746 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:22:30.747 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:22:30.747 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:22:30.747 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:22:30.747 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:22:30.747 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:22:30.747 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:22:30.748 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:30.748 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:22:30.749 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:30.785 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 92 2026-02-18T08:22:30.785 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 95 2026-02-18T08:22:30.785 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:22:30.786 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=381, tag_tid=27, entry_tid=1], [object_number=380, tag_tid=27, entry_tid=0], [object_number=382, tag_tid=26, entry_tid=46994], [object_number=383, tag_tid=26, entry_tid=46991]]], state=connected] 2026-02-18T08:22:30.786 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=371, tag_tid=26, entry_tid=34303], [object_number=370, tag_tid=26, entry_tid=34302], [object_number=369, tag_tid=26, entry_tid=34301], [object_number=368, tag_tid=26, entry_tid=34300]]], state=connected] 2026-02-18T08:22:30.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:30.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=381, tag_tid=27, entry_tid=1' 2026-02-18T08:22:30.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:22:30.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:22:30.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:22:30.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:22:30.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:22:30.793 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:22:30.793 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:22:30.793 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:22:30.793 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:22:30.793 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:22:30.793 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:22:30.794 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:22:30.794 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:30.794 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:22:30.795 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:30.829 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 92 2026-02-18T08:22:30.829 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 95 2026-02-18T08:22:30.829 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:22:30.830 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=381, tag_tid=27, entry_tid=1], [object_number=380, tag_tid=27, entry_tid=0], [object_number=382, tag_tid=26, entry_tid=46994], [object_number=383, tag_tid=26, entry_tid=46991]]], state=connected] 2026-02-18T08:22:30.830 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=371, tag_tid=26, entry_tid=34303], [object_number=370, tag_tid=26, entry_tid=34302], [object_number=369, tag_tid=26, entry_tid=34301], [object_number=368, tag_tid=26, entry_tid=34300]]], state=connected] 2026-02-18T08:22:30.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:30.835 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=371, tag_tid=26, entry_tid=34303' 2026-02-18T08:22:30.835 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=381, tag_tid=27, entry_tid=1' -a 'object_number=381, tag_tid=27, entry_tid=1' = 'object_number=371, tag_tid=26, entry_tid=34303' 2026-02-18T08:22:30.835 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=371, tag_tid=26, entry_tid=34303' '!=' 'object_number=369, tag_tid=26, entry_tid=33941' 2026-02-18T08:22:30.835 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:22:30.835 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=371, tag_tid=26, entry_tid=34303' = 'object_number=369, tag_tid=26, entry_tid=33941' 2026-02-18T08:22:30.835 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=371, tag_tid=26, entry_tid=34303' 2026-02-18T08:22:30.836 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=381, tag_tid=27, entry_tid=1' 2026-02-18T08:22:30.836 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:22:30.836 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:22:30.837 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=27 2026-02-18T08:22:30.838 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=371, tag_tid=26, entry_tid=34303' 2026-02-18T08:22:30.838 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:22:30.838 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:22:30.839 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=26 2026-02-18T08:22:30.840 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=381, tag_tid=27, entry_tid=1' 2026-02-18T08:22:30.840 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:22:30.840 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:22:30.841 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:22:30.842 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=371, tag_tid=26, entry_tid=34303' 2026-02-18T08:22:30.842 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:22:30.842 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:22:30.844 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=34303 2026-02-18T08:22:30.844 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 27 = 26 -a 1 -le 34303 2026-02-18T08:22:30.844 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:22:30.844 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:22:30.844 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:22:31.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:22:31.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:22:31.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:22:31.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:22:31.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:22:31.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:22:31.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:22:31.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:22:31.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:22:31.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:31.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:22:31.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:22:31.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:22:31.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:22:31.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:22:31.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:22:31.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:22:31.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:22:31.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:22:31.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:22:31.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:31.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:22:31.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:22:31.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:22:31.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:22:31.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:22:31.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:22:31.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:22:31.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:22:31.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:22:31.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:31.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:22:31.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:22:31.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:22:31.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:22:31.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:22:31.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:22:31.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:22:31.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:22:31.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:22:31.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:22:31.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:31.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:22:31.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:22:31.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:22:31.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:22:31.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:22:31.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:22:31.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:22:31.059 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:22:31.060 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:22:31.060 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:22:31.060 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:22:31.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:22:31.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:22:31.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:22:31.119 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:22:31.119 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:22:31.119 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:22:31.119 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:31.119 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:22:31.119 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:22:31.120 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:22:31.120 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:22:31.120 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:22:31.120 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:22:31.120 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:22:31.120 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:22:31.120 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:22:31.120 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:31.120 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:22:31.120 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:22:31.120 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:22:31.120 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:22:31.121 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:22:31.121 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:22:31.121 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:22:31.121 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:22:31.121 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:22:31.121 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:22:31.121 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:31.121 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:22:31.121 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:22:31.121 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:22:31.121 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:22:31.122 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:22:31.122 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:22:31.122 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:22:31.130 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:22:31.131 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:22:31.131 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:22:31.131 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:22:31.180 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:22:31.180 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:22:31.180 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:22:31.183 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:22:31.183 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:22:31.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:22:31.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:31.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:22:31.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:22:31.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:22:31.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:22:31.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:22:31.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:22:31.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:22:31.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:22:31.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:22:31.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:31.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:22:31.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:22:31.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:22:31.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:22:31.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:22:31.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:22:31.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:22:31.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:22:31.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:22:31.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:22:31.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:31.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:22:31.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:22:31.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:22:31.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:22:31.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:22:31.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:22:31.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:22:31.194 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:22:31.194 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:22:31.194 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:22:31.195 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:22:31.245 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:22:31.245 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:22:31.245 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:22:31.249 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:22:31.249 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:22:31.249 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:22:31.249 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:31.249 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:22:31.249 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:22:31.249 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:22:31.249 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:22:31.249 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:22:31.249 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:22:31.250 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:22:31.250 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:22:31.250 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:22:31.250 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:31.250 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:22:31.250 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:22:31.250 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:22:31.250 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:22:31.250 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:22:31.250 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:22:31.250 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:22:31.251 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:22:31.251 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:22:31.251 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:22:31.251 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:31.251 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:22:31.251 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:22:31.251 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:22:31.251 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:22:31.251 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:22:31.251 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:22:31.251 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:22:31.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:22:31.259 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:22:31.259 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:22:31.259 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:22:31.319 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:22:31.321 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:22:31.321 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:22:31.321 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:22:31.321 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:22:31.321 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:22:31.321 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:22:31.321 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:22:31.322 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:22:31.322 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:22:31.322 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:22:31.322 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:22:31.322 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:22:31.323 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:31.323 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:22:31.323 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:31.351 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 92 2026-02-18T08:22:31.352 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 95 2026-02-18T08:22:31.352 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:22:31.352 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=381, tag_tid=27, entry_tid=1], [object_number=380, tag_tid=27, entry_tid=0], [object_number=382, tag_tid=26, entry_tid=46994], [object_number=383, tag_tid=26, entry_tid=46991]]], state=connected] 2026-02-18T08:22:31.352 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=368, tag_tid=26, entry_tid=34680], [object_number=371, tag_tid=26, entry_tid=34679], [object_number=370, tag_tid=26, entry_tid=34678], [object_number=369, tag_tid=26, entry_tid=34677]]], state=connected] 2026-02-18T08:22:31.354 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:31.355 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=381, tag_tid=27, entry_tid=1' 2026-02-18T08:22:31.356 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:22:31.356 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:22:31.357 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:22:31.357 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:22:31.357 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:22:31.357 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:22:31.357 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:22:31.357 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:22:31.357 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:22:31.357 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:22:31.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:22:31.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:22:31.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:31.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:22:31.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:31.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 92 2026-02-18T08:22:31.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 95 2026-02-18T08:22:31.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:22:31.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=381, tag_tid=27, entry_tid=1], [object_number=380, tag_tid=27, entry_tid=0], [object_number=382, tag_tid=26, entry_tid=46994], [object_number=383, tag_tid=26, entry_tid=46991]]], state=connected] 2026-02-18T08:22:31.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=368, tag_tid=26, entry_tid=34680], [object_number=371, tag_tid=26, entry_tid=34679], [object_number=370, tag_tid=26, entry_tid=34678], [object_number=369, tag_tid=26, entry_tid=34677]]], state=connected] 2026-02-18T08:22:31.389 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:31.390 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=368, tag_tid=26, entry_tid=34680' 2026-02-18T08:22:31.390 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=381, tag_tid=27, entry_tid=1' -a 'object_number=381, tag_tid=27, entry_tid=1' = 'object_number=368, tag_tid=26, entry_tid=34680' 2026-02-18T08:22:31.390 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=368, tag_tid=26, entry_tid=34680' '!=' 'object_number=371, tag_tid=26, entry_tid=34303' 2026-02-18T08:22:31.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:22:31.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=368, tag_tid=26, entry_tid=34680' = 'object_number=371, tag_tid=26, entry_tid=34303' 2026-02-18T08:22:31.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=368, tag_tid=26, entry_tid=34680' 2026-02-18T08:22:31.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=381, tag_tid=27, entry_tid=1' 2026-02-18T08:22:31.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:22:31.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:22:31.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=27 2026-02-18T08:22:31.393 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=368, tag_tid=26, entry_tid=34680' 2026-02-18T08:22:31.393 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:22:31.393 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:22:31.395 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=26 2026-02-18T08:22:31.395 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=381, tag_tid=27, entry_tid=1' 2026-02-18T08:22:31.396 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:22:31.396 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:22:31.397 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:22:31.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=368, tag_tid=26, entry_tid=34680' 2026-02-18T08:22:31.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:22:31.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:22:31.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=34680 2026-02-18T08:22:31.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 27 = 26 -a 1 -le 34680 2026-02-18T08:22:31.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:22:31.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:22:31.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:22:31.602 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:22:31.602 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:22:31.602 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:22:31.602 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:22:31.602 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:22:31.602 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:22:31.602 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:22:31.602 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:22:31.603 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:22:31.603 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:31.603 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:22:31.603 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:22:31.603 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:22:31.603 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:22:31.603 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:22:31.606 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:22:31.606 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:22:31.606 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:22:31.606 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:22:31.606 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:22:31.606 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:31.606 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:22:31.606 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:22:31.606 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:22:31.607 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:22:31.607 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:22:31.607 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:22:31.607 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:22:31.607 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:22:31.607 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:22:31.607 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:31.607 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:22:31.607 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:22:31.607 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:22:31.607 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:22:31.607 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:22:31.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:22:31.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:22:31.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:22:31.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:22:31.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:22:31.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:31.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:22:31.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:22:31.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:22:31.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:22:31.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:22:31.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:22:31.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:22:31.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:22:31.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:22:31.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:22:31.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:22:31.664 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:22:31.664 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:22:31.664 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:22:31.667 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:22:31.667 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:22:31.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:22:31.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:31.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:22:31.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:22:31.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:22:31.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:22:31.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:22:31.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:22:31.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:22:31.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:22:31.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:22:31.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:31.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:22:31.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:22:31.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:22:31.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:22:31.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:22:31.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:22:31.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:22:31.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:22:31.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:22:31.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:22:31.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:31.670 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:22:31.670 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:22:31.670 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:22:31.670 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:22:31.670 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:22:31.670 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:22:31.670 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:22:31.678 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:22:31.679 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:22:31.679 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:22:31.679 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:22:31.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:22:31.728 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:22:31.728 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:22:31.731 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:22:31.731 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:22:31.731 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:22:31.731 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:31.731 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:22:31.731 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:22:31.731 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:22:31.731 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:22:31.731 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:22:31.732 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:22:31.732 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:22:31.732 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:22:31.732 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:22:31.732 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:31.732 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:22:31.732 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:22:31.732 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:22:31.732 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:22:31.732 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:22:31.732 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:22:31.733 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:22:31.733 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:22:31.733 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:22:31.733 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:22:31.733 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:31.733 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:22:31.733 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:22:31.733 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:22:31.733 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:22:31.733 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:22:31.733 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:22:31.734 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:22:31.742 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:22:31.742 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:22:31.742 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:22:31.743 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:22:31.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:22:31.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:22:31.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:22:31.795 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:22:31.795 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:22:31.795 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:22:31.795 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:31.795 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:22:31.795 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:22:31.796 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:22:31.796 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:22:31.796 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:22:31.796 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:22:31.796 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:22:31.796 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:22:31.796 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:22:31.796 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:31.796 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:22:31.796 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:22:31.796 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:22:31.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:22:31.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:22:31.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:22:31.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:22:31.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:22:31.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:22:31.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:22:31.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:31.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:22:31.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:22:31.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:22:31.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:22:31.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:22:31.798 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:22:31.798 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:22:31.806 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:22:31.806 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:22:31.807 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:22:31.807 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:22:31.877 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:22:31.878 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:22:31.878 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:22:31.879 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:22:31.879 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:22:31.879 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:22:31.879 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:22:31.879 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:22:31.879 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:22:31.879 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:22:31.879 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:22:31.879 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:22:31.880 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:22:31.880 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:31.881 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:22:31.881 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:31.918 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 92 2026-02-18T08:22:31.918 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 95 2026-02-18T08:22:31.918 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:22:31.918 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=381, tag_tid=27, entry_tid=1], [object_number=380, tag_tid=27, entry_tid=0], [object_number=382, tag_tid=26, entry_tid=46994], [object_number=383, tag_tid=26, entry_tid=46991]]], state=connected] 2026-02-18T08:22:31.918 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=369, tag_tid=26, entry_tid=35065], [object_number=368, tag_tid=26, entry_tid=35064], [object_number=371, tag_tid=26, entry_tid=35063], [object_number=370, tag_tid=26, entry_tid=35062]]], state=connected] 2026-02-18T08:22:31.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:31.923 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=381, tag_tid=27, entry_tid=1' 2026-02-18T08:22:31.924 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:22:31.924 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:22:31.924 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:22:31.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:22:31.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:22:31.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:22:31.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:22:31.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:22:31.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:22:31.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:22:31.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:22:31.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:22:31.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:31.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:22:31.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:31.956 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 92 2026-02-18T08:22:31.956 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 95 2026-02-18T08:22:31.956 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:22:31.956 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=381, tag_tid=27, entry_tid=1], [object_number=380, tag_tid=27, entry_tid=0], [object_number=382, tag_tid=26, entry_tid=46994], [object_number=383, tag_tid=26, entry_tid=46991]]], state=connected] 2026-02-18T08:22:31.956 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=369, tag_tid=26, entry_tid=35065], [object_number=368, tag_tid=26, entry_tid=35064], [object_number=371, tag_tid=26, entry_tid=35063], [object_number=370, tag_tid=26, entry_tid=35062]]], state=connected] 2026-02-18T08:22:31.958 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:31.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=369, tag_tid=26, entry_tid=35065' 2026-02-18T08:22:31.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=381, tag_tid=27, entry_tid=1' -a 'object_number=381, tag_tid=27, entry_tid=1' = 'object_number=369, tag_tid=26, entry_tid=35065' 2026-02-18T08:22:31.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=369, tag_tid=26, entry_tid=35065' '!=' 'object_number=368, tag_tid=26, entry_tid=34680' 2026-02-18T08:22:31.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:22:31.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=369, tag_tid=26, entry_tid=35065' = 'object_number=368, tag_tid=26, entry_tid=34680' 2026-02-18T08:22:31.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=369, tag_tid=26, entry_tid=35065' 2026-02-18T08:22:31.961 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=381, tag_tid=27, entry_tid=1' 2026-02-18T08:22:31.961 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:22:31.961 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:22:31.962 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=27 2026-02-18T08:22:31.962 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:22:31.962 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=369, tag_tid=26, entry_tid=35065' 2026-02-18T08:22:31.962 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:22:31.963 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=26 2026-02-18T08:22:31.964 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=381, tag_tid=27, entry_tid=1' 2026-02-18T08:22:31.964 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:22:31.964 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:22:31.965 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:22:31.966 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=369, tag_tid=26, entry_tid=35065' 2026-02-18T08:22:31.966 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:22:31.966 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:22:31.968 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=35065 2026-02-18T08:22:31.968 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 27 = 26 -a 1 -le 35065 2026-02-18T08:22:31.968 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:22:31.968 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:22:31.968 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:22:32.170 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:22:32.170 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:22:32.170 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:22:32.170 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:22:32.170 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:22:32.170 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:22:32.170 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:22:32.170 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:22:32.171 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:22:32.171 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:32.171 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:22:32.171 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:22:32.171 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:22:32.171 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:22:32.171 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:22:32.174 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:22:32.174 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:22:32.174 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:22:32.174 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:22:32.174 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:22:32.175 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:32.175 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:22:32.175 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:22:32.175 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:22:32.175 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:22:32.175 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:22:32.175 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:22:32.175 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:22:32.175 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:22:32.175 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:22:32.175 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:32.176 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:22:32.176 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:22:32.176 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:22:32.176 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:22:32.176 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:22:32.176 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:22:32.176 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:22:32.176 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:22:32.176 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:22:32.176 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:22:32.176 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:32.176 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:22:32.177 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:22:32.177 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:22:32.177 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:22:32.177 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:22:32.177 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:22:32.177 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:22:32.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:22:32.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:22:32.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:22:32.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:22:32.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:22:32.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:22:32.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:22:32.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:22:32.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:22:32.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:22:32.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:32.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:22:32.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:22:32.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:22:32.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:22:32.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:22:32.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:22:32.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:22:32.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:22:32.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:22:32.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:32.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:22:32.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:22:32.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:22:32.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:22:32.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:22:32.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:22:32.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:22:32.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:22:32.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:22:32.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:22:32.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:32.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:22:32.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:22:32.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:22:32.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:22:32.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:22:32.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:22:32.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:22:32.249 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:22:32.250 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:22:32.250 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:22:32.250 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:22:32.298 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:22:32.298 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:22:32.298 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:22:32.301 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:22:32.301 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:22:32.302 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:22:32.302 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:32.302 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:22:32.302 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:22:32.302 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:22:32.302 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:22:32.302 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:22:32.302 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:22:32.302 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:22:32.302 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:22:32.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:22:32.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:32.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:22:32.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:22:32.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:22:32.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:22:32.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:22:32.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:22:32.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:22:32.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:22:32.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:22:32.304 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:22:32.304 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:32.304 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:22:32.304 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:22:32.304 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:22:32.304 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:22:32.304 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:22:32.304 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:22:32.304 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:22:32.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:22:32.312 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:22:32.312 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:22:32.312 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:22:32.362 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:22:32.362 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:22:32.362 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:22:32.365 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:22:32.365 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:22:32.365 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:22:32.366 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:32.366 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:22:32.366 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:22:32.366 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:22:32.366 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:22:32.366 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:22:32.366 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:22:32.366 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:22:32.366 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:22:32.366 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:22:32.366 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:32.367 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:22:32.367 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:22:32.367 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:22:32.367 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:22:32.367 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:22:32.367 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:22:32.367 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:22:32.367 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:22:32.367 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:22:32.367 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:22:32.367 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:32.367 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:22:32.367 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:22:32.368 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:22:32.368 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:22:32.368 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:22:32.368 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:22:32.368 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:22:32.375 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:22:32.375 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:22:32.376 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:22:32.376 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:22:32.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:22:32.440 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:22:32.440 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:22:32.440 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:22:32.440 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:22:32.440 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:22:32.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:22:32.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:22:32.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:22:32.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:22:32.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:22:32.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:22:32.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:22:32.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:32.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:22:32.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:32.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 92 2026-02-18T08:22:32.472 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 95 2026-02-18T08:22:32.472 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:22:32.472 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=381, tag_tid=27, entry_tid=1], [object_number=380, tag_tid=27, entry_tid=0], [object_number=382, tag_tid=26, entry_tid=46994], [object_number=383, tag_tid=26, entry_tid=46991]]], state=connected] 2026-02-18T08:22:32.472 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=370, tag_tid=26, entry_tid=35414], [object_number=369, tag_tid=26, entry_tid=35413], [object_number=368, tag_tid=26, entry_tid=35412], [object_number=371, tag_tid=26, entry_tid=35411]]], state=connected] 2026-02-18T08:22:32.474 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:32.476 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=381, tag_tid=27, entry_tid=1' 2026-02-18T08:22:32.477 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:22:32.477 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:22:32.477 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:22:32.477 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:22:32.477 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:22:32.478 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:22:32.478 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:22:32.478 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:22:32.478 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:22:32.478 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:22:32.478 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:22:32.478 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:22:32.479 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:32.479 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:22:32.479 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:32.506 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 92 2026-02-18T08:22:32.506 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 95 2026-02-18T08:22:32.506 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:22:32.506 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=381, tag_tid=27, entry_tid=1], [object_number=380, tag_tid=27, entry_tid=0], [object_number=382, tag_tid=26, entry_tid=46994], [object_number=383, tag_tid=26, entry_tid=46991]]], state=connected] 2026-02-18T08:22:32.507 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=370, tag_tid=26, entry_tid=35414], [object_number=369, tag_tid=26, entry_tid=35413], [object_number=368, tag_tid=26, entry_tid=35412], [object_number=371, tag_tid=26, entry_tid=35411]]], state=connected] 2026-02-18T08:22:32.509 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:32.511 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=370, tag_tid=26, entry_tid=35414' 2026-02-18T08:22:32.511 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=381, tag_tid=27, entry_tid=1' -a 'object_number=381, tag_tid=27, entry_tid=1' = 'object_number=370, tag_tid=26, entry_tid=35414' 2026-02-18T08:22:32.511 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=370, tag_tid=26, entry_tid=35414' '!=' 'object_number=369, tag_tid=26, entry_tid=35065' 2026-02-18T08:22:32.511 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:22:32.511 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=370, tag_tid=26, entry_tid=35414' = 'object_number=369, tag_tid=26, entry_tid=35065' 2026-02-18T08:22:32.511 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=370, tag_tid=26, entry_tid=35414' 2026-02-18T08:22:32.512 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=381, tag_tid=27, entry_tid=1' 2026-02-18T08:22:32.512 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:22:32.512 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:22:32.513 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=27 2026-02-18T08:22:32.514 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=370, tag_tid=26, entry_tid=35414' 2026-02-18T08:22:32.514 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:22:32.514 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:22:32.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=26 2026-02-18T08:22:32.516 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=381, tag_tid=27, entry_tid=1' 2026-02-18T08:22:32.516 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:22:32.516 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:22:32.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:22:32.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=370, tag_tid=26, entry_tid=35414' 2026-02-18T08:22:32.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:22:32.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:22:32.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=35414 2026-02-18T08:22:32.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 27 = 26 -a 1 -le 35414 2026-02-18T08:22:32.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:22:32.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:22:32.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:22:32.721 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:22:32.722 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:22:32.722 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:22:32.722 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:22:32.722 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:22:32.722 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:22:32.722 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:22:32.722 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:22:32.722 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:22:32.722 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:32.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:22:32.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:22:32.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:22:32.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:22:32.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:22:32.725 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:22:32.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:22:32.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:22:32.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:22:32.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:22:32.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:32.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:22:32.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:22:32.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:22:32.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:22:32.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:22:32.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:22:32.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:22:32.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:22:32.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:22:32.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:32.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:22:32.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:22:32.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:22:32.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:22:32.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:22:32.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:22:32.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:22:32.728 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:22:32.728 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:22:32.728 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:22:32.728 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:32.728 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:22:32.728 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:22:32.728 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:22:32.728 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:22:32.728 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:22:32.728 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:22:32.728 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:22:32.737 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:22:32.737 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:22:32.737 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:22:32.737 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:22:32.800 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:22:32.800 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:22:32.800 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:22:32.803 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:22:32.803 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:22:32.804 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:22:32.804 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:32.804 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:22:32.804 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:22:32.804 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:22:32.804 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:22:32.804 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:22:32.804 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:22:32.804 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:22:32.805 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:22:32.805 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:22:32.805 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:32.805 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:22:32.805 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:22:32.805 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:22:32.805 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:22:32.805 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:22:32.805 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:22:32.805 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:22:32.805 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:22:32.806 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:22:32.806 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:22:32.806 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:32.806 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:22:32.806 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:22:32.806 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:22:32.806 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:22:32.806 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:22:32.806 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:22:32.806 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:22:32.815 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:22:32.815 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:22:32.815 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:22:32.815 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:22:32.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:22:32.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:22:32.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:22:32.874 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:22:32.874 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:22:32.874 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:22:32.875 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:32.875 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:22:32.875 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:22:32.875 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:22:32.875 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:22:32.875 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:22:32.875 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:22:32.875 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:22:32.875 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:22:32.875 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:22:32.875 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:32.875 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:22:32.876 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:22:32.876 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:22:32.876 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:22:32.876 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:22:32.876 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:22:32.876 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:22:32.876 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:22:32.876 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:22:32.876 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:22:32.876 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:32.876 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:22:32.876 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:22:32.877 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:22:32.877 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:22:32.877 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:22:32.877 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:22:32.877 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:22:32.886 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:22:32.887 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:22:32.887 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:22:32.887 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:22:32.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:22:32.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:22:32.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:22:32.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:22:32.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:22:32.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:22:32.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:32.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:22:32.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:22:32.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:22:32.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:22:32.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:22:32.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:22:32.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:22:32.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:22:32.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:22:32.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:32.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:22:32.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:22:32.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:22:32.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:22:32.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:22:32.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:22:32.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:22:32.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:22:32.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:22:32.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:22:32.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:32.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:22:32.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:22:32.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:22:32.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:22:32.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:22:32.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:22:32.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:22:32.950 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:22:32.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:22:32.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:22:32.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:22:33.010 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:22:33.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:22:33.012 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:22:33.012 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:22:33.012 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:22:33.012 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:22:33.012 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:22:33.012 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:22:33.012 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:22:33.012 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:22:33.012 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:22:33.012 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:22:33.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:22:33.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:33.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:22:33.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:33.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 92 2026-02-18T08:22:33.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 95 2026-02-18T08:22:33.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:22:33.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=381, tag_tid=27, entry_tid=1], [object_number=380, tag_tid=27, entry_tid=0], [object_number=382, tag_tid=26, entry_tid=46994], [object_number=383, tag_tid=26, entry_tid=46991]]], state=connected] 2026-02-18T08:22:33.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=370, tag_tid=26, entry_tid=35778], [object_number=369, tag_tid=26, entry_tid=35777], [object_number=368, tag_tid=26, entry_tid=35776], [object_number=371, tag_tid=26, entry_tid=35775]]], state=connected] 2026-02-18T08:22:33.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:33.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=381, tag_tid=27, entry_tid=1' 2026-02-18T08:22:33.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:22:33.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:22:33.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:22:33.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:22:33.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:22:33.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:22:33.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:22:33.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:22:33.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:22:33.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:22:33.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:22:33.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:22:33.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:33.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:22:33.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:33.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 92 2026-02-18T08:22:33.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 95 2026-02-18T08:22:33.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:22:33.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=381, tag_tid=27, entry_tid=1], [object_number=380, tag_tid=27, entry_tid=0], [object_number=382, tag_tid=26, entry_tid=46994], [object_number=383, tag_tid=26, entry_tid=46991]]], state=connected] 2026-02-18T08:22:33.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=370, tag_tid=26, entry_tid=35778], [object_number=369, tag_tid=26, entry_tid=35777], [object_number=368, tag_tid=26, entry_tid=35776], [object_number=371, tag_tid=26, entry_tid=35775]]], state=connected] 2026-02-18T08:22:33.093 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:33.094 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=370, tag_tid=26, entry_tid=35778' 2026-02-18T08:22:33.094 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=381, tag_tid=27, entry_tid=1' -a 'object_number=381, tag_tid=27, entry_tid=1' = 'object_number=370, tag_tid=26, entry_tid=35778' 2026-02-18T08:22:33.095 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=370, tag_tid=26, entry_tid=35778' '!=' 'object_number=370, tag_tid=26, entry_tid=35414' 2026-02-18T08:22:33.095 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:22:33.095 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=370, tag_tid=26, entry_tid=35778' = 'object_number=370, tag_tid=26, entry_tid=35414' 2026-02-18T08:22:33.095 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=370, tag_tid=26, entry_tid=35778' 2026-02-18T08:22:33.095 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=381, tag_tid=27, entry_tid=1' 2026-02-18T08:22:33.096 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:22:33.096 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:22:33.096 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=27 2026-02-18T08:22:33.097 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=370, tag_tid=26, entry_tid=35778' 2026-02-18T08:22:33.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:22:33.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:22:33.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=26 2026-02-18T08:22:33.100 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=381, tag_tid=27, entry_tid=1' 2026-02-18T08:22:33.100 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:22:33.100 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:22:33.101 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:22:33.102 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=370, tag_tid=26, entry_tid=35778' 2026-02-18T08:22:33.102 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:22:33.102 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:22:33.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=35778 2026-02-18T08:22:33.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 27 = 26 -a 1 -le 35778 2026-02-18T08:22:33.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:22:33.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:22:33.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:22:33.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:22:33.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:22:33.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:22:33.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:22:33.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:22:33.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:22:33.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:22:33.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:22:33.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:22:33.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:33.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:22:33.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:22:33.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:22:33.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:22:33.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:22:33.310 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:22:33.310 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:22:33.310 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:22:33.310 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:22:33.310 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:22:33.310 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:33.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:22:33.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:22:33.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:22:33.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:22:33.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:22:33.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:22:33.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:22:33.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:22:33.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:22:33.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:33.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:22:33.312 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:22:33.312 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:22:33.312 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:22:33.312 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:22:33.312 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:22:33.312 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:22:33.312 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:22:33.312 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:22:33.312 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:22:33.312 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:33.312 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:22:33.312 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:22:33.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:22:33.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:22:33.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:22:33.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:22:33.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:22:33.320 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:22:33.321 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:22:33.321 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:22:33.321 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:22:33.380 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:22:33.381 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:22:33.381 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:22:33.383 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:22:33.383 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:22:33.383 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:22:33.384 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:33.384 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:22:33.384 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:22:33.384 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:22:33.384 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:22:33.384 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:22:33.384 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:22:33.384 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:22:33.384 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:22:33.384 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:22:33.384 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:33.384 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:22:33.385 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:22:33.385 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:22:33.385 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:22:33.385 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:22:33.385 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:22:33.385 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:22:33.385 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:22:33.385 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:22:33.385 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:22:33.385 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:33.385 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:22:33.385 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:22:33.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:22:33.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:22:33.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:22:33.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:22:33.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:22:33.395 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:22:33.396 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:22:33.396 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:22:33.396 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:22:33.452 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:22:33.452 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:22:33.452 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:22:33.455 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:22:33.455 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:22:33.455 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:22:33.455 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:33.455 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:22:33.455 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:22:33.455 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:22:33.455 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:22:33.455 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:22:33.455 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:22:33.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:22:33.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:22:33.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:22:33.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:33.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:22:33.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:22:33.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:22:33.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:22:33.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:22:33.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:22:33.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:22:33.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:22:33.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:22:33.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:22:33.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:33.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:22:33.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:22:33.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:22:33.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:22:33.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:22:33.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:22:33.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:22:33.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:22:33.467 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:22:33.467 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:22:33.467 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:22:33.514 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:22:33.514 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:22:33.514 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:22:33.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:22:33.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:22:33.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:22:33.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:33.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:22:33.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:22:33.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:22:33.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:22:33.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:22:33.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:22:33.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:22:33.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:22:33.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:22:33.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:33.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:22:33.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:22:33.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:22:33.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:22:33.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:22:33.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:22:33.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:22:33.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:22:33.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:22:33.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:22:33.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:33.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:22:33.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:22:33.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:22:33.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:22:33.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:22:33.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:22:33.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:22:33.527 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:22:33.528 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:22:33.528 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:22:33.528 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:22:33.594 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:22:33.596 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:22:33.596 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:22:33.596 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:22:33.596 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:22:33.596 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:22:33.596 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:22:33.596 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:22:33.596 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:22:33.596 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:22:33.596 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:22:33.596 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:22:33.597 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:22:33.597 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:33.598 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:22:33.598 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:33.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 92 2026-02-18T08:22:33.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 95 2026-02-18T08:22:33.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:22:33.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=381, tag_tid=27, entry_tid=1], [object_number=380, tag_tid=27, entry_tid=0], [object_number=382, tag_tid=26, entry_tid=46994], [object_number=383, tag_tid=26, entry_tid=46991]]], state=connected] 2026-02-18T08:22:33.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=371, tag_tid=26, entry_tid=36143], [object_number=370, tag_tid=26, entry_tid=36142], [object_number=369, tag_tid=26, entry_tid=36141], [object_number=368, tag_tid=26, entry_tid=36140]]], state=connected] 2026-02-18T08:22:33.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:33.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=381, tag_tid=27, entry_tid=1' 2026-02-18T08:22:33.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:22:33.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:22:33.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:22:33.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:22:33.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:22:33.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:22:33.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:22:33.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:22:33.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:22:33.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:22:33.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:22:33.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:22:33.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:33.635 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:22:33.635 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:33.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 92 2026-02-18T08:22:33.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 95 2026-02-18T08:22:33.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:22:33.670 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=381, tag_tid=27, entry_tid=1], [object_number=380, tag_tid=27, entry_tid=0], [object_number=382, tag_tid=26, entry_tid=46994], [object_number=383, tag_tid=26, entry_tid=46991]]], state=connected] 2026-02-18T08:22:33.670 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=371, tag_tid=26, entry_tid=36143], [object_number=370, tag_tid=26, entry_tid=36142], [object_number=369, tag_tid=26, entry_tid=36141], [object_number=368, tag_tid=26, entry_tid=36140]]], state=connected] 2026-02-18T08:22:33.673 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:33.675 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=371, tag_tid=26, entry_tid=36143' 2026-02-18T08:22:33.675 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=381, tag_tid=27, entry_tid=1' -a 'object_number=381, tag_tid=27, entry_tid=1' = 'object_number=371, tag_tid=26, entry_tid=36143' 2026-02-18T08:22:33.675 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=371, tag_tid=26, entry_tid=36143' '!=' 'object_number=370, tag_tid=26, entry_tid=35778' 2026-02-18T08:22:33.676 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:22:33.676 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=371, tag_tid=26, entry_tid=36143' = 'object_number=370, tag_tid=26, entry_tid=35778' 2026-02-18T08:22:33.676 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=371, tag_tid=26, entry_tid=36143' 2026-02-18T08:22:33.676 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=381, tag_tid=27, entry_tid=1' 2026-02-18T08:22:33.677 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:22:33.677 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:22:33.677 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=27 2026-02-18T08:22:33.678 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=371, tag_tid=26, entry_tid=36143' 2026-02-18T08:22:33.678 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:22:33.678 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:22:33.680 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=26 2026-02-18T08:22:33.681 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=381, tag_tid=27, entry_tid=1' 2026-02-18T08:22:33.681 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:22:33.681 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:22:33.682 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:22:33.683 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=371, tag_tid=26, entry_tid=36143' 2026-02-18T08:22:33.683 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:22:33.683 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:22:33.684 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=36143 2026-02-18T08:22:33.684 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 27 = 26 -a 1 -le 36143 2026-02-18T08:22:33.685 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:22:33.685 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:22:33.685 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:22:33.886 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:22:33.886 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:22:33.886 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:22:33.886 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:22:33.887 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:22:33.887 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:22:33.887 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:22:33.887 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:22:33.887 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:22:33.887 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:33.887 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:22:33.887 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:22:33.887 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:22:33.887 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:22:33.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:22:33.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:22:33.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:22:33.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:22:33.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:22:33.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:22:33.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:33.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:22:33.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:22:33.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:22:33.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:22:33.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:22:33.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:22:33.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:22:33.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:22:33.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:22:33.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:33.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:22:33.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:22:33.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:22:33.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:22:33.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:22:33.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:22:33.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:22:33.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:22:33.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:22:33.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:22:33.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:33.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:22:33.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:22:33.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:22:33.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:22:33.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:22:33.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:22:33.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:22:33.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:22:33.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:22:33.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:22:33.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:22:33.952 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:22:33.952 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:22:33.952 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:22:33.955 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:22:33.955 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:22:33.955 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:22:33.955 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:33.955 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:22:33.956 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:22:33.956 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:22:33.956 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:22:33.956 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:22:33.956 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:22:33.956 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:22:33.956 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:22:33.956 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:22:33.956 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:33.956 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:22:33.956 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:22:33.956 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:22:33.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:22:33.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:22:33.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:22:33.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:22:33.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:22:33.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:22:33.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:22:33.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:33.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:22:33.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:22:33.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:22:33.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:22:33.958 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:22:33.958 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:22:33.958 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:22:33.967 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:22:33.967 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:22:33.967 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:22:33.967 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:22:34.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:22:34.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:22:34.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:22:34.028 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:22:34.028 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:22:34.028 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:22:34.028 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:34.028 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:22:34.028 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:22:34.029 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:22:34.029 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:22:34.029 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:22:34.029 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:22:34.029 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:22:34.029 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:22:34.029 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:22:34.029 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:34.029 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:22:34.029 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:22:34.029 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:22:34.029 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:22:34.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:22:34.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:22:34.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:22:34.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:22:34.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:22:34.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:22:34.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:34.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:22:34.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:22:34.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:22:34.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:22:34.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:22:34.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:22:34.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:22:34.040 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:22:34.040 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:22:34.040 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:22:34.041 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:22:34.097 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:22:34.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:22:34.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:22:34.100 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:22:34.100 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:22:34.100 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:22:34.101 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:34.101 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:22:34.101 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:22:34.101 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:22:34.101 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:22:34.101 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:22:34.101 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:22:34.101 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:22:34.101 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:22:34.101 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:22:34.101 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:34.101 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:22:34.102 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:22:34.102 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:22:34.102 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:22:34.102 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:22:34.102 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:22:34.102 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:22:34.102 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:22:34.102 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:22:34.102 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:22:34.102 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:34.102 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:22:34.102 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:22:34.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:22:34.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:22:34.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:22:34.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:22:34.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:22:34.111 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:22:34.111 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:22:34.112 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:22:34.112 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:22:34.170 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:22:34.171 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:22:34.171 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:22:34.171 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:22:34.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:22:34.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:22:34.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:22:34.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:22:34.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:22:34.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:22:34.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:22:34.173 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:22:34.173 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:22:34.173 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:34.174 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:22:34.174 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:34.202 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 92 2026-02-18T08:22:34.202 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 95 2026-02-18T08:22:34.202 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:22:34.202 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=381, tag_tid=27, entry_tid=1], [object_number=380, tag_tid=27, entry_tid=0], [object_number=382, tag_tid=26, entry_tid=46994], [object_number=383, tag_tid=26, entry_tid=46991]]], state=connected] 2026-02-18T08:22:34.202 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=371, tag_tid=26, entry_tid=36491], [object_number=370, tag_tid=26, entry_tid=36490], [object_number=369, tag_tid=26, entry_tid=36489], [object_number=368, tag_tid=26, entry_tid=36488]]], state=connected] 2026-02-18T08:22:34.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:34.206 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=381, tag_tid=27, entry_tid=1' 2026-02-18T08:22:34.207 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:22:34.207 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:22:34.207 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:22:34.208 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:22:34.208 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:22:34.208 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:22:34.208 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:22:34.208 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:22:34.208 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:22:34.208 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:22:34.209 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:22:34.209 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:22:34.209 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:34.210 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:22:34.210 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:34.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 92 2026-02-18T08:22:34.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 95 2026-02-18T08:22:34.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:22:34.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=381, tag_tid=27, entry_tid=1], [object_number=380, tag_tid=27, entry_tid=0], [object_number=382, tag_tid=26, entry_tid=46994], [object_number=383, tag_tid=26, entry_tid=46991]]], state=connected] 2026-02-18T08:22:34.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=371, tag_tid=26, entry_tid=36491], [object_number=370, tag_tid=26, entry_tid=36490], [object_number=369, tag_tid=26, entry_tid=36489], [object_number=368, tag_tid=26, entry_tid=36488]]], state=connected] 2026-02-18T08:22:34.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:34.245 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=371, tag_tid=26, entry_tid=36491' 2026-02-18T08:22:34.245 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=381, tag_tid=27, entry_tid=1' -a 'object_number=381, tag_tid=27, entry_tid=1' = 'object_number=371, tag_tid=26, entry_tid=36491' 2026-02-18T08:22:34.245 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=371, tag_tid=26, entry_tid=36491' '!=' 'object_number=371, tag_tid=26, entry_tid=36143' 2026-02-18T08:22:34.245 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:22:34.245 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=371, tag_tid=26, entry_tid=36491' = 'object_number=371, tag_tid=26, entry_tid=36143' 2026-02-18T08:22:34.245 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=371, tag_tid=26, entry_tid=36491' 2026-02-18T08:22:34.246 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=381, tag_tid=27, entry_tid=1' 2026-02-18T08:22:34.246 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:22:34.246 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:22:34.247 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=27 2026-02-18T08:22:34.248 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=371, tag_tid=26, entry_tid=36491' 2026-02-18T08:22:34.248 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:22:34.248 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:22:34.249 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=26 2026-02-18T08:22:34.250 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=381, tag_tid=27, entry_tid=1' 2026-02-18T08:22:34.250 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:22:34.250 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:22:34.251 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:22:34.252 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=371, tag_tid=26, entry_tid=36491' 2026-02-18T08:22:34.252 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:22:34.252 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:22:34.254 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=36491 2026-02-18T08:22:34.254 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 27 = 26 -a 1 -le 36491 2026-02-18T08:22:34.254 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:22:34.254 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:22:34.254 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:22:34.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:22:34.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:22:34.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:22:34.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:22:34.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:22:34.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:22:34.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:22:34.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:22:34.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:22:34.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:34.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:22:34.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:22:34.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:22:34.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:22:34.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:22:34.460 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:22:34.460 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:22:34.460 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:22:34.460 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:22:34.460 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:22:34.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:34.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:22:34.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:22:34.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:22:34.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:22:34.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:22:34.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:22:34.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:22:34.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:22:34.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:22:34.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:34.462 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:22:34.462 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:22:34.462 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:22:34.462 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:22:34.462 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:22:34.462 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:22:34.462 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:22:34.462 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:22:34.462 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:22:34.462 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:22:34.462 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:34.462 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:22:34.463 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:22:34.463 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:22:34.463 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:22:34.463 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:22:34.463 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:22:34.463 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:22:34.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:22:34.472 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:22:34.472 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:22:34.472 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:22:34.532 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:22:34.532 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:22:34.532 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:22:34.535 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:22:34.535 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:22:34.535 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:22:34.535 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:34.535 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:22:34.535 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:22:34.535 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:22:34.535 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:22:34.535 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:22:34.535 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:22:34.536 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:22:34.536 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:22:34.536 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:22:34.536 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:34.536 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:22:34.536 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:22:34.536 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:22:34.536 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:22:34.536 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:22:34.536 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:22:34.536 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:22:34.536 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:22:34.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:22:34.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:22:34.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:34.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:22:34.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:22:34.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:22:34.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:22:34.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:22:34.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:22:34.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:22:34.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:22:34.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:22:34.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:22:34.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:22:34.603 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:22:34.603 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:22:34.604 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:22:34.607 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:22:34.607 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:22:34.607 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:22:34.607 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:34.607 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:22:34.607 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:22:34.607 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:22:34.607 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:22:34.607 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:22:34.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:22:34.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:22:34.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:22:34.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:22:34.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:34.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:22:34.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:22:34.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:22:34.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:22:34.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:22:34.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:22:34.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:22:34.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:22:34.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:22:34.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:22:34.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:34.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:22:34.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:22:34.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:22:34.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:22:34.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:22:34.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:22:34.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:22:34.618 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:22:34.618 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:22:34.619 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:22:34.619 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:22:34.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:22:34.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:22:34.670 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:22:34.673 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:22:34.673 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:22:34.673 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:22:34.673 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:34.673 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:22:34.673 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:22:34.673 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:22:34.674 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:22:34.674 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:22:34.674 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:22:34.674 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:22:34.674 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:22:34.674 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:22:34.674 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:34.674 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:22:34.674 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:22:34.674 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:22:34.674 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:22:34.674 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:22:34.675 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:22:34.675 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:22:34.675 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:22:34.675 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:22:34.675 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:22:34.675 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:34.675 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:22:34.675 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:22:34.675 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:22:34.675 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:22:34.675 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:22:34.675 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:22:34.676 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:22:34.683 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:22:34.684 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:22:34.684 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:22:34.684 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:22:34.753 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:22:34.754 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:22:34.754 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:22:34.755 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:22:34.755 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:22:34.755 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:22:34.755 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:22:34.755 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:22:34.755 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:22:34.755 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:22:34.755 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:22:34.756 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:22:34.756 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:22:34.756 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:34.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:22:34.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:34.787 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 92 2026-02-18T08:22:34.787 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 95 2026-02-18T08:22:34.787 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:22:34.787 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=381, tag_tid=27, entry_tid=1], [object_number=380, tag_tid=27, entry_tid=0], [object_number=382, tag_tid=26, entry_tid=46994], [object_number=383, tag_tid=26, entry_tid=46991]]], state=connected] 2026-02-18T08:22:34.787 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=371, tag_tid=26, entry_tid=36855], [object_number=370, tag_tid=26, entry_tid=36854], [object_number=369, tag_tid=26, entry_tid=36853], [object_number=368, tag_tid=26, entry_tid=36852]]], state=connected] 2026-02-18T08:22:34.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:34.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=381, tag_tid=27, entry_tid=1' 2026-02-18T08:22:34.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:22:34.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:22:34.793 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:22:34.793 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:22:34.793 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:22:34.793 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:22:34.793 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:22:34.793 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:22:34.793 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:22:34.793 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:22:34.793 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:22:34.794 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:22:34.794 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:34.794 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:22:34.795 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:34.823 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 92 2026-02-18T08:22:34.823 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 95 2026-02-18T08:22:34.823 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:22:34.823 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=381, tag_tid=27, entry_tid=1], [object_number=380, tag_tid=27, entry_tid=0], [object_number=382, tag_tid=26, entry_tid=46994], [object_number=383, tag_tid=26, entry_tid=46991]]], state=connected] 2026-02-18T08:22:34.823 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=371, tag_tid=26, entry_tid=36855], [object_number=370, tag_tid=26, entry_tid=36854], [object_number=369, tag_tid=26, entry_tid=36853], [object_number=368, tag_tid=26, entry_tid=36852]]], state=connected] 2026-02-18T08:22:34.825 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:34.827 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=371, tag_tid=26, entry_tid=36855' 2026-02-18T08:22:34.828 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=381, tag_tid=27, entry_tid=1' -a 'object_number=381, tag_tid=27, entry_tid=1' = 'object_number=371, tag_tid=26, entry_tid=36855' 2026-02-18T08:22:34.828 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=371, tag_tid=26, entry_tid=36855' '!=' 'object_number=371, tag_tid=26, entry_tid=36491' 2026-02-18T08:22:34.828 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:22:34.828 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=371, tag_tid=26, entry_tid=36855' = 'object_number=371, tag_tid=26, entry_tid=36491' 2026-02-18T08:22:34.828 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=371, tag_tid=26, entry_tid=36855' 2026-02-18T08:22:34.829 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=381, tag_tid=27, entry_tid=1' 2026-02-18T08:22:34.829 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:22:34.829 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:22:34.830 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=27 2026-02-18T08:22:34.830 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=371, tag_tid=26, entry_tid=36855' 2026-02-18T08:22:34.830 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:22:34.830 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:22:34.832 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=26 2026-02-18T08:22:34.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=381, tag_tid=27, entry_tid=1' 2026-02-18T08:22:34.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:22:34.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:22:34.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:22:34.835 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=371, tag_tid=26, entry_tid=36855' 2026-02-18T08:22:34.835 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:22:34.835 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:22:34.836 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=36855 2026-02-18T08:22:34.837 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 27 = 26 -a 1 -le 36855 2026-02-18T08:22:34.837 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:22:34.837 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:22:34.837 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:22:35.038 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:22:35.038 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:22:35.039 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:22:35.039 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:22:35.039 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:22:35.039 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:22:35.039 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:22:35.039 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:22:35.039 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:22:35.039 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:35.039 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:22:35.039 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:22:35.039 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:22:35.040 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:22:35.040 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:22:35.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:22:35.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:22:35.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:22:35.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:22:35.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:22:35.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:35.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:22:35.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:22:35.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:22:35.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:22:35.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:22:35.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:22:35.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:22:35.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:22:35.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:22:35.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:35.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:22:35.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:22:35.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:22:35.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:22:35.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:22:35.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:22:35.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:22:35.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:22:35.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:22:35.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:22:35.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:35.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:22:35.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:22:35.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:22:35.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:22:35.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:22:35.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:22:35.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:22:35.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:22:35.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:22:35.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:22:35.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:22:35.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:22:35.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:22:35.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:22:35.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:22:35.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:22:35.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:22:35.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:35.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:22:35.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:22:35.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:22:35.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:22:35.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:22:35.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:22:35.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:22:35.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:22:35.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:22:35.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:35.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:22:35.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:22:35.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:22:35.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:22:35.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:22:35.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:22:35.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:22:35.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:22:35.110 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:22:35.110 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:22:35.110 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:35.110 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:22:35.110 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:22:35.110 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:22:35.110 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:22:35.110 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:22:35.110 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:22:35.110 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:22:35.120 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:22:35.120 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:22:35.120 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:22:35.120 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:22:35.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:22:35.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:22:35.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:22:35.171 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:22:35.171 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:22:35.171 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:22:35.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:35.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:22:35.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:22:35.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:22:35.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:22:35.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:22:35.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:22:35.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:22:35.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:22:35.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:22:35.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:35.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:22:35.173 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:22:35.173 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:22:35.173 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:22:35.173 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:22:35.173 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:22:35.173 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:22:35.173 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:22:35.173 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:22:35.173 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:22:35.173 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:35.173 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:22:35.173 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:22:35.174 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:22:35.174 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:22:35.174 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:22:35.174 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:22:35.174 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:22:35.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:22:35.183 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:22:35.183 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:22:35.183 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:22:35.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:22:35.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:22:35.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:22:35.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:22:35.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:22:35.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:22:35.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:35.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:22:35.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:22:35.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:22:35.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:22:35.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:22:35.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:22:35.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:22:35.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:22:35.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:22:35.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:35.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:22:35.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:22:35.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:22:35.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:22:35.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:22:35.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:22:35.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:22:35.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:22:35.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:22:35.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:22:35.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:35.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:22:35.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:22:35.238 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:22:35.238 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:22:35.238 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:22:35.238 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:22:35.238 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:22:35.245 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:22:35.246 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:22:35.246 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:22:35.246 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:22:35.310 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:22:35.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:22:35.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:22:35.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:22:35.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:22:35.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:22:35.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:22:35.312 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:22:35.312 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:22:35.312 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:22:35.312 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:22:35.312 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:22:35.312 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:22:35.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:35.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:22:35.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:35.351 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 93 2026-02-18T08:22:35.351 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 95 2026-02-18T08:22:35.351 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:22:35.351 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=381, tag_tid=27, entry_tid=1], [object_number=380, tag_tid=27, entry_tid=0], [object_number=382, tag_tid=26, entry_tid=46994], [object_number=383, tag_tid=26, entry_tid=46991]]], state=connected] 2026-02-18T08:22:35.352 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=375, tag_tid=26, entry_tid=37111], [object_number=374, tag_tid=26, entry_tid=37110], [object_number=373, tag_tid=26, entry_tid=37109], [object_number=372, tag_tid=26, entry_tid=37108]]], state=connected] 2026-02-18T08:22:35.355 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:35.356 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=381, tag_tid=27, entry_tid=1' 2026-02-18T08:22:35.357 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:22:35.357 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:22:35.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:22:35.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:22:35.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:22:35.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:22:35.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:22:35.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:22:35.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:22:35.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:22:35.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:22:35.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:22:35.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:35.360 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:22:35.360 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:35.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 93 2026-02-18T08:22:35.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 95 2026-02-18T08:22:35.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:22:35.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=381, tag_tid=27, entry_tid=1], [object_number=380, tag_tid=27, entry_tid=0], [object_number=382, tag_tid=26, entry_tid=46994], [object_number=383, tag_tid=26, entry_tid=46991]]], state=connected] 2026-02-18T08:22:35.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=375, tag_tid=26, entry_tid=37111], [object_number=374, tag_tid=26, entry_tid=37110], [object_number=373, tag_tid=26, entry_tid=37109], [object_number=372, tag_tid=26, entry_tid=37108]]], state=connected] 2026-02-18T08:22:35.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:35.403 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=375, tag_tid=26, entry_tid=37111' 2026-02-18T08:22:35.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=381, tag_tid=27, entry_tid=1' -a 'object_number=381, tag_tid=27, entry_tid=1' = 'object_number=375, tag_tid=26, entry_tid=37111' 2026-02-18T08:22:35.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=375, tag_tid=26, entry_tid=37111' '!=' 'object_number=371, tag_tid=26, entry_tid=36855' 2026-02-18T08:22:35.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:22:35.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=375, tag_tid=26, entry_tid=37111' = 'object_number=371, tag_tid=26, entry_tid=36855' 2026-02-18T08:22:35.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=375, tag_tid=26, entry_tid=37111' 2026-02-18T08:22:35.405 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=381, tag_tid=27, entry_tid=1' 2026-02-18T08:22:35.405 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:22:35.405 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:22:35.406 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=27 2026-02-18T08:22:35.407 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=375, tag_tid=26, entry_tid=37111' 2026-02-18T08:22:35.407 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:22:35.407 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:22:35.408 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=26 2026-02-18T08:22:35.409 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=381, tag_tid=27, entry_tid=1' 2026-02-18T08:22:35.409 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:22:35.409 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:22:35.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:22:35.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=375, tag_tid=26, entry_tid=37111' 2026-02-18T08:22:35.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:22:35.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:22:35.412 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=37111 2026-02-18T08:22:35.413 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 27 = 26 -a 1 -le 37111 2026-02-18T08:22:35.413 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:22:35.413 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:22:35.413 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:22:35.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:22:35.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:22:35.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:22:35.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:22:35.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:22:35.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:22:35.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:22:35.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:22:35.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:22:35.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:35.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:22:35.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:22:35.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:22:35.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:22:35.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:22:35.618 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:22:35.619 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:22:35.619 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:22:35.619 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:22:35.619 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:22:35.619 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:35.619 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:22:35.619 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:22:35.619 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:22:35.619 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:22:35.619 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:22:35.619 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:22:35.620 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:22:35.620 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:22:35.620 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:22:35.620 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:35.620 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:22:35.620 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:22:35.620 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:22:35.620 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:22:35.620 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:22:35.620 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:22:35.620 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:22:35.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:22:35.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:22:35.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:22:35.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:35.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:22:35.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:22:35.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:22:35.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:22:35.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:22:35.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:22:35.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:22:35.629 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:22:35.629 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:22:35.629 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:22:35.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:22:35.679 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:22:35.679 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:22:35.679 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:22:35.682 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:22:35.682 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:22:35.682 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:22:35.682 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:35.682 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:22:35.682 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:22:35.682 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:22:35.683 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:22:35.683 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:22:35.683 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:22:35.683 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:22:35.683 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:22:35.683 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:22:35.683 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:35.683 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:22:35.683 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:22:35.683 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:22:35.683 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:22:35.684 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:22:35.684 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:22:35.684 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:22:35.684 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:22:35.684 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:22:35.684 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:22:35.684 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:35.684 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:22:35.684 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:22:35.684 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:22:35.684 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:22:35.684 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:22:35.685 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:22:35.685 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:22:35.693 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:22:35.694 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:22:35.694 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:22:35.694 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:22:35.743 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:22:35.743 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:22:35.743 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:22:35.746 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:22:35.746 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:22:35.746 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:22:35.746 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:35.746 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:22:35.746 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:22:35.746 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:22:35.747 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:22:35.747 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:22:35.747 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:22:35.747 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:22:35.747 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:22:35.747 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:22:35.747 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:35.747 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:22:35.747 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:22:35.747 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:22:35.747 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:22:35.748 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:22:35.748 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:22:35.748 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:22:35.748 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:22:35.748 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:22:35.748 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:22:35.748 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:35.748 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:22:35.748 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:22:35.748 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:22:35.748 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:22:35.748 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:22:35.749 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:22:35.749 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:22:35.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:22:35.758 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:22:35.758 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:22:35.758 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:22:35.808 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:22:35.808 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:22:35.808 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:22:35.811 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:22:35.811 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:22:35.811 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:22:35.811 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:35.811 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:22:35.812 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:22:35.812 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:22:35.812 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:22:35.812 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:22:35.812 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:22:35.812 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:22:35.812 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:22:35.812 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:22:35.812 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:35.812 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:22:35.812 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:22:35.813 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:22:35.813 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:22:35.813 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:22:35.813 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:22:35.813 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:22:35.813 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:22:35.813 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:22:35.813 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:22:35.813 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:35.813 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:22:35.813 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:22:35.813 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:22:35.813 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:22:35.814 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:22:35.814 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:22:35.814 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:22:35.822 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:22:35.822 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:22:35.823 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:22:35.823 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:22:35.894 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:22:35.895 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:22:35.895 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:22:35.895 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:22:35.896 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:22:35.896 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:22:35.896 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:22:35.896 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:22:35.896 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:22:35.896 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:22:35.896 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:22:35.896 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:22:35.896 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:22:35.897 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:35.897 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:22:35.897 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:35.928 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 93 2026-02-18T08:22:35.928 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 95 2026-02-18T08:22:35.928 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:22:35.928 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=381, tag_tid=27, entry_tid=1], [object_number=380, tag_tid=27, entry_tid=0], [object_number=382, tag_tid=26, entry_tid=46994], [object_number=383, tag_tid=26, entry_tid=46991]]], state=connected] 2026-02-18T08:22:35.928 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=373, tag_tid=26, entry_tid=37465], [object_number=372, tag_tid=26, entry_tid=37464], [object_number=375, tag_tid=26, entry_tid=37463], [object_number=374, tag_tid=26, entry_tid=37462]]], state=connected] 2026-02-18T08:22:35.931 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:35.932 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=381, tag_tid=27, entry_tid=1' 2026-02-18T08:22:35.933 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:22:35.933 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:22:35.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:22:35.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:22:35.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:22:35.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:22:35.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:22:35.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:22:35.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:22:35.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:22:35.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:22:35.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:22:35.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:35.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:22:35.936 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:35.963 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 93 2026-02-18T08:22:35.963 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 95 2026-02-18T08:22:35.963 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:22:35.963 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=381, tag_tid=27, entry_tid=1], [object_number=380, tag_tid=27, entry_tid=0], [object_number=382, tag_tid=26, entry_tid=46994], [object_number=383, tag_tid=26, entry_tid=46991]]], state=connected] 2026-02-18T08:22:35.964 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=373, tag_tid=26, entry_tid=37465], [object_number=372, tag_tid=26, entry_tid=37464], [object_number=375, tag_tid=26, entry_tid=37463], [object_number=374, tag_tid=26, entry_tid=37462]]], state=connected] 2026-02-18T08:22:35.966 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:35.968 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=373, tag_tid=26, entry_tid=37465' 2026-02-18T08:22:35.968 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=381, tag_tid=27, entry_tid=1' -a 'object_number=381, tag_tid=27, entry_tid=1' = 'object_number=373, tag_tid=26, entry_tid=37465' 2026-02-18T08:22:35.968 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=373, tag_tid=26, entry_tid=37465' '!=' 'object_number=375, tag_tid=26, entry_tid=37111' 2026-02-18T08:22:35.968 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:22:35.969 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=373, tag_tid=26, entry_tid=37465' = 'object_number=375, tag_tid=26, entry_tid=37111' 2026-02-18T08:22:35.969 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=373, tag_tid=26, entry_tid=37465' 2026-02-18T08:22:35.969 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=381, tag_tid=27, entry_tid=1' 2026-02-18T08:22:35.969 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:22:35.970 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:22:35.970 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=27 2026-02-18T08:22:35.971 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=373, tag_tid=26, entry_tid=37465' 2026-02-18T08:22:35.971 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:22:35.971 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:22:35.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=26 2026-02-18T08:22:35.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=381, tag_tid=27, entry_tid=1' 2026-02-18T08:22:35.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:22:35.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:22:35.974 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:22:35.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=373, tag_tid=26, entry_tid=37465' 2026-02-18T08:22:35.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:22:35.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:22:35.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=37465 2026-02-18T08:22:35.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 27 = 26 -a 1 -le 37465 2026-02-18T08:22:35.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:22:35.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:22:35.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:22:36.178 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:22:36.178 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:22:36.178 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:22:36.178 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:22:36.179 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:22:36.179 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:22:36.179 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:22:36.179 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:22:36.179 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:22:36.179 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:36.179 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:22:36.179 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:22:36.179 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:22:36.179 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:22:36.179 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:22:36.180 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:22:36.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:22:36.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:22:36.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:22:36.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:22:36.183 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:36.183 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:22:36.183 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:22:36.183 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:22:36.183 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:22:36.183 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:22:36.183 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:22:36.183 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:22:36.183 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:22:36.183 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:22:36.183 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:36.183 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:22:36.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:22:36.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:22:36.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:22:36.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:22:36.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:22:36.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:22:36.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:22:36.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:22:36.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:22:36.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:36.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:22:36.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:22:36.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:22:36.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:22:36.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:22:36.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:22:36.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:22:36.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:22:36.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:22:36.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:22:36.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:22:36.255 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:22:36.255 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:22:36.255 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:22:36.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:22:36.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:22:36.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:22:36.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:36.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:22:36.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:22:36.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:22:36.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:22:36.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:22:36.259 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:22:36.259 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:22:36.259 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:22:36.259 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:22:36.259 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:36.259 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:22:36.259 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:22:36.259 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:22:36.259 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:22:36.259 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:22:36.259 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:22:36.259 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:22:36.260 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:22:36.260 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:22:36.260 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:22:36.260 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:36.260 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:22:36.260 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:22:36.260 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:22:36.260 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:22:36.260 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:22:36.260 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:22:36.260 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:22:36.270 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:22:36.270 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:22:36.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:22:36.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:22:36.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:22:36.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:22:36.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:22:36.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:22:36.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:22:36.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:22:36.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:36.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:22:36.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:22:36.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:22:36.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:22:36.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:22:36.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:22:36.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:22:36.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:22:36.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:22:36.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:36.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:22:36.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:22:36.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:22:36.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:22:36.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:22:36.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:22:36.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:22:36.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:22:36.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:22:36.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:22:36.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:36.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:22:36.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:22:36.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:22:36.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:22:36.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:22:36.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:22:36.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:22:36.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:22:36.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:22:36.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:22:36.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:22:36.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:22:36.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:22:36.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:22:36.394 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:22:36.394 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:22:36.394 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:22:36.395 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:36.395 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:22:36.395 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:22:36.395 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:22:36.395 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:22:36.395 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:22:36.395 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:22:36.395 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:22:36.395 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:22:36.395 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:22:36.395 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:36.395 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:22:36.396 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:22:36.396 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:22:36.396 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:22:36.396 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:22:36.396 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:22:36.396 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:22:36.396 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:22:36.396 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:22:36.396 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:22:36.396 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:36.396 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:22:36.396 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:22:36.396 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:22:36.397 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:22:36.397 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:22:36.397 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:22:36.397 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:22:36.406 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:22:36.407 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:22:36.407 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:22:36.407 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:22:36.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:22:36.467 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:22:36.467 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:22:36.467 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:22:36.468 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:22:36.468 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:22:36.468 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:22:36.468 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:22:36.468 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:22:36.468 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:22:36.468 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:22:36.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:22:36.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:22:36.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:36.470 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:22:36.470 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:36.504 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 93 2026-02-18T08:22:36.504 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 95 2026-02-18T08:22:36.505 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:22:36.505 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=381, tag_tid=27, entry_tid=1], [object_number=380, tag_tid=27, entry_tid=0], [object_number=382, tag_tid=26, entry_tid=46994], [object_number=383, tag_tid=26, entry_tid=46991]]], state=connected] 2026-02-18T08:22:36.505 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=372, tag_tid=26, entry_tid=37796], [object_number=375, tag_tid=26, entry_tid=37795], [object_number=374, tag_tid=26, entry_tid=37794], [object_number=373, tag_tid=26, entry_tid=37793]]], state=connected] 2026-02-18T08:22:36.508 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:36.510 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=381, tag_tid=27, entry_tid=1' 2026-02-18T08:22:36.511 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:22:36.511 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:22:36.511 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:22:36.511 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:22:36.511 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:22:36.511 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:22:36.512 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:22:36.512 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:22:36.512 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:22:36.512 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:22:36.512 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:22:36.512 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:22:36.513 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:36.513 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:22:36.513 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:36.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 93 2026-02-18T08:22:36.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 95 2026-02-18T08:22:36.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:22:36.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=381, tag_tid=27, entry_tid=1], [object_number=380, tag_tid=27, entry_tid=0], [object_number=382, tag_tid=26, entry_tid=46994], [object_number=383, tag_tid=26, entry_tid=46991]]], state=connected] 2026-02-18T08:22:36.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=372, tag_tid=26, entry_tid=37796], [object_number=375, tag_tid=26, entry_tid=37795], [object_number=374, tag_tid=26, entry_tid=37794], [object_number=373, tag_tid=26, entry_tid=37793]]], state=connected] 2026-02-18T08:22:36.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:36.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=372, tag_tid=26, entry_tid=37796' 2026-02-18T08:22:36.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=381, tag_tid=27, entry_tid=1' -a 'object_number=381, tag_tid=27, entry_tid=1' = 'object_number=372, tag_tid=26, entry_tid=37796' 2026-02-18T08:22:36.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=372, tag_tid=26, entry_tid=37796' '!=' 'object_number=373, tag_tid=26, entry_tid=37465' 2026-02-18T08:22:36.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:22:36.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=372, tag_tid=26, entry_tid=37796' = 'object_number=373, tag_tid=26, entry_tid=37465' 2026-02-18T08:22:36.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=372, tag_tid=26, entry_tid=37796' 2026-02-18T08:22:36.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=381, tag_tid=27, entry_tid=1' 2026-02-18T08:22:36.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:22:36.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:22:36.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=27 2026-02-18T08:22:36.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=372, tag_tid=26, entry_tid=37796' 2026-02-18T08:22:36.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:22:36.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:22:36.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=26 2026-02-18T08:22:36.551 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=381, tag_tid=27, entry_tid=1' 2026-02-18T08:22:36.551 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:22:36.551 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:22:36.553 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:22:36.553 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=372, tag_tid=26, entry_tid=37796' 2026-02-18T08:22:36.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:22:36.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:22:36.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=37796 2026-02-18T08:22:36.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 27 = 26 -a 1 -le 37796 2026-02-18T08:22:36.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:22:36.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:22:36.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:22:36.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:22:36.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:22:36.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:22:36.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:22:36.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:22:36.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:22:36.758 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:22:36.758 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:22:36.758 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:22:36.758 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:36.758 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:22:36.758 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:22:36.758 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:22:36.758 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:22:36.758 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:22:36.761 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:22:36.761 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:22:36.761 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:22:36.761 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:22:36.761 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:22:36.761 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:36.762 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:22:36.762 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:22:36.762 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:22:36.762 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:22:36.762 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:22:36.762 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:22:36.762 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:22:36.762 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:22:36.762 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:22:36.762 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:36.762 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:22:36.763 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:22:36.763 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:22:36.763 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:22:36.763 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:22:36.763 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:22:36.763 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:22:36.763 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:22:36.763 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:22:36.763 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:22:36.763 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:36.763 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:22:36.763 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:22:36.763 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:22:36.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:22:36.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:22:36.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:22:36.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:22:36.772 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:22:36.772 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:22:36.772 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:22:36.772 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:22:36.829 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:22:36.829 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:22:36.829 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:22:36.832 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:22:36.832 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:22:36.832 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:22:36.832 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:36.832 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:22:36.832 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:22:36.832 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:22:36.832 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:22:36.832 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:22:36.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:22:36.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:22:36.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:22:36.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:22:36.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:36.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:22:36.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:22:36.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:22:36.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:22:36.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:22:36.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:22:36.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:22:36.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:22:36.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:22:36.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:22:36.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:36.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:22:36.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:22:36.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:22:36.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:22:36.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:22:36.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:22:36.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:22:36.843 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:22:36.843 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:22:36.843 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:22:36.843 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:22:36.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:22:36.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:22:36.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:22:36.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:22:36.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:22:36.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:22:36.904 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:36.904 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:22:36.904 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:22:36.904 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:22:36.904 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:22:36.904 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:22:36.904 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:22:36.904 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:22:36.904 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:22:36.904 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:22:36.904 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:36.904 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:22:36.905 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:22:36.905 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:22:36.905 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:22:36.905 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:22:36.905 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:22:36.905 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:22:36.905 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:22:36.905 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:22:36.905 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:22:36.905 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:36.905 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:22:36.905 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:22:36.905 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:22:36.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:22:36.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:22:36.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:22:36.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:22:36.914 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:22:36.915 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:22:36.915 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:22:36.915 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:22:36.971 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:22:36.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:22:36.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:22:36.974 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:22:36.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:22:36.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:22:36.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:36.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:22:36.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:22:36.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:22:36.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:22:36.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:22:36.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:22:36.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:22:36.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:22:36.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:22:36.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:36.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:22:36.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:22:36.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:22:36.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:22:36.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:22:36.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:22:36.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:22:36.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:22:36.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:22:36.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:22:36.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:36.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:22:36.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:22:36.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:22:36.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:22:36.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:22:36.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:22:36.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:22:36.986 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:22:36.986 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:22:36.987 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:22:36.987 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:22:37.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:22:37.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:22:37.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:22:37.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:22:37.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:22:37.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:22:37.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:22:37.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:22:37.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:22:37.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:22:37.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:22:37.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:22:37.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:22:37.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:37.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:22:37.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:37.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 93 2026-02-18T08:22:37.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 95 2026-02-18T08:22:37.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:22:37.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=381, tag_tid=27, entry_tid=1], [object_number=380, tag_tid=27, entry_tid=0], [object_number=382, tag_tid=26, entry_tid=46994], [object_number=383, tag_tid=26, entry_tid=46991]]], state=connected] 2026-02-18T08:22:37.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=374, tag_tid=26, entry_tid=38150], [object_number=373, tag_tid=26, entry_tid=38149], [object_number=372, tag_tid=26, entry_tid=38148], [object_number=375, tag_tid=26, entry_tid=38147]]], state=connected] 2026-02-18T08:22:37.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:37.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=381, tag_tid=27, entry_tid=1' 2026-02-18T08:22:37.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:22:37.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:22:37.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:22:37.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:22:37.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:22:37.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:22:37.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:22:37.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:22:37.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:22:37.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:22:37.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:22:37.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:22:37.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:37.086 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:22:37.086 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:37.121 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 93 2026-02-18T08:22:37.121 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 95 2026-02-18T08:22:37.121 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:22:37.121 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=381, tag_tid=27, entry_tid=1], [object_number=380, tag_tid=27, entry_tid=0], [object_number=382, tag_tid=26, entry_tid=46994], [object_number=383, tag_tid=26, entry_tid=46991]]], state=connected] 2026-02-18T08:22:37.121 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=374, tag_tid=26, entry_tid=38150], [object_number=373, tag_tid=26, entry_tid=38149], [object_number=372, tag_tid=26, entry_tid=38148], [object_number=375, tag_tid=26, entry_tid=38147]]], state=connected] 2026-02-18T08:22:37.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:37.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=374, tag_tid=26, entry_tid=38150' 2026-02-18T08:22:37.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=381, tag_tid=27, entry_tid=1' -a 'object_number=381, tag_tid=27, entry_tid=1' = 'object_number=374, tag_tid=26, entry_tid=38150' 2026-02-18T08:22:37.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=374, tag_tid=26, entry_tid=38150' '!=' 'object_number=372, tag_tid=26, entry_tid=37796' 2026-02-18T08:22:37.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:22:37.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=374, tag_tid=26, entry_tid=38150' = 'object_number=372, tag_tid=26, entry_tid=37796' 2026-02-18T08:22:37.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=374, tag_tid=26, entry_tid=38150' 2026-02-18T08:22:37.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=381, tag_tid=27, entry_tid=1' 2026-02-18T08:22:37.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:22:37.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:22:37.129 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=27 2026-02-18T08:22:37.129 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=374, tag_tid=26, entry_tid=38150' 2026-02-18T08:22:37.129 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:22:37.130 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:22:37.131 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=26 2026-02-18T08:22:37.132 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=381, tag_tid=27, entry_tid=1' 2026-02-18T08:22:37.132 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:22:37.132 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:22:37.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:22:37.134 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=374, tag_tid=26, entry_tid=38150' 2026-02-18T08:22:37.134 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:22:37.134 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:22:37.136 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=38150 2026-02-18T08:22:37.136 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 27 = 26 -a 1 -le 38150 2026-02-18T08:22:37.136 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:22:37.136 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:22:37.136 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:22:37.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:22:37.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:22:37.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:22:37.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:22:37.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:22:37.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:22:37.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:22:37.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:22:37.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:22:37.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:37.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:22:37.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:22:37.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:22:37.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:22:37.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:22:37.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:22:37.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:22:37.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:22:37.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:22:37.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:22:37.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:37.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:22:37.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:22:37.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:22:37.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:22:37.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:22:37.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:22:37.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:22:37.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:22:37.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:22:37.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:37.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:22:37.344 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:22:37.344 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:22:37.344 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:22:37.344 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:22:37.344 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:22:37.344 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:22:37.344 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:22:37.344 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:22:37.344 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:22:37.344 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:37.344 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:22:37.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:22:37.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:22:37.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:22:37.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:22:37.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:22:37.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:22:37.353 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:22:37.353 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:22:37.353 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:22:37.353 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:22:37.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:22:37.412 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:22:37.412 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:22:37.415 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:22:37.415 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:22:37.415 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:22:37.415 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:37.415 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:22:37.415 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:22:37.415 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:22:37.415 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:22:37.416 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:22:37.416 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:22:37.416 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:22:37.416 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:22:37.416 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:22:37.416 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:37.416 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:22:37.416 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:22:37.416 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:22:37.416 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:22:37.416 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:22:37.416 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:22:37.417 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:22:37.417 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:22:37.417 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:22:37.417 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:22:37.417 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:37.417 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:22:37.417 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:22:37.417 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:22:37.417 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:22:37.417 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:22:37.417 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:22:37.417 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:22:37.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:22:37.427 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:22:37.427 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:22:37.427 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:22:37.489 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:22:37.489 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:22:37.489 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:22:37.492 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:22:37.492 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:22:37.492 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:22:37.492 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:37.492 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:22:37.492 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:22:37.492 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:22:37.492 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:22:37.493 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:22:37.493 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:22:37.493 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:22:37.493 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:22:37.493 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:22:37.493 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:37.493 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:22:37.493 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:22:37.493 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:22:37.493 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:22:37.493 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:22:37.493 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:22:37.493 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:22:37.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:22:37.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:22:37.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:22:37.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:37.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:22:37.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:22:37.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:22:37.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:22:37.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:22:37.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:22:37.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:22:37.503 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:22:37.503 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:22:37.503 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:22:37.503 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:22:37.551 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:22:37.551 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:22:37.551 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:22:37.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:22:37.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:22:37.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:22:37.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:37.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:22:37.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:22:37.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:22:37.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:22:37.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:22:37.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:22:37.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:22:37.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:22:37.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:22:37.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:37.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:22:37.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:22:37.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:22:37.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:22:37.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:22:37.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:22:37.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:22:37.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:22:37.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:22:37.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:22:37.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:37.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:22:37.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:22:37.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:22:37.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:22:37.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:22:37.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:22:37.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:22:37.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:22:37.565 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:22:37.565 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:22:37.565 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:22:37.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:22:37.628 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:22:37.628 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:22:37.628 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:22:37.629 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:22:37.629 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:22:37.629 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:22:37.629 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:22:37.629 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:22:37.629 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:22:37.629 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:22:37.629 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:22:37.629 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:22:37.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:37.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:22:37.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:37.660 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 93 2026-02-18T08:22:37.660 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 95 2026-02-18T08:22:37.660 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:22:37.660 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=381, tag_tid=27, entry_tid=1], [object_number=380, tag_tid=27, entry_tid=0], [object_number=382, tag_tid=26, entry_tid=46994], [object_number=383, tag_tid=26, entry_tid=46991]]], state=connected] 2026-02-18T08:22:37.660 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=374, tag_tid=26, entry_tid=38478], [object_number=373, tag_tid=26, entry_tid=38477], [object_number=372, tag_tid=26, entry_tid=38476], [object_number=375, tag_tid=26, entry_tid=38475]]], state=connected] 2026-02-18T08:22:37.662 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:37.664 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=381, tag_tid=27, entry_tid=1' 2026-02-18T08:22:37.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:22:37.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:22:37.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:22:37.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:22:37.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:22:37.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:22:37.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:22:37.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:22:37.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:22:37.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:22:37.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:22:37.667 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:22:37.667 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:37.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:22:37.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:37.696 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 93 2026-02-18T08:22:37.697 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 95 2026-02-18T08:22:37.697 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:22:37.697 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=381, tag_tid=27, entry_tid=1], [object_number=380, tag_tid=27, entry_tid=0], [object_number=382, tag_tid=26, entry_tid=46994], [object_number=383, tag_tid=26, entry_tid=46991]]], state=connected] 2026-02-18T08:22:37.697 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=374, tag_tid=26, entry_tid=38478], [object_number=373, tag_tid=26, entry_tid=38477], [object_number=372, tag_tid=26, entry_tid=38476], [object_number=375, tag_tid=26, entry_tid=38475]]], state=connected] 2026-02-18T08:22:37.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:37.701 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=374, tag_tid=26, entry_tid=38478' 2026-02-18T08:22:37.701 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=381, tag_tid=27, entry_tid=1' -a 'object_number=381, tag_tid=27, entry_tid=1' = 'object_number=374, tag_tid=26, entry_tid=38478' 2026-02-18T08:22:37.701 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=374, tag_tid=26, entry_tid=38478' '!=' 'object_number=374, tag_tid=26, entry_tid=38150' 2026-02-18T08:22:37.701 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:22:37.701 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=374, tag_tid=26, entry_tid=38478' = 'object_number=374, tag_tid=26, entry_tid=38150' 2026-02-18T08:22:37.701 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=374, tag_tid=26, entry_tid=38478' 2026-02-18T08:22:37.702 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=381, tag_tid=27, entry_tid=1' 2026-02-18T08:22:37.702 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:22:37.702 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:22:37.703 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=27 2026-02-18T08:22:37.704 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=374, tag_tid=26, entry_tid=38478' 2026-02-18T08:22:37.704 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:22:37.704 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:22:37.705 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=26 2026-02-18T08:22:37.706 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=381, tag_tid=27, entry_tid=1' 2026-02-18T08:22:37.706 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:22:37.706 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:22:37.707 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:22:37.708 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=374, tag_tid=26, entry_tid=38478' 2026-02-18T08:22:37.708 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:22:37.708 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:22:37.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=38478 2026-02-18T08:22:37.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 27 = 26 -a 1 -le 38478 2026-02-18T08:22:37.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:22:37.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:22:37.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:22:37.912 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:22:37.912 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:22:37.912 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:22:37.912 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:22:37.912 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:22:37.912 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:22:37.912 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:22:37.912 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:22:37.913 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:22:37.913 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:37.913 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:22:37.913 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:22:37.913 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:22:37.913 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:22:37.913 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:22:37.913 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:22:37.916 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:22:37.916 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:22:37.916 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:22:37.916 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:22:37.916 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:37.916 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:22:37.916 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:22:37.917 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:22:37.917 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:22:37.917 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:22:37.917 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:22:37.917 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:22:37.917 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:22:37.917 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:22:37.917 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:37.917 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:22:37.917 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:22:37.917 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:22:37.917 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:22:37.917 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:22:37.918 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:22:37.918 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:22:37.918 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:22:37.918 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:22:37.918 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:22:37.918 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:37.918 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:22:37.918 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:22:37.918 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:22:37.918 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:22:37.918 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:22:37.918 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:22:37.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:22:37.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:22:37.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:22:37.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:22:37.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:22:37.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:22:37.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:22:37.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:22:37.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:22:37.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:22:37.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:22:37.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:37.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:22:37.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:22:37.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:22:37.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:22:37.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:22:37.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:22:37.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:22:37.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:22:37.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:22:37.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:37.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:22:37.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:22:37.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:22:37.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:22:37.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:22:37.982 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:22:37.982 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:22:37.982 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:22:37.982 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:22:37.982 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:22:37.982 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:37.982 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:22:37.982 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:22:37.982 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:22:37.982 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:22:37.982 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:22:37.982 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:22:37.982 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:22:37.990 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:22:37.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:22:37.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:22:37.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:22:38.040 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:22:38.040 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:22:38.040 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:22:38.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:22:38.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:22:38.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:22:38.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:38.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:22:38.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:22:38.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:22:38.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:22:38.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:22:38.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:22:38.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:22:38.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:22:38.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:22:38.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:38.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:22:38.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:22:38.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:22:38.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:22:38.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:22:38.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:22:38.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:22:38.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:22:38.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:22:38.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:22:38.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:38.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:22:38.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:22:38.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:22:38.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:22:38.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:22:38.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:22:38.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:22:38.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:22:38.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:22:38.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:22:38.055 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:22:38.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:22:38.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:22:38.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:22:38.106 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:22:38.106 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:22:38.106 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:22:38.106 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:38.106 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:22:38.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:22:38.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:22:38.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:22:38.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:22:38.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:22:38.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:22:38.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:22:38.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:22:38.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:38.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:22:38.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:22:38.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:22:38.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:22:38.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:22:38.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:22:38.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:22:38.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:22:38.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:22:38.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:22:38.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:38.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:22:38.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:22:38.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:22:38.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:22:38.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:22:38.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:22:38.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:22:38.118 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:22:38.118 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:22:38.118 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:22:38.119 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:22:38.178 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:22:38.179 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:22:38.179 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:22:38.179 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:22:38.180 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:22:38.180 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:22:38.180 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:22:38.180 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:22:38.180 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:22:38.180 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:22:38.180 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:22:38.180 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:22:38.180 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:22:38.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:38.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:22:38.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:38.413 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 93 2026-02-18T08:22:38.413 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 95 2026-02-18T08:22:38.413 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:22:38.413 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=381, tag_tid=27, entry_tid=1], [object_number=380, tag_tid=27, entry_tid=0], [object_number=382, tag_tid=26, entry_tid=46994], [object_number=383, tag_tid=26, entry_tid=46991]]], state=connected] 2026-02-18T08:22:38.414 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=375, tag_tid=26, entry_tid=38811], [object_number=374, tag_tid=26, entry_tid=38810], [object_number=373, tag_tid=26, entry_tid=38809], [object_number=372, tag_tid=26, entry_tid=38808]]], state=connected] 2026-02-18T08:22:38.416 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:38.417 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=381, tag_tid=27, entry_tid=1' 2026-02-18T08:22:38.419 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:22:38.419 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:22:38.419 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:22:38.419 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:22:38.419 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:22:38.419 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:22:38.419 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:22:38.419 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:22:38.420 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:22:38.420 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:22:38.420 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:22:38.420 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:22:38.420 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:38.421 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:22:38.421 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:38.449 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 93 2026-02-18T08:22:38.449 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 95 2026-02-18T08:22:38.449 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:22:38.449 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=381, tag_tid=27, entry_tid=1], [object_number=380, tag_tid=27, entry_tid=0], [object_number=382, tag_tid=26, entry_tid=46994], [object_number=383, tag_tid=26, entry_tid=46991]]], state=connected] 2026-02-18T08:22:38.449 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=375, tag_tid=26, entry_tid=38811], [object_number=374, tag_tid=26, entry_tid=38810], [object_number=373, tag_tid=26, entry_tid=38809], [object_number=372, tag_tid=26, entry_tid=38808]]], state=connected] 2026-02-18T08:22:38.452 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:38.453 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=375, tag_tid=26, entry_tid=38811' 2026-02-18T08:22:38.454 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=381, tag_tid=27, entry_tid=1' -a 'object_number=381, tag_tid=27, entry_tid=1' = 'object_number=375, tag_tid=26, entry_tid=38811' 2026-02-18T08:22:38.454 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=375, tag_tid=26, entry_tid=38811' '!=' 'object_number=374, tag_tid=26, entry_tid=38478' 2026-02-18T08:22:38.454 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:22:38.454 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=375, tag_tid=26, entry_tid=38811' = 'object_number=374, tag_tid=26, entry_tid=38478' 2026-02-18T08:22:38.454 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=375, tag_tid=26, entry_tid=38811' 2026-02-18T08:22:38.455 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=381, tag_tid=27, entry_tid=1' 2026-02-18T08:22:38.455 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:22:38.455 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:22:38.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=27 2026-02-18T08:22:38.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=375, tag_tid=26, entry_tid=38811' 2026-02-18T08:22:38.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:22:38.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:22:38.458 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=26 2026-02-18T08:22:38.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=381, tag_tid=27, entry_tid=1' 2026-02-18T08:22:38.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:22:38.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:22:38.460 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:22:38.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=375, tag_tid=26, entry_tid=38811' 2026-02-18T08:22:38.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:22:38.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:22:38.462 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=38811 2026-02-18T08:22:38.463 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 27 = 26 -a 1 -le 38811 2026-02-18T08:22:38.463 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:22:38.463 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:22:38.463 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:22:38.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:22:38.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:22:38.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:22:38.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:22:38.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:22:38.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:22:38.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:22:38.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:22:38.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:22:38.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:38.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:22:38.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:22:38.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:22:38.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:22:38.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:22:38.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:22:38.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:22:38.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:22:38.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:22:38.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:22:38.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:38.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:22:38.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:22:38.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:22:38.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:22:38.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:22:38.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:22:38.670 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:22:38.670 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:22:38.670 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:22:38.670 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:38.670 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:22:38.670 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:22:38.670 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:22:38.670 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:22:38.670 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:22:38.670 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:22:38.670 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:22:38.670 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:22:38.671 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:22:38.671 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:22:38.671 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:38.671 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:22:38.671 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:22:38.671 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:22:38.671 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:22:38.671 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:22:38.671 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:22:38.671 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:22:38.679 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:22:38.680 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:22:38.680 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:22:38.680 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:22:38.728 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:22:38.729 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:22:38.729 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:22:38.732 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:22:38.732 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:22:38.732 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:22:38.732 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:38.732 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:22:38.732 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:22:38.732 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:22:38.732 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:22:38.732 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:22:38.732 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:22:38.733 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:22:38.733 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:22:38.733 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:22:38.733 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:38.733 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:22:38.733 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:22:38.733 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:22:38.733 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:22:38.733 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:22:38.733 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:22:38.733 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:22:38.733 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:22:38.734 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:22:38.734 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:22:38.734 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:38.734 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:22:38.734 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:22:38.734 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:22:38.734 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:22:38.734 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:22:38.734 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:22:38.734 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:22:38.743 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:22:38.743 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:22:38.743 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:22:38.743 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:22:38.799 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:22:38.799 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:22:38.800 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:22:38.803 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:22:38.803 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:22:38.803 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:22:38.803 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:38.803 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:22:38.803 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:22:38.803 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:22:38.803 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:22:38.803 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:22:38.804 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:22:38.804 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:22:38.804 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:22:38.804 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:22:38.804 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:38.804 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:22:38.804 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:22:38.804 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:22:38.804 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:22:38.804 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:22:38.804 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:22:38.804 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:22:38.805 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:22:38.805 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:22:38.805 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:22:38.805 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:38.805 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:22:38.805 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:22:38.805 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:22:38.805 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:22:38.805 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:22:38.805 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:22:38.805 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:22:38.814 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:22:38.814 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:22:38.814 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:22:38.814 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:22:38.862 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:22:38.863 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:22:38.863 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:22:38.865 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:22:38.866 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:22:38.866 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:22:38.866 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:38.866 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:22:38.866 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:22:38.866 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:22:38.866 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:22:38.866 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:22:38.866 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:22:38.866 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:22:38.866 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:22:38.867 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:22:38.867 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:38.867 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:22:38.867 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:22:38.867 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:22:38.867 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:22:38.867 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:22:38.867 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:22:38.867 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:22:38.867 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:22:38.867 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:22:38.867 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:22:38.868 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:38.868 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:22:38.868 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:22:38.868 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:22:38.868 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:22:38.868 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:22:38.868 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:22:38.868 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:22:38.875 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:22:38.876 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:22:38.876 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:22:38.876 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:22:38.936 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:22:38.936 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:22:38.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:22:38.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:22:38.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:22:38.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:22:38.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:22:38.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:22:38.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:22:38.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:22:38.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:22:38.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:22:38.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:22:38.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:38.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:22:38.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:38.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 93 2026-02-18T08:22:38.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 95 2026-02-18T08:22:38.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:22:38.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=381, tag_tid=27, entry_tid=1], [object_number=380, tag_tid=27, entry_tid=0], [object_number=382, tag_tid=26, entry_tid=46994], [object_number=383, tag_tid=26, entry_tid=46991]]], state=connected] 2026-02-18T08:22:38.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=374, tag_tid=26, entry_tid=39274], [object_number=373, tag_tid=26, entry_tid=39273], [object_number=372, tag_tid=26, entry_tid=39272], [object_number=375, tag_tid=26, entry_tid=39271]]], state=connected] 2026-02-18T08:22:38.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:38.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=381, tag_tid=27, entry_tid=1' 2026-02-18T08:22:38.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:22:38.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:22:38.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:22:38.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:22:38.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:22:38.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:22:38.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:22:38.982 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:22:38.982 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:22:38.982 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:22:38.982 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:22:38.982 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:22:38.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:38.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:22:38.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:39.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 93 2026-02-18T08:22:39.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 95 2026-02-18T08:22:39.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:22:39.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=381, tag_tid=27, entry_tid=1], [object_number=380, tag_tid=27, entry_tid=0], [object_number=382, tag_tid=26, entry_tid=46994], [object_number=383, tag_tid=26, entry_tid=46991]]], state=connected] 2026-02-18T08:22:39.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=374, tag_tid=26, entry_tid=39274], [object_number=373, tag_tid=26, entry_tid=39273], [object_number=372, tag_tid=26, entry_tid=39272], [object_number=375, tag_tid=26, entry_tid=39271]]], state=connected] 2026-02-18T08:22:39.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:39.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=374, tag_tid=26, entry_tid=39274' 2026-02-18T08:22:39.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=381, tag_tid=27, entry_tid=1' -a 'object_number=381, tag_tid=27, entry_tid=1' = 'object_number=374, tag_tid=26, entry_tid=39274' 2026-02-18T08:22:39.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=374, tag_tid=26, entry_tid=39274' '!=' 'object_number=375, tag_tid=26, entry_tid=38811' 2026-02-18T08:22:39.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:22:39.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=374, tag_tid=26, entry_tid=39274' = 'object_number=375, tag_tid=26, entry_tid=38811' 2026-02-18T08:22:39.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=374, tag_tid=26, entry_tid=39274' 2026-02-18T08:22:39.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=381, tag_tid=27, entry_tid=1' 2026-02-18T08:22:39.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:22:39.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:22:39.018 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=27 2026-02-18T08:22:39.019 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=374, tag_tid=26, entry_tid=39274' 2026-02-18T08:22:39.019 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:22:39.019 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:22:39.020 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=26 2026-02-18T08:22:39.021 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=381, tag_tid=27, entry_tid=1' 2026-02-18T08:22:39.021 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:22:39.021 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:22:39.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:22:39.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=374, tag_tid=26, entry_tid=39274' 2026-02-18T08:22:39.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:22:39.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:22:39.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=39274 2026-02-18T08:22:39.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 27 = 26 -a 1 -le 39274 2026-02-18T08:22:39.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:22:39.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:22:39.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:22:39.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:22:39.227 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:22:39.227 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:22:39.227 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:22:39.227 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:22:39.227 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:22:39.227 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:22:39.227 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:22:39.227 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:22:39.227 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:39.227 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:22:39.227 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:22:39.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:22:39.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:22:39.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:22:39.231 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:22:39.231 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:22:39.231 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:22:39.231 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:22:39.231 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:22:39.231 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:39.231 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:22:39.231 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:22:39.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:22:39.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:22:39.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:22:39.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:22:39.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:22:39.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:22:39.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:22:39.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:39.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:22:39.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:22:39.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:22:39.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:22:39.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:22:39.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:22:39.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:22:39.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:22:39.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:22:39.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:22:39.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:39.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:22:39.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:22:39.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:22:39.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:22:39.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:22:39.234 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:22:39.234 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:22:39.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:22:39.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:22:39.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:22:39.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:22:39.292 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:22:39.293 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:22:39.293 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:22:39.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:22:39.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:22:39.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:22:39.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:39.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:22:39.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:22:39.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:22:39.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:22:39.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:22:39.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:22:39.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:22:39.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:22:39.297 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:22:39.297 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:39.297 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:22:39.297 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:22:39.297 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:22:39.297 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:22:39.297 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:22:39.297 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:22:39.297 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:22:39.297 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:22:39.297 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:22:39.297 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:22:39.298 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:39.298 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:22:39.298 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:22:39.298 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:22:39.298 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:22:39.298 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:22:39.298 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:22:39.298 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:22:39.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:22:39.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:22:39.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:22:39.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:22:39.356 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:22:39.356 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:22:39.356 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:22:39.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:22:39.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:22:39.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:22:39.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:39.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:22:39.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:22:39.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:22:39.360 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:22:39.360 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:22:39.360 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:22:39.360 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:22:39.360 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:22:39.360 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:22:39.360 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:39.360 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:22:39.360 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:22:39.360 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:22:39.360 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:22:39.360 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:22:39.361 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:22:39.361 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:22:39.361 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:22:39.361 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:22:39.361 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:22:39.361 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:39.361 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:22:39.361 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:22:39.361 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:22:39.361 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:22:39.361 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:22:39.361 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:22:39.361 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:22:39.370 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:22:39.371 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:22:39.371 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:22:39.371 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:22:39.419 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:22:39.419 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:22:39.419 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:22:39.422 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:22:39.422 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:22:39.422 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:22:39.422 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:39.422 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:22:39.422 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:22:39.423 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:22:39.423 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:22:39.423 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:22:39.423 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:22:39.423 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:22:39.423 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:22:39.423 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:22:39.423 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:39.423 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:22:39.423 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:22:39.423 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:22:39.423 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:22:39.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:22:39.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:22:39.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:22:39.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:22:39.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:22:39.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:22:39.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:39.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:22:39.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:22:39.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:22:39.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:22:39.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:22:39.425 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:22:39.425 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:22:39.433 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:22:39.434 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:22:39.434 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:22:39.434 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:22:39.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:22:39.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:22:39.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:22:39.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:22:39.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:22:39.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:22:39.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:22:39.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:22:39.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:22:39.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:22:39.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:22:39.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:22:39.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:22:39.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:39.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:22:39.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:39.525 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 93 2026-02-18T08:22:39.526 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 95 2026-02-18T08:22:39.526 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:22:39.526 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=381, tag_tid=27, entry_tid=1], [object_number=380, tag_tid=27, entry_tid=0], [object_number=382, tag_tid=26, entry_tid=46994], [object_number=383, tag_tid=26, entry_tid=46991]]], state=connected] 2026-02-18T08:22:39.526 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=373, tag_tid=26, entry_tid=39609], [object_number=372, tag_tid=26, entry_tid=39608], [object_number=375, tag_tid=26, entry_tid=39607], [object_number=374, tag_tid=26, entry_tid=39606]]], state=connected] 2026-02-18T08:22:39.528 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:39.530 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=381, tag_tid=27, entry_tid=1' 2026-02-18T08:22:39.531 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:22:39.531 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:22:39.531 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:22:39.531 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:22:39.531 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:22:39.532 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:22:39.532 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:22:39.532 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:22:39.532 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:22:39.532 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:22:39.532 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:22:39.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:22:39.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:39.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:22:39.534 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:39.562 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 93 2026-02-18T08:22:39.562 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 95 2026-02-18T08:22:39.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:22:39.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=381, tag_tid=27, entry_tid=1], [object_number=380, tag_tid=27, entry_tid=0], [object_number=382, tag_tid=26, entry_tid=46994], [object_number=383, tag_tid=26, entry_tid=46991]]], state=connected] 2026-02-18T08:22:39.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=373, tag_tid=26, entry_tid=39609], [object_number=372, tag_tid=26, entry_tid=39608], [object_number=375, tag_tid=26, entry_tid=39607], [object_number=374, tag_tid=26, entry_tid=39606]]], state=connected] 2026-02-18T08:22:39.565 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:39.567 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=373, tag_tid=26, entry_tid=39609' 2026-02-18T08:22:39.567 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=381, tag_tid=27, entry_tid=1' -a 'object_number=381, tag_tid=27, entry_tid=1' = 'object_number=373, tag_tid=26, entry_tid=39609' 2026-02-18T08:22:39.567 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=373, tag_tid=26, entry_tid=39609' '!=' 'object_number=374, tag_tid=26, entry_tid=39274' 2026-02-18T08:22:39.567 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:22:39.567 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=373, tag_tid=26, entry_tid=39609' = 'object_number=374, tag_tid=26, entry_tid=39274' 2026-02-18T08:22:39.567 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=373, tag_tid=26, entry_tid=39609' 2026-02-18T08:22:39.568 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=381, tag_tid=27, entry_tid=1' 2026-02-18T08:22:39.568 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:22:39.568 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:22:39.569 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=27 2026-02-18T08:22:39.569 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=373, tag_tid=26, entry_tid=39609' 2026-02-18T08:22:39.570 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:22:39.570 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:22:39.571 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=26 2026-02-18T08:22:39.572 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=381, tag_tid=27, entry_tid=1' 2026-02-18T08:22:39.572 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:22:39.572 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:22:39.573 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:22:39.574 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=373, tag_tid=26, entry_tid=39609' 2026-02-18T08:22:39.574 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:22:39.574 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:22:39.576 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=39609 2026-02-18T08:22:39.576 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 27 = 26 -a 1 -le 39609 2026-02-18T08:22:39.576 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:22:39.576 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:22:39.576 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:22:39.777 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:22:39.777 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:22:39.777 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:22:39.778 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:22:39.778 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:22:39.778 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:22:39.778 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:22:39.778 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:22:39.778 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:22:39.778 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:39.778 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:22:39.778 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:22:39.778 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:22:39.778 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:22:39.779 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:22:39.781 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:22:39.781 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:22:39.781 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:22:39.782 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:22:39.782 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:22:39.782 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:39.782 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:22:39.782 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:22:39.782 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:22:39.782 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:22:39.782 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:22:39.782 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:22:39.782 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:22:39.782 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:22:39.782 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:22:39.783 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:39.783 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:22:39.783 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:22:39.783 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:22:39.783 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:22:39.783 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:22:39.783 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:22:39.783 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:22:39.783 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:22:39.783 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:22:39.783 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:22:39.783 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:39.784 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:22:39.784 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:22:39.784 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:22:39.784 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:22:39.784 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:22:39.784 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:22:39.784 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:22:39.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:22:39.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:22:39.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:22:39.793 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:22:39.844 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:22:39.844 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:22:39.844 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:22:39.847 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:22:39.847 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:22:39.847 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:22:39.848 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:39.848 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:22:39.848 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:22:39.848 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:22:39.848 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:22:39.848 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:22:39.848 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:22:39.848 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:22:39.848 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:22:39.848 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:22:39.848 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:39.848 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:22:39.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:22:39.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:22:39.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:22:39.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:22:39.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:22:39.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:22:39.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:22:39.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:22:39.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:22:39.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:39.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:22:39.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:22:39.850 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:22:39.850 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:22:39.850 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:22:39.850 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:22:39.850 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:22:39.858 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:22:39.859 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:22:39.859 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:22:39.859 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:22:39.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:22:39.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:22:39.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:22:39.924 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:22:39.924 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:22:39.924 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:22:39.924 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:39.924 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:22:39.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:22:39.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:22:39.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:22:39.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:22:39.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:22:39.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:22:39.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:22:39.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:22:39.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:39.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:22:39.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:22:39.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:22:39.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:22:39.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:22:39.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:22:39.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:22:39.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:22:39.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:22:39.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:22:39.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:39.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:22:39.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:22:39.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:22:39.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:22:39.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:22:39.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:22:39.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:22:39.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:22:39.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:22:39.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:22:39.936 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:22:39.985 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:22:39.985 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:22:39.985 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:22:39.988 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:22:39.989 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:22:39.989 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:22:39.989 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:39.989 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:22:39.989 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:22:39.989 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:22:39.989 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:22:39.989 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:22:39.989 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:22:39.989 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:22:39.989 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:22:39.990 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:22:39.990 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:39.990 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:22:39.990 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:22:39.990 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:22:39.990 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:22:39.990 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:22:39.990 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:22:39.990 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:22:39.990 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:22:39.990 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:22:39.990 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:22:39.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:39.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:22:39.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:22:39.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:22:39.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:22:39.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:22:39.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:22:39.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:22:39.999 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:22:39.999 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:22:39.999 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:22:39.999 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:22:40.059 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:22:40.060 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:22:40.060 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:22:40.060 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:22:40.060 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:22:40.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:22:40.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:22:40.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:22:40.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:22:40.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:22:40.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:22:40.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:22:40.062 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:22:40.062 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:40.062 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:22:40.062 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:40.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 93 2026-02-18T08:22:40.092 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 95 2026-02-18T08:22:40.092 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:22:40.092 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=381, tag_tid=27, entry_tid=1], [object_number=380, tag_tid=27, entry_tid=0], [object_number=382, tag_tid=26, entry_tid=46994], [object_number=383, tag_tid=26, entry_tid=46991]]], state=connected] 2026-02-18T08:22:40.092 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=373, tag_tid=26, entry_tid=39957], [object_number=372, tag_tid=26, entry_tid=39956], [object_number=375, tag_tid=26, entry_tid=39955], [object_number=374, tag_tid=26, entry_tid=39954]]], state=connected] 2026-02-18T08:22:40.094 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:40.096 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=381, tag_tid=27, entry_tid=1' 2026-02-18T08:22:40.097 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:22:40.097 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:22:40.097 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:22:40.097 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:22:40.097 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:22:40.097 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:22:40.097 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:22:40.097 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:22:40.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:22:40.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:22:40.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:22:40.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:22:40.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:40.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:22:40.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:40.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 93 2026-02-18T08:22:40.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 95 2026-02-18T08:22:40.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:22:40.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=381, tag_tid=27, entry_tid=1], [object_number=380, tag_tid=27, entry_tid=0], [object_number=382, tag_tid=26, entry_tid=46994], [object_number=383, tag_tid=26, entry_tid=46991]]], state=connected] 2026-02-18T08:22:40.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=373, tag_tid=26, entry_tid=39957], [object_number=372, tag_tid=26, entry_tid=39956], [object_number=375, tag_tid=26, entry_tid=39955], [object_number=374, tag_tid=26, entry_tid=39954]]], state=connected] 2026-02-18T08:22:40.130 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:40.132 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=373, tag_tid=26, entry_tid=39957' 2026-02-18T08:22:40.132 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=381, tag_tid=27, entry_tid=1' -a 'object_number=381, tag_tid=27, entry_tid=1' = 'object_number=373, tag_tid=26, entry_tid=39957' 2026-02-18T08:22:40.132 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=373, tag_tid=26, entry_tid=39957' '!=' 'object_number=373, tag_tid=26, entry_tid=39609' 2026-02-18T08:22:40.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:22:40.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=373, tag_tid=26, entry_tid=39957' = 'object_number=373, tag_tid=26, entry_tid=39609' 2026-02-18T08:22:40.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=373, tag_tid=26, entry_tid=39957' 2026-02-18T08:22:40.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=381, tag_tid=27, entry_tid=1' 2026-02-18T08:22:40.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:22:40.134 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:22:40.134 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=27 2026-02-18T08:22:40.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=373, tag_tid=26, entry_tid=39957' 2026-02-18T08:22:40.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:22:40.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:22:40.136 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=26 2026-02-18T08:22:40.137 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=381, tag_tid=27, entry_tid=1' 2026-02-18T08:22:40.137 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:22:40.138 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:22:40.139 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:22:40.140 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=373, tag_tid=26, entry_tid=39957' 2026-02-18T08:22:40.140 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:22:40.140 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:22:40.141 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=39957 2026-02-18T08:22:40.141 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 27 = 26 -a 1 -le 39957 2026-02-18T08:22:40.141 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:22:40.142 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:22:40.142 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:22:40.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:22:40.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:22:40.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:22:40.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:22:40.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:22:40.344 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:22:40.344 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:22:40.344 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:22:40.344 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:22:40.344 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:40.344 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:22:40.344 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:22:40.344 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:22:40.344 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:22:40.344 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:22:40.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:22:40.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:22:40.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:22:40.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:22:40.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:22:40.348 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:40.348 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:22:40.348 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:22:40.348 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:22:40.348 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:22:40.348 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:22:40.348 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:22:40.348 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:22:40.348 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:22:40.348 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:22:40.348 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:40.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:22:40.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:22:40.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:22:40.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:22:40.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:22:40.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:22:40.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:22:40.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:22:40.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:22:40.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:22:40.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:40.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:22:40.350 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:22:40.350 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:22:40.350 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:22:40.350 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:22:40.350 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:22:40.350 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:22:40.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:22:40.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:22:40.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:22:40.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:22:40.409 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:22:40.409 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:22:40.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:22:40.412 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:22:40.413 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:22:40.413 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:22:40.413 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:40.413 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:22:40.413 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:22:40.413 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:22:40.413 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:22:40.413 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:22:40.413 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:22:40.413 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:22:40.413 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:22:40.414 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:22:40.414 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:40.414 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:22:40.414 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:22:40.414 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:22:40.414 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:22:40.414 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:22:40.414 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:22:40.414 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:22:40.414 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:22:40.414 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:22:40.414 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:22:40.414 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:40.415 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:22:40.415 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:22:40.415 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:22:40.415 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:22:40.415 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:22:40.415 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:22:40.415 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:22:40.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:22:40.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:22:40.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:22:40.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:22:40.481 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:22:40.481 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:22:40.481 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:22:40.484 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:22:40.484 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:22:40.484 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:22:40.484 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:40.485 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:22:40.485 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:22:40.485 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:22:40.485 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:22:40.485 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:22:40.485 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:22:40.485 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:22:40.485 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:22:40.485 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:22:40.485 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:40.485 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:22:40.485 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:22:40.485 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:22:40.486 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:22:40.486 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:22:40.486 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:22:40.486 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:22:40.486 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:22:40.486 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:22:40.486 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:22:40.486 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:40.486 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:22:40.486 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:22:40.486 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:22:40.486 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:22:40.487 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:22:40.487 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:22:40.487 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:22:40.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:22:40.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:22:40.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:22:40.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:22:40.553 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:22:40.553 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:22:40.553 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:22:40.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:22:40.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:22:40.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:22:40.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:40.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:22:40.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:22:40.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:22:40.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:22:40.557 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:22:40.557 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:22:40.557 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:22:40.557 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:22:40.557 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:22:40.557 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:40.557 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:22:40.557 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:22:40.557 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:22:40.557 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:22:40.557 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:22:40.557 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:22:40.558 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:22:40.558 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:22:40.558 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:22:40.558 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:22:40.558 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:40.558 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:22:40.558 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:22:40.558 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:22:40.558 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:22:40.558 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:22:40.558 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:22:40.558 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:22:40.566 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:22:40.567 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:22:40.567 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:22:40.567 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:22:40.635 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:22:40.637 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:22:40.637 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:22:40.637 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:22:40.637 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:22:40.637 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:22:40.637 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:22:40.637 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:22:40.637 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:22:40.637 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:22:40.637 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:22:40.638 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:22:40.638 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:22:40.638 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:40.638 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:22:40.638 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:40.678 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 93 2026-02-18T08:22:40.678 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 95 2026-02-18T08:22:40.678 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:22:40.678 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=381, tag_tid=27, entry_tid=1], [object_number=380, tag_tid=27, entry_tid=0], [object_number=382, tag_tid=26, entry_tid=46994], [object_number=383, tag_tid=26, entry_tid=46991]]], state=connected] 2026-02-18T08:22:40.679 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=372, tag_tid=26, entry_tid=40348], [object_number=375, tag_tid=26, entry_tid=40347], [object_number=374, tag_tid=26, entry_tid=40346], [object_number=373, tag_tid=26, entry_tid=40345]]], state=connected] 2026-02-18T08:22:40.682 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:40.684 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=381, tag_tid=27, entry_tid=1' 2026-02-18T08:22:40.685 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:22:40.685 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:22:40.685 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:22:40.685 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:22:40.685 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:22:40.685 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:22:40.686 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:22:40.686 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:22:40.686 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:22:40.686 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:22:40.686 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:22:40.686 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:22:40.687 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:40.687 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:22:40.688 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:40.725 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 93 2026-02-18T08:22:40.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 95 2026-02-18T08:22:40.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:22:40.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=381, tag_tid=27, entry_tid=1], [object_number=380, tag_tid=27, entry_tid=0], [object_number=382, tag_tid=26, entry_tid=46994], [object_number=383, tag_tid=26, entry_tid=46991]]], state=connected] 2026-02-18T08:22:40.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=372, tag_tid=26, entry_tid=40348], [object_number=375, tag_tid=26, entry_tid=40347], [object_number=374, tag_tid=26, entry_tid=40346], [object_number=373, tag_tid=26, entry_tid=40345]]], state=connected] 2026-02-18T08:22:40.729 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:40.731 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=372, tag_tid=26, entry_tid=40348' 2026-02-18T08:22:40.732 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=381, tag_tid=27, entry_tid=1' -a 'object_number=381, tag_tid=27, entry_tid=1' = 'object_number=372, tag_tid=26, entry_tid=40348' 2026-02-18T08:22:40.732 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=372, tag_tid=26, entry_tid=40348' '!=' 'object_number=373, tag_tid=26, entry_tid=39957' 2026-02-18T08:22:40.732 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:22:40.732 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=372, tag_tid=26, entry_tid=40348' = 'object_number=373, tag_tid=26, entry_tid=39957' 2026-02-18T08:22:40.732 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=372, tag_tid=26, entry_tid=40348' 2026-02-18T08:22:40.733 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=381, tag_tid=27, entry_tid=1' 2026-02-18T08:22:40.733 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:22:40.733 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:22:40.734 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=27 2026-02-18T08:22:40.735 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=372, tag_tid=26, entry_tid=40348' 2026-02-18T08:22:40.735 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:22:40.735 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:22:40.736 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=26 2026-02-18T08:22:40.737 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=381, tag_tid=27, entry_tid=1' 2026-02-18T08:22:40.737 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:22:40.737 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:22:40.738 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:22:40.739 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=372, tag_tid=26, entry_tid=40348' 2026-02-18T08:22:40.739 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:22:40.739 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:22:40.741 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=40348 2026-02-18T08:22:40.741 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 27 = 26 -a 1 -le 40348 2026-02-18T08:22:40.741 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:22:40.741 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:22:40.742 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:22:40.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:22:40.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:22:40.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:22:40.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:22:40.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:22:40.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:22:40.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:22:40.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:22:40.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:22:40.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:40.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:22:40.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:22:40.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:22:40.944 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:22:40.944 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:22:40.946 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:22:40.946 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:22:40.946 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:22:40.947 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:22:40.947 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:22:40.947 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:40.947 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:22:40.947 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:22:40.947 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:22:40.947 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:22:40.947 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:22:40.947 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:22:40.947 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:22:40.947 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:22:40.947 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:22:40.948 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:40.948 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:22:40.948 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:22:40.948 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:22:40.948 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:22:40.948 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:22:40.948 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:22:40.948 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:22:40.948 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:22:40.948 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:22:40.948 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:22:40.948 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:40.949 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:22:40.949 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:22:40.949 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:22:40.949 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:22:40.949 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:22:40.949 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:22:40.949 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:22:40.956 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:22:40.956 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:22:40.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:22:40.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:22:41.004 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:22:41.004 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:22:41.005 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:22:41.007 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:22:41.007 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:22:41.007 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:22:41.008 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:41.008 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:22:41.008 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:22:41.008 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:22:41.008 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:22:41.008 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:22:41.008 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:22:41.008 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:22:41.008 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:22:41.008 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:22:41.008 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:41.008 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:22:41.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:22:41.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:22:41.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:22:41.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:22:41.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:22:41.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:22:41.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:22:41.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:22:41.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:22:41.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:41.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:22:41.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:22:41.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:22:41.010 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:22:41.010 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:22:41.010 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:22:41.010 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:22:41.018 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:22:41.019 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:22:41.019 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:22:41.019 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:22:41.068 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:22:41.068 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:22:41.068 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:22:41.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:22:41.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:22:41.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:22:41.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:41.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:22:41.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:22:41.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:22:41.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:22:41.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:22:41.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:22:41.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:22:41.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:22:41.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:22:41.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:41.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:22:41.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:22:41.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:22:41.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:22:41.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:22:41.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:22:41.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:22:41.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:22:41.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:22:41.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:22:41.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:41.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:22:41.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:22:41.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:22:41.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:22:41.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:22:41.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:22:41.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:22:41.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:22:41.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:22:41.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:22:41.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:22:41.132 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:22:41.132 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:22:41.132 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:22:41.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:22:41.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:22:41.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:22:41.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:41.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:22:41.136 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:22:41.136 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:22:41.136 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:22:41.136 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:22:41.136 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:22:41.136 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:22:41.136 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:22:41.136 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:22:41.136 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:41.136 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:22:41.136 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:22:41.136 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:22:41.137 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:22:41.137 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:22:41.137 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:22:41.137 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:22:41.137 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:22:41.137 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:22:41.137 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:22:41.137 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:41.137 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:22:41.137 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:22:41.137 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:22:41.137 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:22:41.137 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:22:41.138 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:22:41.138 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:22:41.146 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:22:41.147 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:22:41.147 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:22:41.147 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:22:41.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:22:41.217 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:22:41.217 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:22:41.218 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:22:41.218 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:22:41.218 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:22:41.218 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:22:41.218 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:22:41.218 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:22:41.218 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:22:41.218 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:22:41.219 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:22:41.219 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:22:41.219 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:41.220 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:22:41.220 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:41.256 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 93 2026-02-18T08:22:41.256 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 95 2026-02-18T08:22:41.256 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:22:41.256 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=381, tag_tid=27, entry_tid=1], [object_number=380, tag_tid=27, entry_tid=0], [object_number=382, tag_tid=26, entry_tid=46994], [object_number=383, tag_tid=26, entry_tid=46991]]], state=connected] 2026-02-18T08:22:41.256 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=375, tag_tid=26, entry_tid=40723], [object_number=374, tag_tid=26, entry_tid=40722], [object_number=373, tag_tid=26, entry_tid=40721], [object_number=372, tag_tid=26, entry_tid=40720]]], state=connected] 2026-02-18T08:22:41.260 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:41.261 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=381, tag_tid=27, entry_tid=1' 2026-02-18T08:22:41.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:22:41.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:22:41.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:22:41.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:22:41.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:22:41.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:22:41.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:22:41.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:22:41.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:22:41.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:22:41.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:22:41.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:22:41.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:41.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:22:41.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:41.302 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 93 2026-02-18T08:22:41.302 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 95 2026-02-18T08:22:41.302 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:22:41.302 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=381, tag_tid=27, entry_tid=1], [object_number=380, tag_tid=27, entry_tid=0], [object_number=382, tag_tid=26, entry_tid=46994], [object_number=383, tag_tid=26, entry_tid=46991]]], state=connected] 2026-02-18T08:22:41.302 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=375, tag_tid=26, entry_tid=40723], [object_number=374, tag_tid=26, entry_tid=40722], [object_number=373, tag_tid=26, entry_tid=40721], [object_number=372, tag_tid=26, entry_tid=40720]]], state=connected] 2026-02-18T08:22:41.304 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:41.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=375, tag_tid=26, entry_tid=40723' 2026-02-18T08:22:41.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=381, tag_tid=27, entry_tid=1' -a 'object_number=381, tag_tid=27, entry_tid=1' = 'object_number=375, tag_tid=26, entry_tid=40723' 2026-02-18T08:22:41.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=375, tag_tid=26, entry_tid=40723' '!=' 'object_number=372, tag_tid=26, entry_tid=40348' 2026-02-18T08:22:41.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:22:41.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=375, tag_tid=26, entry_tid=40723' = 'object_number=372, tag_tid=26, entry_tid=40348' 2026-02-18T08:22:41.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=375, tag_tid=26, entry_tid=40723' 2026-02-18T08:22:41.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=381, tag_tid=27, entry_tid=1' 2026-02-18T08:22:41.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:22:41.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:22:41.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=27 2026-02-18T08:22:41.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=375, tag_tid=26, entry_tid=40723' 2026-02-18T08:22:41.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:22:41.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:22:41.310 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=26 2026-02-18T08:22:41.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=381, tag_tid=27, entry_tid=1' 2026-02-18T08:22:41.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:22:41.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:22:41.312 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:22:41.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=375, tag_tid=26, entry_tid=40723' 2026-02-18T08:22:41.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:22:41.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:22:41.315 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=40723 2026-02-18T08:22:41.315 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 27 = 26 -a 1 -le 40723 2026-02-18T08:22:41.315 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:22:41.315 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:22:41.315 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:22:41.516 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:22:41.516 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:22:41.516 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:22:41.516 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:22:41.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:22:41.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:22:41.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:22:41.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:22:41.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:22:41.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:41.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:22:41.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:22:41.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:22:41.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:22:41.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:22:41.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:22:41.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:22:41.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:22:41.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:22:41.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:22:41.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:41.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:22:41.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:22:41.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:22:41.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:22:41.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:22:41.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:22:41.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:22:41.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:22:41.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:22:41.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:41.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:22:41.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:22:41.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:22:41.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:22:41.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:22:41.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:22:41.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:22:41.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:22:41.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:22:41.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:22:41.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:41.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:22:41.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:22:41.523 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:22:41.523 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:22:41.523 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:22:41.523 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:22:41.523 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:22:41.531 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:22:41.532 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:22:41.532 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:22:41.532 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:22:41.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:22:41.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:22:41.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:22:41.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:22:41.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:22:41.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:22:41.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:41.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:22:41.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:22:41.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:22:41.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:22:41.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:22:41.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:22:41.594 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:22:41.594 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:22:41.594 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:22:41.594 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:41.594 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:22:41.594 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:22:41.594 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:22:41.594 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:22:41.594 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:22:41.594 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:22:41.594 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:22:41.594 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:22:41.595 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:22:41.595 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:22:41.595 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:41.595 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:22:41.595 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:22:41.595 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:22:41.595 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:22:41.595 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:22:41.595 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:22:41.595 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:22:41.604 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:22:41.604 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:22:41.605 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:22:41.605 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:22:41.654 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:22:41.654 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:22:41.654 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:22:41.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:22:41.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:22:41.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:22:41.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:41.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:22:41.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:22:41.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:22:41.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:22:41.658 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:22:41.658 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:22:41.658 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:22:41.658 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:22:41.658 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:22:41.658 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:41.658 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:22:41.658 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:22:41.658 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:22:41.658 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:22:41.658 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:22:41.658 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:22:41.659 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:22:41.659 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:22:41.659 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:22:41.659 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:22:41.659 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:41.659 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:22:41.659 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:22:41.659 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:22:41.659 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:22:41.659 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:22:41.659 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:22:41.659 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:22:41.667 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:22:41.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:22:41.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:22:41.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:22:41.717 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:22:41.717 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:22:41.717 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:22:41.720 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:22:41.720 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:22:41.720 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:22:41.720 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:41.720 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:22:41.721 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:22:41.721 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:22:41.721 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:22:41.721 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:22:41.721 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:22:41.721 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:22:41.721 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:22:41.721 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:22:41.721 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:41.721 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:22:41.721 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:22:41.721 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:22:41.722 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:22:41.722 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:22:41.722 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:22:41.722 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:22:41.722 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:22:41.722 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:22:41.722 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:22:41.722 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:41.722 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:22:41.722 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:22:41.722 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:22:41.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:22:41.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:22:41.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:22:41.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:22:41.731 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:22:41.731 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:22:41.731 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:22:41.731 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:22:41.804 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:22:41.805 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:22:41.805 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:22:41.805 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:22:41.805 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:22:41.805 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:22:41.806 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:22:41.806 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:22:41.806 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:22:41.806 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:22:41.806 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:22:41.806 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:22:41.806 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:22:41.807 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:41.807 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:22:41.808 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:41.836 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 93 2026-02-18T08:22:41.836 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 95 2026-02-18T08:22:41.836 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:22:41.836 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=381, tag_tid=27, entry_tid=1], [object_number=380, tag_tid=27, entry_tid=0], [object_number=382, tag_tid=26, entry_tid=46994], [object_number=383, tag_tid=26, entry_tid=46991]]], state=connected] 2026-02-18T08:22:41.836 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=372, tag_tid=26, entry_tid=41100], [object_number=375, tag_tid=26, entry_tid=41099], [object_number=374, tag_tid=26, entry_tid=41098], [object_number=373, tag_tid=26, entry_tid=41097]]], state=connected] 2026-02-18T08:22:41.839 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:41.840 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=381, tag_tid=27, entry_tid=1' 2026-02-18T08:22:41.841 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:22:41.841 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:22:41.841 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:22:41.842 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:22:41.842 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:22:41.842 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:22:41.842 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:22:41.842 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:22:41.842 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:22:41.842 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:22:41.842 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:22:41.843 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:22:41.843 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:41.843 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:22:41.843 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:41.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 93 2026-02-18T08:22:41.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 95 2026-02-18T08:22:41.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:22:41.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=381, tag_tid=27, entry_tid=1], [object_number=380, tag_tid=27, entry_tid=0], [object_number=382, tag_tid=26, entry_tid=46994], [object_number=383, tag_tid=26, entry_tid=46991]]], state=connected] 2026-02-18T08:22:41.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=372, tag_tid=26, entry_tid=41100], [object_number=375, tag_tid=26, entry_tid=41099], [object_number=374, tag_tid=26, entry_tid=41098], [object_number=373, tag_tid=26, entry_tid=41097]]], state=connected] 2026-02-18T08:22:41.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:41.875 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=372, tag_tid=26, entry_tid=41100' 2026-02-18T08:22:41.875 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=381, tag_tid=27, entry_tid=1' -a 'object_number=381, tag_tid=27, entry_tid=1' = 'object_number=372, tag_tid=26, entry_tid=41100' 2026-02-18T08:22:41.875 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=372, tag_tid=26, entry_tid=41100' '!=' 'object_number=375, tag_tid=26, entry_tid=40723' 2026-02-18T08:22:41.875 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:22:41.876 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=372, tag_tid=26, entry_tid=41100' = 'object_number=375, tag_tid=26, entry_tid=40723' 2026-02-18T08:22:41.876 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=372, tag_tid=26, entry_tid=41100' 2026-02-18T08:22:41.876 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=381, tag_tid=27, entry_tid=1' 2026-02-18T08:22:41.876 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:22:41.876 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:22:41.877 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=27 2026-02-18T08:22:41.878 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=372, tag_tid=26, entry_tid=41100' 2026-02-18T08:22:41.878 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:22:41.878 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:22:41.879 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=26 2026-02-18T08:22:41.880 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=381, tag_tid=27, entry_tid=1' 2026-02-18T08:22:41.880 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:22:41.880 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:22:41.881 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:22:41.882 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=372, tag_tid=26, entry_tid=41100' 2026-02-18T08:22:41.882 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:22:41.882 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:22:41.884 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=41100 2026-02-18T08:22:41.884 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 27 = 26 -a 1 -le 41100 2026-02-18T08:22:41.884 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:22:41.884 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:22:41.884 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:22:42.086 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:22:42.086 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:22:42.086 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:22:42.086 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:22:42.086 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:22:42.086 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:22:42.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:22:42.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:22:42.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:22:42.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:42.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:22:42.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:22:42.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:22:42.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:22:42.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:22:42.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:22:42.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:22:42.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:22:42.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:22:42.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:22:42.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:42.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:22:42.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:22:42.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:22:42.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:22:42.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:22:42.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:22:42.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:22:42.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:22:42.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:22:42.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:42.092 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:22:42.092 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:22:42.092 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:22:42.092 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:22:42.092 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:22:42.092 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:22:42.092 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:22:42.092 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:22:42.092 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:22:42.092 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:22:42.092 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:42.092 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:22:42.093 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:22:42.093 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:22:42.093 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:22:42.093 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:22:42.093 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:22:42.093 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:22:42.100 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:22:42.100 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:22:42.101 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:22:42.101 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:22:42.149 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:22:42.149 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:22:42.149 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:22:42.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:22:42.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:22:42.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:22:42.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:42.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:22:42.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:22:42.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:22:42.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:22:42.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:22:42.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:22:42.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:22:42.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:22:42.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:22:42.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:42.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:22:42.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:22:42.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:22:42.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:22:42.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:22:42.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:22:42.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:22:42.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:22:42.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:22:42.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:22:42.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:42.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:22:42.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:22:42.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:22:42.155 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:22:42.155 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:22:42.155 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:22:42.155 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:22:42.163 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:22:42.163 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:22:42.163 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:22:42.163 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:22:42.218 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:22:42.218 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:22:42.218 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:22:42.220 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:22:42.221 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:22:42.221 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:22:42.221 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:42.221 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:22:42.221 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:22:42.221 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:22:42.221 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:22:42.221 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:22:42.221 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:22:42.221 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:22:42.221 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:22:42.222 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:22:42.222 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:42.222 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:22:42.222 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:22:42.222 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:22:42.222 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:22:42.222 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:22:42.222 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:22:42.222 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:22:42.222 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:22:42.222 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:22:42.223 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:22:42.223 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:42.223 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:22:42.223 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:22:42.223 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:22:42.223 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:22:42.223 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:22:42.223 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:22:42.223 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:22:42.231 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:22:42.231 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:22:42.231 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:22:42.231 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:22:42.279 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:22:42.279 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:22:42.279 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:22:42.282 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:22:42.282 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:22:42.282 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:22:42.283 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:42.283 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:22:42.283 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:22:42.283 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:22:42.283 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:22:42.283 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:22:42.283 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:22:42.283 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:22:42.283 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:22:42.283 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:22:42.283 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:42.284 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:22:42.284 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:22:42.284 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:22:42.284 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:22:42.284 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:22:42.284 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:22:42.284 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:22:42.284 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:22:42.284 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:22:42.284 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:22:42.284 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:42.284 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:22:42.285 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:22:42.285 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:22:42.285 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:22:42.285 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:22:42.285 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:22:42.285 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:22:42.293 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:22:42.294 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:22:42.294 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:22:42.294 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:22:42.355 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:22:42.356 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:22:42.356 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:22:42.357 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:22:42.357 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:22:42.357 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:22:42.357 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:22:42.357 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:22:42.357 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:22:42.357 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:22:42.357 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:22:42.357 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:22:42.357 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:22:42.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:42.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:42.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:22:42.387 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 94 2026-02-18T08:22:42.387 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 95 2026-02-18T08:22:42.388 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:22:42.388 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=381, tag_tid=27, entry_tid=1], [object_number=380, tag_tid=27, entry_tid=0], [object_number=382, tag_tid=26, entry_tid=46994], [object_number=383, tag_tid=26, entry_tid=46991]]], state=connected] 2026-02-18T08:22:42.388 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=376, tag_tid=26, entry_tid=41340], [object_number=379, tag_tid=26, entry_tid=41339], [object_number=378, tag_tid=26, entry_tid=41338], [object_number=377, tag_tid=26, entry_tid=41337]]], state=connected] 2026-02-18T08:22:42.390 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:42.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=381, tag_tid=27, entry_tid=1' 2026-02-18T08:22:42.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:22:42.393 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:22:42.393 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:22:42.393 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:22:42.393 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:22:42.393 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:22:42.393 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:22:42.393 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:22:42.393 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:22:42.393 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:22:42.394 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:22:42.394 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:22:42.395 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:42.395 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:22:42.395 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:42.422 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 94 2026-02-18T08:22:42.422 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 95 2026-02-18T08:22:42.422 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:22:42.422 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=381, tag_tid=27, entry_tid=1], [object_number=380, tag_tid=27, entry_tid=0], [object_number=382, tag_tid=26, entry_tid=46994], [object_number=383, tag_tid=26, entry_tid=46991]]], state=connected] 2026-02-18T08:22:42.422 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=376, tag_tid=26, entry_tid=41340], [object_number=379, tag_tid=26, entry_tid=41339], [object_number=378, tag_tid=26, entry_tid=41338], [object_number=377, tag_tid=26, entry_tid=41337]]], state=connected] 2026-02-18T08:22:42.425 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:42.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=376, tag_tid=26, entry_tid=41340' 2026-02-18T08:22:42.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=381, tag_tid=27, entry_tid=1' -a 'object_number=381, tag_tid=27, entry_tid=1' = 'object_number=376, tag_tid=26, entry_tid=41340' 2026-02-18T08:22:42.427 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=376, tag_tid=26, entry_tid=41340' '!=' 'object_number=372, tag_tid=26, entry_tid=41100' 2026-02-18T08:22:42.427 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:22:42.427 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=376, tag_tid=26, entry_tid=41340' = 'object_number=372, tag_tid=26, entry_tid=41100' 2026-02-18T08:22:42.427 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=376, tag_tid=26, entry_tid=41340' 2026-02-18T08:22:42.427 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=381, tag_tid=27, entry_tid=1' 2026-02-18T08:22:42.427 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:22:42.428 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:22:42.428 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=27 2026-02-18T08:22:42.429 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=376, tag_tid=26, entry_tid=41340' 2026-02-18T08:22:42.429 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:22:42.429 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:22:42.431 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=26 2026-02-18T08:22:42.432 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=381, tag_tid=27, entry_tid=1' 2026-02-18T08:22:42.432 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:22:42.432 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:22:42.433 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:22:42.434 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=376, tag_tid=26, entry_tid=41340' 2026-02-18T08:22:42.434 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:22:42.434 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:22:42.436 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=41340 2026-02-18T08:22:42.436 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 27 = 26 -a 1 -le 41340 2026-02-18T08:22:42.436 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:22:42.436 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:22:42.436 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:22:42.637 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:22:42.638 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:22:42.638 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:22:42.638 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:22:42.638 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:22:42.638 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:22:42.638 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:22:42.638 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:22:42.638 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:22:42.638 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:42.638 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:22:42.639 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:22:42.639 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:22:42.639 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:22:42.639 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:22:42.639 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:22:42.642 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:22:42.642 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:22:42.642 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:22:42.642 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:22:42.642 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:42.642 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:22:42.642 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:22:42.642 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:22:42.642 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:22:42.642 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:22:42.643 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:22:42.643 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:22:42.643 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:22:42.643 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:22:42.643 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:42.643 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:22:42.643 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:22:42.643 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:22:42.643 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:22:42.643 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:22:42.643 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:22:42.643 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:22:42.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:22:42.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:22:42.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:22:42.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:42.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:22:42.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:22:42.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:22:42.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:22:42.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:22:42.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:22:42.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:22:42.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:22:42.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:22:42.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:22:42.653 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:22:42.700 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:22:42.700 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:22:42.700 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:22:42.703 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:22:42.703 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:22:42.703 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:22:42.703 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:42.703 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:22:42.704 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:22:42.704 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:22:42.704 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:22:42.704 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:22:42.704 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:22:42.704 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:22:42.704 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:22:42.704 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:22:42.704 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:42.704 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:22:42.704 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:22:42.704 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:22:42.705 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:22:42.705 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:22:42.705 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:22:42.705 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:22:42.705 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:22:42.705 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:22:42.705 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:22:42.705 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:42.705 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:22:42.705 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:22:42.705 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:22:42.705 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:22:42.706 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:22:42.706 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:22:42.706 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:22:42.714 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:22:42.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:22:42.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:22:42.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:22:42.763 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:22:42.763 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:22:42.763 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:22:42.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:22:42.766 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:22:42.766 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:22:42.766 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:42.766 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:22:42.766 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:22:42.766 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:22:42.766 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:22:42.766 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:22:42.766 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:22:42.766 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:22:42.766 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:22:42.767 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:22:42.767 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:42.767 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:22:42.767 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:22:42.767 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:22:42.767 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:22:42.767 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:22:42.767 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:22:42.767 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:22:42.767 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:22:42.767 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:22:42.767 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:22:42.767 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:42.768 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:22:42.768 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:22:42.768 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:22:42.768 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:22:42.768 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:22:42.768 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:22:42.768 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:22:42.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:22:42.776 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:22:42.776 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:22:42.776 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:22:42.824 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:22:42.824 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:22:42.824 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:22:42.827 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:22:42.828 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:22:42.828 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:22:42.828 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:42.828 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:22:42.828 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:22:42.828 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:22:42.828 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:22:42.828 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:22:42.828 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:22:42.828 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:22:42.829 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:22:42.829 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:22:42.829 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:42.829 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:22:42.829 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:22:42.829 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:22:42.829 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:22:42.829 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:22:42.830 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:22:42.830 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:22:42.830 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:22:42.830 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:22:42.830 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:22:42.830 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:42.830 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:22:42.831 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:22:42.831 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:22:42.831 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:22:42.831 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:22:42.831 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:22:42.831 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:22:42.839 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:22:42.839 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:22:42.839 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:22:42.839 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:22:42.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:22:42.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:22:42.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:22:42.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:22:42.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:22:42.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:22:42.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:22:42.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:22:42.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:22:42.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:22:42.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:22:42.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:22:42.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:22:42.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:42.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:22:42.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:42.933 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 94 2026-02-18T08:22:42.933 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 95 2026-02-18T08:22:42.933 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:22:42.933 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=381, tag_tid=27, entry_tid=1], [object_number=380, tag_tid=27, entry_tid=0], [object_number=382, tag_tid=26, entry_tid=46994], [object_number=383, tag_tid=26, entry_tid=46991]]], state=connected] 2026-02-18T08:22:42.933 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=377, tag_tid=26, entry_tid=41669], [object_number=376, tag_tid=26, entry_tid=41668], [object_number=379, tag_tid=26, entry_tid=41667], [object_number=378, tag_tid=26, entry_tid=41666]]], state=connected] 2026-02-18T08:22:42.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:42.936 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=381, tag_tid=27, entry_tid=1' 2026-02-18T08:22:42.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:22:42.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:22:42.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:22:42.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:22:42.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:22:42.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:22:42.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:22:42.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:22:42.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:22:42.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:22:42.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:22:42.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:22:42.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:42.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:22:42.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:42.969 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 94 2026-02-18T08:22:42.970 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 95 2026-02-18T08:22:42.970 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:22:42.970 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=381, tag_tid=27, entry_tid=1], [object_number=380, tag_tid=27, entry_tid=0], [object_number=382, tag_tid=26, entry_tid=46994], [object_number=383, tag_tid=26, entry_tid=46991]]], state=connected] 2026-02-18T08:22:42.970 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=377, tag_tid=26, entry_tid=41669], [object_number=376, tag_tid=26, entry_tid=41668], [object_number=379, tag_tid=26, entry_tid=41667], [object_number=378, tag_tid=26, entry_tid=41666]]], state=connected] 2026-02-18T08:22:42.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:42.974 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=377, tag_tid=26, entry_tid=41669' 2026-02-18T08:22:42.974 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=381, tag_tid=27, entry_tid=1' -a 'object_number=381, tag_tid=27, entry_tid=1' = 'object_number=377, tag_tid=26, entry_tid=41669' 2026-02-18T08:22:42.974 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=377, tag_tid=26, entry_tid=41669' '!=' 'object_number=376, tag_tid=26, entry_tid=41340' 2026-02-18T08:22:42.974 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:22:42.974 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=377, tag_tid=26, entry_tid=41669' = 'object_number=376, tag_tid=26, entry_tid=41340' 2026-02-18T08:22:42.974 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=377, tag_tid=26, entry_tid=41669' 2026-02-18T08:22:42.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=381, tag_tid=27, entry_tid=1' 2026-02-18T08:22:42.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:22:42.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:22:42.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=27 2026-02-18T08:22:42.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=377, tag_tid=26, entry_tid=41669' 2026-02-18T08:22:42.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:22:42.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:22:42.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=26 2026-02-18T08:22:42.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=381, tag_tid=27, entry_tid=1' 2026-02-18T08:22:42.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:22:42.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:22:42.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:22:42.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=377, tag_tid=26, entry_tid=41669' 2026-02-18T08:22:42.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:22:42.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:22:42.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=41669 2026-02-18T08:22:42.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 27 = 26 -a 1 -le 41669 2026-02-18T08:22:42.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:22:42.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:22:42.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:22:43.190 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:22:43.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:22:43.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:22:43.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:22:43.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:22:43.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:22:43.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:22:43.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:22:43.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:22:43.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:43.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:22:43.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:22:43.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:22:43.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:22:43.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:22:43.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:22:43.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:22:43.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:22:43.194 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:22:43.194 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:22:43.194 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:43.194 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:22:43.194 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:22:43.194 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:22:43.195 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:22:43.195 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:22:43.195 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:22:43.195 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:22:43.195 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:22:43.195 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:22:43.196 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:43.196 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:22:43.196 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:22:43.196 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:22:43.196 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:22:43.197 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:22:43.197 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:22:43.197 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:22:43.197 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:22:43.197 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:22:43.197 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:22:43.198 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:43.198 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:22:43.198 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:22:43.198 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:22:43.198 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:22:43.198 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:22:43.199 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:22:43.199 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:22:43.200 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:22:43.200 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:22:43.200 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:22:43.201 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:22:43.256 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:22:43.257 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:22:43.257 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:22:43.259 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:22:43.259 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:22:43.259 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:22:43.259 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:43.259 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:22:43.259 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:22:43.259 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:22:43.259 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:22:43.259 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:22:43.259 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:22:43.259 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:22:43.259 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:22:43.259 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:22:43.259 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:43.259 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:22:43.259 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:22:43.259 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:22:43.260 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:22:43.260 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:22:43.260 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:22:43.260 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:22:43.260 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:22:43.260 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:22:43.260 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:22:43.260 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:43.260 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:22:43.260 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:22:43.260 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:22:43.260 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:22:43.260 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:22:43.260 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:22:43.260 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:22:43.270 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:22:43.270 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:22:43.270 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:22:43.270 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:22:43.327 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:22:43.327 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:22:43.327 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:22:43.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:22:43.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:22:43.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:22:43.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:43.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:22:43.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:22:43.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:22:43.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:22:43.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:22:43.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:22:43.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:22:43.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:22:43.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:22:43.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:43.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:22:43.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:22:43.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:22:43.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:22:43.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:22:43.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:22:43.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:22:43.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:22:43.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:22:43.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:22:43.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:43.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:22:43.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:22:43.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:22:43.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:22:43.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:22:43.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:22:43.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:22:43.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:22:43.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:22:43.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:22:43.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:22:43.399 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:22:43.399 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:22:43.399 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:22:43.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:22:43.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:22:43.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:22:43.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:43.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:22:43.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:22:43.403 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:22:43.403 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:22:43.403 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:22:43.403 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:22:43.403 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:22:43.403 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:22:43.403 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:22:43.403 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:43.403 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:22:43.403 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:22:43.403 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:22:43.403 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:22:43.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:22:43.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:22:43.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:22:43.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:22:43.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:22:43.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:22:43.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:43.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:22:43.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:22:43.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:22:43.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:22:43.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:22:43.405 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:22:43.405 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:22:43.412 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:22:43.413 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:22:43.413 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:22:43.413 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:22:43.472 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:22:43.474 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:22:43.474 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:22:43.474 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:22:43.474 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:22:43.474 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:22:43.474 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:22:43.474 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:22:43.474 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:22:43.474 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:22:43.474 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:22:43.474 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:22:43.475 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:22:43.475 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:43.475 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:22:43.475 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:43.504 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 94 2026-02-18T08:22:43.504 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 95 2026-02-18T08:22:43.504 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:22:43.505 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=381, tag_tid=27, entry_tid=1], [object_number=380, tag_tid=27, entry_tid=0], [object_number=382, tag_tid=26, entry_tid=46994], [object_number=383, tag_tid=26, entry_tid=46991]]], state=connected] 2026-02-18T08:22:43.505 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=378, tag_tid=26, entry_tid=42010], [object_number=377, tag_tid=26, entry_tid=42009], [object_number=376, tag_tid=26, entry_tid=42008], [object_number=379, tag_tid=26, entry_tid=42007]]], state=connected] 2026-02-18T08:22:43.507 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:43.509 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=381, tag_tid=27, entry_tid=1' 2026-02-18T08:22:43.510 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:22:43.510 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:22:43.510 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:22:43.510 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:22:43.510 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:22:43.510 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:22:43.510 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:22:43.510 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:22:43.511 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:22:43.511 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:22:43.511 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:22:43.511 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:22:43.512 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:43.512 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:22:43.512 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:43.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 94 2026-02-18T08:22:43.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 95 2026-02-18T08:22:43.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:22:43.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=381, tag_tid=27, entry_tid=1], [object_number=380, tag_tid=27, entry_tid=0], [object_number=382, tag_tid=26, entry_tid=46994], [object_number=383, tag_tid=26, entry_tid=46991]]], state=connected] 2026-02-18T08:22:43.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=378, tag_tid=26, entry_tid=42010], [object_number=377, tag_tid=26, entry_tid=42009], [object_number=376, tag_tid=26, entry_tid=42008], [object_number=379, tag_tid=26, entry_tid=42007]]], state=connected] 2026-02-18T08:22:43.552 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:43.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=378, tag_tid=26, entry_tid=42010' 2026-02-18T08:22:43.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=381, tag_tid=27, entry_tid=1' -a 'object_number=381, tag_tid=27, entry_tid=1' = 'object_number=378, tag_tid=26, entry_tid=42010' 2026-02-18T08:22:43.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=378, tag_tid=26, entry_tid=42010' '!=' 'object_number=377, tag_tid=26, entry_tid=41669' 2026-02-18T08:22:43.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:22:43.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=378, tag_tid=26, entry_tid=42010' = 'object_number=377, tag_tid=26, entry_tid=41669' 2026-02-18T08:22:43.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=378, tag_tid=26, entry_tid=42010' 2026-02-18T08:22:43.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=381, tag_tid=27, entry_tid=1' 2026-02-18T08:22:43.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:22:43.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:22:43.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=27 2026-02-18T08:22:43.557 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=378, tag_tid=26, entry_tid=42010' 2026-02-18T08:22:43.557 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:22:43.557 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:22:43.558 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=26 2026-02-18T08:22:43.559 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=381, tag_tid=27, entry_tid=1' 2026-02-18T08:22:43.559 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:22:43.559 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:22:43.560 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:22:43.561 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=378, tag_tid=26, entry_tid=42010' 2026-02-18T08:22:43.561 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:22:43.562 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:22:43.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=42010 2026-02-18T08:22:43.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 27 = 26 -a 1 -le 42010 2026-02-18T08:22:43.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:22:43.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:22:43.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:22:43.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:22:43.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:22:43.766 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:22:43.766 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:22:43.766 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:22:43.766 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:22:43.766 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:22:43.766 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:22:43.766 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:22:43.766 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:43.766 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:22:43.766 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:22:43.766 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:22:43.767 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:22:43.767 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:22:43.767 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:22:43.769 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:22:43.770 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:22:43.770 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:22:43.770 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:22:43.770 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:43.770 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:22:43.770 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:22:43.770 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:22:43.770 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:22:43.770 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:22:43.770 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:22:43.770 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:22:43.771 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:22:43.771 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:22:43.771 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:43.771 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:22:43.771 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:22:43.771 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:22:43.771 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:22:43.771 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:22:43.771 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:22:43.771 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:22:43.771 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:22:43.771 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:22:43.772 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:22:43.772 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:43.772 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:22:43.772 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:22:43.772 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:22:43.772 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:22:43.772 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:22:43.772 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:22:43.772 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:22:43.779 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:22:43.780 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:22:43.780 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:22:43.780 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:22:43.838 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:22:43.838 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:22:43.838 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:22:43.841 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:22:43.841 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:22:43.841 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:22:43.841 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:43.841 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:22:43.841 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:22:43.841 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:22:43.841 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:22:43.842 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:22:43.842 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:22:43.842 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:22:43.842 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:22:43.842 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:22:43.842 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:43.842 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:22:43.842 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:22:43.842 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:22:43.842 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:22:43.842 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:22:43.842 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:22:43.843 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:22:43.843 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:22:43.843 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:22:43.843 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:22:43.843 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:43.843 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:22:43.843 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:22:43.843 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:22:43.843 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:22:43.843 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:22:43.843 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:22:43.843 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:22:43.851 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:22:43.852 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:22:43.852 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:22:43.852 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:22:43.910 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:22:43.910 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:22:43.910 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:22:43.913 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:22:43.913 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:22:43.913 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:22:43.913 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:43.913 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:22:43.913 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:22:43.913 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:22:43.913 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:22:43.913 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:22:43.913 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:22:43.914 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:22:43.914 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:22:43.914 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:22:43.914 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:43.914 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:22:43.914 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:22:43.914 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:22:43.914 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:22:43.914 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:22:43.914 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:22:43.914 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:22:43.914 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:22:43.915 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:22:43.915 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:22:43.915 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:43.915 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:22:43.915 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:22:43.915 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:22:43.915 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:22:43.915 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:22:43.915 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:22:43.915 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:22:43.923 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:22:43.924 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:22:43.924 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:22:43.924 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:22:43.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:22:43.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:22:43.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:22:43.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:22:43.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:22:43.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:22:43.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:43.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:22:43.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:22:43.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:22:43.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:22:43.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:22:43.985 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:22:43.985 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:22:43.985 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:22:43.985 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:22:43.985 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:43.985 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:22:43.985 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:22:43.985 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:22:43.985 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:22:43.985 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:22:43.985 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:22:43.985 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:22:43.986 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:22:43.986 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:22:43.986 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:22:43.986 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:43.986 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:22:43.986 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:22:43.986 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:22:43.986 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:22:43.986 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:22:43.986 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:22:43.986 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:22:43.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:22:43.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:22:43.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:22:43.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:22:44.062 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:22:44.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:22:44.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:22:44.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:22:44.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:22:44.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:22:44.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:22:44.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:22:44.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:22:44.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:22:44.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:22:44.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:22:44.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:22:44.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:44.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:22:44.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:44.096 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 94 2026-02-18T08:22:44.096 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 95 2026-02-18T08:22:44.096 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:22:44.096 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=381, tag_tid=27, entry_tid=1], [object_number=380, tag_tid=27, entry_tid=0], [object_number=382, tag_tid=26, entry_tid=46994], [object_number=383, tag_tid=26, entry_tid=46991]]], state=connected] 2026-02-18T08:22:44.097 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=379, tag_tid=26, entry_tid=42407], [object_number=378, tag_tid=26, entry_tid=42406], [object_number=377, tag_tid=26, entry_tid=42405], [object_number=376, tag_tid=26, entry_tid=42404]]], state=connected] 2026-02-18T08:22:44.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:44.101 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=381, tag_tid=27, entry_tid=1' 2026-02-18T08:22:44.102 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:22:44.102 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:22:44.102 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:22:44.102 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:22:44.102 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:22:44.102 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:22:44.102 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:22:44.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:22:44.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:22:44.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:22:44.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:22:44.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:22:44.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:44.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:22:44.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:44.140 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 94 2026-02-18T08:22:44.140 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 95 2026-02-18T08:22:44.140 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:22:44.140 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=381, tag_tid=27, entry_tid=1], [object_number=380, tag_tid=27, entry_tid=0], [object_number=382, tag_tid=26, entry_tid=46994], [object_number=383, tag_tid=26, entry_tid=46991]]], state=connected] 2026-02-18T08:22:44.140 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=379, tag_tid=26, entry_tid=42407], [object_number=378, tag_tid=26, entry_tid=42406], [object_number=377, tag_tid=26, entry_tid=42405], [object_number=376, tag_tid=26, entry_tid=42404]]], state=connected] 2026-02-18T08:22:44.144 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:44.146 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=379, tag_tid=26, entry_tid=42407' 2026-02-18T08:22:44.146 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=381, tag_tid=27, entry_tid=1' -a 'object_number=381, tag_tid=27, entry_tid=1' = 'object_number=379, tag_tid=26, entry_tid=42407' 2026-02-18T08:22:44.146 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=379, tag_tid=26, entry_tid=42407' '!=' 'object_number=378, tag_tid=26, entry_tid=42010' 2026-02-18T08:22:44.146 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:22:44.146 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=379, tag_tid=26, entry_tid=42407' = 'object_number=378, tag_tid=26, entry_tid=42010' 2026-02-18T08:22:44.146 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=379, tag_tid=26, entry_tid=42407' 2026-02-18T08:22:44.147 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=381, tag_tid=27, entry_tid=1' 2026-02-18T08:22:44.147 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:22:44.147 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:22:44.148 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=27 2026-02-18T08:22:44.149 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=379, tag_tid=26, entry_tid=42407' 2026-02-18T08:22:44.149 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:22:44.149 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:22:44.150 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=26 2026-02-18T08:22:44.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=381, tag_tid=27, entry_tid=1' 2026-02-18T08:22:44.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:22:44.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:22:44.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:22:44.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=379, tag_tid=26, entry_tid=42407' 2026-02-18T08:22:44.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:22:44.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:22:44.155 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=42407 2026-02-18T08:22:44.155 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 27 = 26 -a 1 -le 42407 2026-02-18T08:22:44.156 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:22:44.156 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:22:44.156 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:22:44.357 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:22:44.357 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:22:44.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:22:44.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:22:44.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:22:44.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:22:44.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:22:44.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:22:44.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:22:44.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:44.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:22:44.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:22:44.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:22:44.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:22:44.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:22:44.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:22:44.362 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:22:44.362 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:22:44.362 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:22:44.362 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:22:44.362 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:44.362 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:22:44.362 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:22:44.362 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:22:44.363 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:22:44.363 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:22:44.363 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:22:44.363 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:22:44.363 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:22:44.363 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:22:44.363 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:44.363 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:22:44.363 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:22:44.363 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:22:44.363 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:22:44.363 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:22:44.364 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:22:44.364 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:22:44.364 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:22:44.364 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:22:44.364 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:22:44.364 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:44.364 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:22:44.364 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:22:44.364 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:22:44.364 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:22:44.364 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:22:44.364 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:22:44.365 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:22:44.371 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:22:44.372 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:22:44.372 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:22:44.372 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:22:44.421 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:22:44.421 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:22:44.421 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:22:44.423 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:22:44.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:22:44.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:22:44.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:44.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:22:44.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:22:44.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:22:44.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:22:44.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:22:44.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:22:44.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:22:44.425 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:22:44.425 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:22:44.425 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:44.425 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:22:44.425 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:22:44.425 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:22:44.425 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:22:44.425 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:22:44.425 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:22:44.425 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:22:44.425 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:22:44.425 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:22:44.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:22:44.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:44.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:22:44.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:22:44.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:22:44.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:22:44.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:22:44.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:22:44.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:22:44.434 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:22:44.435 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:22:44.435 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:22:44.435 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:22:44.491 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:22:44.491 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:22:44.492 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:22:44.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:22:44.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:22:44.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:22:44.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:44.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:22:44.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:22:44.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:22:44.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:22:44.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:22:44.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:22:44.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:22:44.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:22:44.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:22:44.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:44.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:22:44.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:22:44.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:22:44.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:22:44.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:22:44.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:22:44.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:22:44.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:22:44.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:22:44.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:22:44.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:44.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:22:44.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:22:44.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:22:44.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:22:44.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:22:44.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:22:44.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:22:44.506 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:22:44.507 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:22:44.507 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:22:44.507 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:22:44.565 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:22:44.565 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:22:44.565 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:22:44.568 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:22:44.568 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:22:44.569 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:22:44.569 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:44.569 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:22:44.569 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:22:44.569 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:22:44.569 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:22:44.569 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:22:44.569 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:22:44.569 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:22:44.569 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:22:44.569 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:22:44.570 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:44.570 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:22:44.570 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:22:44.570 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:22:44.570 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:22:44.570 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:22:44.570 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:22:44.570 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:22:44.570 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:22:44.570 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:22:44.570 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:22:44.570 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:44.571 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:22:44.571 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:22:44.571 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:22:44.571 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:22:44.571 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:22:44.571 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:22:44.571 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:22:44.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:22:44.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:22:44.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:22:44.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:22:44.648 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:22:44.649 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:22:44.649 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:22:44.649 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:22:44.649 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:22:44.649 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:22:44.649 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:22:44.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:22:44.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:22:44.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:22:44.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:22:44.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:22:44.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:22:44.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:44.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:22:44.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:44.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 94 2026-02-18T08:22:44.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 95 2026-02-18T08:22:44.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:22:44.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=381, tag_tid=27, entry_tid=1], [object_number=380, tag_tid=27, entry_tid=0], [object_number=382, tag_tid=26, entry_tid=46994], [object_number=383, tag_tid=26, entry_tid=46991]]], state=connected] 2026-02-18T08:22:44.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=377, tag_tid=26, entry_tid=42777], [object_number=376, tag_tid=26, entry_tid=42776], [object_number=379, tag_tid=26, entry_tid=42775], [object_number=378, tag_tid=26, entry_tid=42774]]], state=connected] 2026-02-18T08:22:44.692 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:44.693 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=381, tag_tid=27, entry_tid=1' 2026-02-18T08:22:44.695 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:22:44.695 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:22:44.695 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:22:44.695 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:22:44.695 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:22:44.695 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:22:44.695 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:22:44.695 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:22:44.695 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:22:44.696 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:22:44.696 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:22:44.696 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:22:44.696 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:44.696 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:22:44.697 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:44.725 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 94 2026-02-18T08:22:44.725 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 95 2026-02-18T08:22:44.725 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:22:44.725 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=381, tag_tid=27, entry_tid=1], [object_number=380, tag_tid=27, entry_tid=0], [object_number=382, tag_tid=26, entry_tid=46994], [object_number=383, tag_tid=26, entry_tid=46991]]], state=connected] 2026-02-18T08:22:44.725 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=377, tag_tid=26, entry_tid=42777], [object_number=376, tag_tid=26, entry_tid=42776], [object_number=379, tag_tid=26, entry_tid=42775], [object_number=378, tag_tid=26, entry_tid=42774]]], state=connected] 2026-02-18T08:22:44.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:44.729 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=377, tag_tid=26, entry_tid=42777' 2026-02-18T08:22:44.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=381, tag_tid=27, entry_tid=1' -a 'object_number=381, tag_tid=27, entry_tid=1' = 'object_number=377, tag_tid=26, entry_tid=42777' 2026-02-18T08:22:44.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=377, tag_tid=26, entry_tid=42777' '!=' 'object_number=379, tag_tid=26, entry_tid=42407' 2026-02-18T08:22:44.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:22:44.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=377, tag_tid=26, entry_tid=42777' = 'object_number=379, tag_tid=26, entry_tid=42407' 2026-02-18T08:22:44.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=377, tag_tid=26, entry_tid=42777' 2026-02-18T08:22:44.731 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=381, tag_tid=27, entry_tid=1' 2026-02-18T08:22:44.731 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:22:44.731 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:22:44.731 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=27 2026-02-18T08:22:44.732 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=377, tag_tid=26, entry_tid=42777' 2026-02-18T08:22:44.732 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:22:44.732 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:22:44.734 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=26 2026-02-18T08:22:44.735 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=381, tag_tid=27, entry_tid=1' 2026-02-18T08:22:44.735 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:22:44.735 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:22:44.736 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:22:44.737 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=377, tag_tid=26, entry_tid=42777' 2026-02-18T08:22:44.737 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:22:44.737 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:22:44.738 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=42777 2026-02-18T08:22:44.738 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 27 = 26 -a 1 -le 42777 2026-02-18T08:22:44.738 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:22:44.739 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:22:44.739 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:22:44.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:22:44.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:22:44.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:22:44.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:22:44.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:22:44.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:22:44.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:22:44.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:22:44.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:22:44.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:44.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:22:44.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:22:44.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:22:44.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:22:44.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:22:44.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:22:44.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:22:44.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:22:44.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:22:44.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:22:44.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:44.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:22:44.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:22:44.946 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:22:44.946 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:22:44.946 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:22:44.946 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:22:44.946 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:22:44.946 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:22:44.946 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:22:44.946 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:44.946 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:22:44.946 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:22:44.946 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:22:44.946 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:22:44.947 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:22:44.947 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:22:44.947 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:22:44.947 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:22:44.947 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:22:44.947 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:22:44.947 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:44.947 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:22:44.947 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:22:44.947 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:22:44.947 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:22:44.947 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:22:44.947 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:22:44.948 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:22:44.955 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:22:44.956 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:22:44.956 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:22:44.956 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:22:45.019 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:22:45.019 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:22:45.019 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:22:45.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:22:45.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:22:45.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:22:45.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:45.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:22:45.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:22:45.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:22:45.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:22:45.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:22:45.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:22:45.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:22:45.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:22:45.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:22:45.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:45.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:22:45.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:22:45.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:22:45.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:22:45.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:22:45.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:22:45.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:22:45.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:22:45.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:22:45.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:22:45.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:45.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:22:45.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:22:45.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:22:45.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:22:45.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:22:45.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:22:45.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:22:45.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:22:45.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:22:45.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:22:45.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:22:45.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:22:45.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:22:45.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:22:45.086 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:22:45.086 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:22:45.086 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:22:45.086 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:45.086 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:22:45.086 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:22:45.086 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:22:45.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:22:45.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:22:45.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:22:45.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:22:45.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:22:45.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:22:45.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:45.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:22:45.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:22:45.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:22:45.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:22:45.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:22:45.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:22:45.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:22:45.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:22:45.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:22:45.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:22:45.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:45.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:22:45.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:22:45.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:22:45.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:22:45.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:22:45.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:22:45.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:22:45.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:22:45.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:22:45.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:22:45.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:22:45.148 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:22:45.148 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:22:45.148 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:22:45.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:22:45.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:22:45.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:22:45.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:45.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:22:45.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:22:45.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:22:45.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:22:45.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:22:45.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:22:45.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:22:45.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:22:45.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:22:45.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:45.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:22:45.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:22:45.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:22:45.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:22:45.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:22:45.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:22:45.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:22:45.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:22:45.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:22:45.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:22:45.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:45.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:22:45.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:22:45.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:22:45.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:22:45.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:22:45.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:22:45.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:22:45.162 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:22:45.163 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:22:45.163 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:22:45.163 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:22:45.229 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:22:45.230 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:22:45.230 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:22:45.230 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:22:45.230 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:22:45.231 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:22:45.231 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:22:45.231 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:22:45.231 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:22:45.231 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:22:45.231 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:22:45.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:22:45.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:22:45.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:45.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:22:45.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:45.261 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 94 2026-02-18T08:22:45.261 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 95 2026-02-18T08:22:45.261 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:22:45.261 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=381, tag_tid=27, entry_tid=1], [object_number=380, tag_tid=27, entry_tid=0], [object_number=382, tag_tid=26, entry_tid=46994], [object_number=383, tag_tid=26, entry_tid=46991]]], state=connected] 2026-02-18T08:22:45.261 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=377, tag_tid=26, entry_tid=43153], [object_number=376, tag_tid=26, entry_tid=43152], [object_number=379, tag_tid=26, entry_tid=43151], [object_number=378, tag_tid=26, entry_tid=43150]]], state=connected] 2026-02-18T08:22:45.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:45.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=381, tag_tid=27, entry_tid=1' 2026-02-18T08:22:45.266 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:22:45.266 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:22:45.266 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:22:45.266 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:22:45.266 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:22:45.266 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:22:45.266 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:22:45.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:22:45.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:22:45.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:22:45.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:22:45.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:22:45.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:45.269 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:22:45.269 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:45.297 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 94 2026-02-18T08:22:45.297 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 95 2026-02-18T08:22:45.297 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:22:45.297 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=381, tag_tid=27, entry_tid=1], [object_number=380, tag_tid=27, entry_tid=0], [object_number=382, tag_tid=26, entry_tid=46994], [object_number=383, tag_tid=26, entry_tid=46991]]], state=connected] 2026-02-18T08:22:45.297 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=377, tag_tid=26, entry_tid=43153], [object_number=376, tag_tid=26, entry_tid=43152], [object_number=379, tag_tid=26, entry_tid=43151], [object_number=378, tag_tid=26, entry_tid=43150]]], state=connected] 2026-02-18T08:22:45.300 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:45.302 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=377, tag_tid=26, entry_tid=43153' 2026-02-18T08:22:45.302 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=381, tag_tid=27, entry_tid=1' -a 'object_number=381, tag_tid=27, entry_tid=1' = 'object_number=377, tag_tid=26, entry_tid=43153' 2026-02-18T08:22:45.302 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=377, tag_tid=26, entry_tid=43153' '!=' 'object_number=377, tag_tid=26, entry_tid=42777' 2026-02-18T08:22:45.302 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:22:45.302 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=377, tag_tid=26, entry_tid=43153' = 'object_number=377, tag_tid=26, entry_tid=42777' 2026-02-18T08:22:45.302 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=377, tag_tid=26, entry_tid=43153' 2026-02-18T08:22:45.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=381, tag_tid=27, entry_tid=1' 2026-02-18T08:22:45.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:22:45.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:22:45.304 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=27 2026-02-18T08:22:45.304 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=377, tag_tid=26, entry_tid=43153' 2026-02-18T08:22:45.304 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:22:45.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:22:45.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=26 2026-02-18T08:22:45.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=381, tag_tid=27, entry_tid=1' 2026-02-18T08:22:45.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:22:45.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:22:45.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:22:45.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=377, tag_tid=26, entry_tid=43153' 2026-02-18T08:22:45.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:22:45.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:22:45.310 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=43153 2026-02-18T08:22:45.310 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 27 = 26 -a 1 -le 43153 2026-02-18T08:22:45.310 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:22:45.310 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:22:45.310 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:22:45.512 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:22:45.512 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:22:45.512 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:22:45.512 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:22:45.512 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:22:45.513 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:22:45.513 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:22:45.513 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:22:45.513 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:22:45.513 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:45.513 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:22:45.513 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:22:45.513 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:22:45.513 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:22:45.513 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:22:45.516 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:22:45.516 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:22:45.516 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:22:45.516 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:22:45.516 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:22:45.516 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:45.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:22:45.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:22:45.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:22:45.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:22:45.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:22:45.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:22:45.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:22:45.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:22:45.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:22:45.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:45.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:22:45.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:22:45.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:22:45.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:22:45.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:22:45.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:22:45.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:22:45.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:22:45.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:22:45.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:22:45.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:45.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:22:45.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:22:45.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:22:45.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:22:45.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:22:45.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:22:45.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:22:45.527 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:22:45.527 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:22:45.527 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:22:45.527 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:22:45.576 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:22:45.576 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:22:45.577 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:22:45.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:22:45.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:22:45.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:22:45.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:45.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:22:45.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:22:45.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:22:45.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:22:45.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:22:45.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:22:45.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:22:45.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:22:45.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:22:45.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:45.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:22:45.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:22:45.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:22:45.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:22:45.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:22:45.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:22:45.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:22:45.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:22:45.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:22:45.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:22:45.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:45.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:22:45.582 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:22:45.582 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:22:45.582 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:22:45.582 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:22:45.582 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:22:45.582 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:22:45.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:22:45.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:22:45.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:22:45.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:22:45.643 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:22:45.643 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:22:45.643 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:22:45.646 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:22:45.646 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:22:45.646 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:22:45.646 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:45.646 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:22:45.646 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:22:45.646 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:22:45.646 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:22:45.646 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:22:45.647 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:22:45.647 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:22:45.647 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:22:45.647 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:22:45.647 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:45.647 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:22:45.647 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:22:45.647 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:22:45.647 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:22:45.647 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:22:45.647 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:22:45.647 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:22:45.648 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:22:45.648 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:22:45.648 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:22:45.648 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:45.648 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:22:45.648 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:22:45.648 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:22:45.648 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:22:45.648 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:22:45.648 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:22:45.648 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:22:45.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:22:45.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:22:45.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:22:45.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:22:45.707 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:22:45.707 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:22:45.707 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:22:45.709 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:22:45.709 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:22:45.709 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:22:45.709 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:45.709 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:22:45.709 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:22:45.709 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:22:45.709 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:22:45.709 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:22:45.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:22:45.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:22:45.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:22:45.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:22:45.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:45.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:22:45.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:22:45.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:22:45.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:22:45.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:22:45.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:22:45.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:22:45.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:22:45.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:22:45.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:22:45.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:45.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:22:45.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:22:45.711 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:22:45.711 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:22:45.711 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:22:45.711 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:22:45.711 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:22:45.719 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:22:45.720 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:22:45.720 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:22:45.720 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:22:45.778 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:22:45.779 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:22:45.779 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:22:45.779 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:22:45.779 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:22:45.779 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:22:45.779 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:22:45.779 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:22:45.779 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:22:45.779 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:22:45.779 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:22:45.780 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:22:45.780 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:22:45.781 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:45.781 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:22:45.782 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:45.810 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 94 2026-02-18T08:22:45.810 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 95 2026-02-18T08:22:45.810 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:22:45.811 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=381, tag_tid=27, entry_tid=1], [object_number=380, tag_tid=27, entry_tid=0], [object_number=382, tag_tid=26, entry_tid=46994], [object_number=383, tag_tid=26, entry_tid=46991]]], state=connected] 2026-02-18T08:22:45.811 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=379, tag_tid=26, entry_tid=43519], [object_number=378, tag_tid=26, entry_tid=43518], [object_number=377, tag_tid=26, entry_tid=43517], [object_number=376, tag_tid=26, entry_tid=43516]]], state=connected] 2026-02-18T08:22:45.813 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:45.815 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=381, tag_tid=27, entry_tid=1' 2026-02-18T08:22:45.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:22:45.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:22:45.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:22:45.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:22:45.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:22:45.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:22:45.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:22:45.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:22:45.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:22:45.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:22:45.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:22:45.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:22:45.818 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:45.818 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:22:45.818 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:45.846 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 94 2026-02-18T08:22:45.846 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 95 2026-02-18T08:22:45.846 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:22:45.846 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=381, tag_tid=27, entry_tid=1], [object_number=380, tag_tid=27, entry_tid=0], [object_number=382, tag_tid=26, entry_tid=46994], [object_number=383, tag_tid=26, entry_tid=46991]]], state=connected] 2026-02-18T08:22:45.846 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=379, tag_tid=26, entry_tid=43519], [object_number=378, tag_tid=26, entry_tid=43518], [object_number=377, tag_tid=26, entry_tid=43517], [object_number=376, tag_tid=26, entry_tid=43516]]], state=connected] 2026-02-18T08:22:45.848 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:45.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=379, tag_tid=26, entry_tid=43519' 2026-02-18T08:22:45.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=381, tag_tid=27, entry_tid=1' -a 'object_number=381, tag_tid=27, entry_tid=1' = 'object_number=379, tag_tid=26, entry_tid=43519' 2026-02-18T08:22:45.850 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=379, tag_tid=26, entry_tid=43519' '!=' 'object_number=377, tag_tid=26, entry_tid=43153' 2026-02-18T08:22:45.850 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:22:45.850 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=379, tag_tid=26, entry_tid=43519' = 'object_number=377, tag_tid=26, entry_tid=43153' 2026-02-18T08:22:45.850 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=379, tag_tid=26, entry_tid=43519' 2026-02-18T08:22:45.850 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=381, tag_tid=27, entry_tid=1' 2026-02-18T08:22:45.850 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:22:45.850 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:22:45.852 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=27 2026-02-18T08:22:45.852 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=379, tag_tid=26, entry_tid=43519' 2026-02-18T08:22:45.853 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:22:45.853 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:22:45.854 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=26 2026-02-18T08:22:45.854 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=381, tag_tid=27, entry_tid=1' 2026-02-18T08:22:45.855 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:22:45.855 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:22:45.856 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:22:45.857 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=379, tag_tid=26, entry_tid=43519' 2026-02-18T08:22:45.857 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:22:45.857 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:22:45.859 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=43519 2026-02-18T08:22:45.859 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 27 = 26 -a 1 -le 43519 2026-02-18T08:22:45.859 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:22:45.859 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:22:45.859 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:22:46.060 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:22:46.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:22:46.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:22:46.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:22:46.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:22:46.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:22:46.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:22:46.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:22:46.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:22:46.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:46.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:22:46.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:22:46.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:22:46.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:22:46.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:22:46.062 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:22:46.062 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:22:46.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:22:46.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:22:46.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:22:46.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:46.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:22:46.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:22:46.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:22:46.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:22:46.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:22:46.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:22:46.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:22:46.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:22:46.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:22:46.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:46.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:22:46.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:22:46.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:22:46.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:22:46.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:22:46.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:22:46.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:22:46.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:22:46.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:22:46.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:22:46.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:46.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:22:46.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:22:46.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:22:46.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:22:46.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:22:46.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:22:46.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:22:46.075 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:22:46.076 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:22:46.076 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:22:46.076 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:22:46.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:22:46.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:22:46.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:22:46.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:22:46.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:22:46.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:22:46.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:46.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:22:46.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:22:46.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:22:46.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:22:46.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:22:46.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:22:46.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:22:46.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:22:46.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:22:46.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:46.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:22:46.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:22:46.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:22:46.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:22:46.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:22:46.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:22:46.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:22:46.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:22:46.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:22:46.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:22:46.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:46.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:22:46.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:22:46.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:22:46.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:22:46.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:22:46.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:22:46.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:22:46.139 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:22:46.139 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:22:46.139 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:22:46.139 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:22:46.188 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:22:46.188 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:22:46.188 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:22:46.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:22:46.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:22:46.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:22:46.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:46.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:22:46.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:22:46.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:22:46.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:22:46.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:22:46.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:22:46.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:22:46.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:22:46.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:22:46.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:46.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:22:46.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:22:46.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:22:46.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:22:46.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:22:46.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:22:46.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:22:46.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:22:46.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:22:46.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:22:46.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:46.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:22:46.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:22:46.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:22:46.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:22:46.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:22:46.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:22:46.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:22:46.203 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:22:46.203 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:22:46.204 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:22:46.204 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:22:46.253 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:22:46.253 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:22:46.253 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:22:46.255 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:22:46.256 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:22:46.256 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:22:46.256 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:46.256 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:22:46.256 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:22:46.256 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:22:46.256 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:22:46.256 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:22:46.256 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:22:46.256 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:22:46.257 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:22:46.257 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:22:46.257 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:46.257 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:22:46.257 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:22:46.257 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:22:46.257 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:22:46.257 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:22:46.257 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:22:46.257 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:22:46.257 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:22:46.257 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:22:46.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:22:46.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:46.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:22:46.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:22:46.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:22:46.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:22:46.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:22:46.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:22:46.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:22:46.266 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:22:46.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:22:46.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:22:46.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:22:46.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:22:46.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:22:46.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:22:46.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:22:46.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:22:46.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:22:46.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:22:46.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:22:46.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:22:46.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:22:46.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:22:46.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:22:46.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:22:46.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:46.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:22:46.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:46.365 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 94 2026-02-18T08:22:46.365 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 95 2026-02-18T08:22:46.365 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:22:46.366 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=381, tag_tid=27, entry_tid=1], [object_number=380, tag_tid=27, entry_tid=0], [object_number=382, tag_tid=26, entry_tid=46994], [object_number=383, tag_tid=26, entry_tid=46991]]], state=connected] 2026-02-18T08:22:46.366 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=379, tag_tid=26, entry_tid=43859], [object_number=378, tag_tid=26, entry_tid=43858], [object_number=377, tag_tid=26, entry_tid=43857], [object_number=376, tag_tid=26, entry_tid=43856]]], state=connected] 2026-02-18T08:22:46.368 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:46.370 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=381, tag_tid=27, entry_tid=1' 2026-02-18T08:22:46.371 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:22:46.371 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:22:46.371 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:22:46.371 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:22:46.371 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:22:46.371 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:22:46.372 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:22:46.372 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:22:46.372 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:22:46.372 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:22:46.372 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:22:46.372 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:22:46.373 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:46.373 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:22:46.373 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:46.409 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 94 2026-02-18T08:22:46.409 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 95 2026-02-18T08:22:46.409 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:22:46.409 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=381, tag_tid=27, entry_tid=1], [object_number=380, tag_tid=27, entry_tid=0], [object_number=382, tag_tid=26, entry_tid=46994], [object_number=383, tag_tid=26, entry_tid=46991]]], state=connected] 2026-02-18T08:22:46.409 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=379, tag_tid=26, entry_tid=43859], [object_number=378, tag_tid=26, entry_tid=43858], [object_number=377, tag_tid=26, entry_tid=43857], [object_number=376, tag_tid=26, entry_tid=43856]]], state=connected] 2026-02-18T08:22:46.413 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:46.415 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=379, tag_tid=26, entry_tid=43859' 2026-02-18T08:22:46.415 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=381, tag_tid=27, entry_tid=1' -a 'object_number=381, tag_tid=27, entry_tid=1' = 'object_number=379, tag_tid=26, entry_tid=43859' 2026-02-18T08:22:46.415 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=379, tag_tid=26, entry_tid=43859' '!=' 'object_number=379, tag_tid=26, entry_tid=43519' 2026-02-18T08:22:46.415 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:22:46.415 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=379, tag_tid=26, entry_tid=43859' = 'object_number=379, tag_tid=26, entry_tid=43519' 2026-02-18T08:22:46.415 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=379, tag_tid=26, entry_tid=43859' 2026-02-18T08:22:46.416 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=381, tag_tid=27, entry_tid=1' 2026-02-18T08:22:46.416 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:22:46.416 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:22:46.417 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=27 2026-02-18T08:22:46.418 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=379, tag_tid=26, entry_tid=43859' 2026-02-18T08:22:46.418 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:22:46.418 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:22:46.419 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=26 2026-02-18T08:22:46.420 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=381, tag_tid=27, entry_tid=1' 2026-02-18T08:22:46.420 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:22:46.420 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:22:46.422 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:22:46.423 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=379, tag_tid=26, entry_tid=43859' 2026-02-18T08:22:46.423 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:22:46.423 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:22:46.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=43859 2026-02-18T08:22:46.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 27 = 26 -a 1 -le 43859 2026-02-18T08:22:46.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:22:46.425 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:22:46.425 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:22:46.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:22:46.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:22:46.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:22:46.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:22:46.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:22:46.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:22:46.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:22:46.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:22:46.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:22:46.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:46.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:22:46.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:22:46.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:22:46.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:22:46.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:22:46.628 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:22:46.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:22:46.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:22:46.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:22:46.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:22:46.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:46.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:22:46.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:22:46.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:22:46.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:22:46.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:22:46.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:22:46.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:22:46.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:22:46.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:22:46.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:46.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:22:46.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:22:46.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:22:46.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:22:46.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:22:46.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:22:46.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:22:46.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:22:46.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:22:46.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:22:46.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:46.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:22:46.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:22:46.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:22:46.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:22:46.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:22:46.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:22:46.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:22:46.640 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:22:46.641 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:22:46.641 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:22:46.641 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:22:46.692 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:22:46.693 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:22:46.693 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:22:46.695 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:22:46.695 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:22:46.695 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:22:46.696 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:46.696 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:22:46.696 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:22:46.696 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:22:46.696 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:22:46.696 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:22:46.696 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:22:46.696 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:22:46.696 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:22:46.696 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:22:46.696 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:46.696 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:22:46.697 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:22:46.697 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:22:46.697 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:22:46.697 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:22:46.697 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:22:46.697 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:22:46.697 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:22:46.697 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:22:46.697 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:22:46.697 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:46.697 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:22:46.697 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:22:46.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:22:46.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:22:46.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:22:46.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:22:46.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:22:46.706 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:22:46.707 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:22:46.707 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:22:46.707 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:22:46.756 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:22:46.756 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:22:46.756 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:22:46.760 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:22:46.760 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:22:46.760 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:22:46.760 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:46.760 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:22:46.760 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:22:46.760 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:22:46.760 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:22:46.760 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:22:46.760 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:22:46.761 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:22:46.761 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:22:46.761 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:22:46.761 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:46.761 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:22:46.761 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:22:46.761 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:22:46.761 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:22:46.761 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:22:46.761 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:22:46.761 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:22:46.761 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:22:46.761 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:22:46.762 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:22:46.762 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:46.762 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:22:46.762 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:22:46.762 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:22:46.762 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:22:46.762 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:22:46.762 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:22:46.762 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:22:46.770 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:22:46.771 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:22:46.771 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:22:46.771 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:22:46.818 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:22:46.819 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:22:46.819 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:22:46.822 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:22:46.822 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:22:46.822 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:22:46.822 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:46.822 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:22:46.822 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:22:46.822 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:22:46.822 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:22:46.823 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:22:46.823 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:22:46.823 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:22:46.823 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:22:46.823 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:22:46.823 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:46.823 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:22:46.823 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:22:46.823 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:22:46.823 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:22:46.824 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:22:46.824 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:22:46.824 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:22:46.824 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:22:46.824 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:22:46.824 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:22:46.824 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:46.824 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:22:46.824 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:22:46.824 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:22:46.824 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:22:46.824 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:22:46.824 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:22:46.825 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:22:46.832 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:22:46.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:22:46.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:22:46.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:22:46.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:22:46.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:22:46.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:22:46.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:22:46.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:22:46.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:22:46.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:22:46.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:22:46.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:22:46.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:22:46.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:22:46.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:22:46.894 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:22:46.894 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:46.895 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:22:46.895 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:46.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 94 2026-02-18T08:22:46.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 95 2026-02-18T08:22:46.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:22:46.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=381, tag_tid=27, entry_tid=1], [object_number=380, tag_tid=27, entry_tid=0], [object_number=382, tag_tid=26, entry_tid=46994], [object_number=383, tag_tid=26, entry_tid=46991]]], state=connected] 2026-02-18T08:22:46.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=378, tag_tid=26, entry_tid=44214], [object_number=377, tag_tid=26, entry_tid=44213], [object_number=376, tag_tid=26, entry_tid=44212], [object_number=379, tag_tid=26, entry_tid=44211]]], state=connected] 2026-02-18T08:22:46.928 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:46.929 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=381, tag_tid=27, entry_tid=1' 2026-02-18T08:22:46.930 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:22:46.931 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:22:46.931 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:22:46.931 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:22:46.931 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:22:46.931 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:22:46.931 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:22:46.931 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:22:46.931 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:22:46.931 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:22:46.932 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:22:46.932 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:22:46.932 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:46.933 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:22:46.933 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:46.970 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 94 2026-02-18T08:22:46.970 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 95 2026-02-18T08:22:46.970 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:22:46.970 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=381, tag_tid=27, entry_tid=1], [object_number=380, tag_tid=27, entry_tid=0], [object_number=382, tag_tid=26, entry_tid=46994], [object_number=383, tag_tid=26, entry_tid=46991]]], state=connected] 2026-02-18T08:22:46.970 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=378, tag_tid=26, entry_tid=44214], [object_number=377, tag_tid=26, entry_tid=44213], [object_number=376, tag_tid=26, entry_tid=44212], [object_number=379, tag_tid=26, entry_tid=44211]]], state=connected] 2026-02-18T08:22:46.974 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:46.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=378, tag_tid=26, entry_tid=44214' 2026-02-18T08:22:46.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=381, tag_tid=27, entry_tid=1' -a 'object_number=381, tag_tid=27, entry_tid=1' = 'object_number=378, tag_tid=26, entry_tid=44214' 2026-02-18T08:22:46.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=378, tag_tid=26, entry_tid=44214' '!=' 'object_number=379, tag_tid=26, entry_tid=43859' 2026-02-18T08:22:46.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:22:46.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=378, tag_tid=26, entry_tid=44214' = 'object_number=379, tag_tid=26, entry_tid=43859' 2026-02-18T08:22:46.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=378, tag_tid=26, entry_tid=44214' 2026-02-18T08:22:46.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=381, tag_tid=27, entry_tid=1' 2026-02-18T08:22:46.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:22:46.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:22:46.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=27 2026-02-18T08:22:46.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=378, tag_tid=26, entry_tid=44214' 2026-02-18T08:22:46.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:22:46.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:22:46.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=26 2026-02-18T08:22:46.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=381, tag_tid=27, entry_tid=1' 2026-02-18T08:22:46.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:22:46.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:22:46.982 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:22:46.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=378, tag_tid=26, entry_tid=44214' 2026-02-18T08:22:46.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:22:46.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:22:46.985 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=44214 2026-02-18T08:22:46.985 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 27 = 26 -a 1 -le 44214 2026-02-18T08:22:46.985 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:22:46.985 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:22:46.985 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:22:47.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:22:47.187 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:22:47.187 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:22:47.187 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:22:47.187 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:22:47.187 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:22:47.187 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:22:47.187 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:22:47.187 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:22:47.188 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:47.188 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:22:47.188 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:22:47.188 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:22:47.188 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:22:47.188 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:22:47.188 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:22:47.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:22:47.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:22:47.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:22:47.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:22:47.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:47.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:22:47.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:22:47.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:22:47.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:22:47.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:22:47.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:22:47.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:22:47.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:22:47.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:22:47.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:47.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:22:47.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:22:47.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:22:47.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:22:47.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:22:47.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:22:47.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:22:47.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:22:47.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:22:47.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:22:47.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:47.194 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:22:47.194 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:22:47.194 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:22:47.194 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:22:47.194 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:22:47.194 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:22:47.194 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:22:47.202 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:22:47.202 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:22:47.202 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:22:47.202 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:22:47.253 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:22:47.253 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:22:47.253 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:22:47.256 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:22:47.256 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:22:47.256 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:22:47.256 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:47.256 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:22:47.257 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:22:47.257 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:22:47.257 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:22:47.257 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:22:47.257 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:22:47.257 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:22:47.257 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:22:47.257 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:22:47.257 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:47.257 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:22:47.257 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:22:47.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:22:47.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:22:47.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:22:47.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:22:47.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:22:47.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:22:47.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:22:47.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:22:47.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:47.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:22:47.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:22:47.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:22:47.259 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:22:47.259 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:22:47.259 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:22:47.259 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:22:47.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:22:47.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:22:47.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:22:47.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:22:47.326 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:22:47.326 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:22:47.326 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:22:47.329 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:22:47.329 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:22:47.329 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:22:47.329 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:47.329 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:22:47.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:22:47.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:22:47.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:22:47.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:22:47.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:22:47.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:22:47.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:22:47.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:22:47.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:47.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:22:47.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:22:47.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:22:47.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:22:47.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:22:47.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:22:47.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:22:47.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:22:47.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:22:47.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:22:47.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:47.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:22:47.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:22:47.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:22:47.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:22:47.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:22:47.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:22:47.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:22:47.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:22:47.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:22:47.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:22:47.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:22:47.388 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:22:47.388 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:22:47.388 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:22:47.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:22:47.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:22:47.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:22:47.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:47.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:22:47.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:22:47.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:22:47.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:22:47.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:22:47.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:22:47.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:22:47.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:22:47.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:22:47.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:47.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:22:47.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:22:47.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:22:47.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:22:47.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:22:47.393 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:22:47.393 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:22:47.393 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:22:47.393 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:22:47.393 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:22:47.393 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:47.393 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:22:47.393 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:22:47.393 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:22:47.393 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:22:47.393 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:22:47.393 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:22:47.394 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:22:47.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:22:47.403 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:22:47.403 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:22:47.403 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:22:47.463 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:22:47.463 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:22:47.463 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:22:47.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:22:47.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:22:47.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:22:47.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:22:47.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:22:47.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:22:47.465 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:22:47.465 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:22:47.465 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:22:47.465 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:22:47.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:47.467 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:22:47.467 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:47.493 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 94 2026-02-18T08:22:47.493 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 95 2026-02-18T08:22:47.493 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:22:47.493 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=381, tag_tid=27, entry_tid=1], [object_number=380, tag_tid=27, entry_tid=0], [object_number=382, tag_tid=26, entry_tid=46994], [object_number=383, tag_tid=26, entry_tid=46991]]], state=connected] 2026-02-18T08:22:47.493 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=378, tag_tid=26, entry_tid=44582], [object_number=377, tag_tid=26, entry_tid=44581], [object_number=376, tag_tid=26, entry_tid=44580], [object_number=379, tag_tid=26, entry_tid=44579]]], state=connected] 2026-02-18T08:22:47.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:47.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=381, tag_tid=27, entry_tid=1' 2026-02-18T08:22:47.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:22:47.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:22:47.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:22:47.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:22:47.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:22:47.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:22:47.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:22:47.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:22:47.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:22:47.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:22:47.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:22:47.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:22:47.500 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:47.501 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:22:47.501 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:47.528 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 94 2026-02-18T08:22:47.529 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 95 2026-02-18T08:22:47.529 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:22:47.529 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=381, tag_tid=27, entry_tid=1], [object_number=380, tag_tid=27, entry_tid=0], [object_number=382, tag_tid=26, entry_tid=46994], [object_number=383, tag_tid=26, entry_tid=46991]]], state=connected] 2026-02-18T08:22:47.529 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=378, tag_tid=26, entry_tid=44582], [object_number=377, tag_tid=26, entry_tid=44581], [object_number=376, tag_tid=26, entry_tid=44580], [object_number=379, tag_tid=26, entry_tid=44579]]], state=connected] 2026-02-18T08:22:47.532 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:47.534 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=378, tag_tid=26, entry_tid=44582' 2026-02-18T08:22:47.534 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=381, tag_tid=27, entry_tid=1' -a 'object_number=381, tag_tid=27, entry_tid=1' = 'object_number=378, tag_tid=26, entry_tid=44582' 2026-02-18T08:22:47.534 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=378, tag_tid=26, entry_tid=44582' '!=' 'object_number=378, tag_tid=26, entry_tid=44214' 2026-02-18T08:22:47.534 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:22:47.534 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=378, tag_tid=26, entry_tid=44582' = 'object_number=378, tag_tid=26, entry_tid=44214' 2026-02-18T08:22:47.534 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=378, tag_tid=26, entry_tid=44582' 2026-02-18T08:22:47.535 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=381, tag_tid=27, entry_tid=1' 2026-02-18T08:22:47.535 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:22:47.535 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:22:47.536 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=27 2026-02-18T08:22:47.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=378, tag_tid=26, entry_tid=44582' 2026-02-18T08:22:47.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:22:47.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:22:47.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=26 2026-02-18T08:22:47.539 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=381, tag_tid=27, entry_tid=1' 2026-02-18T08:22:47.539 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:22:47.539 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:22:47.540 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:22:47.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=378, tag_tid=26, entry_tid=44582' 2026-02-18T08:22:47.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:22:47.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:22:47.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=44582 2026-02-18T08:22:47.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 27 = 26 -a 1 -le 44582 2026-02-18T08:22:47.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:22:47.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:22:47.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:22:47.744 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:22:47.745 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:22:47.745 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:22:47.745 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:22:47.745 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:22:47.745 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:22:47.745 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:22:47.745 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:22:47.745 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:22:47.746 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:47.746 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:22:47.746 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:22:47.746 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:22:47.746 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:22:47.746 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:22:47.746 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:22:47.749 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:22:47.749 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:22:47.749 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:22:47.749 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:22:47.749 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:47.749 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:22:47.749 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:22:47.749 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:22:47.750 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:22:47.750 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:22:47.750 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:22:47.750 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:22:47.750 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:22:47.750 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:22:47.751 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:47.751 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:22:47.751 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:22:47.751 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:22:47.751 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:22:47.751 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:22:47.751 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:22:47.751 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:22:47.751 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:22:47.751 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:22:47.752 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:22:47.752 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:47.752 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:22:47.752 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:22:47.752 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:22:47.752 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:22:47.752 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:22:47.752 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:22:47.753 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:22:47.759 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:22:47.759 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:22:47.759 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:22:47.760 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:22:47.811 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:22:47.811 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:22:47.811 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:22:47.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:22:47.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:22:47.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:22:47.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:47.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:22:47.817 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:22:47.817 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:22:47.817 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:22:47.817 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:22:47.817 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:22:47.817 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:22:47.818 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:22:47.818 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:22:47.818 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:47.818 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:22:47.818 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:22:47.819 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:22:47.819 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:22:47.819 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:22:47.819 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:22:47.819 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:22:47.819 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:22:47.820 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:22:47.820 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:22:47.820 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:47.820 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:22:47.820 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:22:47.820 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:22:47.820 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:22:47.821 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:22:47.821 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:22:47.821 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:22:47.826 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:22:47.826 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:22:47.826 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:22:47.827 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:22:47.876 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:22:47.876 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:22:47.876 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:22:47.877 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:22:47.877 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:22:47.877 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:22:47.877 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:47.877 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:22:47.877 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:22:47.877 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:22:47.878 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:22:47.878 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:22:47.878 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:22:47.878 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:22:47.878 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:22:47.878 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:22:47.878 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:47.878 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:22:47.878 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:22:47.878 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:22:47.878 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:22:47.878 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:22:47.878 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:22:47.878 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:22:47.878 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:22:47.878 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:22:47.878 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:22:47.878 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:47.878 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:22:47.878 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:22:47.878 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:22:47.878 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:22:47.878 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:22:47.878 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:22:47.878 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:22:47.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:22:47.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:22:47.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:22:47.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:22:47.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:22:47.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:22:47.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:22:47.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:22:47.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:22:47.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:22:47.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:47.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:22:47.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:22:47.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:22:47.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:22:47.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:22:47.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:22:47.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:22:47.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:22:47.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:22:47.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:47.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:22:47.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:22:47.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:22:47.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:22:47.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:22:47.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:22:47.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:22:47.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:22:47.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:22:47.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:22:47.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:47.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:22:47.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:22:47.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:22:47.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:22:47.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:22:47.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:22:47.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:22:47.954 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:22:47.954 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:22:47.954 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:22:47.954 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:22:48.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:22:48.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:22:48.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:22:48.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:22:48.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:22:48.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:22:48.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:22:48.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:22:48.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:22:48.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:22:48.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:22:48.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:22:48.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:22:48.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:48.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:22:48.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:48.056 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 94 2026-02-18T08:22:48.056 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 95 2026-02-18T08:22:48.056 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:22:48.056 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=381, tag_tid=27, entry_tid=1], [object_number=380, tag_tid=27, entry_tid=0], [object_number=382, tag_tid=26, entry_tid=46994], [object_number=383, tag_tid=26, entry_tid=46991]]], state=connected] 2026-02-18T08:22:48.056 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=377, tag_tid=26, entry_tid=44921], [object_number=376, tag_tid=26, entry_tid=44920], [object_number=379, tag_tid=26, entry_tid=44919], [object_number=378, tag_tid=26, entry_tid=44918]]], state=connected] 2026-02-18T08:22:48.060 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:48.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=381, tag_tid=27, entry_tid=1' 2026-02-18T08:22:48.062 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:22:48.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:22:48.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:22:48.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:22:48.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:22:48.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:22:48.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:22:48.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:22:48.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:22:48.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:22:48.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:22:48.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:22:48.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:48.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:22:48.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:48.101 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 94 2026-02-18T08:22:48.101 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 95 2026-02-18T08:22:48.101 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:22:48.102 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=381, tag_tid=27, entry_tid=1], [object_number=380, tag_tid=27, entry_tid=0], [object_number=382, tag_tid=26, entry_tid=46994], [object_number=383, tag_tid=26, entry_tid=46991]]], state=connected] 2026-02-18T08:22:48.102 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=377, tag_tid=26, entry_tid=44921], [object_number=376, tag_tid=26, entry_tid=44920], [object_number=379, tag_tid=26, entry_tid=44919], [object_number=378, tag_tid=26, entry_tid=44918]]], state=connected] 2026-02-18T08:22:48.105 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:48.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=377, tag_tid=26, entry_tid=44921' 2026-02-18T08:22:48.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=381, tag_tid=27, entry_tid=1' -a 'object_number=381, tag_tid=27, entry_tid=1' = 'object_number=377, tag_tid=26, entry_tid=44921' 2026-02-18T08:22:48.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=377, tag_tid=26, entry_tid=44921' '!=' 'object_number=378, tag_tid=26, entry_tid=44582' 2026-02-18T08:22:48.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:22:48.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=377, tag_tid=26, entry_tid=44921' = 'object_number=378, tag_tid=26, entry_tid=44582' 2026-02-18T08:22:48.110 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=377, tag_tid=26, entry_tid=44921' 2026-02-18T08:22:48.110 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=381, tag_tid=27, entry_tid=1' 2026-02-18T08:22:48.110 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:22:48.110 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:22:48.111 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=27 2026-02-18T08:22:48.112 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=377, tag_tid=26, entry_tid=44921' 2026-02-18T08:22:48.112 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:22:48.112 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:22:48.113 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=26 2026-02-18T08:22:48.113 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=381, tag_tid=27, entry_tid=1' 2026-02-18T08:22:48.114 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:22:48.114 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:22:48.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:22:48.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=377, tag_tid=26, entry_tid=44921' 2026-02-18T08:22:48.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:22:48.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:22:48.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=44921 2026-02-18T08:22:48.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 27 = 26 -a 1 -le 44921 2026-02-18T08:22:48.118 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:22:48.118 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:22:48.118 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:22:48.318 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:22:48.319 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:22:48.319 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:22:48.319 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:22:48.319 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:22:48.319 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:22:48.319 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:22:48.319 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:22:48.319 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:22:48.319 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:48.319 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:22:48.319 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:22:48.319 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:22:48.319 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:22:48.319 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:22:48.321 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:22:48.321 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:22:48.321 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:22:48.321 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:22:48.322 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:22:48.322 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:48.322 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:22:48.322 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:22:48.322 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:22:48.322 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:22:48.322 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:22:48.322 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:22:48.322 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:22:48.322 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:22:48.322 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:22:48.322 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:48.322 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:22:48.322 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:22:48.322 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:22:48.322 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:22:48.323 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:22:48.323 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:22:48.323 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:22:48.323 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:22:48.323 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:22:48.323 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:22:48.323 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:48.323 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:22:48.323 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:22:48.323 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:22:48.323 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:22:48.323 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:22:48.323 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:22:48.323 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:22:48.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:22:48.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:22:48.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:22:48.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:22:48.383 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:22:48.383 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:22:48.383 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:22:48.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:22:48.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:22:48.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:22:48.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:48.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:22:48.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:22:48.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:22:48.387 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:22:48.387 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:22:48.387 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:22:48.387 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:22:48.387 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:22:48.387 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:22:48.387 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:48.387 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:22:48.387 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:22:48.387 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:22:48.387 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:22:48.387 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:22:48.387 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:22:48.388 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:22:48.388 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:22:48.388 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:22:48.388 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:22:48.388 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:48.388 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:22:48.388 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:22:48.388 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:22:48.388 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:22:48.388 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:22:48.388 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:22:48.388 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:22:48.396 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:22:48.397 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:22:48.397 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:22:48.397 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:22:48.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:22:48.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:22:48.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:22:48.448 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:22:48.448 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:22:48.448 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:22:48.448 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:48.448 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:22:48.448 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:22:48.448 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:22:48.449 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:22:48.449 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:22:48.449 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:22:48.449 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:22:48.449 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:22:48.449 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:22:48.449 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:48.449 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:22:48.449 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:22:48.449 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:22:48.449 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:22:48.449 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:22:48.449 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:22:48.450 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:22:48.450 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:22:48.450 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:22:48.450 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:22:48.450 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:48.450 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:22:48.450 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:22:48.450 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:22:48.450 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:22:48.450 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:22:48.450 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:22:48.450 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:22:48.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:22:48.460 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:22:48.460 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:22:48.460 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:22:48.506 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:22:48.506 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:22:48.506 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:22:48.509 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:22:48.509 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:22:48.509 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:22:48.509 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:48.509 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:22:48.509 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:22:48.510 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:22:48.510 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:22:48.510 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:22:48.510 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:22:48.510 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:22:48.510 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:22:48.510 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:22:48.510 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:48.510 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:22:48.510 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:22:48.511 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:22:48.511 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:22:48.511 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:22:48.511 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:22:48.511 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:22:48.511 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:22:48.511 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:22:48.511 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:22:48.511 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:48.511 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:22:48.511 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:22:48.512 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:22:48.512 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:22:48.512 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:22:48.512 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:22:48.512 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:22:48.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:22:48.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:22:48.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:22:48.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:22:48.577 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:22:48.577 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:22:48.577 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:22:48.577 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:22:48.578 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:22:48.578 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:22:48.578 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:22:48.578 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:22:48.578 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:22:48.578 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:22:48.578 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:22:48.578 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:22:48.578 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:22:48.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:48.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:22:48.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:48.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 94 2026-02-18T08:22:48.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 95 2026-02-18T08:22:48.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:22:48.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=381, tag_tid=27, entry_tid=1], [object_number=380, tag_tid=27, entry_tid=0], [object_number=382, tag_tid=26, entry_tid=46994], [object_number=383, tag_tid=26, entry_tid=46991]]], state=connected] 2026-02-18T08:22:48.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=378, tag_tid=26, entry_tid=45262], [object_number=377, tag_tid=26, entry_tid=45261], [object_number=376, tag_tid=26, entry_tid=45260], [object_number=379, tag_tid=26, entry_tid=45259]]], state=connected] 2026-02-18T08:22:48.611 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:48.613 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=381, tag_tid=27, entry_tid=1' 2026-02-18T08:22:48.613 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:22:48.613 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:22:48.613 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:22:48.613 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:22:48.613 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:22:48.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:22:48.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:22:48.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:22:48.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:22:48.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:22:48.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:22:48.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:22:48.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:48.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:22:48.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:48.643 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 94 2026-02-18T08:22:48.643 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 95 2026-02-18T08:22:48.643 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:22:48.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=381, tag_tid=27, entry_tid=1], [object_number=380, tag_tid=27, entry_tid=0], [object_number=382, tag_tid=26, entry_tid=46994], [object_number=383, tag_tid=26, entry_tid=46991]]], state=connected] 2026-02-18T08:22:48.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=378, tag_tid=26, entry_tid=45262], [object_number=377, tag_tid=26, entry_tid=45261], [object_number=376, tag_tid=26, entry_tid=45260], [object_number=379, tag_tid=26, entry_tid=45259]]], state=connected] 2026-02-18T08:22:48.646 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:48.648 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=378, tag_tid=26, entry_tid=45262' 2026-02-18T08:22:48.648 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=381, tag_tid=27, entry_tid=1' -a 'object_number=381, tag_tid=27, entry_tid=1' = 'object_number=378, tag_tid=26, entry_tid=45262' 2026-02-18T08:22:48.648 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=378, tag_tid=26, entry_tid=45262' '!=' 'object_number=377, tag_tid=26, entry_tid=44921' 2026-02-18T08:22:48.648 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:22:48.648 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=378, tag_tid=26, entry_tid=45262' = 'object_number=377, tag_tid=26, entry_tid=44921' 2026-02-18T08:22:48.648 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=378, tag_tid=26, entry_tid=45262' 2026-02-18T08:22:48.649 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=381, tag_tid=27, entry_tid=1' 2026-02-18T08:22:48.649 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:22:48.649 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:22:48.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=27 2026-02-18T08:22:48.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=378, tag_tid=26, entry_tid=45262' 2026-02-18T08:22:48.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:22:48.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:22:48.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=26 2026-02-18T08:22:48.653 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=381, tag_tid=27, entry_tid=1' 2026-02-18T08:22:48.653 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:22:48.653 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:22:48.654 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:22:48.655 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=378, tag_tid=26, entry_tid=45262' 2026-02-18T08:22:48.655 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:22:48.655 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:22:48.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=45262 2026-02-18T08:22:48.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 27 = 26 -a 1 -le 45262 2026-02-18T08:22:48.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:22:48.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:22:48.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:22:48.858 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:22:48.858 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:22:48.858 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:22:48.858 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:22:48.858 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:22:48.858 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:22:48.858 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:22:48.858 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:22:48.858 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:22:48.859 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:48.859 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:22:48.859 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:22:48.859 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:22:48.859 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:22:48.859 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:22:48.859 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:22:48.859 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:22:48.861 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:22:48.861 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:22:48.861 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:22:48.861 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:48.861 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:22:48.861 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:22:48.861 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:22:48.861 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:22:48.861 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:22:48.861 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:22:48.861 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:22:48.861 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:22:48.861 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:22:48.861 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:48.861 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:22:48.861 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:22:48.861 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:22:48.861 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:22:48.861 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:22:48.861 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:22:48.861 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:22:48.861 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:22:48.861 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:22:48.861 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:22:48.861 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:48.861 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:22:48.861 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:22:48.861 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:22:48.861 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:22:48.861 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:22:48.862 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:22:48.862 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:22:48.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:22:48.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:22:48.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:22:48.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:22:48.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:22:48.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:22:48.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:22:48.922 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:22:48.922 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:22:48.922 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:22:48.922 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:48.922 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:22:48.922 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:22:48.922 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:22:48.922 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:22:48.922 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:22:48.922 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:22:48.922 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:22:48.922 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:22:48.922 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:22:48.922 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:48.922 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:22:48.922 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:22:48.922 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:22:48.923 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:22:48.923 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:22:48.923 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:22:48.923 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:22:48.923 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:22:48.923 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:22:48.923 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:22:48.923 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:48.923 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:22:48.923 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:22:48.923 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:22:48.923 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:22:48.923 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:22:48.923 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:22:48.923 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:22:48.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:22:48.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:22:48.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:22:48.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:22:48.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:22:48.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:22:48.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:22:49.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:22:49.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:22:49.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:22:49.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:49.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:22:49.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:22:49.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:22:49.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:22:49.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:22:49.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:22:49.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:22:49.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:22:49.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:22:49.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:49.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:22:49.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:22:49.003 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:22:49.003 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:22:49.003 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:22:49.003 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:22:49.003 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:22:49.003 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:22:49.003 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:22:49.003 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:22:49.003 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:49.003 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:22:49.003 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:22:49.003 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:22:49.004 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:22:49.004 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:22:49.004 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:22:49.004 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:22:49.012 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:22:49.012 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:22:49.012 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:22:49.012 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:22:49.069 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:22:49.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:22:49.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:22:49.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:22:49.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:22:49.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:22:49.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:49.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:22:49.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:22:49.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:22:49.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:22:49.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:22:49.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:22:49.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:22:49.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:22:49.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:22:49.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:49.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:22:49.074 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:22:49.074 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:22:49.074 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:22:49.074 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:22:49.074 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:22:49.074 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:22:49.074 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:22:49.074 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:22:49.074 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:22:49.074 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:49.074 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:22:49.074 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:22:49.075 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:22:49.075 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:22:49.075 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:22:49.075 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:22:49.075 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:22:49.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:22:49.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:22:49.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:22:49.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:22:49.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:22:49.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:22:49.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:22:49.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:22:49.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:22:49.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:22:49.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:22:49.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:22:49.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:22:49.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:22:49.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:22:49.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:22:49.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:22:49.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:49.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:22:49.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:49.183 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 95 2026-02-18T08:22:49.183 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 95 2026-02-18T08:22:49.183 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:22:49.183 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=381, tag_tid=27, entry_tid=1], [object_number=380, tag_tid=27, entry_tid=0], [object_number=382, tag_tid=26, entry_tid=46994], [object_number=383, tag_tid=26, entry_tid=46991]]], state=connected] 2026-02-18T08:22:49.183 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=381, tag_tid=26, entry_tid=45577], [object_number=380, tag_tid=26, entry_tid=45576], [object_number=383, tag_tid=26, entry_tid=45575], [object_number=382, tag_tid=26, entry_tid=45574]]], state=connected] 2026-02-18T08:22:49.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:49.187 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=381, tag_tid=27, entry_tid=1' 2026-02-18T08:22:49.188 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:22:49.188 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:22:49.188 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:22:49.188 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:22:49.188 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:22:49.188 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:22:49.188 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:22:49.188 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:22:49.188 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:22:49.188 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:22:49.189 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:22:49.189 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:22:49.190 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:49.190 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:22:49.190 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:49.220 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 95 2026-02-18T08:22:49.220 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 95 2026-02-18T08:22:49.220 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:22:49.220 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=381, tag_tid=27, entry_tid=1], [object_number=380, tag_tid=27, entry_tid=0], [object_number=382, tag_tid=26, entry_tid=46994], [object_number=383, tag_tid=26, entry_tid=46991]]], state=connected] 2026-02-18T08:22:49.220 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=381, tag_tid=26, entry_tid=45577], [object_number=380, tag_tid=26, entry_tid=45576], [object_number=383, tag_tid=26, entry_tid=45575], [object_number=382, tag_tid=26, entry_tid=45574]]], state=connected] 2026-02-18T08:22:49.223 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:49.224 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=381, tag_tid=26, entry_tid=45577' 2026-02-18T08:22:49.224 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=381, tag_tid=27, entry_tid=1' -a 'object_number=381, tag_tid=27, entry_tid=1' = 'object_number=381, tag_tid=26, entry_tid=45577' 2026-02-18T08:22:49.224 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=381, tag_tid=26, entry_tid=45577' '!=' 'object_number=378, tag_tid=26, entry_tid=45262' 2026-02-18T08:22:49.224 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:22:49.224 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=381, tag_tid=26, entry_tid=45577' = 'object_number=378, tag_tid=26, entry_tid=45262' 2026-02-18T08:22:49.225 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=381, tag_tid=26, entry_tid=45577' 2026-02-18T08:22:49.225 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=381, tag_tid=27, entry_tid=1' 2026-02-18T08:22:49.225 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:22:49.225 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:22:49.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=27 2026-02-18T08:22:49.227 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=381, tag_tid=26, entry_tid=45577' 2026-02-18T08:22:49.227 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:22:49.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:22:49.229 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=26 2026-02-18T08:22:49.229 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=381, tag_tid=27, entry_tid=1' 2026-02-18T08:22:49.229 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:22:49.230 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:22:49.231 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:22:49.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=381, tag_tid=26, entry_tid=45577' 2026-02-18T08:22:49.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:22:49.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:22:49.234 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=45577 2026-02-18T08:22:49.234 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 27 = 26 -a 1 -le 45577 2026-02-18T08:22:49.234 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:22:49.234 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:22:49.234 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:22:49.436 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:22:49.436 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:22:49.437 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:22:49.437 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:22:49.437 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:22:49.437 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:22:49.437 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:22:49.437 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:22:49.437 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:22:49.437 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:49.437 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:22:49.437 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:22:49.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:22:49.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:22:49.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:22:49.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:22:49.440 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:22:49.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:22:49.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:22:49.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:22:49.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:49.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:22:49.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:22:49.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:22:49.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:22:49.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:22:49.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:22:49.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:22:49.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:22:49.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:22:49.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:49.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:22:49.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:22:49.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:22:49.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:22:49.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:22:49.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:22:49.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:22:49.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:22:49.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:22:49.443 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:22:49.443 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:49.443 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:22:49.443 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:22:49.443 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:22:49.443 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:22:49.443 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:22:49.443 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:22:49.443 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:22:49.451 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:22:49.451 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:22:49.451 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:22:49.452 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:22:49.500 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:22:49.500 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:22:49.500 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:22:49.503 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:22:49.503 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:22:49.503 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:22:49.503 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:49.503 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:22:49.503 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:22:49.504 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:22:49.504 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:22:49.504 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:22:49.504 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:22:49.504 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:22:49.504 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:22:49.504 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:22:49.504 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:49.504 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:22:49.504 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:22:49.504 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:22:49.504 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:22:49.505 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:22:49.505 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:22:49.505 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:22:49.505 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:22:49.505 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:22:49.505 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:22:49.505 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:49.505 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:22:49.505 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:22:49.505 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:22:49.505 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:22:49.505 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:22:49.505 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:22:49.506 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:22:49.514 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:22:49.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:22:49.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:22:49.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:22:49.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:22:49.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:22:49.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:22:49.567 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:22:49.567 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:22:49.567 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:22:49.567 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:49.567 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:22:49.567 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:22:49.567 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:22:49.567 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:22:49.567 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:22:49.568 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:22:49.568 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:22:49.568 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:22:49.568 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:22:49.568 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:49.568 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:22:49.568 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:22:49.568 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:22:49.568 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:22:49.568 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:22:49.568 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:22:49.568 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:22:49.569 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:22:49.569 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:22:49.569 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:22:49.569 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:49.569 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:22:49.569 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:22:49.569 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:22:49.569 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:22:49.569 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:22:49.569 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:22:49.569 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:22:49.578 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:22:49.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:22:49.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:22:49.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:22:49.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:22:49.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:22:49.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:22:49.629 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:22:49.629 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:22:49.629 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:22:49.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:49.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:22:49.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:22:49.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:22:49.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:22:49.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:22:49.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:22:49.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:22:49.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:22:49.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:22:49.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:49.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:22:49.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:22:49.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:22:49.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:22:49.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:22:49.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:22:49.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:22:49.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:22:49.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:22:49.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:22:49.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:49.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:22:49.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:22:49.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:22:49.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:22:49.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:22:49.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:22:49.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:22:49.639 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:22:49.640 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:22:49.640 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:22:49.640 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:22:49.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:22:49.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:22:49.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:22:49.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:22:49.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:22:49.700 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:22:49.700 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:22:49.700 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:22:49.700 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:22:49.700 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:22:49.701 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:22:49.701 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:22:49.701 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:22:49.701 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:49.702 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:22:49.702 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:49.731 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 95 2026-02-18T08:22:49.731 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 95 2026-02-18T08:22:49.731 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:22:49.731 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=381, tag_tid=27, entry_tid=1], [object_number=380, tag_tid=27, entry_tid=0], [object_number=382, tag_tid=26, entry_tid=46994], [object_number=383, tag_tid=26, entry_tid=46991]]], state=connected] 2026-02-18T08:22:49.731 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=381, tag_tid=26, entry_tid=45949], [object_number=380, tag_tid=26, entry_tid=45948], [object_number=383, tag_tid=26, entry_tid=45947], [object_number=382, tag_tid=26, entry_tid=45946]]], state=connected] 2026-02-18T08:22:49.734 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:49.735 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=381, tag_tid=27, entry_tid=1' 2026-02-18T08:22:49.736 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:22:49.736 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:22:49.736 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:22:49.737 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:22:49.737 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:22:49.737 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:22:49.737 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:22:49.737 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:22:49.737 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:22:49.737 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:22:49.738 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:22:49.738 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:22:49.739 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:49.739 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:22:49.739 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:49.773 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 95 2026-02-18T08:22:49.774 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 95 2026-02-18T08:22:49.774 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:22:49.774 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=381, tag_tid=27, entry_tid=1], [object_number=380, tag_tid=27, entry_tid=0], [object_number=382, tag_tid=26, entry_tid=46994], [object_number=383, tag_tid=26, entry_tid=46991]]], state=connected] 2026-02-18T08:22:49.774 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=381, tag_tid=26, entry_tid=45949], [object_number=380, tag_tid=26, entry_tid=45948], [object_number=383, tag_tid=26, entry_tid=45947], [object_number=382, tag_tid=26, entry_tid=45946]]], state=connected] 2026-02-18T08:22:49.777 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:49.779 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=381, tag_tid=26, entry_tid=45949' 2026-02-18T08:22:49.779 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=381, tag_tid=27, entry_tid=1' -a 'object_number=381, tag_tid=27, entry_tid=1' = 'object_number=381, tag_tid=26, entry_tid=45949' 2026-02-18T08:22:49.779 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=381, tag_tid=26, entry_tid=45949' '!=' 'object_number=381, tag_tid=26, entry_tid=45577' 2026-02-18T08:22:49.780 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:22:49.780 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=381, tag_tid=26, entry_tid=45949' = 'object_number=381, tag_tid=26, entry_tid=45577' 2026-02-18T08:22:49.780 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=381, tag_tid=26, entry_tid=45949' 2026-02-18T08:22:49.780 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=381, tag_tid=27, entry_tid=1' 2026-02-18T08:22:49.780 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:22:49.781 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:22:49.781 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=27 2026-02-18T08:22:49.782 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=381, tag_tid=26, entry_tid=45949' 2026-02-18T08:22:49.782 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:22:49.782 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:22:49.784 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=26 2026-02-18T08:22:49.784 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=381, tag_tid=27, entry_tid=1' 2026-02-18T08:22:49.784 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:22:49.785 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:22:49.786 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:22:49.787 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=381, tag_tid=26, entry_tid=45949' 2026-02-18T08:22:49.787 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:22:49.787 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:22:49.788 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=45949 2026-02-18T08:22:49.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 27 = 26 -a 1 -le 45949 2026-02-18T08:22:49.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:22:49.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:22:49.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:22:49.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:22:49.992 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:22:49.992 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:22:49.992 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:22:49.992 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:22:49.992 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:22:49.992 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:22:49.992 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:22:49.992 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:22:49.992 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:49.992 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:22:49.993 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:22:49.993 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:22:49.993 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:22:49.993 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:22:49.993 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:22:49.993 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:22:49.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:22:49.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:22:49.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:22:49.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:49.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:22:49.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:22:49.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:22:49.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:22:49.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:22:49.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:22:49.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:22:49.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:22:49.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:22:49.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:49.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:22:49.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:22:49.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:22:49.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:22:49.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:22:49.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:22:49.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:22:49.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:22:49.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:22:49.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:22:49.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:49.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:22:49.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:22:49.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:22:49.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:22:49.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:22:49.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:22:49.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:22:50.004 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:22:50.005 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:22:50.005 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:22:50.005 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:22:50.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:22:50.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:22:50.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:22:50.056 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:22:50.056 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:22:50.056 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:22:50.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:50.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:22:50.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:22:50.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:22:50.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:22:50.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:22:50.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:22:50.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:22:50.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:22:50.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:22:50.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:50.058 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:22:50.058 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:22:50.058 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:22:50.058 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:22:50.058 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:22:50.058 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:22:50.058 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:22:50.058 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:22:50.058 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:22:50.059 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:22:50.059 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:50.059 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:22:50.059 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:22:50.059 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:22:50.059 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:22:50.059 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:22:50.059 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:22:50.059 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:22:50.067 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:22:50.067 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:22:50.067 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:22:50.068 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:22:50.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:22:50.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:22:50.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:22:50.119 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:22:50.119 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:22:50.119 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:22:50.120 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:50.120 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:22:50.120 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:22:50.120 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:22:50.120 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:22:50.120 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:22:50.120 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:22:50.120 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:22:50.120 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:22:50.120 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:22:50.121 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:50.121 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:22:50.121 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:22:50.121 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:22:50.121 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:22:50.121 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:22:50.121 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:22:50.121 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:22:50.121 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:22:50.121 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:22:50.122 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:22:50.122 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:50.122 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:22:50.122 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:22:50.122 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:22:50.122 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:22:50.122 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:22:50.122 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:22:50.122 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:22:50.130 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:22:50.130 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:22:50.130 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:22:50.131 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:22:50.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:22:50.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:22:50.187 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:22:50.189 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:22:50.189 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:22:50.189 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:22:50.190 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:50.190 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:22:50.190 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:22:50.190 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:22:50.190 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:22:50.190 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:22:50.190 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:22:50.190 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:22:50.190 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:22:50.190 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:22:50.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:50.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:22:50.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:22:50.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:22:50.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:22:50.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:22:50.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:22:50.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:22:50.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:22:50.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:22:50.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:22:50.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:50.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:22:50.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:22:50.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:22:50.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:22:50.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:22:50.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:22:50.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:22:50.200 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:22:50.200 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:22:50.200 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:22:50.200 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:22:50.259 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:22:50.261 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:22:50.261 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:22:50.261 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:22:50.261 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:22:50.261 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:22:50.261 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:22:50.261 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:22:50.261 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:22:50.261 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:22:50.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:22:50.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:22:50.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:22:50.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:50.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:22:50.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:50.291 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 95 2026-02-18T08:22:50.291 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 95 2026-02-18T08:22:50.291 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:22:50.291 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=381, tag_tid=27, entry_tid=1], [object_number=380, tag_tid=27, entry_tid=0], [object_number=382, tag_tid=26, entry_tid=46994], [object_number=383, tag_tid=26, entry_tid=46991]]], state=connected] 2026-02-18T08:22:50.291 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=381, tag_tid=26, entry_tid=46305], [object_number=380, tag_tid=26, entry_tid=46304], [object_number=383, tag_tid=26, entry_tid=46303], [object_number=382, tag_tid=26, entry_tid=46302]]], state=connected] 2026-02-18T08:22:50.294 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:50.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=381, tag_tid=27, entry_tid=1' 2026-02-18T08:22:50.297 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:22:50.297 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:22:50.297 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:22:50.297 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:22:50.297 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:22:50.297 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:22:50.297 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:22:50.297 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:22:50.297 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:22:50.297 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:22:50.298 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:22:50.298 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:22:50.298 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:50.299 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:22:50.299 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:50.327 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 95 2026-02-18T08:22:50.327 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 95 2026-02-18T08:22:50.327 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:22:50.327 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=381, tag_tid=27, entry_tid=1], [object_number=380, tag_tid=27, entry_tid=0], [object_number=382, tag_tid=26, entry_tid=46994], [object_number=383, tag_tid=26, entry_tid=46991]]], state=connected] 2026-02-18T08:22:50.327 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=381, tag_tid=26, entry_tid=46305], [object_number=380, tag_tid=26, entry_tid=46304], [object_number=383, tag_tid=26, entry_tid=46303], [object_number=382, tag_tid=26, entry_tid=46302]]], state=connected] 2026-02-18T08:22:50.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:50.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=381, tag_tid=26, entry_tid=46305' 2026-02-18T08:22:50.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=381, tag_tid=27, entry_tid=1' -a 'object_number=381, tag_tid=27, entry_tid=1' = 'object_number=381, tag_tid=26, entry_tid=46305' 2026-02-18T08:22:50.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=381, tag_tid=26, entry_tid=46305' '!=' 'object_number=381, tag_tid=26, entry_tid=45949' 2026-02-18T08:22:50.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:22:50.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=381, tag_tid=26, entry_tid=46305' = 'object_number=381, tag_tid=26, entry_tid=45949' 2026-02-18T08:22:50.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=381, tag_tid=26, entry_tid=46305' 2026-02-18T08:22:50.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=381, tag_tid=27, entry_tid=1' 2026-02-18T08:22:50.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:22:50.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:22:50.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=27 2026-02-18T08:22:50.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=381, tag_tid=26, entry_tid=46305' 2026-02-18T08:22:50.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:22:50.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:22:50.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=26 2026-02-18T08:22:50.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=381, tag_tid=27, entry_tid=1' 2026-02-18T08:22:50.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:22:50.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:22:50.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:22:50.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=381, tag_tid=26, entry_tid=46305' 2026-02-18T08:22:50.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:22:50.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:22:50.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=46305 2026-02-18T08:22:50.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 27 = 26 -a 1 -le 46305 2026-02-18T08:22:50.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:22:50.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:22:50.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:22:50.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:22:50.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:22:50.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:22:50.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:22:50.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:22:50.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:22:50.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:22:50.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:22:50.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:22:50.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:50.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:22:50.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:22:50.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:22:50.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:22:50.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:22:50.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:22:50.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:22:50.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:22:50.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:22:50.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:22:50.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:50.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:22:50.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:22:50.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:22:50.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:22:50.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:22:50.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:22:50.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:22:50.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:22:50.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:22:50.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:50.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:22:50.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:22:50.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:22:50.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:22:50.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:22:50.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:22:50.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:22:50.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:22:50.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:22:50.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:22:50.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:50.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:22:50.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:22:50.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:22:50.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:22:50.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:22:50.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:22:50.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:22:50.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:22:50.557 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:22:50.557 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:22:50.557 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:22:50.606 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:22:50.606 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:22:50.606 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:22:50.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:22:50.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:22:50.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:22:50.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:50.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:22:50.610 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:22:50.610 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:22:50.610 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:22:50.610 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:22:50.610 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:22:50.610 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:22:50.610 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:22:50.610 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:22:50.610 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:50.610 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:22:50.610 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:22:50.610 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:22:50.611 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:22:50.611 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:22:50.611 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:22:50.611 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:22:50.611 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:22:50.611 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:22:50.611 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:22:50.611 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:50.611 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:22:50.611 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:22:50.611 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:22:50.611 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:22:50.612 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:22:50.612 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:22:50.612 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:22:50.620 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:22:50.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:22:50.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:22:50.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:22:50.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:22:50.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:22:50.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:22:50.672 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:22:50.672 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:22:50.672 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:22:50.672 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:50.672 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:22:50.672 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:22:50.672 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:22:50.672 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:22:50.672 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:22:50.673 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:22:50.673 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:22:50.673 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:22:50.673 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:22:50.673 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:50.673 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:22:50.673 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:22:50.673 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:22:50.673 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:22:50.673 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:22:50.673 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:22:50.673 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:22:50.674 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:22:50.674 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:22:50.674 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:22:50.674 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:50.674 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:22:50.674 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:22:50.674 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:22:50.674 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:22:50.674 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:22:50.674 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:22:50.674 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:22:50.683 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:22:50.683 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:22:50.684 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:22:50.684 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:22:50.740 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:22:50.740 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:22:50.740 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:22:50.743 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:22:50.744 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:22:50.744 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:22:50.744 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:50.744 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:22:50.744 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:22:50.744 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:22:50.744 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:22:50.744 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:22:50.744 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:22:50.745 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:22:50.745 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:22:50.745 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:22:50.745 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:50.745 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:22:50.745 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:22:50.745 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:22:50.745 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:22:50.745 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:22:50.745 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:22:50.745 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:22:50.745 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:22:50.746 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:22:50.746 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:22:50.746 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:50.746 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:22:50.746 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:22:50.746 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:22:50.746 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:22:50.746 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:22:50.746 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:22:50.746 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:22:50.754 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:22:50.755 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:22:50.755 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:22:50.755 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:22:50.822 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:22:50.823 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:22:50.824 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:22:50.824 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:22:50.824 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:22:50.824 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:22:50.824 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:22:50.824 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:22:50.824 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:22:50.824 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:22:50.824 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:22:50.825 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:22:50.825 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:22:50.825 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:50.826 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:22:50.826 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:50.854 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 95 2026-02-18T08:22:50.854 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 95 2026-02-18T08:22:50.854 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:22:50.854 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=381, tag_tid=27, entry_tid=1], [object_number=380, tag_tid=27, entry_tid=0], [object_number=382, tag_tid=26, entry_tid=46994], [object_number=383, tag_tid=26, entry_tid=46991]]], state=connected] 2026-02-18T08:22:50.854 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=383, tag_tid=26, entry_tid=46651], [object_number=382, tag_tid=26, entry_tid=46650], [object_number=381, tag_tid=26, entry_tid=46649], [object_number=380, tag_tid=26, entry_tid=46648]]], state=connected] 2026-02-18T08:22:50.856 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:50.858 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=381, tag_tid=27, entry_tid=1' 2026-02-18T08:22:50.859 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:22:50.859 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:22:50.859 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:22:50.859 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:22:50.860 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:22:50.860 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:22:50.860 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:22:50.860 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:22:50.860 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:22:50.860 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:22:50.860 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:22:50.860 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:22:50.861 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:50.861 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:22:50.861 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:50.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 95 2026-02-18T08:22:50.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 95 2026-02-18T08:22:50.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:22:50.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=381, tag_tid=27, entry_tid=1], [object_number=380, tag_tid=27, entry_tid=0], [object_number=382, tag_tid=26, entry_tid=46994], [object_number=383, tag_tid=26, entry_tid=46991]]], state=connected] 2026-02-18T08:22:50.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=383, tag_tid=26, entry_tid=46651], [object_number=382, tag_tid=26, entry_tid=46650], [object_number=381, tag_tid=26, entry_tid=46649], [object_number=380, tag_tid=26, entry_tid=46648]]], state=connected] 2026-02-18T08:22:50.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:50.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=383, tag_tid=26, entry_tid=46651' 2026-02-18T08:22:50.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=381, tag_tid=27, entry_tid=1' -a 'object_number=381, tag_tid=27, entry_tid=1' = 'object_number=383, tag_tid=26, entry_tid=46651' 2026-02-18T08:22:50.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=383, tag_tid=26, entry_tid=46651' '!=' 'object_number=381, tag_tid=26, entry_tid=46305' 2026-02-18T08:22:50.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:22:50.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=383, tag_tid=26, entry_tid=46651' = 'object_number=381, tag_tid=26, entry_tid=46305' 2026-02-18T08:22:50.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=383, tag_tid=26, entry_tid=46651' 2026-02-18T08:22:50.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=381, tag_tid=27, entry_tid=1' 2026-02-18T08:22:50.894 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:22:50.894 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:22:50.894 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=27 2026-02-18T08:22:50.895 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=383, tag_tid=26, entry_tid=46651' 2026-02-18T08:22:50.895 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:22:50.895 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:22:50.896 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=26 2026-02-18T08:22:50.897 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=381, tag_tid=27, entry_tid=1' 2026-02-18T08:22:50.897 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:22:50.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:22:50.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:22:50.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=383, tag_tid=26, entry_tid=46651' 2026-02-18T08:22:50.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:22:50.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:22:50.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=46651 2026-02-18T08:22:50.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 27 = 26 -a 1 -le 46651 2026-02-18T08:22:50.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:22:50.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:22:50.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:22:51.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:22:51.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:22:51.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:22:51.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:22:51.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:22:51.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:22:51.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:22:51.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:22:51.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:22:51.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:51.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:22:51.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:22:51.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:22:51.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:22:51.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:22:51.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:22:51.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:22:51.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:22:51.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:22:51.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:22:51.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:51.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:22:51.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:22:51.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:22:51.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:22:51.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:22:51.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:22:51.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:22:51.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:22:51.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:22:51.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:51.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:22:51.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:22:51.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:22:51.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:22:51.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:22:51.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:22:51.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:22:51.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:22:51.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:22:51.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:22:51.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:51.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:22:51.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:22:51.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:22:51.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:22:51.110 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:22:51.110 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:22:51.110 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:22:51.119 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:22:51.119 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:22:51.119 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:22:51.119 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:22:51.167 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:22:51.167 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:22:51.167 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:22:51.170 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:22:51.170 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:22:51.170 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:22:51.170 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:51.170 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:22:51.170 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:22:51.171 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:22:51.171 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:22:51.171 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:22:51.171 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:22:51.171 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:22:51.171 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:22:51.171 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:22:51.171 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:51.171 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:22:51.171 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:22:51.171 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:22:51.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:22:51.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:22:51.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:22:51.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:22:51.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:22:51.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:22:51.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:22:51.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:51.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:22:51.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:22:51.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:22:51.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:22:51.173 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:22:51.173 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:22:51.173 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:22:51.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:22:51.183 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:22:51.183 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:22:51.183 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:22:51.231 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:22:51.231 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:22:51.231 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:22:51.234 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:22:51.234 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:22:51.234 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:22:51.234 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:51.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:22:51.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:22:51.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:22:51.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:22:51.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:22:51.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:22:51.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:22:51.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:22:51.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:22:51.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:51.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:22:51.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:22:51.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:22:51.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:22:51.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:22:51.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:22:51.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:22:51.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:22:51.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:22:51.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:22:51.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:51.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:22:51.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:22:51.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:22:51.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:22:51.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:22:51.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:22:51.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:22:51.244 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:22:51.245 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:22:51.245 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:22:51.245 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:22:51.293 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:22:51.293 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:22:51.293 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:22:51.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:22:51.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:22:51.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:22:51.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:51.297 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:22:51.297 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:22:51.297 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:22:51.297 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:22:51.297 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:22:51.297 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:22:51.297 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:22:51.297 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:22:51.297 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:22:51.297 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:51.298 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:22:51.298 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:22:51.298 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:22:51.298 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:22:51.298 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:22:51.298 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:22:51.298 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:22:51.298 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:22:51.298 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:22:51.298 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:22:51.298 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:51.299 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:22:51.299 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:22:51.299 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:22:51.299 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:22:51.299 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:22:51.299 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:22:51.299 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:22:51.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:22:51.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:22:51.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:22:51.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:22:51.367 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:22:51.368 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:22:51.368 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:22:51.368 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:22:51.368 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:22:51.368 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:22:51.368 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:22:51.368 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:22:51.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:22:51.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:22:51.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:22:51.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:22:51.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:22:51.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:51.370 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:22:51.370 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:51.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 95 2026-02-18T08:22:51.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 95 2026-02-18T08:22:51.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:22:51.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=381, tag_tid=27, entry_tid=1], [object_number=380, tag_tid=27, entry_tid=0], [object_number=382, tag_tid=26, entry_tid=46994], [object_number=383, tag_tid=26, entry_tid=46991]]], state=connected] 2026-02-18T08:22:51.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=381, tag_tid=26, entry_tid=46989], [object_number=380, tag_tid=26, entry_tid=46988], [object_number=383, tag_tid=26, entry_tid=46987], [object_number=382, tag_tid=26, entry_tid=46986]]], state=connected] 2026-02-18T08:22:51.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:51.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=381, tag_tid=27, entry_tid=1' 2026-02-18T08:22:51.403 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:22:51.403 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:22:51.403 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:22:51.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:22:51.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:22:51.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:22:51.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:22:51.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:22:51.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:22:51.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:22:51.405 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:22:51.405 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:22:51.405 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:51.406 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:22:51.406 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:51.435 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 95 2026-02-18T08:22:51.435 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 95 2026-02-18T08:22:51.435 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:22:51.436 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=381, tag_tid=27, entry_tid=1], [object_number=380, tag_tid=27, entry_tid=0], [object_number=382, tag_tid=26, entry_tid=46994], [object_number=383, tag_tid=26, entry_tid=46991]]], state=connected] 2026-02-18T08:22:51.436 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=381, tag_tid=26, entry_tid=46989], [object_number=380, tag_tid=26, entry_tid=46988], [object_number=383, tag_tid=26, entry_tid=46987], [object_number=382, tag_tid=26, entry_tid=46986]]], state=connected] 2026-02-18T08:22:51.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:51.439 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=381, tag_tid=26, entry_tid=46989' 2026-02-18T08:22:51.439 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=381, tag_tid=27, entry_tid=1' -a 'object_number=381, tag_tid=27, entry_tid=1' = 'object_number=381, tag_tid=26, entry_tid=46989' 2026-02-18T08:22:51.439 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=381, tag_tid=26, entry_tid=46989' '!=' 'object_number=383, tag_tid=26, entry_tid=46651' 2026-02-18T08:22:51.440 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:22:51.440 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=381, tag_tid=26, entry_tid=46989' = 'object_number=383, tag_tid=26, entry_tid=46651' 2026-02-18T08:22:51.440 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=381, tag_tid=26, entry_tid=46989' 2026-02-18T08:22:51.440 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=381, tag_tid=27, entry_tid=1' 2026-02-18T08:22:51.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:22:51.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:22:51.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=27 2026-02-18T08:22:51.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=381, tag_tid=26, entry_tid=46989' 2026-02-18T08:22:51.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:22:51.443 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:22:51.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=26 2026-02-18T08:22:51.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=381, tag_tid=27, entry_tid=1' 2026-02-18T08:22:51.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:22:51.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:22:51.446 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:22:51.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=381, tag_tid=26, entry_tid=46989' 2026-02-18T08:22:51.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:22:51.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:22:51.449 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=46989 2026-02-18T08:22:51.449 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 27 = 26 -a 1 -le 46989 2026-02-18T08:22:51.449 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:22:51.449 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:22:51.449 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:22:51.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:22:51.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:22:51.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:22:51.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:22:51.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:22:51.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:22:51.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:22:51.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:22:51.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:22:51.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:51.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:22:51.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:22:51.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:22:51.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:22:51.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:22:51.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:22:51.655 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:22:51.655 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:22:51.655 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:22:51.655 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:22:51.655 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:51.655 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:22:51.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:22:51.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:22:51.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:22:51.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:22:51.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:22:51.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:22:51.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:22:51.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:22:51.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:51.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:22:51.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:22:51.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:22:51.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:22:51.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:22:51.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:22:51.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:22:51.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:22:51.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:22:51.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:22:51.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:51.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:22:51.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:22:51.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:22:51.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:22:51.658 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:22:51.658 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:22:51.658 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:22:51.664 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:22:51.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:22:51.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:22:51.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:22:51.724 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:22:51.725 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:22:51.725 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:22:51.728 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:22:51.728 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:22:51.728 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:22:51.728 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:51.728 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:22:51.729 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:22:51.729 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:22:51.729 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:22:51.729 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:22:51.729 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:22:51.729 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:22:51.729 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:22:51.729 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:22:51.729 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:51.729 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:22:51.729 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:22:51.729 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:22:51.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:22:51.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:22:51.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:22:51.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:22:51.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:22:51.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:22:51.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:22:51.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:51.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:22:51.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:22:51.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:22:51.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:22:51.731 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:22:51.731 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:22:51.731 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:22:51.739 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:22:51.739 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:22:51.740 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:22:51.740 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:22:51.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:22:51.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:22:51.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:22:51.800 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:22:51.800 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:22:51.800 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:22:51.800 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:51.801 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:22:51.801 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:22:51.801 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:22:51.801 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:22:51.801 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:22:51.801 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:22:51.801 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:22:51.801 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:22:51.801 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:22:51.801 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:51.801 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:22:51.801 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:22:51.802 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:22:51.802 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:22:51.802 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:22:51.802 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:22:51.802 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:22:51.802 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:22:51.802 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:22:51.802 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:22:51.802 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:51.802 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:22:51.802 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:22:51.803 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:22:51.803 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:22:51.803 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:22:51.803 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:22:51.803 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:22:51.811 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:22:51.811 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:22:51.812 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:22:51.812 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:22:51.868 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:22:51.868 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:22:51.869 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:22:51.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:22:51.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:22:51.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:22:51.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:51.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:22:51.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:22:51.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:22:51.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:22:51.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:22:51.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:22:51.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:22:51.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:22:51.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:22:51.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:51.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:22:51.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:22:51.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:22:51.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:22:51.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:22:51.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:22:51.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:22:51.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:22:51.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:22:51.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:22:51.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:51.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:22:51.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:22:51.874 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:22:51.874 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:22:51.874 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:22:51.874 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:22:51.874 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:22:51.882 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:22:51.883 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:22:51.883 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:22:51.883 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:22:51.950 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:22:51.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:22:51.952 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:22:51.952 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:22:51.952 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:22:51.952 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:22:51.952 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:22:51.952 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:22:51.952 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:22:51.952 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:22:51.952 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:22:51.952 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:22:51.952 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:22:51.953 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:51.954 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:22:51.954 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:51.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 95 2026-02-18T08:22:51.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 95 2026-02-18T08:22:51.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:22:51.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=381, tag_tid=27, entry_tid=1], [object_number=380, tag_tid=27, entry_tid=0], [object_number=382, tag_tid=26, entry_tid=46994], [object_number=383, tag_tid=26, entry_tid=46991]]], state=connected] 2026-02-18T08:22:51.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=382, tag_tid=26, entry_tid=46994], [object_number=381, tag_tid=26, entry_tid=46993], [object_number=380, tag_tid=26, entry_tid=46992], [object_number=383, tag_tid=26, entry_tid=46991]]], state=connected] 2026-02-18T08:22:51.987 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:51.988 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=381, tag_tid=27, entry_tid=1' 2026-02-18T08:22:51.989 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:22:51.990 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:22:51.990 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:22:51.990 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:22:51.990 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:22:51.990 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:22:51.990 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:22:51.990 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:22:51.990 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:22:51.990 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:22:51.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:22:51.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:22:51.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:51.992 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:22:51.992 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:52.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 95 2026-02-18T08:22:52.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 95 2026-02-18T08:22:52.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:22:52.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=381, tag_tid=27, entry_tid=1], [object_number=380, tag_tid=27, entry_tid=0], [object_number=382, tag_tid=26, entry_tid=46994], [object_number=383, tag_tid=26, entry_tid=46991]]], state=connected] 2026-02-18T08:22:52.027 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=382, tag_tid=26, entry_tid=46994], [object_number=381, tag_tid=26, entry_tid=46993], [object_number=380, tag_tid=26, entry_tid=46992], [object_number=383, tag_tid=26, entry_tid=46991]]], state=connected] 2026-02-18T08:22:52.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:52.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=382, tag_tid=26, entry_tid=46994' 2026-02-18T08:22:52.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=381, tag_tid=27, entry_tid=1' -a 'object_number=381, tag_tid=27, entry_tid=1' = 'object_number=382, tag_tid=26, entry_tid=46994' 2026-02-18T08:22:52.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=382, tag_tid=26, entry_tid=46994' '!=' 'object_number=381, tag_tid=26, entry_tid=46989' 2026-02-18T08:22:52.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:22:52.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=382, tag_tid=26, entry_tid=46994' = 'object_number=381, tag_tid=26, entry_tid=46989' 2026-02-18T08:22:52.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=382, tag_tid=26, entry_tid=46994' 2026-02-18T08:22:52.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=381, tag_tid=27, entry_tid=1' 2026-02-18T08:22:52.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:22:52.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:22:52.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=27 2026-02-18T08:22:52.035 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=382, tag_tid=26, entry_tid=46994' 2026-02-18T08:22:52.035 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:22:52.035 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:22:52.036 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=26 2026-02-18T08:22:52.037 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=381, tag_tid=27, entry_tid=1' 2026-02-18T08:22:52.037 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:22:52.038 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:22:52.039 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:22:52.039 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=382, tag_tid=26, entry_tid=46994' 2026-02-18T08:22:52.039 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:22:52.040 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:22:52.041 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=46994 2026-02-18T08:22:52.041 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 27 = 26 -a 1 -le 46994 2026-02-18T08:22:52.041 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:22:52.041 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:22:52.041 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:22:52.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:22:52.244 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:22:52.244 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:22:52.244 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:22:52.244 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:22:52.244 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:22:52.244 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:22:52.245 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:22:52.245 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:22:52.245 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:52.245 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:22:52.245 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:22:52.245 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:22:52.245 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:22:52.245 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:22:52.246 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:22:52.246 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:22:52.247 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:22:52.247 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:22:52.247 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:22:52.247 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:52.247 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:22:52.247 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:22:52.247 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:22:52.247 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:22:52.247 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:22:52.247 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:22:52.247 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:22:52.248 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:22:52.248 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:22:52.248 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:52.248 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:22:52.248 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:22:52.248 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:22:52.248 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:22:52.248 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:22:52.248 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:22:52.248 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:22:52.248 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:22:52.248 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:22:52.248 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:22:52.248 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:52.248 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:22:52.248 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:22:52.248 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:22:52.248 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:22:52.248 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:22:52.248 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:22:52.248 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:22:52.256 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:22:52.256 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:22:52.256 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:22:52.256 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:22:52.320 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:22:52.320 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:22:52.321 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:22:52.323 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:22:52.324 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:22:52.324 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:22:52.324 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:52.324 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:22:52.324 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:22:52.324 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:22:52.324 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:22:52.324 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:22:52.324 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:22:52.324 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:22:52.325 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:22:52.325 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:22:52.325 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:52.325 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:22:52.325 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:22:52.325 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:22:52.325 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:22:52.325 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:22:52.325 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:22:52.325 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:22:52.325 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:22:52.326 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:22:52.326 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:22:52.326 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:52.326 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:22:52.326 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:22:52.326 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:22:52.326 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:22:52.326 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:22:52.326 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:22:52.326 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:22:52.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:22:52.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:22:52.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:22:52.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:22:52.393 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:22:52.393 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:22:52.393 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:22:52.396 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:22:52.396 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:22:52.396 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:22:52.396 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:52.396 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:22:52.396 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:22:52.396 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:22:52.396 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:22:52.397 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:22:52.397 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:22:52.397 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:22:52.397 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:22:52.397 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:22:52.397 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:52.397 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:22:52.397 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:22:52.397 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:22:52.397 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:22:52.397 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:22:52.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:22:52.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:22:52.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:22:52.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:22:52.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:22:52.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:52.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:22:52.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:22:52.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:22:52.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:22:52.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:22:52.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:22:52.399 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:22:52.406 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:22:52.407 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:22:52.407 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:22:52.407 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:22:52.465 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:22:52.465 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:22:52.465 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:22:52.468 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:22:52.468 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:22:52.468 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:22:52.468 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:52.468 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:22:52.468 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:22:52.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:22:52.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:22:52.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:22:52.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:22:52.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:22:52.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:22:52.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:22:52.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:52.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:22:52.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:22:52.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:22:52.470 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:22:52.470 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:22:52.470 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:22:52.470 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:22:52.470 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:22:52.470 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:22:52.470 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:22:52.470 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:52.470 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:22:52.470 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:22:52.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:22:52.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:22:52.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:22:52.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:22:52.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:22:52.478 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:22:52.479 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:22:52.479 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:22:52.479 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:22:52.534 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:22:52.535 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:22:52.535 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:22:52.536 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:22:52.536 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:22:52.536 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:22:52.536 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:22:52.536 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:22:52.536 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:22:52.536 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:22:52.536 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:22:52.536 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:22:52.536 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:22:52.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:52.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:22:52.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:52.573 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 95 2026-02-18T08:22:52.573 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 95 2026-02-18T08:22:52.573 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:22:52.573 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=381, tag_tid=27, entry_tid=1], [object_number=380, tag_tid=27, entry_tid=0], [object_number=382, tag_tid=26, entry_tid=46994], [object_number=383, tag_tid=26, entry_tid=46991]]], state=connected] 2026-02-18T08:22:52.573 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=382, tag_tid=26, entry_tid=46994], [object_number=381, tag_tid=26, entry_tid=46993], [object_number=380, tag_tid=26, entry_tid=46992], [object_number=383, tag_tid=26, entry_tid=46991]]], state=connected] 2026-02-18T08:22:52.577 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:52.578 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=381, tag_tid=27, entry_tid=1' 2026-02-18T08:22:52.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:22:52.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:22:52.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:22:52.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:22:52.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:22:52.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:22:52.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:22:52.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:22:52.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:22:52.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:22:52.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:22:52.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:22:52.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:52.582 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:22:52.582 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:52.618 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 95 2026-02-18T08:22:52.618 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 95 2026-02-18T08:22:52.618 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:22:52.618 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=381, tag_tid=27, entry_tid=1], [object_number=380, tag_tid=27, entry_tid=0], [object_number=382, tag_tid=26, entry_tid=46994], [object_number=383, tag_tid=26, entry_tid=46991]]], state=connected] 2026-02-18T08:22:52.618 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=382, tag_tid=26, entry_tid=46994], [object_number=381, tag_tid=26, entry_tid=46993], [object_number=380, tag_tid=26, entry_tid=46992], [object_number=383, tag_tid=26, entry_tid=46991]]], state=connected] 2026-02-18T08:22:52.622 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:52.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=382, tag_tid=26, entry_tid=46994' 2026-02-18T08:22:52.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=381, tag_tid=27, entry_tid=1' -a 'object_number=381, tag_tid=27, entry_tid=1' = 'object_number=382, tag_tid=26, entry_tid=46994' 2026-02-18T08:22:52.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=382, tag_tid=26, entry_tid=46994' '!=' 'object_number=382, tag_tid=26, entry_tid=46994' 2026-02-18T08:22:52.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:22:52.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.4 2026-02-18T08:22:53.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:22:53.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:22:53.027 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:22:53.027 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:22:53.027 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:22:53.027 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:22:53.027 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:22:53.027 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:22:53.027 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:22:53.027 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:53.027 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:22:53.027 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:22:53.027 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:22:53.028 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:22:53.028 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:22:53.028 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:22:53.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:22:53.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:22:53.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:22:53.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:22:53.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:53.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:22:53.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:22:53.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:22:53.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:22:53.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:22:53.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:22:53.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:22:53.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:22:53.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:22:53.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:53.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:22:53.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:22:53.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:22:53.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:22:53.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:22:53.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:22:53.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:22:53.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:22:53.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:22:53.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:22:53.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:53.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:22:53.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:22:53.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:22:53.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:22:53.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:22:53.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:22:53.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:22:53.040 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:22:53.040 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:22:53.041 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:22:53.041 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:22:53.092 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:22:53.092 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:22:53.092 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:22:53.095 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:22:53.095 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:22:53.095 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:22:53.095 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:53.096 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:22:53.096 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:22:53.096 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:22:53.096 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:22:53.096 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:22:53.096 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:22:53.096 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:22:53.096 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:22:53.096 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:22:53.096 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:53.096 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:22:53.096 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:22:53.097 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:22:53.097 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:22:53.097 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:22:53.097 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:22:53.097 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:22:53.097 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:22:53.097 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:22:53.097 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:22:53.097 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:53.097 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:22:53.097 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:22:53.097 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:22:53.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:22:53.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:22:53.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:22:53.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:22:53.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:22:53.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:22:53.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:22:53.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:22:53.164 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:22:53.165 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:22:53.165 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:22:53.167 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:22:53.167 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:22:53.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:22:53.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:53.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:22:53.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:22:53.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:22:53.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:22:53.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:22:53.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:22:53.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:22:53.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:22:53.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:22:53.169 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:53.169 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:22:53.169 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:22:53.169 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:22:53.169 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:22:53.169 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:22:53.169 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:22:53.169 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:22:53.169 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:22:53.169 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:22:53.169 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:22:53.169 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:53.169 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:22:53.170 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:22:53.170 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:22:53.170 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:22:53.170 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:22:53.170 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:22:53.170 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:22:53.179 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:22:53.180 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:22:53.180 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:22:53.180 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:22:53.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:22:53.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:22:53.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:22:53.231 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:22:53.231 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:22:53.231 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:22:53.231 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:53.231 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:22:53.231 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:22:53.231 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:22:53.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:22:53.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:22:53.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:22:53.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:22:53.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:22:53.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:22:53.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:53.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:22:53.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:22:53.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:22:53.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:22:53.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:22:53.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:22:53.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:22:53.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:22:53.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:22:53.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:22:53.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:53.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:22:53.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:22:53.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:22:53.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:22:53.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:22:53.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:22:53.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:22:53.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:22:53.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:22:53.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:22:53.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:22:53.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:22:53.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:22:53.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:22:53.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:22:53.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:22:53.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:22:53.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:22:53.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:22:53.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:22:53.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:22:53.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:22:53.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:22:53.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:22:53.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:53.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:22:53.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:53.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 95 2026-02-18T08:22:53.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 95 2026-02-18T08:22:53.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:22:53.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=381, tag_tid=27, entry_tid=1], [object_number=380, tag_tid=27, entry_tid=0], [object_number=382, tag_tid=26, entry_tid=46994], [object_number=383, tag_tid=26, entry_tid=46991]]], state=connected] 2026-02-18T08:22:53.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=382, tag_tid=26, entry_tid=46994], [object_number=381, tag_tid=26, entry_tid=46993], [object_number=380, tag_tid=26, entry_tid=46992], [object_number=383, tag_tid=26, entry_tid=46991]]], state=connected] 2026-02-18T08:22:53.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:53.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=381, tag_tid=27, entry_tid=1' 2026-02-18T08:22:53.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:22:53.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:22:53.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:22:53.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:22:53.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:22:53.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:22:53.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:22:53.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:22:53.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:22:53.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:22:53.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:22:53.344 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:22:53.344 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:53.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:22:53.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:53.382 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 95 2026-02-18T08:22:53.382 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 95 2026-02-18T08:22:53.382 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:22:53.382 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=381, tag_tid=27, entry_tid=1], [object_number=380, tag_tid=27, entry_tid=0], [object_number=382, tag_tid=26, entry_tid=46994], [object_number=383, tag_tid=26, entry_tid=46991]]], state=connected] 2026-02-18T08:22:53.382 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=382, tag_tid=26, entry_tid=46994], [object_number=381, tag_tid=26, entry_tid=46993], [object_number=380, tag_tid=26, entry_tid=46992], [object_number=383, tag_tid=26, entry_tid=46991]]], state=connected] 2026-02-18T08:22:53.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:53.387 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=382, tag_tid=26, entry_tid=46994' 2026-02-18T08:22:53.387 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=381, tag_tid=27, entry_tid=1' -a 'object_number=381, tag_tid=27, entry_tid=1' = 'object_number=382, tag_tid=26, entry_tid=46994' 2026-02-18T08:22:53.387 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=382, tag_tid=26, entry_tid=46994' '!=' 'object_number=382, tag_tid=26, entry_tid=46994' 2026-02-18T08:22:53.387 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:22:53.387 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.8 2026-02-18T08:22:54.189 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:22:54.190 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:22:54.190 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:22:54.190 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:22:54.190 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:22:54.190 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:22:54.190 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:22:54.190 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:22:54.190 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:22:54.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:54.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:22:54.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:22:54.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:22:54.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:22:54.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:22:54.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:22:54.194 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:22:54.194 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:22:54.195 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:22:54.195 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:22:54.195 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:54.195 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:22:54.195 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:22:54.195 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:22:54.195 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:22:54.195 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:22:54.195 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:22:54.196 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:22:54.196 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:22:54.196 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:22:54.196 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:54.196 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:22:54.196 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:22:54.196 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:22:54.196 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:22:54.196 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:22:54.196 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:22:54.196 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:22:54.197 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:22:54.197 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:22:54.197 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:22:54.197 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:54.197 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:22:54.197 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:22:54.197 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:22:54.197 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:22:54.197 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:22:54.197 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:22:54.197 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:22:54.204 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:22:54.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:22:54.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:22:54.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:22:54.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:22:54.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:22:54.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:22:54.266 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:22:54.266 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:22:54.266 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:22:54.266 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:54.266 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:22:54.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:22:54.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:22:54.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:22:54.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:22:54.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:22:54.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:22:54.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:22:54.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:22:54.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:54.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:22:54.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:22:54.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:22:54.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:22:54.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:22:54.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:22:54.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:22:54.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:22:54.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:22:54.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:22:54.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:54.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:22:54.269 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:22:54.269 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:22:54.269 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:22:54.269 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:22:54.269 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:22:54.269 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:22:54.276 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:22:54.277 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:22:54.277 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:22:54.277 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:22:54.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:22:54.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:22:54.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:22:54.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:22:54.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:22:54.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:22:54.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:54.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:22:54.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:22:54.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:22:54.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:22:54.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:22:54.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:22:54.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:22:54.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:22:54.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:22:54.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:54.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:22:54.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:22:54.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:22:54.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:22:54.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:22:54.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:22:54.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:22:54.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:22:54.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:22:54.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:22:54.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:54.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:22:54.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:22:54.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:22:54.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:22:54.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:22:54.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:22:54.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:22:54.348 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:22:54.348 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:22:54.348 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:22:54.348 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:22:54.406 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:22:54.406 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:22:54.406 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:22:54.409 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:22:54.409 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:22:54.409 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:22:54.409 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:54.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:22:54.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:22:54.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:22:54.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:22:54.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:22:54.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:22:54.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:22:54.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:22:54.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:22:54.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:54.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:22:54.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:22:54.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:22:54.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:22:54.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:22:54.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:22:54.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:22:54.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:22:54.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:22:54.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:22:54.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:54.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:22:54.412 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:22:54.412 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:22:54.412 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:22:54.412 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:22:54.412 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:22:54.412 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:22:54.419 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:22:54.419 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:22:54.420 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:22:54.420 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:22:54.482 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:22:54.484 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:22:54.484 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:22:54.484 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:22:54.484 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:22:54.485 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:22:54.485 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:22:54.485 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:22:54.485 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:22:54.485 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:22:54.485 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:22:54.485 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:22:54.485 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:22:54.486 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:54.486 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:22:54.486 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:54.514 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 95 2026-02-18T08:22:54.514 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 95 2026-02-18T08:22:54.514 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:22:54.514 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=381, tag_tid=27, entry_tid=1], [object_number=380, tag_tid=27, entry_tid=0], [object_number=382, tag_tid=26, entry_tid=46994], [object_number=383, tag_tid=26, entry_tid=46991]]], state=connected] 2026-02-18T08:22:54.514 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=382, tag_tid=26, entry_tid=46994], [object_number=381, tag_tid=26, entry_tid=46993], [object_number=380, tag_tid=26, entry_tid=46992], [object_number=383, tag_tid=26, entry_tid=46991]]], state=connected] 2026-02-18T08:22:54.516 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:54.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=381, tag_tid=27, entry_tid=1' 2026-02-18T08:22:54.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:22:54.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:22:54.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:22:54.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:22:54.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:22:54.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:22:54.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:22:54.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:22:54.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:22:54.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:22:54.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:22:54.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:22:54.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:54.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:22:54.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:54.557 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 95 2026-02-18T08:22:54.557 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 95 2026-02-18T08:22:54.558 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:22:54.558 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=381, tag_tid=27, entry_tid=1], [object_number=380, tag_tid=27, entry_tid=0], [object_number=382, tag_tid=26, entry_tid=46994], [object_number=383, tag_tid=26, entry_tid=46991]]], state=connected] 2026-02-18T08:22:54.558 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=382, tag_tid=26, entry_tid=46994], [object_number=381, tag_tid=26, entry_tid=46993], [object_number=380, tag_tid=26, entry_tid=46992], [object_number=383, tag_tid=26, entry_tid=46991]]], state=connected] 2026-02-18T08:22:54.561 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:54.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=382, tag_tid=26, entry_tid=46994' 2026-02-18T08:22:54.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=381, tag_tid=27, entry_tid=1' -a 'object_number=381, tag_tid=27, entry_tid=1' = 'object_number=382, tag_tid=26, entry_tid=46994' 2026-02-18T08:22:54.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=382, tag_tid=26, entry_tid=46994' '!=' 'object_number=382, tag_tid=26, entry_tid=46994' 2026-02-18T08:22:54.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:22:54.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1.6 2026-02-18T08:22:56.165 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:22:56.165 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:22:56.166 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:22:56.166 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:22:56.166 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:22:56.166 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:22:56.166 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:22:56.166 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:22:56.166 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:22:56.166 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:56.166 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:22:56.166 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:22:56.167 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:22:56.167 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:22:56.167 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:22:56.167 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:22:56.167 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:22:56.170 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:22:56.170 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:22:56.170 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:22:56.170 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:56.170 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:22:56.170 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:22:56.170 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:22:56.171 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:22:56.171 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:22:56.171 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:22:56.171 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:22:56.171 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:22:56.171 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:22:56.171 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:56.171 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:22:56.171 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:22:56.171 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:22:56.171 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:22:56.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:22:56.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:22:56.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:22:56.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:22:56.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:22:56.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:22:56.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:56.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:22:56.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:22:56.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:22:56.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:22:56.173 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:22:56.173 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:22:56.173 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:22:56.179 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:22:56.179 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:22:56.180 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:22:56.180 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:22:56.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:22:56.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:22:56.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:22:56.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:22:56.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:22:56.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:22:56.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:56.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:22:56.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:22:56.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:22:56.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:22:56.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:22:56.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:22:56.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:22:56.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:22:56.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:22:56.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:56.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:22:56.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:22:56.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:22:56.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:22:56.234 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:22:56.234 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:22:56.234 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:22:56.234 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:22:56.234 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:22:56.234 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:22:56.234 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:56.234 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:22:56.234 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:22:56.234 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:22:56.234 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:22:56.234 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:22:56.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:22:56.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:22:56.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:22:56.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:22:56.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:22:56.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:22:56.292 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:22:56.293 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:22:56.293 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:22:56.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:22:56.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:22:56.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:22:56.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:56.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:22:56.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:22:56.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:22:56.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:22:56.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:22:56.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:22:56.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:22:56.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:22:56.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:22:56.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:56.297 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:22:56.297 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:22:56.297 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:22:56.297 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:22:56.297 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:22:56.297 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:22:56.297 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:22:56.297 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:22:56.297 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:22:56.297 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:22:56.297 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:56.297 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:22:56.298 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:22:56.298 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:22:56.298 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:22:56.298 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:22:56.298 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:22:56.298 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:22:56.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:22:56.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:22:56.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:22:56.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:22:56.355 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:22:56.355 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:22:56.355 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:22:56.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:22:56.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:22:56.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:22:56.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:56.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:22:56.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:22:56.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:22:56.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:22:56.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:22:56.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:22:56.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:22:56.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:22:56.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:22:56.360 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:56.360 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:22:56.360 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:22:56.360 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:22:56.360 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:22:56.360 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:22:56.360 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:22:56.360 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:22:56.360 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:22:56.360 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:22:56.360 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:22:56.360 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:56.361 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:22:56.361 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:22:56.361 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:22:56.361 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:22:56.361 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:22:56.361 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:22:56.361 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:22:56.370 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:22:56.371 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:22:56.371 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:22:56.371 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:22:56.427 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:22:56.428 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:22:56.429 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:22:56.429 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:22:56.429 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:22:56.429 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:22:56.429 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:22:56.429 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:22:56.429 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:22:56.429 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:22:56.429 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:22:56.430 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:22:56.430 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:22:56.430 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:56.431 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:22:56.431 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:56.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 95 2026-02-18T08:22:56.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 95 2026-02-18T08:22:56.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:22:56.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=381, tag_tid=27, entry_tid=1], [object_number=380, tag_tid=27, entry_tid=0], [object_number=382, tag_tid=26, entry_tid=46994], [object_number=383, tag_tid=26, entry_tid=46991]]], state=connected] 2026-02-18T08:22:56.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=382, tag_tid=26, entry_tid=46994], [object_number=381, tag_tid=26, entry_tid=46993], [object_number=380, tag_tid=26, entry_tid=46992], [object_number=383, tag_tid=26, entry_tid=46991]]], state=connected] 2026-02-18T08:22:56.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:56.463 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=381, tag_tid=27, entry_tid=1' 2026-02-18T08:22:56.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:22:56.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:22:56.465 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:22:56.465 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:22:56.465 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:22:56.465 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:22:56.465 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:22:56.465 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:22:56.465 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:22:56.465 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:22:56.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:22:56.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:22:56.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:56.467 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:22:56.467 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:56.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 95 2026-02-18T08:22:56.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 95 2026-02-18T08:22:56.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:22:56.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=381, tag_tid=27, entry_tid=1], [object_number=380, tag_tid=27, entry_tid=0], [object_number=382, tag_tid=26, entry_tid=46994], [object_number=383, tag_tid=26, entry_tid=46991]]], state=connected] 2026-02-18T08:22:56.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=382, tag_tid=26, entry_tid=46994], [object_number=381, tag_tid=26, entry_tid=46993], [object_number=380, tag_tid=26, entry_tid=46992], [object_number=383, tag_tid=26, entry_tid=46991]]], state=connected] 2026-02-18T08:22:56.503 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:56.505 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=382, tag_tid=26, entry_tid=46994' 2026-02-18T08:22:56.505 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=381, tag_tid=27, entry_tid=1' -a 'object_number=381, tag_tid=27, entry_tid=1' = 'object_number=382, tag_tid=26, entry_tid=46994' 2026-02-18T08:22:56.505 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=382, tag_tid=26, entry_tid=46994' '!=' 'object_number=382, tag_tid=26, entry_tid=46994' 2026-02-18T08:22:56.505 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:22:56.505 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 2 2026-02-18T08:22:58.507 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:22:58.507 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:22:58.507 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:22:58.507 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:22:58.507 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:22:58.507 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:22:58.508 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:22:58.508 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:22:58.508 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:22:58.508 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:58.508 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:22:58.508 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:22:58.508 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:22:58.508 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:22:58.508 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:22:58.511 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:22:58.511 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:22:58.511 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:22:58.511 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:22:58.511 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:22:58.511 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:58.512 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:22:58.512 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:22:58.512 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:22:58.512 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:22:58.512 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:22:58.512 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:22:58.512 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:22:58.512 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:22:58.512 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:22:58.512 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:58.512 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:22:58.512 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:22:58.513 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:22:58.513 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:22:58.513 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:22:58.513 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:22:58.513 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:22:58.513 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:22:58.513 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:22:58.513 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:22:58.513 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:58.513 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:22:58.513 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:22:58.513 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:22:58.514 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:22:58.514 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:22:58.514 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:22:58.514 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:22:58.523 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:22:58.524 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:22:58.524 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:22:58.524 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:22:58.572 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:22:58.572 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:22:58.573 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:22:58.575 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:22:58.575 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:22:58.576 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:22:58.576 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:58.576 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:22:58.576 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:22:58.576 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:22:58.576 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:22:58.576 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:22:58.576 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:22:58.576 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:22:58.576 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:22:58.576 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:22:58.577 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:58.577 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:22:58.577 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:22:58.577 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:22:58.577 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:22:58.577 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:22:58.577 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:22:58.577 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:22:58.577 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:22:58.577 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:22:58.577 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:22:58.577 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:58.578 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:22:58.578 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:22:58.578 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:22:58.578 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:22:58.578 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:22:58.578 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:22:58.578 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:22:58.587 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:22:58.587 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:22:58.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:22:58.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:22:58.636 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:22:58.636 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:22:58.636 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:22:58.639 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:22:58.639 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:22:58.639 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:22:58.639 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:58.639 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:22:58.639 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:22:58.639 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:22:58.639 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:22:58.639 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:22:58.640 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:22:58.640 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:22:58.640 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:22:58.640 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:22:58.640 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:58.640 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:22:58.640 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:22:58.640 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:22:58.640 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:22:58.640 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:22:58.640 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:22:58.640 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:22:58.641 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:22:58.641 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:22:58.641 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:22:58.641 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:58.641 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:22:58.641 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:22:58.641 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:22:58.641 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:22:58.641 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:22:58.641 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:22:58.641 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:22:58.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:22:58.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:22:58.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:22:58.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:22:58.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:22:58.700 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:22:58.700 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:22:58.702 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:22:58.702 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:22:58.703 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:22:58.703 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:22:58.703 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:22:58.703 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:22:58.703 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:22:58.703 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:22:58.703 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:22:58.703 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:22:58.703 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:22:58.703 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:22:58.704 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:22:58.704 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:58.704 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:22:58.704 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:22:58.704 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:22:58.704 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:22:58.704 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:22:58.704 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:22:58.704 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:22:58.704 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:22:58.704 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:22:58.704 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:22:58.704 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:22:58.705 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:22:58.705 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:22:58.705 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:22:58.705 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:22:58.705 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:22:58.705 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:22:58.705 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:22:58.714 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:22:58.714 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:22:58.714 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:22:58.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:22:58.773 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:22:58.773 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:22:58.774 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:22:58.774 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:22:58.774 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:22:58.774 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:22:58.774 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:22:58.774 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:22:58.774 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:22:58.774 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:22:58.774 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:22:58.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:22:58.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:22:58.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:58.776 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:22:58.776 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:58.814 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 95 2026-02-18T08:22:58.814 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 95 2026-02-18T08:22:58.814 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:22:58.814 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=381, tag_tid=27, entry_tid=1], [object_number=380, tag_tid=27, entry_tid=0], [object_number=382, tag_tid=26, entry_tid=46994], [object_number=383, tag_tid=26, entry_tid=46991]]], state=connected] 2026-02-18T08:22:58.814 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=381, tag_tid=27, entry_tid=1], [object_number=380, tag_tid=27, entry_tid=0], [object_number=382, tag_tid=26, entry_tid=46994], [object_number=383, tag_tid=26, entry_tid=46991]]], state=connected] 2026-02-18T08:22:58.818 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:58.819 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=381, tag_tid=27, entry_tid=1' 2026-02-18T08:22:58.820 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:22:58.820 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:22:58.820 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:22:58.821 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:22:58.821 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:22:58.821 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:22:58.821 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:22:58.821 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:22:58.821 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:22:58.821 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:22:58.822 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:22:58.822 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:22:58.822 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:58.823 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:22:58.823 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:58.858 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 95 2026-02-18T08:22:58.858 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 95 2026-02-18T08:22:58.858 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:22:58.858 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=381, tag_tid=27, entry_tid=1], [object_number=380, tag_tid=27, entry_tid=0], [object_number=382, tag_tid=26, entry_tid=46994], [object_number=383, tag_tid=26, entry_tid=46991]]], state=connected] 2026-02-18T08:22:58.859 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=381, tag_tid=27, entry_tid=1], [object_number=380, tag_tid=27, entry_tid=0], [object_number=382, tag_tid=26, entry_tid=46994], [object_number=383, tag_tid=26, entry_tid=46991]]], state=connected] 2026-02-18T08:22:58.862 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:22:58.864 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=381, tag_tid=27, entry_tid=1' 2026-02-18T08:22:58.864 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=381, tag_tid=27, entry_tid=1' -a 'object_number=381, tag_tid=27, entry_tid=1' = 'object_number=381, tag_tid=27, entry_tid=1' 2026-02-18T08:22:58.864 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:22:58.864 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_snap_present cluster1 mirror test snap9 2026-02-18T08:22:58.865 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:22:58.865 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:22:58.865 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:22:58.865 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap9 2026-02-18T08:22:58.865 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-18T08:22:58.865 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-18T08:22:58.865 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1 2026-02-18T08:22:59.865 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster1 info mirror/test@snap9 2026-02-18T08:22:59.897 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:rbd image 'test': 2026-02-18T08:22:59.897 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: size 512 MiB in 128 objects 2026-02-18T08:22:59.897 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: order 22 (4 MiB objects) 2026-02-18T08:22:59.897 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: snapshot_count: 17 2026-02-18T08:22:59.897 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: id: 10a74aa8ebc0 2026-02-18T08:22:59.897 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: block_name_prefix: rbd_data.10a74aa8ebc0 2026-02-18T08:22:59.897 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: format: 2 2026-02-18T08:22:59.897 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: features: layering, exclusive-lock, journaling 2026-02-18T08:22:59.897 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: op_features: 2026-02-18T08:22:59.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: flags: 2026-02-18T08:22:59.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: create_timestamp: Wed Feb 18 08:10:48 2026 2026-02-18T08:22:59.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: access_timestamp: Wed Feb 18 08:20:28 2026 2026-02-18T08:22:59.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: modify_timestamp: Wed Feb 18 08:22:37 2026 2026-02-18T08:22:59.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: protected: False 2026-02-18T08:22:59.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: journal: 10a74aa8ebc0 2026-02-18T08:22:59.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring state: enabled 2026-02-18T08:22:59.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring mode: journal 2026-02-18T08:22:59.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring global id: 0b548823-d751-43df-a076-4efac2da4d3d 2026-02-18T08:22:59.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring primary: false 2026-02-18T08:22:59.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:22:59.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n snap8-clean ']' 2026-02-18T08:22:59.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ compare_image_snaps mirror test snap8-clean 2026-02-18T08:22:59.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:22:59.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:22:59.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap8-clean 2026-02-18T08:22:59.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local ret=0 2026-02-18T08:22:59.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local rmt_export=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.export 2026-02-18T08:22:59.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loc_export=/tmp/tmp.vybh1m9yJF/cluster1-mirror-test.export 2026-02-18T08:22:59.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.export /tmp/tmp.vybh1m9yJF/cluster1-mirror-test.export 2026-02-18T08:22:59.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 -p mirror export test@snap8-clean /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.export 2026-02-18T08:23:00.379 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-18T08:23:00.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster1 -p mirror export test@snap8-clean /tmp/tmp.vybh1m9yJF/cluster1-mirror-test.export 2026-02-18T08:23:01.218 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-18T08:23:01.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmp /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.export /tmp/tmp.vybh1m9yJF/cluster1-mirror-test.export 2026-02-18T08:23:01.421 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.export /tmp/tmp.vybh1m9yJF/cluster1-mirror-test.export 2026-02-18T08:23:01.529 INFO:tasks.rbd_mirror_thrash:kill cluster2.client.mirror.0 2026-02-18T08:23:01.529 INFO:tasks.rbd_mirror.cluster2.client.mirror.0:Sent signal 15 2026-02-18T08:23:01.530 INFO:tasks.rbd_mirror_thrash:waiting for 0.7907422271851594 secs before reviving daemons 2026-02-18T08:23:01.530 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.trial163.stderr:2026-02-18T08:23:01.522+0000 7fe99d0eb640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.0 (PID: 66317) UID: 1000 2026-02-18T08:23:01.565 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:23:01.565 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ compare_image_snaps mirror test snap9 2026-02-18T08:23:01.565 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:23:01.565 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:23:01.566 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap9 2026-02-18T08:23:01.566 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local ret=0 2026-02-18T08:23:01.566 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local rmt_export=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.export 2026-02-18T08:23:01.566 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loc_export=/tmp/tmp.vybh1m9yJF/cluster1-mirror-test.export 2026-02-18T08:23:01.566 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.export /tmp/tmp.vybh1m9yJF/cluster1-mirror-test.export 2026-02-18T08:23:01.566 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 -p mirror export test@snap9 /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.export 2026-02-18T08:23:02.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-18T08:23:02.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster1 -p mirror export test@snap9 /tmp/tmp.vybh1m9yJF/cluster1-mirror-test.export 2026-02-18T08:23:02.322 INFO:tasks.rbd_mirror_thrash:waiting for cluster2.client.mirror.0 2026-02-18T08:23:02.322 INFO:teuthology.orchestra.run:waiting for 600 2026-02-18T08:23:02.322 INFO:tasks.rbd_mirror_thrash:reviving cluster2.client.mirror.0 2026-02-18T08:23:02.322 INFO:tasks.rbd_mirror.cluster2.client.mirror.0:Restarting daemon 2026-02-18T08:23:02.323 DEBUG:teuthology.orchestra.run.trial163:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.0 2026-02-18T08:23:02.324 INFO:tasks.rbd_mirror.cluster2.client.mirror.0:Started 2026-02-18T08:23:02.325 INFO:tasks.rbd_mirror_thrash:waiting for 87.01128040033421 secs before thrashing 2026-02-18T08:23:02.706 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-18T08:23:02.724 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmp /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.export /tmp/tmp.vybh1m9yJF/cluster1-mirror-test.export 2026-02-18T08:23:02.911 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.export /tmp/tmp.vybh1m9yJF/cluster1-mirror-test.export 2026-02-18T08:23:03.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:23:03.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ clean_snap_name=snap9-clean 2026-02-18T08:23:03.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_snap cluster2 mirror test snap9-clean 2026-02-18T08:23:03.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:23:03.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:23:03.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:23:03.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap9-clean 2026-02-18T08:23:03.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 -p mirror snap create test@snap9-clean --debug-rbd=20 --debug-journaler=20 2026-02-18T08:23:03.759 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 10` 2026-02-18T08:23:03.759 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ stress_write_image cluster2 mirror test 2026-02-18T08:23:03.759 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:23:03.759 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:23:03.759 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:23:03.759 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-18T08:23:03.761 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local duration=12 2026-02-18T08:23:03.761 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set +e 2026-02-18T08:23:03.761 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ timeout 12s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror test --debug-rbd=20 --debug-journaler=20 2026-02-18T08:23:04.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:100 IOs, 24672256 bytes 2026-02-18T08:23:04.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:200 IOs, 25268224 bytes 2026-02-18T08:23:04.578 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:300 IOs, 39107584 bytes 2026-02-18T08:23:04.933 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:400 IOs, 75755520 bytes 2026-02-18T08:23:06.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:500 IOs, 165747712 bytes 2026-02-18T08:23:07.270 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:600 IOs, 203334656 bytes 2026-02-18T08:23:08.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:700 IOs, 275658752 bytes 2026-02-18T08:23:08.821 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:800 IOs, 310428672 bytes 2026-02-18T08:23:10.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:900 IOs, 414974976 bytes 2026-02-18T08:23:10.646 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1000 IOs, 421087232 bytes 2026-02-18T08:23:10.705 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1100 IOs, 438824960 bytes 2026-02-18T08:23:11.103 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.0 with skip (0.630844967125108) > weight (0.25) 2026-02-18T08:23:11.103 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.1 with skip (0.5775867201411051) > weight (0.25) 2026-02-18T08:23:11.103 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.2 with skip (0.6553766291670033) > weight (0.25) 2026-02-18T08:23:11.103 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.3 with skip (0.9473635868355944) > weight (0.25) 2026-02-18T08:23:11.103 INFO:tasks.rbd_mirror_thrash:waiting for 103.50131175766771 secs before thrashing 2026-02-18T08:23:11.414 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1200 IOs, 466919424 bytes 2026-02-18T08:23:12.282 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1300 IOs, 549199872 bytes 2026-02-18T08:23:13.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1400 IOs, 580962304 bytes 2026-02-18T08:23:13.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1500 IOs, 599522304 bytes 2026-02-18T08:23:13.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1600 IOs, 610324480 bytes 2026-02-18T08:23:14.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1700 IOs, 664333312 bytes 2026-02-18T08:23:15.238 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1800 IOs, 719943680 bytes 2026-02-18T08:23:15.772 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ error_code=124 2026-02-18T08:23:15.773 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set -e 2026-02-18T08:23:15.773 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 124 -eq 124 ']' 2026-02-18T08:23:15.773 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:23:15.773 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+replaying 2026-02-18T08:23:15.773 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:23:15.773 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:23:15.773 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:23:15.773 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local state_pattern=up+replaying 2026-02-18T08:23:15.773 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local description_pattern= 2026-02-18T08:23:15.773 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local service_pattern= 2026-02-18T08:23:15.774 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:23:15.774 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1 2026-02-18T08:23:16.773 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test_status_in_pool_dir cluster1 mirror test up+replaying '' '' 2026-02-18T08:23:16.773 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:23:16.773 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:23:16.773 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:23:16.773 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local state_pattern=up+replaying 2026-02-18T08:23:16.773 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local description_pattern= 2026-02-18T08:23:16.773 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local service_pattern= 2026-02-18T08:23:16.773 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local status 2026-02-18T08:23:16.773 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ CEPH_ARGS= 2026-02-18T08:23:16.773 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-18T08:23:16.814 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ status='test: 2026-02-18T08:23:16.814 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: global_id: 0b548823-d751-43df-a076-4efac2da4d3d 2026-02-18T08:23:16.814 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: state: up+replaying 2026-02-18T08:23:16.814 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: description: replaying, {"bytes_per_second":4135103.47,"entries_behind_primary":17705,"entries_per_second":262.03,"non_primary_position":{"entry_tid":451,"object_number":383,"tag_tid":29},"primary_position":{"entry_tid":18156,"object_number":396,"tag_tid":29},"seconds_until_synced":67} 2026-02-18T08:23:16.814 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: service: mirror.3 on trial163 2026-02-18T08:23:16.814 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: last_update: 2026-02-18 08:23:16 2026-02-18T08:23:16.814 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: peer_sites: 2026-02-18T08:23:16.814 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: name: cluster2 2026-02-18T08:23:16.814 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: state: up+stopped 2026-02-18T08:23:16.814 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: description: local image is primary 2026-02-18T08:23:16.814 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: last_update: 2026-02-18 08:23:16' 2026-02-18T08:23:16.814 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ grep '^ state: .*up+replaying' 2026-02-18T08:23:16.815 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: state: up+replaying 2026-02-18T08:23:16.815 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ grep '^ description: .*' 2026-02-18T08:23:16.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: description: replaying, {"bytes_per_second":4135103.47,"entries_behind_primary":17705,"entries_per_second":262.03,"non_primary_position":{"entry_tid":451,"object_number":383,"tag_tid":29},"primary_position":{"entry_tid":18156,"object_number":396,"tag_tid":29},"seconds_until_synced":67} 2026-02-18T08:23:16.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n '' ']' 2026-02-18T08:23:16.817 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ echo up+replaying 2026-02-18T08:23:16.817 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ grep '^up+' 2026-02-18T08:23:16.818 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:up+replaying 2026-02-18T08:23:16.818 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ grep 'service: *mirror..* on ' 2026-02-18T08:23:16.819 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: service: mirror.3 on trial163 2026-02-18T08:23:16.820 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local last_update 2026-02-18T08:23:16.820 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-18T08:23:16.822 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_update='2026-02-18 08:23:16' 2026-02-18T08:23:16.822 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+replaying '2026-02-18 08:23:16' 2026-02-18T08:23:16.822 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:23:16.822 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:23:16.822 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:23:16.822 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local state_pattern=up+replaying 2026-02-18T08:23:16.822 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'prev_last_update=2026-02-18 08:23:16' 2026-02-18T08:23:16.822 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local status 2026-02-18T08:23:16.822 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ CEPH_ARGS= 2026-02-18T08:23:16.822 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-18T08:23:16.866 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ status='OKOKOK142194268mirror.3trial16320.2.0-670-gadc19233falseOK43304333mirror.2trial16320.2.0-670-gadc19233falseOK43534356mirror.1trial16320.2.0-670-gadc19233trueOK44374440mirror.0trial16320.2.0-670-gadc19233falseOKtest0b548823-d751-43df-a076-4efac2da4d3dup+replayingreplaying, {"bytes_per_second":4135103.47,"entries_behind_primary":17705,"entries_per_second":262.03,"non_primary_position":{"entry_tid":451,"object_number":383,"tag_tid":29},"primary_position":{"entry_tid":18156,"object_number":396,"tag_tid":29},"seconds_until_synced":67}42194268mirror.3trial1632026-02-18 08:23:16cluster2c5e4f85e-ec31-430b-a71d-56e000d0de0bup+stoppedlocal image is primary2026-02-18 08:23:16' 2026-02-18T08:23:16.866 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local last_update state 2026-02-18T08:23:16.866 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-18T08:23:16.870 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_update='2026-02-18 08:23:16' 2026-02-18T08:23:16.870 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-18T08:23:16.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ state=up+replaying 2026-02-18T08:23:16.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ echo up+replaying 2026-02-18T08:23:16.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ grep up+replaying 2026-02-18T08:23:16.874 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:up+replaying 2026-02-18T08:23:16.875 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:23:16.875 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:23:16.875 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ snap_name=snap10 2026-02-18T08:23:16.875 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_snap cluster2 mirror test snap10 2026-02-18T08:23:16.875 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:23:16.875 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:23:16.875 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:23:16.875 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap10 2026-02-18T08:23:16.875 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 -p mirror snap create test@snap10 --debug-rbd=20 --debug-journaler=20 2026-02-18T08:23:47.800 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_replay_started cluster1 mirror test 2026-02-18T08:23:47.801 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:23:47.801 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:23:47.801 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:23:47.801 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_replay_state cluster1 mirror test started 2026-02-18T08:23:47.801 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:23:47.801 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:23:47.801 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:23:47.801 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local state=started 2026-02-18T08:23:47.802 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-18T08:23:47.802 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:23:47.802 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1 2026-02-18T08:23:48.801 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test_image_replay_state cluster1 mirror test started 2026-02-18T08:23:48.801 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:23:48.801 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:23:48.801 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:23:48.802 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local test_state=started 2026-02-18T08:23:48.802 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local status_result 2026-02-18T08:23:48.802 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_state=stopped 2026-02-18T08:23:48.802 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-18T08:23:48.802 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -i state 2026-02-18T08:23:48.802 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_instance=cluster1 2026-02-18T08:23:48.802 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-18T08:23:48.802 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:23:48.802 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance=cluster1 2026-02-18T08:23:48.802 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local loop_instance 2026-02-18T08:23:48.802 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:23:48.802 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sleep 0 2026-02-18T08:23:48.803 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:23:48.803 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ seq 0 3 2026-02-18T08:23:48.804 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:23:48.805 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-18T08:23:48.805 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:0 2026-02-18T08:23:48.805 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-18T08:23:48.805 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:23:48.805 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:23:48.805 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:23:48.805 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-18T08:23:48.805 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:23:48.805 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:23:48.805 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:23:48.805 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:23:48.805 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-18T08:23:48.805 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:23:48.805 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:23:48.805 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:23:48.806 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:23:48.806 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:0 2026-02-18T08:23:48.806 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-18T08:23:48.806 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-18T08:23:48.806 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:23:48.806 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:0 2026-02-18T08:23:48.806 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T08:23:48.806 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-18T08:23:48.806 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:23:48.806 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:23:48.806 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-18T08:23:48.806 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-18T08:23:48.806 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-18T08:23:48.806 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=0 2026-02-18T08:23:48.806 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=0 2026-02-18T08:23:48.806 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:23:48.817 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:23:48.817 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:23:48.817 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:23:48.817 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-18T08:23:48.878 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:23:48.878 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status : get status for rbd mirror 2026-02-18T08:23:48.878 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:23:48.881 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:23:48.881 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test 2026-02-18T08:23:48.881 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:1 2026-02-18T08:23:48.881 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-18T08:23:48.881 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:23:48.881 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:23:48.881 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:23:48.881 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-18T08:23:48.882 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:23:48.882 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:23:48.882 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:23:48.882 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:23:48.882 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-18T08:23:48.882 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:23:48.882 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:23:48.882 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:23:48.882 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:23:48.882 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:1 2026-02-18T08:23:48.882 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-18T08:23:48.882 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-18T08:23:48.883 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:23:48.883 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:1 2026-02-18T08:23:48.883 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T08:23:48.883 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-18T08:23:48.883 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:23:48.883 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:23:48.883 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-18T08:23:48.883 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-18T08:23:48.883 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-18T08:23:48.883 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=1 2026-02-18T08:23:48.883 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=1 2026-02-18T08:23:48.883 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:23:48.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:23:48.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:23:48.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:23:48.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test 2026-02-18T08:23:48.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:23:48.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status : get status for rbd mirror 2026-02-18T08:23:48.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:23:48.944 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:23:48.944 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/test 2026-02-18T08:23:48.944 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:2 2026-02-18T08:23:48.944 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-18T08:23:48.944 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:23:48.944 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:23:48.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:23:48.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-18T08:23:48.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:23:48.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:23:48.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:23:48.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:23:48.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-18T08:23:48.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:23:48.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:23:48.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:23:48.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:23:48.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:2 2026-02-18T08:23:48.946 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-18T08:23:48.946 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-18T08:23:48.946 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:23:48.946 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:2 2026-02-18T08:23:48.946 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T08:23:48.946 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-18T08:23:48.946 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:23:48.946 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:23:48.946 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-18T08:23:48.946 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-18T08:23:48.946 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-18T08:23:48.946 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=2 2026-02-18T08:23:48.947 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=2 2026-02-18T08:23:48.947 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:23:48.956 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:23:48.956 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:23:48.956 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:23:48.956 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/test 2026-02-18T08:23:49.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:23:49.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status : get status for rbd mirror 2026-02-18T08:23:49.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:23:49.020 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:23:49.020 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/test 2026-02-18T08:23:49.020 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:3 2026-02-18T08:23:49.020 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-18T08:23:49.020 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:23:49.020 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:23:49.020 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:23:49.020 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-18T08:23:49.020 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:23:49.021 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:23:49.021 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:23:49.021 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:23:49.021 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-18T08:23:49.021 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:23:49.021 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:23:49.021 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:23:49.021 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:23:49.021 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:3 2026-02-18T08:23:49.021 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-18T08:23:49.021 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-18T08:23:49.021 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:23:49.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:3 2026-02-18T08:23:49.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T08:23:49.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-18T08:23:49.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:23:49.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:23:49.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-18T08:23:49.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-18T08:23:49.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-18T08:23:49.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=3 2026-02-18T08:23:49.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=3 2026-02-18T08:23:49.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:23:49.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:23:49.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:23:49.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:23:49.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/test 2026-02-18T08:23:49.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ return 0 2026-02-18T08:23:49.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ status_result=' "state": "Replaying"' 2026-02-18T08:23:49.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ echo ' "state": "Replaying"' 2026-02-18T08:23:49.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ grep -i Replaying 2026-02-18T08:23:49.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: "state": "Replaying" 2026-02-18T08:23:49.092 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ current_state=started 2026-02-18T08:23:49.092 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test started = started 2026-02-18T08:23:49.092 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:23:49.092 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror test 2026-02-18T08:23:49.093 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_cluster=cluster1 2026-02-18T08:23:49.093 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:23:49.093 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_pool=mirror 2026-02-18T08:23:49.093 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local remote_pool=mirror 2026-02-18T08:23:49.093 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:23:49.093 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' journal = journal ']' 2026-02-18T08:23:49.093 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror mirror test 2026-02-18T08:23:49.093 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_cluster=cluster1 2026-02-18T08:23:49.093 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:23:49.093 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_pool=mirror 2026-02-18T08:23:49.093 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local remote_pool=mirror 2026-02-18T08:23:49.093 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:23:49.094 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-02-18T08:23:49.094 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-02-18T08:23:49.094 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:23:49.094 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:23:49.094 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:23:49.293 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:23:49.294 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:23:49.294 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:23:49.294 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:23:49.294 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:23:49.294 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:23:49.294 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:23:49.294 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:23:49.294 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:23:49.294 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:23:49.294 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:23:49.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:23:49.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:23:49.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:23:49.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:23:49.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:23:49.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:23:49.297 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:23:49.297 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:23:49.298 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:23:49.298 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:23:49.298 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:23:49.298 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:23:49.298 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:23:49.298 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:23:49.298 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:23:49.298 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:23:49.298 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:23:49.298 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:23:49.298 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:23:49.298 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:23:49.299 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:23:49.299 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:23:49.299 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:23:49.299 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:23:49.299 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:23:49.299 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:23:49.299 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:23:49.299 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:23:49.299 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:23:49.299 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:23:49.299 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:23:49.299 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:23:49.300 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:23:49.300 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:23:49.300 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:23:49.300 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:23:49.300 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:23:49.300 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:23:49.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:23:49.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:23:49.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:23:49.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:23:49.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:23:49.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:23:49.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:23:49.360 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:23:49.360 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:23:49.361 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:23:49.361 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:23:49.361 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:23:49.361 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:23:49.361 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:23:49.361 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:23:49.361 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:23:49.361 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:23:49.361 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:23:49.361 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:23:49.361 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:23:49.362 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:23:49.362 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:23:49.362 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:23:49.362 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:23:49.362 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:23:49.362 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:23:49.362 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:23:49.362 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:23:49.362 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:23:49.362 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:23:49.362 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:23:49.362 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:23:49.363 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:23:49.363 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:23:49.363 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:23:49.363 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:23:49.363 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:23:49.363 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:23:49.363 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:23:49.371 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:23:49.372 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:23:49.372 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:23:49.372 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:23:49.429 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:23:49.429 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:23:49.429 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:23:49.432 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:23:49.432 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:23:49.432 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:23:49.432 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:23:49.432 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:23:49.432 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:23:49.432 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:23:49.433 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:23:49.433 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:23:49.433 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:23:49.433 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:23:49.433 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:23:49.433 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:23:49.433 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:23:49.433 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:23:49.433 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:23:49.433 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:23:49.433 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:23:49.433 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:23:49.434 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:23:49.434 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:23:49.434 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:23:49.434 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:23:49.434 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:23:49.434 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:23:49.434 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:23:49.434 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:23:49.434 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:23:49.434 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:23:49.434 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:23:49.434 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:23:49.435 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:23:49.443 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:23:49.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:23:49.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:23:49.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:23:49.501 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:23:49.501 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:23:49.501 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:23:49.504 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:23:49.504 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:23:49.504 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:23:49.504 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:23:49.504 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:23:49.504 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:23:49.504 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:23:49.504 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:23:49.504 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:23:49.505 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:23:49.505 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:23:49.505 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:23:49.505 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:23:49.505 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:23:49.505 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:23:49.505 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:23:49.505 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:23:49.505 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:23:49.505 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:23:49.505 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:23:49.505 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:23:49.506 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:23:49.506 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:23:49.506 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:23:49.506 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:23:49.506 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:23:49.506 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:23:49.506 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:23:49.506 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:23:49.506 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:23:49.506 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:23:49.506 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:23:49.514 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:23:49.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:23:49.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:23:49.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:23:49.574 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:23:49.576 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:23:49.576 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:23:49.576 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:23:49.576 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:23:49.576 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:23:49.576 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:23:49.576 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:23:49.576 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:23:49.576 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:23:49.576 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:23:49.577 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:23:49.577 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:23:49.577 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:23:49.578 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:23:49.578 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:23:49.606 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 101 2026-02-18T08:23:49.606 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 106 2026-02-18T08:23:49.606 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:23:49.606 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=425, tag_tid=30, entry_tid=1], [object_number=424, tag_tid=30, entry_tid=0], [object_number=427, tag_tid=29, entry_tid=45931], [object_number=426, tag_tid=29, entry_tid=45930]]], state=connected] 2026-02-18T08:23:49.606 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=405, tag_tid=29, entry_tid=24713], [object_number=404, tag_tid=29, entry_tid=24712], [object_number=407, tag_tid=29, entry_tid=24711], [object_number=406, tag_tid=29, entry_tid=24710]]], state=connected] 2026-02-18T08:23:49.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:23:49.610 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=425, tag_tid=30, entry_tid=1' 2026-02-18T08:23:49.611 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:23:49.611 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:23:49.612 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:23:49.612 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:23:49.612 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:23:49.612 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:23:49.612 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:23:49.612 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:23:49.612 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:23:49.612 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:23:49.612 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:23:49.613 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:23:49.613 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:23:49.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:23:49.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:23:49.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 101 2026-02-18T08:23:49.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 106 2026-02-18T08:23:49.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:23:49.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=425, tag_tid=30, entry_tid=1], [object_number=424, tag_tid=30, entry_tid=0], [object_number=427, tag_tid=29, entry_tid=45931], [object_number=426, tag_tid=29, entry_tid=45930]]], state=connected] 2026-02-18T08:23:49.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=405, tag_tid=29, entry_tid=24713], [object_number=404, tag_tid=29, entry_tid=24712], [object_number=407, tag_tid=29, entry_tid=24711], [object_number=406, tag_tid=29, entry_tid=24710]]], state=connected] 2026-02-18T08:23:49.654 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:23:49.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=405, tag_tid=29, entry_tid=24713' 2026-02-18T08:23:49.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=425, tag_tid=30, entry_tid=1' -a 'object_number=425, tag_tid=30, entry_tid=1' = 'object_number=405, tag_tid=29, entry_tid=24713' 2026-02-18T08:23:49.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=405, tag_tid=29, entry_tid=24713' '!=' '' 2026-02-18T08:23:49.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:23:49.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=405, tag_tid=29, entry_tid=24713' = '' 2026-02-18T08:23:49.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=405, tag_tid=29, entry_tid=24713' 2026-02-18T08:23:49.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=425, tag_tid=30, entry_tid=1' 2026-02-18T08:23:49.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:23:49.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:23:49.658 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=30 2026-02-18T08:23:49.659 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=405, tag_tid=29, entry_tid=24713' 2026-02-18T08:23:49.659 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:23:49.659 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:23:49.660 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=29 2026-02-18T08:23:49.661 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=425, tag_tid=30, entry_tid=1' 2026-02-18T08:23:49.661 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:23:49.661 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:23:49.663 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:23:49.663 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=405, tag_tid=29, entry_tid=24713' 2026-02-18T08:23:49.664 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:23:49.664 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:23:49.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=24713 2026-02-18T08:23:49.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 30 = 29 -a 1 -le 24713 2026-02-18T08:23:49.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:23:49.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:23:49.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:23:49.867 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:23:49.868 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:23:49.868 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:23:49.868 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:23:49.868 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:23:49.868 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:23:49.868 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:23:49.868 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:23:49.868 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:23:49.868 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:23:49.869 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:23:49.869 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:23:49.869 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:23:49.869 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:23:49.869 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:23:49.869 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:23:49.869 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:23:49.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:23:49.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:23:49.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:23:49.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:23:49.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:23:49.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:23:49.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:23:49.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:23:49.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:23:49.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:23:49.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:23:49.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:23:49.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:23:49.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:23:49.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:23:49.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:23:49.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:23:49.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:23:49.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:23:49.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:23:49.874 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:23:49.874 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:23:49.874 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:23:49.874 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:23:49.874 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:23:49.874 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:23:49.874 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:23:49.874 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:23:49.874 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:23:49.874 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:23:49.874 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:23:49.875 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:23:49.881 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:23:49.881 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:23:49.881 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:23:49.881 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:23:49.931 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:23:49.931 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:23:49.931 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:23:49.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:23:49.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:23:49.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:23:49.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:23:49.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:23:49.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:23:49.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:23:49.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:23:49.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:23:49.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:23:49.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:23:49.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:23:49.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:23:49.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:23:49.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:23:49.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:23:49.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:23:49.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:23:49.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:23:49.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:23:49.936 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:23:49.936 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:23:49.936 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:23:49.936 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:23:49.936 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:23:49.936 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:23:49.936 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:23:49.936 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:23:49.936 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:23:49.936 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:23:49.936 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:23:49.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:23:49.944 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:23:49.944 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:23:49.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:23:49.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:23:49.993 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:23:49.993 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:23:49.993 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:23:49.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:23:49.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:23:49.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:23:49.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:23:49.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:23:49.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:23:49.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:23:49.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:23:49.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:23:49.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:23:49.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:23:49.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:23:49.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:23:49.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:23:49.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:23:49.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:23:49.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:23:49.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:23:49.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:23:49.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:23:49.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:23:49.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:23:49.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:23:49.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:23:49.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:23:49.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:23:49.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:23:49.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:23:49.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:23:49.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:23:49.999 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:23:49.999 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:23:50.006 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:23:50.007 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:23:50.007 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:23:50.007 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:23:50.056 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:23:50.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:23:50.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:23:50.060 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:23:50.060 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:23:50.060 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:23:50.060 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:23:50.060 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:23:50.060 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:23:50.060 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:23:50.060 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:23:50.060 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:23:50.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:23:50.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:23:50.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:23:50.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:23:50.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:23:50.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:23:50.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:23:50.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:23:50.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:23:50.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:23:50.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:23:50.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:23:50.062 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:23:50.062 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:23:50.062 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:23:50.062 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:23:50.062 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:23:50.062 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:23:50.062 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:23:50.062 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:23:50.062 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:23:50.062 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:23:50.062 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:23:50.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:23:50.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:23:50.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:23:50.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:23:50.134 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:23:50.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:23:50.136 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:23:50.136 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:23:50.136 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:23:50.136 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:23:50.136 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:23:50.136 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:23:50.136 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:23:50.136 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:23:50.136 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:23:50.136 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:23:50.136 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:23:50.137 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:23:50.138 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:23:50.138 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:23:50.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 101 2026-02-18T08:23:50.169 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 106 2026-02-18T08:23:50.169 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:23:50.169 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=425, tag_tid=30, entry_tid=1], [object_number=424, tag_tid=30, entry_tid=0], [object_number=427, tag_tid=29, entry_tid=45931], [object_number=426, tag_tid=29, entry_tid=45930]]], state=connected] 2026-02-18T08:23:50.169 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=407, tag_tid=29, entry_tid=25075], [object_number=406, tag_tid=29, entry_tid=25074], [object_number=405, tag_tid=29, entry_tid=25073], [object_number=404, tag_tid=29, entry_tid=25072]]], state=connected] 2026-02-18T08:23:50.171 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:23:50.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=425, tag_tid=30, entry_tid=1' 2026-02-18T08:23:50.173 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:23:50.173 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:23:50.174 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:23:50.174 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:23:50.174 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:23:50.174 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:23:50.174 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:23:50.174 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:23:50.174 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:23:50.174 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:23:50.175 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:23:50.175 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:23:50.176 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:23:50.176 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:23:50.176 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:23:50.203 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 101 2026-02-18T08:23:50.204 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 106 2026-02-18T08:23:50.204 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:23:50.204 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=425, tag_tid=30, entry_tid=1], [object_number=424, tag_tid=30, entry_tid=0], [object_number=427, tag_tid=29, entry_tid=45931], [object_number=426, tag_tid=29, entry_tid=45930]]], state=connected] 2026-02-18T08:23:50.204 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=407, tag_tid=29, entry_tid=25075], [object_number=406, tag_tid=29, entry_tid=25074], [object_number=405, tag_tid=29, entry_tid=25073], [object_number=404, tag_tid=29, entry_tid=25072]]], state=connected] 2026-02-18T08:23:50.206 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:23:50.208 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=407, tag_tid=29, entry_tid=25075' 2026-02-18T08:23:50.208 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=425, tag_tid=30, entry_tid=1' -a 'object_number=425, tag_tid=30, entry_tid=1' = 'object_number=407, tag_tid=29, entry_tid=25075' 2026-02-18T08:23:50.208 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=407, tag_tid=29, entry_tid=25075' '!=' 'object_number=405, tag_tid=29, entry_tid=24713' 2026-02-18T08:23:50.209 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:23:50.209 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=407, tag_tid=29, entry_tid=25075' = 'object_number=405, tag_tid=29, entry_tid=24713' 2026-02-18T08:23:50.209 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=407, tag_tid=29, entry_tid=25075' 2026-02-18T08:23:50.209 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=425, tag_tid=30, entry_tid=1' 2026-02-18T08:23:50.209 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:23:50.209 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:23:50.210 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=30 2026-02-18T08:23:50.211 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=407, tag_tid=29, entry_tid=25075' 2026-02-18T08:23:50.211 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:23:50.211 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:23:50.212 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=29 2026-02-18T08:23:50.213 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=425, tag_tid=30, entry_tid=1' 2026-02-18T08:23:50.213 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:23:50.213 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:23:50.214 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:23:50.215 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=407, tag_tid=29, entry_tid=25075' 2026-02-18T08:23:50.215 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:23:50.215 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:23:50.217 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=25075 2026-02-18T08:23:50.217 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 30 = 29 -a 1 -le 25075 2026-02-18T08:23:50.217 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:23:50.217 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:23:50.218 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:23:50.419 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:23:50.419 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:23:50.419 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:23:50.419 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:23:50.420 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:23:50.420 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:23:50.420 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:23:50.420 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:23:50.420 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:23:50.420 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:23:50.420 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:23:50.420 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:23:50.420 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:23:50.420 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:23:50.420 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:23:50.420 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:23:50.423 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:23:50.423 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:23:50.423 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:23:50.423 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:23:50.423 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:23:50.423 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:23:50.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:23:50.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:23:50.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:23:50.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:23:50.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:23:50.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:23:50.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:23:50.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:23:50.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:23:50.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:23:50.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:23:50.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:23:50.425 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:23:50.425 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:23:50.425 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:23:50.425 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:23:50.425 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:23:50.425 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:23:50.425 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:23:50.425 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:23:50.425 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:23:50.425 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:23:50.425 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:23:50.425 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:23:50.425 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:23:50.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:23:50.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:23:50.432 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:23:50.433 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:23:50.433 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:23:50.433 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:23:50.491 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:23:50.491 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:23:50.491 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:23:50.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:23:50.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:23:50.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:23:50.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:23:50.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:23:50.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:23:50.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:23:50.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:23:50.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:23:50.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:23:50.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:23:50.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:23:50.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:23:50.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:23:50.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:23:50.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:23:50.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:23:50.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:23:50.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:23:50.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:23:50.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:23:50.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:23:50.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:23:50.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:23:50.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:23:50.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:23:50.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:23:50.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:23:50.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:23:50.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:23:50.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:23:50.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:23:50.504 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:23:50.504 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:23:50.505 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:23:50.505 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:23:50.561 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:23:50.561 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:23:50.562 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:23:50.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:23:50.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:23:50.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:23:50.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:23:50.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:23:50.565 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:23:50.565 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:23:50.565 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:23:50.565 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:23:50.565 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:23:50.565 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:23:50.565 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:23:50.565 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:23:50.565 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:23:50.565 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:23:50.565 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:23:50.566 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:23:50.566 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:23:50.566 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:23:50.566 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:23:50.566 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:23:50.566 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:23:50.566 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:23:50.566 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:23:50.566 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:23:50.566 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:23:50.566 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:23:50.566 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:23:50.567 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:23:50.567 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:23:50.567 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:23:50.567 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:23:50.575 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:23:50.576 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:23:50.576 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:23:50.576 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:23:50.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:23:50.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:23:50.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:23:50.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:23:50.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:23:50.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:23:50.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:23:50.635 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:23:50.635 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:23:50.635 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:23:50.635 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:23:50.635 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:23:50.635 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:23:50.635 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:23:50.635 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:23:50.635 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:23:50.635 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:23:50.635 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:23:50.635 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:23:50.636 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:23:50.636 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:23:50.636 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:23:50.636 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:23:50.636 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:23:50.636 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:23:50.636 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:23:50.636 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:23:50.636 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:23:50.636 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:23:50.636 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:23:50.636 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:23:50.637 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:23:50.637 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:23:50.637 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:23:50.637 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:23:50.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:23:50.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:23:50.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:23:50.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:23:50.712 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:23:50.713 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:23:50.713 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:23:50.713 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:23:50.713 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:23:50.713 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:23:50.713 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:23:50.714 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:23:50.714 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:23:50.714 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:23:50.714 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:23:50.714 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:23:50.714 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:23:50.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:23:50.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:23:50.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:23:50.744 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 101 2026-02-18T08:23:50.744 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 106 2026-02-18T08:23:50.744 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:23:50.744 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=425, tag_tid=30, entry_tid=1], [object_number=424, tag_tid=30, entry_tid=0], [object_number=427, tag_tid=29, entry_tid=45931], [object_number=426, tag_tid=29, entry_tid=45930]]], state=connected] 2026-02-18T08:23:50.745 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=406, tag_tid=29, entry_tid=25430], [object_number=405, tag_tid=29, entry_tid=25429], [object_number=404, tag_tid=29, entry_tid=25428], [object_number=407, tag_tid=29, entry_tid=25427]]], state=connected] 2026-02-18T08:23:50.747 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:23:50.749 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=425, tag_tid=30, entry_tid=1' 2026-02-18T08:23:50.750 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:23:50.750 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:23:50.750 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:23:50.750 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:23:50.750 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:23:50.750 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:23:50.750 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:23:50.750 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:23:50.751 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:23:50.751 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:23:50.751 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:23:50.751 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:23:50.752 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:23:50.752 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:23:50.752 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:23:50.779 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 101 2026-02-18T08:23:50.779 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 106 2026-02-18T08:23:50.779 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:23:50.779 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=425, tag_tid=30, entry_tid=1], [object_number=424, tag_tid=30, entry_tid=0], [object_number=427, tag_tid=29, entry_tid=45931], [object_number=426, tag_tid=29, entry_tid=45930]]], state=connected] 2026-02-18T08:23:50.780 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=406, tag_tid=29, entry_tid=25430], [object_number=405, tag_tid=29, entry_tid=25429], [object_number=404, tag_tid=29, entry_tid=25428], [object_number=407, tag_tid=29, entry_tid=25427]]], state=connected] 2026-02-18T08:23:50.782 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:23:50.784 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=406, tag_tid=29, entry_tid=25430' 2026-02-18T08:23:50.784 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=425, tag_tid=30, entry_tid=1' -a 'object_number=425, tag_tid=30, entry_tid=1' = 'object_number=406, tag_tid=29, entry_tid=25430' 2026-02-18T08:23:50.784 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=406, tag_tid=29, entry_tid=25430' '!=' 'object_number=407, tag_tid=29, entry_tid=25075' 2026-02-18T08:23:50.784 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:23:50.784 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=406, tag_tid=29, entry_tid=25430' = 'object_number=407, tag_tid=29, entry_tid=25075' 2026-02-18T08:23:50.785 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=406, tag_tid=29, entry_tid=25430' 2026-02-18T08:23:50.785 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=425, tag_tid=30, entry_tid=1' 2026-02-18T08:23:50.785 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:23:50.785 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:23:50.786 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=30 2026-02-18T08:23:50.787 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=406, tag_tid=29, entry_tid=25430' 2026-02-18T08:23:50.787 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:23:50.787 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:23:50.788 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=29 2026-02-18T08:23:50.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=425, tag_tid=30, entry_tid=1' 2026-02-18T08:23:50.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:23:50.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:23:50.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:23:50.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=406, tag_tid=29, entry_tid=25430' 2026-02-18T08:23:50.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:23:50.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:23:50.793 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=25430 2026-02-18T08:23:50.793 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 30 = 29 -a 1 -le 25430 2026-02-18T08:23:50.793 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:23:50.793 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:23:50.793 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:23:50.994 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:23:50.994 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:23:50.994 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:23:50.994 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:23:50.994 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:23:50.994 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:23:50.994 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:23:50.994 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:23:50.994 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:23:50.994 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:23:50.994 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:23:50.994 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:23:50.994 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:23:50.994 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:23:50.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:23:50.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:23:50.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:23:50.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:23:50.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:23:50.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:23:50.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:23:50.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:23:50.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:23:50.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:23:50.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:23:50.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:23:50.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:23:50.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:23:50.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:23:50.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:23:50.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:23:50.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:23:50.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:23:50.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:23:50.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:23:50.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:23:50.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:23:50.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:23:50.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:23:50.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:23:50.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:23:50.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:23:50.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:23:50.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:23:50.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:23:50.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:23:50.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:23:50.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:23:50.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:23:51.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:23:51.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:23:51.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:23:51.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:23:51.059 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:23:51.059 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:23:51.059 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:23:51.062 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:23:51.062 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:23:51.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:23:51.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:23:51.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:23:51.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:23:51.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:23:51.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:23:51.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:23:51.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:23:51.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:23:51.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:23:51.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:23:51.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:23:51.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:23:51.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:23:51.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:23:51.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:23:51.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:23:51.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:23:51.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:23:51.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:23:51.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:23:51.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:23:51.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:23:51.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:23:51.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:23:51.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:23:51.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:23:51.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:23:51.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:23:51.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:23:51.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:23:51.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:23:51.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:23:51.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:23:51.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:23:51.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:23:51.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:23:51.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:23:51.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:23:51.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:23:51.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:23:51.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:23:51.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:23:51.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:23:51.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:23:51.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:23:51.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:23:51.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:23:51.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:23:51.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:23:51.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:23:51.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:23:51.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:23:51.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:23:51.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:23:51.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:23:51.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:23:51.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:23:51.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:23:51.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:23:51.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:23:51.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:23:51.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:23:51.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:23:51.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:23:51.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:23:51.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:23:51.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:23:51.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:23:51.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:23:51.136 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:23:51.136 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:23:51.136 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:23:51.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:23:51.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:23:51.187 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:23:51.190 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:23:51.190 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:23:51.190 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:23:51.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:23:51.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:23:51.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:23:51.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:23:51.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:23:51.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:23:51.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:23:51.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:23:51.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:23:51.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:23:51.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:23:51.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:23:51.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:23:51.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:23:51.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:23:51.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:23:51.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:23:51.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:23:51.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:23:51.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:23:51.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:23:51.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:23:51.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:23:51.194 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:23:51.194 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:23:51.194 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:23:51.194 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:23:51.194 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:23:51.194 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:23:51.199 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:23:51.200 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:23:51.200 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:23:51.200 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:23:51.260 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:23:51.260 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:23:51.260 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:23:51.261 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:23:51.261 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:23:51.261 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:23:51.261 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:23:51.261 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:23:51.261 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:23:51.261 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:23:51.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:23:51.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:23:51.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:23:51.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:23:51.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:23:51.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:23:51.291 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 101 2026-02-18T08:23:51.292 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 106 2026-02-18T08:23:51.292 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:23:51.292 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=425, tag_tid=30, entry_tid=1], [object_number=424, tag_tid=30, entry_tid=0], [object_number=427, tag_tid=29, entry_tid=45931], [object_number=426, tag_tid=29, entry_tid=45930]]], state=connected] 2026-02-18T08:23:51.292 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=405, tag_tid=29, entry_tid=25777], [object_number=404, tag_tid=29, entry_tid=25776], [object_number=407, tag_tid=29, entry_tid=25775], [object_number=406, tag_tid=29, entry_tid=25774]]], state=connected] 2026-02-18T08:23:51.294 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:23:51.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=425, tag_tid=30, entry_tid=1' 2026-02-18T08:23:51.297 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:23:51.297 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:23:51.297 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:23:51.297 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:23:51.297 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:23:51.297 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:23:51.297 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:23:51.297 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:23:51.298 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:23:51.298 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:23:51.298 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:23:51.298 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:23:51.299 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:23:51.299 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:23:51.299 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:23:51.326 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 101 2026-02-18T08:23:51.326 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 106 2026-02-18T08:23:51.326 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:23:51.326 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=425, tag_tid=30, entry_tid=1], [object_number=424, tag_tid=30, entry_tid=0], [object_number=427, tag_tid=29, entry_tid=45931], [object_number=426, tag_tid=29, entry_tid=45930]]], state=connected] 2026-02-18T08:23:51.326 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=405, tag_tid=29, entry_tid=25777], [object_number=404, tag_tid=29, entry_tid=25776], [object_number=407, tag_tid=29, entry_tid=25775], [object_number=406, tag_tid=29, entry_tid=25774]]], state=connected] 2026-02-18T08:23:51.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:23:51.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=405, tag_tid=29, entry_tid=25777' 2026-02-18T08:23:51.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=425, tag_tid=30, entry_tid=1' -a 'object_number=425, tag_tid=30, entry_tid=1' = 'object_number=405, tag_tid=29, entry_tid=25777' 2026-02-18T08:23:51.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=405, tag_tid=29, entry_tid=25777' '!=' 'object_number=406, tag_tid=29, entry_tid=25430' 2026-02-18T08:23:51.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:23:51.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=405, tag_tid=29, entry_tid=25777' = 'object_number=406, tag_tid=29, entry_tid=25430' 2026-02-18T08:23:51.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=405, tag_tid=29, entry_tid=25777' 2026-02-18T08:23:51.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=425, tag_tid=30, entry_tid=1' 2026-02-18T08:23:51.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:23:51.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:23:51.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=30 2026-02-18T08:23:51.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=405, tag_tid=29, entry_tid=25777' 2026-02-18T08:23:51.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:23:51.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:23:51.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=29 2026-02-18T08:23:51.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=425, tag_tid=30, entry_tid=1' 2026-02-18T08:23:51.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:23:51.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:23:51.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:23:51.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=405, tag_tid=29, entry_tid=25777' 2026-02-18T08:23:51.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:23:51.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:23:51.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=25777 2026-02-18T08:23:51.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 30 = 29 -a 1 -le 25777 2026-02-18T08:23:51.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:23:51.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:23:51.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:23:51.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:23:51.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:23:51.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:23:51.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:23:51.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:23:51.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:23:51.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:23:51.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:23:51.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:23:51.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:23:51.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:23:51.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:23:51.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:23:51.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:23:51.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:23:51.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:23:51.545 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:23:51.545 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:23:51.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:23:51.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:23:51.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:23:51.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:23:51.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:23:51.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:23:51.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:23:51.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:23:51.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:23:51.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:23:51.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:23:51.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:23:51.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:23:51.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:23:51.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:23:51.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:23:51.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:23:51.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:23:51.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:23:51.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:23:51.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:23:51.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:23:51.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:23:51.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:23:51.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:23:51.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:23:51.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:23:51.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:23:51.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:23:51.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:23:51.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:23:51.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:23:51.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:23:51.557 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:23:51.557 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:23:51.606 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:23:51.606 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:23:51.606 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:23:51.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:23:51.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:23:51.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:23:51.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:23:51.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:23:51.610 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:23:51.610 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:23:51.610 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:23:51.610 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:23:51.610 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:23:51.610 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:23:51.610 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:23:51.610 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:23:51.610 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:23:51.610 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:23:51.610 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:23:51.610 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:23:51.611 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:23:51.611 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:23:51.611 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:23:51.611 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:23:51.611 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:23:51.611 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:23:51.611 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:23:51.611 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:23:51.611 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:23:51.611 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:23:51.611 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:23:51.611 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:23:51.612 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:23:51.612 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:23:51.612 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:23:51.620 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:23:51.620 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:23:51.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:23:51.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:23:51.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:23:51.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:23:51.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:23:51.672 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:23:51.672 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:23:51.672 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:23:51.672 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:23:51.672 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:23:51.672 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:23:51.672 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:23:51.673 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:23:51.673 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:23:51.673 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:23:51.673 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:23:51.673 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:23:51.673 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:23:51.673 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:23:51.673 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:23:51.673 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:23:51.673 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:23:51.673 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:23:51.674 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:23:51.674 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:23:51.674 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:23:51.674 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:23:51.674 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:23:51.674 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:23:51.674 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:23:51.674 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:23:51.674 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:23:51.674 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:23:51.674 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:23:51.674 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:23:51.674 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:23:51.675 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:23:51.683 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:23:51.684 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:23:51.684 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:23:51.684 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:23:51.734 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:23:51.734 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:23:51.735 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:23:51.737 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:23:51.737 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:23:51.738 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:23:51.738 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:23:51.738 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:23:51.738 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:23:51.738 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:23:51.738 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:23:51.738 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:23:51.738 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:23:51.738 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:23:51.738 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:23:51.738 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:23:51.738 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:23:51.739 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:23:51.739 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:23:51.739 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:23:51.739 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:23:51.739 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:23:51.739 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:23:51.739 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:23:51.739 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:23:51.739 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:23:51.739 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:23:51.739 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:23:51.739 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:23:51.740 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:23:51.740 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:23:51.740 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:23:51.740 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:23:51.740 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:23:51.740 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:23:51.748 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:23:51.749 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:23:51.749 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:23:51.749 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:23:51.809 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:23:51.811 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:23:51.811 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:23:51.811 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:23:51.811 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:23:51.811 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:23:51.811 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:23:51.811 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:23:51.811 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:23:51.811 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:23:51.811 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:23:51.812 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:23:51.812 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:23:51.812 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:23:51.813 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:23:51.813 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:23:51.851 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 101 2026-02-18T08:23:51.851 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 106 2026-02-18T08:23:51.852 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:23:51.852 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=425, tag_tid=30, entry_tid=1], [object_number=424, tag_tid=30, entry_tid=0], [object_number=427, tag_tid=29, entry_tid=45931], [object_number=426, tag_tid=29, entry_tid=45930]]], state=connected] 2026-02-18T08:23:51.852 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=407, tag_tid=29, entry_tid=26107], [object_number=406, tag_tid=29, entry_tid=26106], [object_number=405, tag_tid=29, entry_tid=26105], [object_number=404, tag_tid=29, entry_tid=26104]]], state=connected] 2026-02-18T08:23:51.855 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:23:51.856 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=425, tag_tid=30, entry_tid=1' 2026-02-18T08:23:51.857 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:23:51.858 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:23:51.858 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:23:51.858 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:23:51.858 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:23:51.858 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:23:51.858 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:23:51.858 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:23:51.858 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:23:51.858 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:23:51.859 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:23:51.859 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:23:51.860 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:23:51.860 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:23:51.860 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:23:51.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 101 2026-02-18T08:23:51.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 106 2026-02-18T08:23:51.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:23:51.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=425, tag_tid=30, entry_tid=1], [object_number=424, tag_tid=30, entry_tid=0], [object_number=427, tag_tid=29, entry_tid=45931], [object_number=426, tag_tid=29, entry_tid=45930]]], state=connected] 2026-02-18T08:23:51.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=407, tag_tid=29, entry_tid=26107], [object_number=406, tag_tid=29, entry_tid=26106], [object_number=405, tag_tid=29, entry_tid=26105], [object_number=404, tag_tid=29, entry_tid=26104]]], state=connected] 2026-02-18T08:23:51.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:23:51.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=407, tag_tid=29, entry_tid=26107' 2026-02-18T08:23:51.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=425, tag_tid=30, entry_tid=1' -a 'object_number=425, tag_tid=30, entry_tid=1' = 'object_number=407, tag_tid=29, entry_tid=26107' 2026-02-18T08:23:51.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=407, tag_tid=29, entry_tid=26107' '!=' 'object_number=405, tag_tid=29, entry_tid=25777' 2026-02-18T08:23:51.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:23:51.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=407, tag_tid=29, entry_tid=26107' = 'object_number=405, tag_tid=29, entry_tid=25777' 2026-02-18T08:23:51.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=407, tag_tid=29, entry_tid=26107' 2026-02-18T08:23:51.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=425, tag_tid=30, entry_tid=1' 2026-02-18T08:23:51.894 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:23:51.894 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:23:51.894 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=30 2026-02-18T08:23:51.895 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=407, tag_tid=29, entry_tid=26107' 2026-02-18T08:23:51.895 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:23:51.895 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:23:51.896 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=29 2026-02-18T08:23:51.897 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=425, tag_tid=30, entry_tid=1' 2026-02-18T08:23:51.897 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:23:51.897 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:23:51.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:23:51.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=407, tag_tid=29, entry_tid=26107' 2026-02-18T08:23:51.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:23:51.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:23:51.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=26107 2026-02-18T08:23:51.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 30 = 29 -a 1 -le 26107 2026-02-18T08:23:51.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:23:51.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:23:51.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:23:52.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:23:52.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:23:52.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:23:52.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:23:52.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:23:52.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:23:52.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:23:52.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:23:52.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:23:52.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:23:52.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:23:52.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:23:52.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:23:52.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:23:52.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:23:52.105 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:23:52.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:23:52.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:23:52.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:23:52.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:23:52.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:23:52.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:23:52.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:23:52.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:23:52.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:23:52.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:23:52.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:23:52.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:23:52.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:23:52.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:23:52.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:23:52.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:23:52.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:23:52.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:23:52.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:23:52.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:23:52.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:23:52.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:23:52.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:23:52.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:23:52.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:23:52.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:23:52.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:23:52.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:23:52.110 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:23:52.110 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:23:52.110 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:23:52.110 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:23:52.110 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:23:52.118 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:23:52.118 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:23:52.118 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:23:52.118 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:23:52.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:23:52.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:23:52.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:23:52.171 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:23:52.171 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:23:52.171 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:23:52.171 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:23:52.171 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:23:52.171 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:23:52.171 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:23:52.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:23:52.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:23:52.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:23:52.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:23:52.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:23:52.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:23:52.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:23:52.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:23:52.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:23:52.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:23:52.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:23:52.173 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:23:52.173 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:23:52.173 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:23:52.173 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:23:52.173 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:23:52.173 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:23:52.173 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:23:52.173 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:23:52.173 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:23:52.173 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:23:52.173 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:23:52.173 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:23:52.174 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:23:52.174 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:23:52.183 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:23:52.183 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:23:52.183 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:23:52.183 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:23:52.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:23:52.234 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:23:52.234 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:23:52.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:23:52.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:23:52.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:23:52.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:23:52.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:23:52.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:23:52.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:23:52.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:23:52.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:23:52.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:23:52.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:23:52.238 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:23:52.238 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:23:52.238 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:23:52.238 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:23:52.238 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:23:52.238 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:23:52.238 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:23:52.238 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:23:52.238 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:23:52.238 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:23:52.238 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:23:52.238 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:23:52.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:23:52.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:23:52.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:23:52.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:23:52.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:23:52.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:23:52.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:23:52.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:23:52.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:23:52.247 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:23:52.248 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:23:52.248 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:23:52.248 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:23:52.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:23:52.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:23:52.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:23:52.310 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:23:52.310 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:23:52.310 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:23:52.310 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:23:52.310 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:23:52.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:23:52.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:23:52.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:23:52.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:23:52.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:23:52.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:23:52.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:23:52.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:23:52.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:23:52.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:23:52.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:23:52.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:23:52.312 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:23:52.312 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:23:52.312 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:23:52.312 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:23:52.312 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:23:52.312 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:23:52.312 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:23:52.312 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:23:52.312 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:23:52.312 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:23:52.312 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:23:52.312 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:23:52.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:23:52.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:23:52.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:23:52.320 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:23:52.321 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:23:52.321 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:23:52.321 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:23:52.383 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:23:52.383 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:23:52.383 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:23:52.384 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:23:52.384 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:23:52.384 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:23:52.384 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:23:52.384 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:23:52.384 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:23:52.384 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:23:52.384 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:23:52.385 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:23:52.385 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:23:52.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:23:52.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:23:52.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:23:52.414 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 101 2026-02-18T08:23:52.415 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 106 2026-02-18T08:23:52.415 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:23:52.415 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=425, tag_tid=30, entry_tid=1], [object_number=424, tag_tid=30, entry_tid=0], [object_number=427, tag_tid=29, entry_tid=45931], [object_number=426, tag_tid=29, entry_tid=45930]]], state=connected] 2026-02-18T08:23:52.415 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=406, tag_tid=29, entry_tid=26422], [object_number=405, tag_tid=29, entry_tid=26421], [object_number=404, tag_tid=29, entry_tid=26420], [object_number=407, tag_tid=29, entry_tid=26419]]], state=connected] 2026-02-18T08:23:52.417 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:23:52.419 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=425, tag_tid=30, entry_tid=1' 2026-02-18T08:23:52.420 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:23:52.420 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:23:52.420 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:23:52.420 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:23:52.420 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:23:52.421 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:23:52.421 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:23:52.421 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:23:52.421 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:23:52.421 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:23:52.421 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:23:52.421 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:23:52.422 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:23:52.422 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:23:52.422 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:23:52.451 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 101 2026-02-18T08:23:52.451 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 106 2026-02-18T08:23:52.451 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:23:52.451 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=425, tag_tid=30, entry_tid=1], [object_number=424, tag_tid=30, entry_tid=0], [object_number=427, tag_tid=29, entry_tid=45931], [object_number=426, tag_tid=29, entry_tid=45930]]], state=connected] 2026-02-18T08:23:52.452 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=406, tag_tid=29, entry_tid=26422], [object_number=405, tag_tid=29, entry_tid=26421], [object_number=404, tag_tid=29, entry_tid=26420], [object_number=407, tag_tid=29, entry_tid=26419]]], state=connected] 2026-02-18T08:23:52.454 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:23:52.455 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=406, tag_tid=29, entry_tid=26422' 2026-02-18T08:23:52.455 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=425, tag_tid=30, entry_tid=1' -a 'object_number=425, tag_tid=30, entry_tid=1' = 'object_number=406, tag_tid=29, entry_tid=26422' 2026-02-18T08:23:52.455 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=406, tag_tid=29, entry_tid=26422' '!=' 'object_number=407, tag_tid=29, entry_tid=26107' 2026-02-18T08:23:52.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:23:52.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=406, tag_tid=29, entry_tid=26422' = 'object_number=407, tag_tid=29, entry_tid=26107' 2026-02-18T08:23:52.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=406, tag_tid=29, entry_tid=26422' 2026-02-18T08:23:52.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=425, tag_tid=30, entry_tid=1' 2026-02-18T08:23:52.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:23:52.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:23:52.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=30 2026-02-18T08:23:52.458 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=406, tag_tid=29, entry_tid=26422' 2026-02-18T08:23:52.458 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:23:52.458 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:23:52.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=29 2026-02-18T08:23:52.460 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=425, tag_tid=30, entry_tid=1' 2026-02-18T08:23:52.460 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:23:52.460 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:23:52.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:23:52.462 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=406, tag_tid=29, entry_tid=26422' 2026-02-18T08:23:52.462 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:23:52.462 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:23:52.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=26422 2026-02-18T08:23:52.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 30 = 29 -a 1 -le 26422 2026-02-18T08:23:52.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:23:52.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:23:52.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:23:52.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:23:52.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:23:52.667 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:23:52.667 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:23:52.667 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:23:52.667 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:23:52.667 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:23:52.667 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:23:52.667 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:23:52.667 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:23:52.667 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:23:52.667 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:23:52.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:23:52.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:23:52.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:23:52.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:23:52.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:23:52.670 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:23:52.671 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:23:52.671 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:23:52.671 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:23:52.671 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:23:52.671 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:23:52.671 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:23:52.671 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:23:52.671 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:23:52.671 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:23:52.671 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:23:52.671 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:23:52.672 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:23:52.672 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:23:52.672 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:23:52.672 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:23:52.672 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:23:52.672 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:23:52.672 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:23:52.672 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:23:52.672 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:23:52.672 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:23:52.672 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:23:52.672 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:23:52.673 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:23:52.673 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:23:52.673 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:23:52.673 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:23:52.673 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:23:52.673 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:23:52.673 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:23:52.673 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:23:52.680 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:23:52.680 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:23:52.681 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:23:52.681 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:23:52.731 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:23:52.731 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:23:52.731 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:23:52.733 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:23:52.734 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:23:52.734 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:23:52.734 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:23:52.734 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:23:52.734 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:23:52.734 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:23:52.734 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:23:52.734 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:23:52.734 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:23:52.734 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:23:52.734 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:23:52.735 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:23:52.735 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:23:52.735 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:23:52.735 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:23:52.735 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:23:52.735 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:23:52.735 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:23:52.735 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:23:52.735 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:23:52.735 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:23:52.735 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:23:52.735 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:23:52.735 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:23:52.736 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:23:52.736 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:23:52.736 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:23:52.736 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:23:52.736 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:23:52.736 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:23:52.736 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:23:52.744 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:23:52.745 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:23:52.745 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:23:52.745 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:23:52.796 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:23:52.796 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:23:52.796 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:23:52.799 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:23:52.799 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:23:52.799 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:23:52.799 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:23:52.799 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:23:52.799 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:23:52.799 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:23:52.799 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:23:52.799 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:23:52.800 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:23:52.800 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:23:52.800 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:23:52.800 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:23:52.800 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:23:52.800 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:23:52.800 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:23:52.800 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:23:52.800 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:23:52.800 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:23:52.800 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:23:52.800 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:23:52.801 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:23:52.801 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:23:52.801 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:23:52.801 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:23:52.801 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:23:52.801 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:23:52.801 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:23:52.801 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:23:52.801 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:23:52.801 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:23:52.801 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:23:52.810 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:23:52.811 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:23:52.811 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:23:52.811 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:23:52.862 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:23:52.863 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:23:52.863 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:23:52.865 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:23:52.866 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:23:52.866 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:23:52.866 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:23:52.866 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:23:52.866 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:23:52.866 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:23:52.866 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:23:52.866 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:23:52.866 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:23:52.866 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:23:52.867 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:23:52.867 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:23:52.867 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:23:52.867 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:23:52.867 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:23:52.867 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:23:52.867 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:23:52.867 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:23:52.867 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:23:52.867 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:23:52.867 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:23:52.867 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:23:52.868 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:23:52.868 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:23:52.868 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:23:52.868 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:23:52.868 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:23:52.868 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:23:52.868 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:23:52.868 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:23:52.868 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:23:52.876 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:23:52.876 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:23:52.876 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:23:52.876 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:23:52.936 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:23:52.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:23:52.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:23:52.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:23:52.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:23:52.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:23:52.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:23:52.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:23:52.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:23:52.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:23:52.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:23:52.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:23:52.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:23:52.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:23:52.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:23:52.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:23:52.969 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 101 2026-02-18T08:23:52.969 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 106 2026-02-18T08:23:52.970 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:23:52.970 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=425, tag_tid=30, entry_tid=1], [object_number=424, tag_tid=30, entry_tid=0], [object_number=427, tag_tid=29, entry_tid=45931], [object_number=426, tag_tid=29, entry_tid=45930]]], state=connected] 2026-02-18T08:23:52.970 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=406, tag_tid=29, entry_tid=26778], [object_number=405, tag_tid=29, entry_tid=26777], [object_number=404, tag_tid=29, entry_tid=26776], [object_number=407, tag_tid=29, entry_tid=26775]]], state=connected] 2026-02-18T08:23:52.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:23:52.974 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=425, tag_tid=30, entry_tid=1' 2026-02-18T08:23:52.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:23:52.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:23:52.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:23:52.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:23:52.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:23:52.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:23:52.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:23:52.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:23:52.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:23:52.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:23:52.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:23:52.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:23:52.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:23:52.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:23:52.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:23:53.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 101 2026-02-18T08:23:53.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 106 2026-02-18T08:23:53.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:23:53.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=425, tag_tid=30, entry_tid=1], [object_number=424, tag_tid=30, entry_tid=0], [object_number=427, tag_tid=29, entry_tid=45931], [object_number=426, tag_tid=29, entry_tid=45930]]], state=connected] 2026-02-18T08:23:53.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=406, tag_tid=29, entry_tid=26778], [object_number=405, tag_tid=29, entry_tid=26777], [object_number=404, tag_tid=29, entry_tid=26776], [object_number=407, tag_tid=29, entry_tid=26775]]], state=connected] 2026-02-18T08:23:53.018 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:23:53.020 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=406, tag_tid=29, entry_tid=26778' 2026-02-18T08:23:53.020 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=425, tag_tid=30, entry_tid=1' -a 'object_number=425, tag_tid=30, entry_tid=1' = 'object_number=406, tag_tid=29, entry_tid=26778' 2026-02-18T08:23:53.020 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=406, tag_tid=29, entry_tid=26778' '!=' 'object_number=406, tag_tid=29, entry_tid=26422' 2026-02-18T08:23:53.020 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:23:53.020 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=406, tag_tid=29, entry_tid=26778' = 'object_number=406, tag_tid=29, entry_tid=26422' 2026-02-18T08:23:53.020 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=406, tag_tid=29, entry_tid=26778' 2026-02-18T08:23:53.021 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=425, tag_tid=30, entry_tid=1' 2026-02-18T08:23:53.021 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:23:53.021 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:23:53.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=30 2026-02-18T08:23:53.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=406, tag_tid=29, entry_tid=26778' 2026-02-18T08:23:53.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:23:53.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:23:53.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=29 2026-02-18T08:23:53.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=425, tag_tid=30, entry_tid=1' 2026-02-18T08:23:53.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:23:53.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:23:53.027 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:23:53.027 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=406, tag_tid=29, entry_tid=26778' 2026-02-18T08:23:53.027 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:23:53.028 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:23:53.029 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=26778 2026-02-18T08:23:53.029 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 30 = 29 -a 1 -le 26778 2026-02-18T08:23:53.029 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:23:53.029 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:23:53.029 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:23:53.230 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:23:53.231 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:23:53.231 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:23:53.231 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:23:53.231 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:23:53.231 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:23:53.231 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:23:53.231 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:23:53.231 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:23:53.231 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:23:53.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:23:53.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:23:53.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:23:53.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:23:53.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:23:53.234 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:23:53.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:23:53.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:23:53.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:23:53.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:23:53.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:23:53.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:23:53.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:23:53.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:23:53.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:23:53.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:23:53.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:23:53.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:23:53.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:23:53.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:23:53.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:23:53.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:23:53.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:23:53.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:23:53.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:23:53.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:23:53.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:23:53.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:23:53.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:23:53.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:23:53.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:23:53.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:23:53.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:23:53.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:23:53.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:23:53.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:23:53.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:23:53.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:23:53.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:23:53.244 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:23:53.245 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:23:53.245 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:23:53.245 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:23:53.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:23:53.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:23:53.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:23:53.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:23:53.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:23:53.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:23:53.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:23:53.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:23:53.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:23:53.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:23:53.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:23:53.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:23:53.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:23:53.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:23:53.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:23:53.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:23:53.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:23:53.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:23:53.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:23:53.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:23:53.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:23:53.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:23:53.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:23:53.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:23:53.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:23:53.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:23:53.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:23:53.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:23:53.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:23:53.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:23:53.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:23:53.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:23:53.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:23:53.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:23:53.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:23:53.316 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:23:53.317 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:23:53.317 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:23:53.317 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:23:53.365 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:23:53.366 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:23:53.366 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:23:53.368 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:23:53.368 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:23:53.368 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:23:53.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:23:53.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:23:53.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:23:53.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:23:53.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:23:53.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:23:53.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:23:53.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:23:53.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:23:53.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:23:53.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:23:53.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:23:53.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:23:53.370 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:23:53.370 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:23:53.370 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:23:53.370 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:23:53.370 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:23:53.370 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:23:53.370 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:23:53.370 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:23:53.370 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:23:53.370 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:23:53.370 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:23:53.370 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:23:53.371 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:23:53.371 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:23:53.371 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:23:53.371 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:23:53.380 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:23:53.381 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:23:53.381 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:23:53.381 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:23:53.429 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:23:53.429 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:23:53.429 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:23:53.432 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:23:53.432 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:23:53.432 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:23:53.432 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:23:53.433 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:23:53.433 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:23:53.433 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:23:53.433 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:23:53.433 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:23:53.433 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:23:53.433 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:23:53.433 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:23:53.433 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:23:53.433 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:23:53.433 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:23:53.434 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:23:53.434 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:23:53.434 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:23:53.434 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:23:53.434 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:23:53.434 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:23:53.434 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:23:53.434 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:23:53.434 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:23:53.434 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:23:53.434 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:23:53.434 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:23:53.435 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:23:53.435 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:23:53.435 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:23:53.435 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:23:53.435 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:23:53.443 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:23:53.443 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:23:53.443 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:23:53.443 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:23:53.503 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:23:53.504 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:23:53.504 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:23:53.504 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:23:53.504 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:23:53.505 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:23:53.505 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:23:53.505 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:23:53.505 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:23:53.505 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:23:53.505 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:23:53.505 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:23:53.505 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:23:53.506 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:23:53.506 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:23:53.506 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:23:53.536 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 101 2026-02-18T08:23:53.536 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 106 2026-02-18T08:23:53.536 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:23:53.536 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=425, tag_tid=30, entry_tid=1], [object_number=424, tag_tid=30, entry_tid=0], [object_number=427, tag_tid=29, entry_tid=45931], [object_number=426, tag_tid=29, entry_tid=45930]]], state=connected] 2026-02-18T08:23:53.536 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=406, tag_tid=29, entry_tid=27010], [object_number=405, tag_tid=29, entry_tid=27009], [object_number=404, tag_tid=29, entry_tid=27008], [object_number=407, tag_tid=29, entry_tid=27007]]], state=connected] 2026-02-18T08:23:53.539 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:23:53.540 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=425, tag_tid=30, entry_tid=1' 2026-02-18T08:23:53.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:23:53.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:23:53.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:23:53.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:23:53.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:23:53.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:23:53.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:23:53.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:23:53.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:23:53.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:23:53.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:23:53.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:23:53.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:23:53.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:23:53.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:23:53.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 101 2026-02-18T08:23:53.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 106 2026-02-18T08:23:53.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:23:53.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=425, tag_tid=30, entry_tid=1], [object_number=424, tag_tid=30, entry_tid=0], [object_number=427, tag_tid=29, entry_tid=45931], [object_number=426, tag_tid=29, entry_tid=45930]]], state=connected] 2026-02-18T08:23:53.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=406, tag_tid=29, entry_tid=27010], [object_number=405, tag_tid=29, entry_tid=27009], [object_number=404, tag_tid=29, entry_tid=27008], [object_number=407, tag_tid=29, entry_tid=27007]]], state=connected] 2026-02-18T08:23:53.585 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:23:53.587 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=406, tag_tid=29, entry_tid=27010' 2026-02-18T08:23:53.587 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=425, tag_tid=30, entry_tid=1' -a 'object_number=425, tag_tid=30, entry_tid=1' = 'object_number=406, tag_tid=29, entry_tid=27010' 2026-02-18T08:23:53.587 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=406, tag_tid=29, entry_tid=27010' '!=' 'object_number=406, tag_tid=29, entry_tid=26778' 2026-02-18T08:23:53.587 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:23:53.587 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=406, tag_tid=29, entry_tid=27010' = 'object_number=406, tag_tid=29, entry_tid=26778' 2026-02-18T08:23:53.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=406, tag_tid=29, entry_tid=27010' 2026-02-18T08:23:53.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=425, tag_tid=30, entry_tid=1' 2026-02-18T08:23:53.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:23:53.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:23:53.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=30 2026-02-18T08:23:53.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=406, tag_tid=29, entry_tid=27010' 2026-02-18T08:23:53.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:23:53.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:23:53.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=29 2026-02-18T08:23:53.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=425, tag_tid=30, entry_tid=1' 2026-02-18T08:23:53.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:23:53.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:23:53.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:23:53.594 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=406, tag_tid=29, entry_tid=27010' 2026-02-18T08:23:53.594 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:23:53.595 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:23:53.596 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=27010 2026-02-18T08:23:53.596 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 30 = 29 -a 1 -le 27010 2026-02-18T08:23:53.596 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:23:53.596 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:23:53.596 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:23:53.798 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:23:53.798 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:23:53.798 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:23:53.799 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:23:53.799 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:23:53.799 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:23:53.799 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:23:53.799 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:23:53.799 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:23:53.799 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:23:53.799 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:23:53.799 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:23:53.799 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:23:53.800 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:23:53.800 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:23:53.800 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:23:53.802 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:23:53.802 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:23:53.802 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:23:53.802 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:23:53.803 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:23:53.803 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:23:53.803 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:23:53.803 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:23:53.803 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:23:53.803 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:23:53.803 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:23:53.803 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:23:53.803 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:23:53.803 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:23:53.803 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:23:53.803 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:23:53.804 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:23:53.804 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:23:53.804 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:23:53.804 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:23:53.804 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:23:53.804 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:23:53.804 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:23:53.804 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:23:53.804 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:23:53.804 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:23:53.804 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:23:53.804 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:23:53.805 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:23:53.805 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:23:53.805 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:23:53.805 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:23:53.805 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:23:53.813 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:23:53.813 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:23:53.813 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:23:53.813 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:23:53.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:23:53.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:23:53.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:23:53.875 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:23:53.875 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:23:53.875 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:23:53.876 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:23:53.876 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:23:53.876 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:23:53.876 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:23:53.876 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:23:53.876 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:23:53.876 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:23:53.876 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:23:53.876 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:23:53.876 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:23:53.876 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:23:53.877 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:23:53.877 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:23:53.877 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:23:53.877 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:23:53.877 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:23:53.877 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:23:53.877 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:23:53.877 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:23:53.877 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:23:53.877 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:23:53.877 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:23:53.877 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:23:53.878 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:23:53.878 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:23:53.878 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:23:53.878 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:23:53.878 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:23:53.878 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:23:53.887 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:23:53.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:23:53.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:23:53.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:23:53.944 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:23:53.944 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:23:53.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:23:53.947 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:23:53.947 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:23:53.947 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:23:53.947 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:23:53.947 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:23:53.947 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:23:53.947 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:23:53.948 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:23:53.948 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:23:53.948 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:23:53.948 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:23:53.948 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:23:53.948 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:23:53.948 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:23:53.948 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:23:53.948 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:23:53.948 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:23:53.948 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:23:53.948 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:23:53.949 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:23:53.949 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:23:53.949 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:23:53.949 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:23:53.949 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:23:53.949 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:23:53.949 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:23:53.949 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:23:53.949 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:23:53.949 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:23:53.949 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:23:53.949 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:23:53.950 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:23:53.958 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:23:53.959 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:23:53.959 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:23:53.959 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:23:54.006 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:23:54.006 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:23:54.006 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:23:54.008 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:23:54.008 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:23:54.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:23:54.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:23:54.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:23:54.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:23:54.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:23:54.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:23:54.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:23:54.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:23:54.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:23:54.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:23:54.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:23:54.010 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:23:54.010 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:23:54.010 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:23:54.010 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:23:54.010 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:23:54.010 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:23:54.010 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:23:54.010 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:23:54.010 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:23:54.010 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:23:54.010 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:23:54.010 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:23:54.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:23:54.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:23:54.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:23:54.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:23:54.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:23:54.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:23:54.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:23:54.019 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:23:54.020 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:23:54.020 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:23:54.020 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:23:54.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:23:54.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:23:54.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:23:54.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:23:54.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:23:54.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:23:54.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:23:54.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:23:54.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:23:54.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:23:54.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:23:54.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:23:54.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:23:54.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:23:54.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:23:54.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:23:54.119 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 101 2026-02-18T08:23:54.119 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 106 2026-02-18T08:23:54.119 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:23:54.119 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=425, tag_tid=30, entry_tid=1], [object_number=424, tag_tid=30, entry_tid=0], [object_number=427, tag_tid=29, entry_tid=45931], [object_number=426, tag_tid=29, entry_tid=45930]]], state=connected] 2026-02-18T08:23:54.120 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=407, tag_tid=29, entry_tid=27387], [object_number=406, tag_tid=29, entry_tid=27386], [object_number=405, tag_tid=29, entry_tid=27385], [object_number=404, tag_tid=29, entry_tid=27384]]], state=connected] 2026-02-18T08:23:54.122 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:23:54.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=425, tag_tid=30, entry_tid=1' 2026-02-18T08:23:54.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:23:54.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:23:54.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:23:54.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:23:54.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:23:54.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:23:54.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:23:54.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:23:54.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:23:54.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:23:54.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:23:54.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:23:54.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:23:54.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:23:54.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:23:54.156 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 101 2026-02-18T08:23:54.156 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 106 2026-02-18T08:23:54.156 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:23:54.156 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=425, tag_tid=30, entry_tid=1], [object_number=424, tag_tid=30, entry_tid=0], [object_number=427, tag_tid=29, entry_tid=45931], [object_number=426, tag_tid=29, entry_tid=45930]]], state=connected] 2026-02-18T08:23:54.157 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=407, tag_tid=29, entry_tid=27387], [object_number=406, tag_tid=29, entry_tid=27386], [object_number=405, tag_tid=29, entry_tid=27385], [object_number=404, tag_tid=29, entry_tid=27384]]], state=connected] 2026-02-18T08:23:54.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:23:54.161 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=407, tag_tid=29, entry_tid=27387' 2026-02-18T08:23:54.161 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=425, tag_tid=30, entry_tid=1' -a 'object_number=425, tag_tid=30, entry_tid=1' = 'object_number=407, tag_tid=29, entry_tid=27387' 2026-02-18T08:23:54.161 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=407, tag_tid=29, entry_tid=27387' '!=' 'object_number=406, tag_tid=29, entry_tid=27010' 2026-02-18T08:23:54.161 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:23:54.161 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=407, tag_tid=29, entry_tid=27387' = 'object_number=406, tag_tid=29, entry_tid=27010' 2026-02-18T08:23:54.161 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=407, tag_tid=29, entry_tid=27387' 2026-02-18T08:23:54.162 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=425, tag_tid=30, entry_tid=1' 2026-02-18T08:23:54.162 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:23:54.162 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:23:54.163 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=30 2026-02-18T08:23:54.164 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=407, tag_tid=29, entry_tid=27387' 2026-02-18T08:23:54.164 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:23:54.164 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:23:54.165 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=29 2026-02-18T08:23:54.166 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=425, tag_tid=30, entry_tid=1' 2026-02-18T08:23:54.166 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:23:54.166 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:23:54.167 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:23:54.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=407, tag_tid=29, entry_tid=27387' 2026-02-18T08:23:54.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:23:54.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:23:54.170 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=27387 2026-02-18T08:23:54.170 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 30 = 29 -a 1 -le 27387 2026-02-18T08:23:54.170 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:23:54.170 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:23:54.170 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:23:54.372 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:23:54.372 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:23:54.372 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:23:54.372 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:23:54.372 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:23:54.372 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:23:54.372 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:23:54.373 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:23:54.373 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:23:54.373 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:23:54.373 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:23:54.373 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:23:54.373 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:23:54.373 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:23:54.373 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:23:54.376 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:23:54.376 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:23:54.376 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:23:54.376 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:23:54.376 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:23:54.376 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:23:54.376 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:23:54.376 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:23:54.376 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:23:54.376 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:23:54.377 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:23:54.377 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:23:54.377 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:23:54.377 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:23:54.377 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:23:54.377 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:23:54.377 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:23:54.377 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:23:54.377 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:23:54.377 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:23:54.377 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:23:54.377 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:23:54.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:23:54.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:23:54.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:23:54.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:23:54.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:23:54.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:23:54.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:23:54.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:23:54.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:23:54.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:23:54.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:23:54.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:23:54.387 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:23:54.388 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:23:54.388 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:23:54.388 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:23:54.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:23:54.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:23:54.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:23:54.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:23:54.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:23:54.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:23:54.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:23:54.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:23:54.448 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:23:54.448 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:23:54.448 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:23:54.448 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:23:54.448 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:23:54.448 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:23:54.448 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:23:54.448 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:23:54.448 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:23:54.448 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:23:54.448 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:23:54.448 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:23:54.449 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:23:54.449 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:23:54.449 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:23:54.449 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:23:54.449 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:23:54.449 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:23:54.449 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:23:54.449 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:23:54.449 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:23:54.449 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:23:54.449 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:23:54.449 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:23:54.450 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:23:54.450 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:23:54.450 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:23:54.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:23:54.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:23:54.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:23:54.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:23:54.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:23:54.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:23:54.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:23:54.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:23:54.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:23:54.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:23:54.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:23:54.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:23:54.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:23:54.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:23:54.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:23:54.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:23:54.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:23:54.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:23:54.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:23:54.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:23:54.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:23:54.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:23:54.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:23:54.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:23:54.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:23:54.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:23:54.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:23:54.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:23:54.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:23:54.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:23:54.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:23:54.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:23:54.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:23:54.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:23:54.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:23:54.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:23:54.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:23:54.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:23:54.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:23:54.528 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:23:54.529 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:23:54.529 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:23:54.529 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:23:54.587 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:23:54.587 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:23:54.587 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:23:54.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:23:54.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:23:54.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:23:54.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:23:54.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:23:54.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:23:54.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:23:54.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:23:54.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:23:54.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:23:54.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:23:54.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:23:54.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:23:54.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:23:54.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:23:54.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:23:54.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:23:54.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:23:54.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:23:54.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:23:54.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:23:54.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:23:54.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:23:54.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:23:54.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:23:54.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:23:54.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:23:54.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:23:54.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:23:54.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:23:54.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:23:54.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:23:54.600 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:23:54.601 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:23:54.601 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:23:54.601 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:23:54.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:23:54.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:23:54.667 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:23:54.667 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:23:54.667 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:23:54.667 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:23:54.667 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:23:54.667 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:23:54.667 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:23:54.667 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:23:54.667 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:23:54.667 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:23:54.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:23:54.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:23:54.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:23:54.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:23:54.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 102 2026-02-18T08:23:54.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 106 2026-02-18T08:23:54.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:23:54.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=425, tag_tid=30, entry_tid=1], [object_number=424, tag_tid=30, entry_tid=0], [object_number=427, tag_tid=29, entry_tid=45931], [object_number=426, tag_tid=29, entry_tid=45930]]], state=connected] 2026-02-18T08:23:54.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=411, tag_tid=29, entry_tid=27647], [object_number=410, tag_tid=29, entry_tid=27646], [object_number=409, tag_tid=29, entry_tid=27645], [object_number=408, tag_tid=29, entry_tid=27644]]], state=connected] 2026-02-18T08:23:54.905 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:23:54.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=425, tag_tid=30, entry_tid=1' 2026-02-18T08:23:54.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:23:54.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:23:54.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:23:54.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:23:54.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:23:54.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:23:54.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:23:54.909 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:23:54.909 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:23:54.909 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:23:54.909 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:23:54.909 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:23:54.910 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:23:54.910 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:23:54.910 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:23:54.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 102 2026-02-18T08:23:54.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 106 2026-02-18T08:23:54.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:23:54.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=425, tag_tid=30, entry_tid=1], [object_number=424, tag_tid=30, entry_tid=0], [object_number=427, tag_tid=29, entry_tid=45931], [object_number=426, tag_tid=29, entry_tid=45930]]], state=connected] 2026-02-18T08:23:54.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=411, tag_tid=29, entry_tid=27647], [object_number=410, tag_tid=29, entry_tid=27646], [object_number=409, tag_tid=29, entry_tid=27645], [object_number=408, tag_tid=29, entry_tid=27644]]], state=connected] 2026-02-18T08:23:54.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:23:54.944 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=411, tag_tid=29, entry_tid=27647' 2026-02-18T08:23:54.944 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=425, tag_tid=30, entry_tid=1' -a 'object_number=425, tag_tid=30, entry_tid=1' = 'object_number=411, tag_tid=29, entry_tid=27647' 2026-02-18T08:23:54.944 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=411, tag_tid=29, entry_tid=27647' '!=' 'object_number=407, tag_tid=29, entry_tid=27387' 2026-02-18T08:23:54.944 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:23:54.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=411, tag_tid=29, entry_tid=27647' = 'object_number=407, tag_tid=29, entry_tid=27387' 2026-02-18T08:23:54.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=411, tag_tid=29, entry_tid=27647' 2026-02-18T08:23:54.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=425, tag_tid=30, entry_tid=1' 2026-02-18T08:23:54.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:23:54.946 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:23:54.946 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=30 2026-02-18T08:23:54.947 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=411, tag_tid=29, entry_tid=27647' 2026-02-18T08:23:54.947 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:23:54.947 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:23:54.949 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=29 2026-02-18T08:23:54.949 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=425, tag_tid=30, entry_tid=1' 2026-02-18T08:23:54.950 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:23:54.950 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:23:54.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:23:54.952 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=411, tag_tid=29, entry_tid=27647' 2026-02-18T08:23:54.952 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:23:54.952 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:23:54.953 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=27647 2026-02-18T08:23:54.954 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 30 = 29 -a 1 -le 27647 2026-02-18T08:23:54.954 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:23:54.954 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:23:54.954 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:23:55.156 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:23:55.156 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:23:55.156 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:23:55.156 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:23:55.156 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:23:55.157 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:23:55.157 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:23:55.157 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:23:55.157 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:23:55.157 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:23:55.157 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:23:55.157 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:23:55.157 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:23:55.157 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:23:55.157 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:23:55.157 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:23:55.158 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:23:55.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:23:55.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:23:55.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:23:55.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:23:55.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:23:55.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:23:55.161 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:23:55.161 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:23:55.161 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:23:55.161 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:23:55.161 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:23:55.161 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:23:55.161 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:23:55.161 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:23:55.161 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:23:55.161 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:23:55.161 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:23:55.161 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:23:55.162 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:23:55.162 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:23:55.162 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:23:55.162 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:23:55.162 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:23:55.162 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:23:55.162 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:23:55.162 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:23:55.162 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:23:55.162 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:23:55.163 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:23:55.163 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:23:55.163 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:23:55.163 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:23:55.170 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:23:55.171 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:23:55.171 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:23:55.171 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:23:55.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:23:55.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:23:55.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:23:55.231 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:23:55.231 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:23:55.231 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:23:55.231 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:23:55.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:23:55.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:23:55.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:23:55.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:23:55.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:23:55.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:23:55.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:23:55.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:23:55.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:23:55.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:23:55.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:23:55.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:23:55.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:23:55.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:23:55.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:23:55.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:23:55.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:23:55.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:23:55.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:23:55.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:23:55.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:23:55.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:23:55.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:23:55.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:23:55.234 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:23:55.234 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:23:55.234 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:23:55.234 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:23:55.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:23:55.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:23:55.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:23:55.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:23:55.300 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:23:55.300 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:23:55.301 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:23:55.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:23:55.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:23:55.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:23:55.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:23:55.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:23:55.304 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:23:55.304 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:23:55.304 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:23:55.304 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:23:55.304 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:23:55.304 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:23:55.304 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:23:55.304 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:23:55.304 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:23:55.304 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:23:55.304 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:23:55.304 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:23:55.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:23:55.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:23:55.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:23:55.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:23:55.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:23:55.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:23:55.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:23:55.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:23:55.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:23:55.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:23:55.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:23:55.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:23:55.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:23:55.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:23:55.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:23:55.315 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:23:55.315 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:23:55.315 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:23:55.315 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:23:55.371 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:23:55.372 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:23:55.372 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:23:55.374 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:23:55.375 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:23:55.375 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:23:55.375 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:23:55.375 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:23:55.375 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:23:55.375 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:23:55.375 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:23:55.375 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:23:55.375 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:23:55.376 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:23:55.376 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:23:55.376 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:23:55.376 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:23:55.376 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:23:55.376 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:23:55.376 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:23:55.376 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:23:55.376 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:23:55.376 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:23:55.376 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:23:55.376 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:23:55.376 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:23:55.377 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:23:55.377 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:23:55.377 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:23:55.377 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:23:55.377 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:23:55.377 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:23:55.377 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:23:55.377 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:23:55.377 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:23:55.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:23:55.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:23:55.387 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:23:55.387 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:23:55.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:23:55.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:23:55.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:23:55.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:23:55.446 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:23:55.446 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:23:55.446 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:23:55.446 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:23:55.446 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:23:55.446 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:23:55.446 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:23:55.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:23:55.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:23:55.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:23:55.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:23:55.448 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:23:55.485 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 102 2026-02-18T08:23:55.486 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 106 2026-02-18T08:23:55.486 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:23:55.486 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=425, tag_tid=30, entry_tid=1], [object_number=424, tag_tid=30, entry_tid=0], [object_number=427, tag_tid=29, entry_tid=45931], [object_number=426, tag_tid=29, entry_tid=45930]]], state=connected] 2026-02-18T08:23:55.486 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=409, tag_tid=29, entry_tid=28089], [object_number=408, tag_tid=29, entry_tid=28088], [object_number=411, tag_tid=29, entry_tid=28087], [object_number=410, tag_tid=29, entry_tid=28086]]], state=connected] 2026-02-18T08:23:55.489 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:23:55.491 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=425, tag_tid=30, entry_tid=1' 2026-02-18T08:23:55.492 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:23:55.492 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:23:55.492 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:23:55.492 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:23:55.492 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:23:55.492 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:23:55.493 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:23:55.493 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:23:55.493 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:23:55.493 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:23:55.493 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:23:55.493 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:23:55.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:23:55.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:23:55.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:23:55.524 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 102 2026-02-18T08:23:55.524 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 106 2026-02-18T08:23:55.524 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:23:55.524 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=425, tag_tid=30, entry_tid=1], [object_number=424, tag_tid=30, entry_tid=0], [object_number=427, tag_tid=29, entry_tid=45931], [object_number=426, tag_tid=29, entry_tid=45930]]], state=connected] 2026-02-18T08:23:55.525 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=409, tag_tid=29, entry_tid=28089], [object_number=408, tag_tid=29, entry_tid=28088], [object_number=411, tag_tid=29, entry_tid=28087], [object_number=410, tag_tid=29, entry_tid=28086]]], state=connected] 2026-02-18T08:23:55.527 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:23:55.529 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=409, tag_tid=29, entry_tid=28089' 2026-02-18T08:23:55.529 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=425, tag_tid=30, entry_tid=1' -a 'object_number=425, tag_tid=30, entry_tid=1' = 'object_number=409, tag_tid=29, entry_tid=28089' 2026-02-18T08:23:55.529 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=409, tag_tid=29, entry_tid=28089' '!=' 'object_number=411, tag_tid=29, entry_tid=27647' 2026-02-18T08:23:55.529 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:23:55.529 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=409, tag_tid=29, entry_tid=28089' = 'object_number=411, tag_tid=29, entry_tid=27647' 2026-02-18T08:23:55.529 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=409, tag_tid=29, entry_tid=28089' 2026-02-18T08:23:55.530 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=425, tag_tid=30, entry_tid=1' 2026-02-18T08:23:55.530 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:23:55.530 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:23:55.531 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=30 2026-02-18T08:23:55.532 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=409, tag_tid=29, entry_tid=28089' 2026-02-18T08:23:55.532 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:23:55.532 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:23:55.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=29 2026-02-18T08:23:55.534 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=425, tag_tid=30, entry_tid=1' 2026-02-18T08:23:55.534 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:23:55.534 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:23:55.535 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:23:55.536 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=409, tag_tid=29, entry_tid=28089' 2026-02-18T08:23:55.536 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:23:55.536 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:23:55.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=28089 2026-02-18T08:23:55.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 30 = 29 -a 1 -le 28089 2026-02-18T08:23:55.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:23:55.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:23:55.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:23:55.740 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:23:55.741 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:23:55.741 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:23:55.741 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:23:55.741 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:23:55.741 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:23:55.741 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:23:55.742 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:23:55.742 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:23:55.742 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:23:55.742 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:23:55.742 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:23:55.742 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:23:55.742 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:23:55.742 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:23:55.742 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:23:55.743 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:23:55.746 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:23:55.746 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:23:55.746 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:23:55.746 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:23:55.746 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:23:55.746 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:23:55.746 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:23:55.746 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:23:55.747 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:23:55.747 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:23:55.747 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:23:55.747 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:23:55.747 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:23:55.747 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:23:55.747 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:23:55.747 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:23:55.747 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:23:55.747 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:23:55.747 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:23:55.748 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:23:55.748 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:23:55.748 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:23:55.748 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:23:55.748 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:23:55.748 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:23:55.748 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:23:55.748 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:23:55.748 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:23:55.748 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:23:55.748 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:23:55.749 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:23:55.749 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:23:55.755 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:23:55.756 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:23:55.756 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:23:55.756 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:23:55.805 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:23:55.806 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:23:55.806 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:23:55.808 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:23:55.809 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:23:55.809 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:23:55.809 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:23:55.809 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:23:55.809 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:23:55.809 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:23:55.809 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:23:55.809 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:23:55.809 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:23:55.810 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:23:55.810 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:23:55.810 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:23:55.810 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:23:55.810 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:23:55.810 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:23:55.810 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:23:55.810 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:23:55.810 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:23:55.810 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:23:55.810 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:23:55.810 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:23:55.811 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:23:55.811 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:23:55.811 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:23:55.811 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:23:55.811 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:23:55.811 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:23:55.811 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:23:55.811 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:23:55.811 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:23:55.811 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:23:55.819 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:23:55.820 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:23:55.820 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:23:55.820 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:23:55.870 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:23:55.870 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:23:55.870 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:23:55.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:23:55.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:23:55.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:23:55.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:23:55.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:23:55.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:23:55.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:23:55.874 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:23:55.874 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:23:55.874 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:23:55.874 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:23:55.874 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:23:55.874 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:23:55.874 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:23:55.874 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:23:55.874 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:23:55.874 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:23:55.874 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:23:55.874 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:23:55.875 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:23:55.875 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:23:55.875 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:23:55.875 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:23:55.875 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:23:55.875 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:23:55.875 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:23:55.875 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:23:55.875 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:23:55.875 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:23:55.875 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:23:55.875 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:23:55.876 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:23:55.883 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:23:55.884 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:23:55.884 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:23:55.884 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:23:55.933 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:23:55.933 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:23:55.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:23:55.936 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:23:55.936 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:23:55.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:23:55.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:23:55.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:23:55.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:23:55.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:23:55.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:23:55.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:23:55.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:23:55.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:23:55.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:23:55.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:23:55.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:23:55.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:23:55.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:23:55.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:23:55.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:23:55.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:23:55.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:23:55.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:23:55.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:23:55.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:23:55.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:23:55.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:23:55.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:23:55.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:23:55.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:23:55.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:23:55.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:23:55.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:23:55.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:23:55.948 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:23:55.948 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:23:55.948 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:23:55.948 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:23:56.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:23:56.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:23:56.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:23:56.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:23:56.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:23:56.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:23:56.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:23:56.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:23:56.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:23:56.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:23:56.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:23:56.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:23:56.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:23:56.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:23:56.027 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:23:56.027 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:23:56.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 102 2026-02-18T08:23:56.055 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 106 2026-02-18T08:23:56.055 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:23:56.055 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=425, tag_tid=30, entry_tid=1], [object_number=424, tag_tid=30, entry_tid=0], [object_number=427, tag_tid=29, entry_tid=45931], [object_number=426, tag_tid=29, entry_tid=45930]]], state=connected] 2026-02-18T08:23:56.055 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=408, tag_tid=29, entry_tid=28436], [object_number=411, tag_tid=29, entry_tid=28435], [object_number=410, tag_tid=29, entry_tid=28434], [object_number=409, tag_tid=29, entry_tid=28433]]], state=connected] 2026-02-18T08:23:56.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:23:56.059 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=425, tag_tid=30, entry_tid=1' 2026-02-18T08:23:56.060 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:23:56.060 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:23:56.060 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:23:56.060 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:23:56.060 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:23:56.060 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:23:56.060 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:23:56.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:23:56.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:23:56.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:23:56.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:23:56.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:23:56.062 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:23:56.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:23:56.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:23:56.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 102 2026-02-18T08:23:56.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 106 2026-02-18T08:23:56.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:23:56.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=425, tag_tid=30, entry_tid=1], [object_number=424, tag_tid=30, entry_tid=0], [object_number=427, tag_tid=29, entry_tid=45931], [object_number=426, tag_tid=29, entry_tid=45930]]], state=connected] 2026-02-18T08:23:56.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=408, tag_tid=29, entry_tid=28436], [object_number=411, tag_tid=29, entry_tid=28435], [object_number=410, tag_tid=29, entry_tid=28434], [object_number=409, tag_tid=29, entry_tid=28433]]], state=connected] 2026-02-18T08:23:56.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:23:56.093 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=408, tag_tid=29, entry_tid=28436' 2026-02-18T08:23:56.093 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=425, tag_tid=30, entry_tid=1' -a 'object_number=425, tag_tid=30, entry_tid=1' = 'object_number=408, tag_tid=29, entry_tid=28436' 2026-02-18T08:23:56.093 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=408, tag_tid=29, entry_tid=28436' '!=' 'object_number=409, tag_tid=29, entry_tid=28089' 2026-02-18T08:23:56.093 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:23:56.093 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=408, tag_tid=29, entry_tid=28436' = 'object_number=409, tag_tid=29, entry_tid=28089' 2026-02-18T08:23:56.093 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=408, tag_tid=29, entry_tid=28436' 2026-02-18T08:23:56.094 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=425, tag_tid=30, entry_tid=1' 2026-02-18T08:23:56.094 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:23:56.094 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:23:56.095 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=30 2026-02-18T08:23:56.096 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=408, tag_tid=29, entry_tid=28436' 2026-02-18T08:23:56.096 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:23:56.096 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:23:56.097 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=29 2026-02-18T08:23:56.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=425, tag_tid=30, entry_tid=1' 2026-02-18T08:23:56.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:23:56.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:23:56.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:23:56.100 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=408, tag_tid=29, entry_tid=28436' 2026-02-18T08:23:56.100 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:23:56.100 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:23:56.102 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=28436 2026-02-18T08:23:56.102 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 30 = 29 -a 1 -le 28436 2026-02-18T08:23:56.102 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:23:56.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:23:56.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:23:56.304 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:23:56.304 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:23:56.304 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:23:56.304 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:23:56.304 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:23:56.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:23:56.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:23:56.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:23:56.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:23:56.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:23:56.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:23:56.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:23:56.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:23:56.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:23:56.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:23:56.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:23:56.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:23:56.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:23:56.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:23:56.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:23:56.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:23:56.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:23:56.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:23:56.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:23:56.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:23:56.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:23:56.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:23:56.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:23:56.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:23:56.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:23:56.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:23:56.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:23:56.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:23:56.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:23:56.310 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:23:56.310 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:23:56.310 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:23:56.310 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:23:56.310 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:23:56.310 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:23:56.310 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:23:56.310 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:23:56.310 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:23:56.310 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:23:56.310 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:23:56.310 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:23:56.310 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:23:56.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:23:56.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:23:56.318 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:23:56.319 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:23:56.319 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:23:56.319 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:23:56.370 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:23:56.370 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:23:56.371 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:23:56.373 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:23:56.373 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:23:56.374 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:23:56.374 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:23:56.374 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:23:56.374 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:23:56.374 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:23:56.374 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:23:56.374 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:23:56.374 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:23:56.374 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:23:56.374 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:23:56.374 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:23:56.375 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:23:56.375 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:23:56.375 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:23:56.375 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:23:56.375 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:23:56.375 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:23:56.375 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:23:56.375 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:23:56.375 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:23:56.375 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:23:56.375 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:23:56.375 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:23:56.376 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:23:56.376 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:23:56.376 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:23:56.376 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:23:56.376 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:23:56.376 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:23:56.376 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:23:56.385 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:23:56.385 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:23:56.385 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:23:56.385 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:23:56.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:23:56.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:23:56.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:23:56.448 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:23:56.448 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:23:56.448 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:23:56.448 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:23:56.448 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:23:56.448 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:23:56.448 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:23:56.448 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:23:56.448 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:23:56.448 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:23:56.449 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:23:56.449 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:23:56.449 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:23:56.449 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:23:56.449 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:23:56.449 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:23:56.449 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:23:56.449 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:23:56.449 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:23:56.449 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:23:56.449 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:23:56.449 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:23:56.450 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:23:56.450 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:23:56.450 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:23:56.450 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:23:56.450 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:23:56.450 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:23:56.450 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:23:56.450 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:23:56.450 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:23:56.450 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:23:56.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:23:56.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:23:56.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:23:56.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:23:56.508 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:23:56.508 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:23:56.508 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:23:56.511 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:23:56.511 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:23:56.511 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:23:56.511 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:23:56.511 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:23:56.511 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:23:56.511 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:23:56.511 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:23:56.512 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:23:56.512 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:23:56.512 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:23:56.512 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:23:56.512 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:23:56.512 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:23:56.512 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:23:56.512 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:23:56.512 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:23:56.512 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:23:56.512 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:23:56.512 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:23:56.513 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:23:56.513 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:23:56.513 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:23:56.513 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:23:56.513 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:23:56.513 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:23:56.513 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:23:56.513 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:23:56.513 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:23:56.513 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:23:56.513 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:23:56.513 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:23:56.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:23:56.523 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:23:56.523 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:23:56.523 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:23:56.586 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:23:56.587 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:23:56.587 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:23:56.587 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:23:56.587 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:23:56.587 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:23:56.587 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:23:56.587 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:23:56.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:23:56.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:23:56.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:23:56.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:23:56.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:23:56.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:23:56.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:23:56.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:23:56.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 102 2026-02-18T08:23:56.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 106 2026-02-18T08:23:56.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:23:56.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=425, tag_tid=30, entry_tid=1], [object_number=424, tag_tid=30, entry_tid=0], [object_number=427, tag_tid=29, entry_tid=45931], [object_number=426, tag_tid=29, entry_tid=45930]]], state=connected] 2026-02-18T08:23:56.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=409, tag_tid=29, entry_tid=28781], [object_number=408, tag_tid=29, entry_tid=28780], [object_number=411, tag_tid=29, entry_tid=28779], [object_number=410, tag_tid=29, entry_tid=28778]]], state=connected] 2026-02-18T08:23:56.620 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:23:56.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=425, tag_tid=30, entry_tid=1' 2026-02-18T08:23:56.622 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:23:56.623 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:23:56.623 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:23:56.623 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:23:56.623 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:23:56.623 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:23:56.623 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:23:56.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:23:56.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:23:56.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:23:56.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:23:56.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:23:56.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:23:56.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:23:56.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:23:56.653 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 102 2026-02-18T08:23:56.653 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 106 2026-02-18T08:23:56.653 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:23:56.653 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=425, tag_tid=30, entry_tid=1], [object_number=424, tag_tid=30, entry_tid=0], [object_number=427, tag_tid=29, entry_tid=45931], [object_number=426, tag_tid=29, entry_tid=45930]]], state=connected] 2026-02-18T08:23:56.653 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=409, tag_tid=29, entry_tid=28781], [object_number=408, tag_tid=29, entry_tid=28780], [object_number=411, tag_tid=29, entry_tid=28779], [object_number=410, tag_tid=29, entry_tid=28778]]], state=connected] 2026-02-18T08:23:56.655 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:23:56.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=409, tag_tid=29, entry_tid=28781' 2026-02-18T08:23:56.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=425, tag_tid=30, entry_tid=1' -a 'object_number=425, tag_tid=30, entry_tid=1' = 'object_number=409, tag_tid=29, entry_tid=28781' 2026-02-18T08:23:56.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=409, tag_tid=29, entry_tid=28781' '!=' 'object_number=408, tag_tid=29, entry_tid=28436' 2026-02-18T08:23:56.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:23:56.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=409, tag_tid=29, entry_tid=28781' = 'object_number=408, tag_tid=29, entry_tid=28436' 2026-02-18T08:23:56.658 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=409, tag_tid=29, entry_tid=28781' 2026-02-18T08:23:56.658 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=425, tag_tid=30, entry_tid=1' 2026-02-18T08:23:56.658 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:23:56.658 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:23:56.659 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=30 2026-02-18T08:23:56.659 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=409, tag_tid=29, entry_tid=28781' 2026-02-18T08:23:56.660 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:23:56.660 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:23:56.661 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=29 2026-02-18T08:23:56.662 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=425, tag_tid=30, entry_tid=1' 2026-02-18T08:23:56.662 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:23:56.662 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:23:56.663 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:23:56.664 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=409, tag_tid=29, entry_tid=28781' 2026-02-18T08:23:56.664 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:23:56.664 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:23:56.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=28781 2026-02-18T08:23:56.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 30 = 29 -a 1 -le 28781 2026-02-18T08:23:56.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:23:56.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:23:56.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:23:56.868 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:23:56.868 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:23:56.868 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:23:56.868 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:23:56.868 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:23:56.868 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:23:56.868 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:23:56.869 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:23:56.869 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:23:56.869 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:23:56.869 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:23:56.869 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:23:56.869 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:23:56.869 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:23:56.869 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:23:56.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:23:56.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:23:56.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:23:56.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:23:56.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:23:56.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:23:56.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:23:56.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:23:56.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:23:56.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:23:56.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:23:56.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:23:56.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:23:56.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:23:56.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:23:56.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:23:56.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:23:56.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:23:56.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:23:56.874 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:23:56.874 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:23:56.874 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:23:56.874 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:23:56.874 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:23:56.874 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:23:56.874 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:23:56.874 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:23:56.874 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:23:56.874 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:23:56.875 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:23:56.875 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:23:56.875 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:23:56.875 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:23:56.875 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:23:56.883 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:23:56.884 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:23:56.884 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:23:56.884 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:23:56.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:23:56.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:23:56.946 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:23:56.949 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:23:56.949 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:23:56.949 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:23:56.949 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:23:56.949 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:23:56.949 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:23:56.949 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:23:56.949 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:23:56.949 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:23:56.950 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:23:56.950 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:23:56.950 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:23:56.950 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:23:56.950 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:23:56.950 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:23:56.950 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:23:56.950 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:23:56.950 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:23:56.950 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:23:56.950 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:23:56.950 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:23:56.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:23:56.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:23:56.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:23:56.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:23:56.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:23:56.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:23:56.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:23:56.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:23:56.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:23:56.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:23:56.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:23:56.961 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:23:56.961 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:23:56.961 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:23:56.961 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:23:57.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:23:57.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:23:57.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:23:57.012 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:23:57.012 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:23:57.012 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:23:57.012 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:23:57.012 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:23:57.012 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:23:57.012 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:23:57.012 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:23:57.012 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:23:57.012 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:23:57.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:23:57.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:23:57.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:23:57.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:23:57.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:23:57.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:23:57.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:23:57.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:23:57.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:23:57.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:23:57.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:23:57.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:23:57.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:23:57.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:23:57.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:23:57.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:23:57.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:23:57.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:23:57.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:23:57.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:23:57.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:23:57.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:23:57.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:23:57.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:23:57.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:23:57.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:23:57.080 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:23:57.080 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:23:57.080 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:23:57.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:23:57.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:23:57.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:23:57.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:23:57.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:23:57.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:23:57.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:23:57.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:23:57.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:23:57.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:23:57.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:23:57.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:23:57.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:23:57.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:23:57.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:23:57.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:23:57.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:23:57.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:23:57.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:23:57.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:23:57.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:23:57.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:23:57.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:23:57.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:23:57.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:23:57.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:23:57.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:23:57.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:23:57.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:23:57.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:23:57.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:23:57.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:23:57.095 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:23:57.095 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:23:57.095 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:23:57.096 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:23:57.157 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:23:57.157 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:23:57.157 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:23:57.157 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:23:57.157 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:23:57.157 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:23:57.157 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:23:57.158 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:23:57.158 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:23:57.158 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:23:57.158 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:23:57.158 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:23:57.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:23:57.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:23:57.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:23:57.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:23:57.188 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 102 2026-02-18T08:23:57.188 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 106 2026-02-18T08:23:57.188 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:23:57.188 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=425, tag_tid=30, entry_tid=1], [object_number=424, tag_tid=30, entry_tid=0], [object_number=427, tag_tid=29, entry_tid=45931], [object_number=426, tag_tid=29, entry_tid=45930]]], state=connected] 2026-02-18T08:23:57.188 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=408, tag_tid=29, entry_tid=29112], [object_number=411, tag_tid=29, entry_tid=29111], [object_number=410, tag_tid=29, entry_tid=29110], [object_number=409, tag_tid=29, entry_tid=29109]]], state=connected] 2026-02-18T08:23:57.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:23:57.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=425, tag_tid=30, entry_tid=1' 2026-02-18T08:23:57.194 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:23:57.194 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:23:57.194 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:23:57.194 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:23:57.194 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:23:57.194 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:23:57.194 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:23:57.194 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:23:57.194 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:23:57.195 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:23:57.195 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:23:57.195 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:23:57.195 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:23:57.196 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:23:57.196 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:23:57.428 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 102 2026-02-18T08:23:57.428 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 106 2026-02-18T08:23:57.428 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:23:57.428 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=425, tag_tid=30, entry_tid=1], [object_number=424, tag_tid=30, entry_tid=0], [object_number=427, tag_tid=29, entry_tid=45931], [object_number=426, tag_tid=29, entry_tid=45930]]], state=connected] 2026-02-18T08:23:57.428 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=408, tag_tid=29, entry_tid=29112], [object_number=411, tag_tid=29, entry_tid=29111], [object_number=410, tag_tid=29, entry_tid=29110], [object_number=409, tag_tid=29, entry_tid=29109]]], state=connected] 2026-02-18T08:23:57.431 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:23:57.433 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=408, tag_tid=29, entry_tid=29112' 2026-02-18T08:23:57.433 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=425, tag_tid=30, entry_tid=1' -a 'object_number=425, tag_tid=30, entry_tid=1' = 'object_number=408, tag_tid=29, entry_tid=29112' 2026-02-18T08:23:57.433 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=408, tag_tid=29, entry_tid=29112' '!=' 'object_number=409, tag_tid=29, entry_tid=28781' 2026-02-18T08:23:57.433 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:23:57.433 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=408, tag_tid=29, entry_tid=29112' = 'object_number=409, tag_tid=29, entry_tid=28781' 2026-02-18T08:23:57.434 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=408, tag_tid=29, entry_tid=29112' 2026-02-18T08:23:57.434 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=425, tag_tid=30, entry_tid=1' 2026-02-18T08:23:57.434 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:23:57.434 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:23:57.435 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=30 2026-02-18T08:23:57.436 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=408, tag_tid=29, entry_tid=29112' 2026-02-18T08:23:57.436 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:23:57.436 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:23:57.437 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=29 2026-02-18T08:23:57.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=425, tag_tid=30, entry_tid=1' 2026-02-18T08:23:57.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:23:57.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:23:57.439 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:23:57.440 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=408, tag_tid=29, entry_tid=29112' 2026-02-18T08:23:57.440 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:23:57.440 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:23:57.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=29112 2026-02-18T08:23:57.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 30 = 29 -a 1 -le 29112 2026-02-18T08:23:57.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:23:57.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:23:57.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:23:57.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:23:57.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:23:57.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:23:57.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:23:57.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:23:57.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:23:57.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:23:57.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:23:57.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:23:57.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:23:57.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:23:57.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:23:57.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:23:57.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:23:57.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:23:57.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:23:57.648 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:23:57.648 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:23:57.648 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:23:57.648 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:23:57.648 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:23:57.648 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:23:57.648 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:23:57.648 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:23:57.648 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:23:57.648 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:23:57.648 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:23:57.649 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:23:57.649 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:23:57.649 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:23:57.649 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:23:57.649 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:23:57.649 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:23:57.649 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:23:57.649 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:23:57.649 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:23:57.649 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:23:57.649 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:23:57.649 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:23:57.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:23:57.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:23:57.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:23:57.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:23:57.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:23:57.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:23:57.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:23:57.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:23:57.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:23:57.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:23:57.659 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:23:57.659 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:23:57.659 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:23:57.659 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:23:57.720 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:23:57.720 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:23:57.720 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:23:57.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:23:57.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:23:57.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:23:57.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:23:57.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:23:57.724 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:23:57.724 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:23:57.724 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:23:57.724 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:23:57.724 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:23:57.724 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:23:57.724 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:23:57.724 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:23:57.724 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:23:57.724 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:23:57.724 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:23:57.725 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:23:57.725 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:23:57.725 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:23:57.725 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:23:57.725 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:23:57.725 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:23:57.725 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:23:57.725 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:23:57.725 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:23:57.725 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:23:57.725 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:23:57.725 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:23:57.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:23:57.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:23:57.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:23:57.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:23:57.734 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:23:57.734 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:23:57.735 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:23:57.735 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:23:57.785 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:23:57.785 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:23:57.785 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:23:57.788 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:23:57.788 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:23:57.788 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:23:57.788 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:23:57.788 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:23:57.788 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:23:57.788 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:23:57.788 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:23:57.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:23:57.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:23:57.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:23:57.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:23:57.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:23:57.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:23:57.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:23:57.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:23:57.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:23:57.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:23:57.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:23:57.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:23:57.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:23:57.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:23:57.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:23:57.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:23:57.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:23:57.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:23:57.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:23:57.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:23:57.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:23:57.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:23:57.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:23:57.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:23:57.798 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:23:57.799 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:23:57.799 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:23:57.799 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:23:57.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:23:57.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:23:57.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:23:57.852 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:23:57.852 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:23:57.852 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:23:57.853 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:23:57.853 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:23:57.853 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:23:57.853 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:23:57.853 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:23:57.853 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:23:57.853 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:23:57.853 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:23:57.853 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:23:57.853 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:23:57.853 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:23:57.854 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:23:57.854 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:23:57.854 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:23:57.854 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:23:57.854 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:23:57.854 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:23:57.854 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:23:57.854 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:23:57.854 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:23:57.854 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:23:57.854 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:23:57.854 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:23:57.854 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:23:57.855 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:23:57.855 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:23:57.855 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:23:57.855 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:23:57.855 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:23:57.862 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:23:57.863 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:23:57.863 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:23:57.863 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:23:57.924 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:23:57.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:23:57.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:23:57.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:23:57.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:23:57.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:23:57.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:23:57.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:23:57.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:23:57.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:23:57.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:23:57.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:23:57.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:23:57.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:23:57.928 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:23:57.928 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:23:57.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 102 2026-02-18T08:23:57.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 106 2026-02-18T08:23:57.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:23:57.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=425, tag_tid=30, entry_tid=1], [object_number=424, tag_tid=30, entry_tid=0], [object_number=427, tag_tid=29, entry_tid=45931], [object_number=426, tag_tid=29, entry_tid=45930]]], state=connected] 2026-02-18T08:23:57.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=408, tag_tid=29, entry_tid=29584], [object_number=411, tag_tid=29, entry_tid=29583], [object_number=410, tag_tid=29, entry_tid=29582], [object_number=409, tag_tid=29, entry_tid=29581]]], state=connected] 2026-02-18T08:23:57.959 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:23:57.961 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=425, tag_tid=30, entry_tid=1' 2026-02-18T08:23:57.962 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:23:57.962 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:23:57.962 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:23:57.962 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:23:57.962 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:23:57.962 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:23:57.963 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:23:57.963 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:23:57.963 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:23:57.963 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:23:57.963 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:23:57.963 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:23:57.964 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:23:57.964 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:23:57.964 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:23:57.992 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 102 2026-02-18T08:23:57.992 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 106 2026-02-18T08:23:57.992 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:23:57.992 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=425, tag_tid=30, entry_tid=1], [object_number=424, tag_tid=30, entry_tid=0], [object_number=427, tag_tid=29, entry_tid=45931], [object_number=426, tag_tid=29, entry_tid=45930]]], state=connected] 2026-02-18T08:23:57.992 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=408, tag_tid=29, entry_tid=29584], [object_number=411, tag_tid=29, entry_tid=29583], [object_number=410, tag_tid=29, entry_tid=29582], [object_number=409, tag_tid=29, entry_tid=29581]]], state=connected] 2026-02-18T08:23:57.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:23:57.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=408, tag_tid=29, entry_tid=29584' 2026-02-18T08:23:57.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=425, tag_tid=30, entry_tid=1' -a 'object_number=425, tag_tid=30, entry_tid=1' = 'object_number=408, tag_tid=29, entry_tid=29584' 2026-02-18T08:23:57.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=408, tag_tid=29, entry_tid=29584' '!=' 'object_number=408, tag_tid=29, entry_tid=29112' 2026-02-18T08:23:57.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:23:57.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=408, tag_tid=29, entry_tid=29584' = 'object_number=408, tag_tid=29, entry_tid=29112' 2026-02-18T08:23:57.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=408, tag_tid=29, entry_tid=29584' 2026-02-18T08:23:57.999 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=425, tag_tid=30, entry_tid=1' 2026-02-18T08:23:57.999 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:23:57.999 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:23:58.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=30 2026-02-18T08:23:58.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=408, tag_tid=29, entry_tid=29584' 2026-02-18T08:23:58.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:23:58.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:23:58.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=29 2026-02-18T08:23:58.003 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=425, tag_tid=30, entry_tid=1' 2026-02-18T08:23:58.003 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:23:58.003 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:23:58.004 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:23:58.005 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=408, tag_tid=29, entry_tid=29584' 2026-02-18T08:23:58.005 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:23:58.005 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:23:58.007 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=29584 2026-02-18T08:23:58.007 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 30 = 29 -a 1 -le 29584 2026-02-18T08:23:58.007 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:23:58.007 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:23:58.008 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:23:58.209 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:23:58.209 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:23:58.209 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:23:58.209 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:23:58.209 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:23:58.209 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:23:58.210 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:23:58.210 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:23:58.210 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:23:58.210 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:23:58.210 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:23:58.210 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:23:58.210 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:23:58.210 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:23:58.210 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:23:58.210 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:23:58.213 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:23:58.213 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:23:58.213 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:23:58.214 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:23:58.214 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:23:58.214 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:23:58.214 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:23:58.214 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:23:58.214 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:23:58.214 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:23:58.214 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:23:58.214 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:23:58.214 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:23:58.214 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:23:58.214 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:23:58.215 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:23:58.215 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:23:58.215 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:23:58.215 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:23:58.215 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:23:58.215 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:23:58.215 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:23:58.215 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:23:58.215 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:23:58.215 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:23:58.215 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:23:58.215 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:23:58.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:23:58.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:23:58.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:23:58.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:23:58.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:23:58.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:23:58.223 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:23:58.224 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:23:58.224 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:23:58.224 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:23:58.274 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:23:58.274 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:23:58.275 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:23:58.277 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:23:58.277 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:23:58.278 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:23:58.278 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:23:58.278 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:23:58.278 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:23:58.278 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:23:58.278 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:23:58.278 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:23:58.278 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:23:58.278 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:23:58.278 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:23:58.278 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:23:58.278 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:23:58.279 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:23:58.279 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:23:58.279 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:23:58.279 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:23:58.279 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:23:58.279 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:23:58.279 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:23:58.279 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:23:58.279 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:23:58.279 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:23:58.279 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:23:58.279 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:23:58.279 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:23:58.280 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:23:58.280 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:23:58.280 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:23:58.280 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:23:58.280 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:23:58.288 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:23:58.288 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:23:58.288 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:23:58.288 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:23:58.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:23:58.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:23:58.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:23:58.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:23:58.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:23:58.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:23:58.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:23:58.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:23:58.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:23:58.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:23:58.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:23:58.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:23:58.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:23:58.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:23:58.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:23:58.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:23:58.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:23:58.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:23:58.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:23:58.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:23:58.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:23:58.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:23:58.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:23:58.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:23:58.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:23:58.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:23:58.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:23:58.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:23:58.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:23:58.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:23:58.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:23:58.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:23:58.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:23:58.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:23:58.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:23:58.351 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:23:58.351 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:23:58.351 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:23:58.352 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:23:58.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:23:58.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:23:58.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:23:58.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:23:58.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:23:58.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:23:58.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:23:58.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:23:58.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:23:58.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:23:58.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:23:58.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:23:58.405 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:23:58.405 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:23:58.405 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:23:58.405 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:23:58.405 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:23:58.405 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:23:58.405 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:23:58.405 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:23:58.405 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:23:58.405 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:23:58.405 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:23:58.405 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:23:58.405 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:23:58.406 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:23:58.406 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:23:58.406 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:23:58.406 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:23:58.406 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:23:58.406 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:23:58.406 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:23:58.406 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:23:58.406 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:23:58.406 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:23:58.415 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:23:58.416 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:23:58.416 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:23:58.416 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:23:58.477 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:23:58.478 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:23:58.478 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:23:58.478 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:23:58.478 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:23:58.478 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:23:58.478 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:23:58.478 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:23:58.478 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:23:58.479 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:23:58.479 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:23:58.479 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:23:58.479 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:23:58.479 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:23:58.480 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:23:58.480 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:23:58.516 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 102 2026-02-18T08:23:58.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 106 2026-02-18T08:23:58.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:23:58.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=425, tag_tid=30, entry_tid=1], [object_number=424, tag_tid=30, entry_tid=0], [object_number=427, tag_tid=29, entry_tid=45931], [object_number=426, tag_tid=29, entry_tid=45930]]], state=connected] 2026-02-18T08:23:58.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=411, tag_tid=29, entry_tid=29947], [object_number=410, tag_tid=29, entry_tid=29946], [object_number=409, tag_tid=29, entry_tid=29945], [object_number=408, tag_tid=29, entry_tid=29944]]], state=connected] 2026-02-18T08:23:58.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:23:58.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=425, tag_tid=30, entry_tid=1' 2026-02-18T08:23:58.523 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:23:58.523 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:23:58.524 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:23:58.524 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:23:58.524 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:23:58.524 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:23:58.524 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:23:58.524 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:23:58.524 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:23:58.524 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:23:58.524 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:23:58.525 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:23:58.525 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:23:58.525 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:23:58.526 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:23:58.562 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 102 2026-02-18T08:23:58.562 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 106 2026-02-18T08:23:58.562 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:23:58.562 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=425, tag_tid=30, entry_tid=1], [object_number=424, tag_tid=30, entry_tid=0], [object_number=427, tag_tid=29, entry_tid=45931], [object_number=426, tag_tid=29, entry_tid=45930]]], state=connected] 2026-02-18T08:23:58.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=411, tag_tid=29, entry_tid=29947], [object_number=410, tag_tid=29, entry_tid=29946], [object_number=409, tag_tid=29, entry_tid=29945], [object_number=408, tag_tid=29, entry_tid=29944]]], state=connected] 2026-02-18T08:23:58.566 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:23:58.568 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=411, tag_tid=29, entry_tid=29947' 2026-02-18T08:23:58.568 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=425, tag_tid=30, entry_tid=1' -a 'object_number=425, tag_tid=30, entry_tid=1' = 'object_number=411, tag_tid=29, entry_tid=29947' 2026-02-18T08:23:58.568 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=411, tag_tid=29, entry_tid=29947' '!=' 'object_number=408, tag_tid=29, entry_tid=29584' 2026-02-18T08:23:58.569 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:23:58.569 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=411, tag_tid=29, entry_tid=29947' = 'object_number=408, tag_tid=29, entry_tid=29584' 2026-02-18T08:23:58.569 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=411, tag_tid=29, entry_tid=29947' 2026-02-18T08:23:58.569 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=425, tag_tid=30, entry_tid=1' 2026-02-18T08:23:58.569 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:23:58.570 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:23:58.570 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=30 2026-02-18T08:23:58.571 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=411, tag_tid=29, entry_tid=29947' 2026-02-18T08:23:58.571 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:23:58.571 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:23:58.573 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=29 2026-02-18T08:23:58.573 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=425, tag_tid=30, entry_tid=1' 2026-02-18T08:23:58.573 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:23:58.574 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:23:58.575 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:23:58.576 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=411, tag_tid=29, entry_tid=29947' 2026-02-18T08:23:58.576 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:23:58.576 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:23:58.577 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=29947 2026-02-18T08:23:58.577 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 30 = 29 -a 1 -le 29947 2026-02-18T08:23:58.578 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:23:58.578 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:23:58.578 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:23:58.779 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:23:58.779 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:23:58.779 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:23:58.780 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:23:58.780 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:23:58.780 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:23:58.780 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:23:58.780 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:23:58.780 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:23:58.780 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:23:58.780 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:23:58.780 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:23:58.780 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:23:58.780 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:23:58.781 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:23:58.783 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:23:58.783 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:23:58.783 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:23:58.784 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:23:58.784 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:23:58.784 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:23:58.784 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:23:58.784 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:23:58.784 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:23:58.784 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:23:58.784 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:23:58.784 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:23:58.784 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:23:58.784 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:23:58.785 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:23:58.785 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:23:58.785 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:23:58.785 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:23:58.785 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:23:58.785 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:23:58.785 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:23:58.785 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:23:58.785 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:23:58.785 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:23:58.785 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:23:58.785 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:23:58.786 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:23:58.786 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:23:58.786 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:23:58.786 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:23:58.786 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:23:58.786 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:23:58.786 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:23:58.786 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:23:58.794 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:23:58.795 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:23:58.795 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:23:58.795 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:23:58.846 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:23:58.846 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:23:58.846 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:23:58.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:23:58.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:23:58.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:23:58.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:23:58.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:23:58.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:23:58.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:23:58.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:23:58.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:23:58.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:23:58.850 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:23:58.850 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:23:58.850 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:23:58.850 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:23:58.850 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:23:58.850 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:23:58.850 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:23:58.850 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:23:58.850 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:23:58.850 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:23:58.850 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:23:58.850 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:23:58.851 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:23:58.851 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:23:58.851 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:23:58.851 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:23:58.851 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:23:58.851 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:23:58.851 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:23:58.851 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:23:58.851 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:23:58.851 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:23:58.859 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:23:58.860 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:23:58.860 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:23:58.860 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:23:58.910 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:23:58.910 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:23:58.910 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:23:58.912 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:23:58.913 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:23:58.913 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:23:58.913 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:23:58.913 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:23:58.913 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:23:58.913 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:23:58.913 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:23:58.913 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:23:58.913 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:23:58.913 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:23:58.913 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:23:58.914 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:23:58.914 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:23:58.914 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:23:58.914 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:23:58.914 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:23:58.914 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:23:58.914 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:23:58.914 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:23:58.914 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:23:58.914 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:23:58.914 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:23:58.914 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:23:58.915 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:23:58.915 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:23:58.915 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:23:58.915 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:23:58.915 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:23:58.915 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:23:58.915 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:23:58.915 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:23:58.923 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:23:58.924 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:23:58.924 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:23:58.924 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:23:58.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:23:58.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:23:58.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:23:58.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:23:58.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:23:58.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:23:58.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:23:58.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:23:58.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:23:58.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:23:58.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:23:58.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:23:58.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:23:58.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:23:58.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:23:58.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:23:58.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:23:58.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:23:58.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:23:58.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:23:58.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:23:58.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:23:58.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:23:58.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:23:58.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:23:58.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:23:58.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:23:58.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:23:58.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:23:58.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:23:58.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:23:58.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:23:58.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:23:58.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:23:58.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:23:58.988 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:23:58.988 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:23:58.988 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:23:58.988 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:23:59.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:23:59.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:23:59.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:23:59.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:23:59.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:23:59.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:23:59.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:23:59.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:23:59.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:23:59.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:23:59.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:23:59.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:23:59.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:23:59.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:23:59.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:23:59.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:23:59.080 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 102 2026-02-18T08:23:59.080 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 106 2026-02-18T08:23:59.080 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:23:59.080 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=425, tag_tid=30, entry_tid=1], [object_number=424, tag_tid=30, entry_tid=0], [object_number=427, tag_tid=29, entry_tid=45931], [object_number=426, tag_tid=29, entry_tid=45930]]], state=connected] 2026-02-18T08:23:59.080 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=411, tag_tid=29, entry_tid=30319], [object_number=410, tag_tid=29, entry_tid=30318], [object_number=409, tag_tid=29, entry_tid=30317], [object_number=408, tag_tid=29, entry_tid=30316]]], state=connected] 2026-02-18T08:23:59.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:23:59.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=425, tag_tid=30, entry_tid=1' 2026-02-18T08:23:59.086 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:23:59.086 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:23:59.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:23:59.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:23:59.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:23:59.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:23:59.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:23:59.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:23:59.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:23:59.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:23:59.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:23:59.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:23:59.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:23:59.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:23:59.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:23:59.122 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 102 2026-02-18T08:23:59.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 106 2026-02-18T08:23:59.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:23:59.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=425, tag_tid=30, entry_tid=1], [object_number=424, tag_tid=30, entry_tid=0], [object_number=427, tag_tid=29, entry_tid=45931], [object_number=426, tag_tid=29, entry_tid=45930]]], state=connected] 2026-02-18T08:23:59.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=411, tag_tid=29, entry_tid=30319], [object_number=410, tag_tid=29, entry_tid=30318], [object_number=409, tag_tid=29, entry_tid=30317], [object_number=408, tag_tid=29, entry_tid=30316]]], state=connected] 2026-02-18T08:23:59.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:23:59.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=411, tag_tid=29, entry_tid=30319' 2026-02-18T08:23:59.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=425, tag_tid=30, entry_tid=1' -a 'object_number=425, tag_tid=30, entry_tid=1' = 'object_number=411, tag_tid=29, entry_tid=30319' 2026-02-18T08:23:59.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=411, tag_tid=29, entry_tid=30319' '!=' 'object_number=411, tag_tid=29, entry_tid=29947' 2026-02-18T08:23:59.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:23:59.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=411, tag_tid=29, entry_tid=30319' = 'object_number=411, tag_tid=29, entry_tid=29947' 2026-02-18T08:23:59.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=411, tag_tid=29, entry_tid=30319' 2026-02-18T08:23:59.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=425, tag_tid=30, entry_tid=1' 2026-02-18T08:23:59.129 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:23:59.129 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:23:59.130 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=30 2026-02-18T08:23:59.131 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=411, tag_tid=29, entry_tid=30319' 2026-02-18T08:23:59.131 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:23:59.131 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:23:59.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=29 2026-02-18T08:23:59.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=425, tag_tid=30, entry_tid=1' 2026-02-18T08:23:59.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:23:59.134 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:23:59.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:23:59.136 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=411, tag_tid=29, entry_tid=30319' 2026-02-18T08:23:59.136 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:23:59.136 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:23:59.137 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=30319 2026-02-18T08:23:59.137 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 30 = 29 -a 1 -le 30319 2026-02-18T08:23:59.137 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:23:59.137 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:23:59.138 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:23:59.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:23:59.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:23:59.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:23:59.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:23:59.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:23:59.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:23:59.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:23:59.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:23:59.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:23:59.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:23:59.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:23:59.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:23:59.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:23:59.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:23:59.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:23:59.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:23:59.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:23:59.344 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:23:59.344 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:23:59.344 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:23:59.344 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:23:59.344 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:23:59.344 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:23:59.344 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:23:59.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:23:59.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:23:59.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:23:59.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:23:59.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:23:59.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:23:59.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:23:59.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:23:59.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:23:59.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:23:59.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:23:59.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:23:59.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:23:59.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:23:59.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:23:59.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:23:59.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:23:59.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:23:59.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:23:59.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:23:59.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:23:59.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:23:59.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:23:59.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:23:59.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:23:59.354 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:23:59.355 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:23:59.355 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:23:59.355 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:23:59.406 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:23:59.406 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:23:59.406 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:23:59.408 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:23:59.409 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:23:59.409 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:23:59.409 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:23:59.409 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:23:59.409 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:23:59.409 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:23:59.409 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:23:59.409 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:23:59.409 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:23:59.409 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:23:59.409 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:23:59.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:23:59.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:23:59.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:23:59.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:23:59.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:23:59.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:23:59.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:23:59.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:23:59.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:23:59.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:23:59.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:23:59.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:23:59.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:23:59.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:23:59.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:23:59.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:23:59.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:23:59.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:23:59.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:23:59.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:23:59.420 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:23:59.420 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:23:59.420 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:23:59.421 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:23:59.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:23:59.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:23:59.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:23:59.472 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:23:59.473 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:23:59.473 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:23:59.473 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:23:59.473 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:23:59.473 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:23:59.473 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:23:59.473 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:23:59.473 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:23:59.473 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:23:59.473 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:23:59.473 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:23:59.473 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:23:59.474 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:23:59.474 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:23:59.474 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:23:59.474 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:23:59.474 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:23:59.474 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:23:59.474 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:23:59.474 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:23:59.474 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:23:59.474 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:23:59.474 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:23:59.474 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:23:59.475 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:23:59.475 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:23:59.475 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:23:59.475 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:23:59.475 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:23:59.475 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:23:59.475 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:23:59.483 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:23:59.483 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:23:59.483 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:23:59.484 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:23:59.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:23:59.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:23:59.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:23:59.536 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:23:59.536 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:23:59.536 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:23:59.536 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:23:59.536 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:23:59.536 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:23:59.536 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:23:59.536 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:23:59.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:23:59.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:23:59.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:23:59.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:23:59.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:23:59.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:23:59.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:23:59.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:23:59.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:23:59.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:23:59.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:23:59.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:23:59.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:23:59.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:23:59.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:23:59.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:23:59.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:23:59.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:23:59.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:23:59.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:23:59.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:23:59.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:23:59.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:23:59.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:23:59.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:23:59.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:23:59.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:23:59.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:23:59.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:23:59.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:23:59.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:23:59.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:23:59.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:23:59.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:23:59.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:23:59.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:23:59.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:23:59.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:23:59.610 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:23:59.610 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:23:59.610 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:23:59.610 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:23:59.611 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:23:59.611 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:23:59.641 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 102 2026-02-18T08:23:59.641 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 106 2026-02-18T08:23:59.642 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:23:59.642 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=425, tag_tid=30, entry_tid=1], [object_number=424, tag_tid=30, entry_tid=0], [object_number=427, tag_tid=29, entry_tid=45931], [object_number=426, tag_tid=29, entry_tid=45930]]], state=connected] 2026-02-18T08:23:59.642 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=411, tag_tid=29, entry_tid=30659], [object_number=410, tag_tid=29, entry_tid=30658], [object_number=409, tag_tid=29, entry_tid=30657], [object_number=408, tag_tid=29, entry_tid=30656]]], state=connected] 2026-02-18T08:23:59.646 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:23:59.647 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=425, tag_tid=30, entry_tid=1' 2026-02-18T08:23:59.648 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:23:59.649 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:23:59.649 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:23:59.649 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:23:59.649 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:23:59.649 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:23:59.649 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:23:59.649 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:23:59.649 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:23:59.649 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:23:59.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:23:59.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:23:59.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:23:59.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:23:59.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:23:59.679 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 102 2026-02-18T08:23:59.679 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 106 2026-02-18T08:23:59.680 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:23:59.680 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=425, tag_tid=30, entry_tid=1], [object_number=424, tag_tid=30, entry_tid=0], [object_number=427, tag_tid=29, entry_tid=45931], [object_number=426, tag_tid=29, entry_tid=45930]]], state=connected] 2026-02-18T08:23:59.680 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=411, tag_tid=29, entry_tid=30659], [object_number=410, tag_tid=29, entry_tid=30658], [object_number=409, tag_tid=29, entry_tid=30657], [object_number=408, tag_tid=29, entry_tid=30656]]], state=connected] 2026-02-18T08:23:59.682 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:23:59.684 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=411, tag_tid=29, entry_tid=30659' 2026-02-18T08:23:59.684 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=425, tag_tid=30, entry_tid=1' -a 'object_number=425, tag_tid=30, entry_tid=1' = 'object_number=411, tag_tid=29, entry_tid=30659' 2026-02-18T08:23:59.684 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=411, tag_tid=29, entry_tid=30659' '!=' 'object_number=411, tag_tid=29, entry_tid=30319' 2026-02-18T08:23:59.685 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:23:59.685 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=411, tag_tid=29, entry_tid=30659' = 'object_number=411, tag_tid=29, entry_tid=30319' 2026-02-18T08:23:59.685 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=411, tag_tid=29, entry_tid=30659' 2026-02-18T08:23:59.685 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=425, tag_tid=30, entry_tid=1' 2026-02-18T08:23:59.685 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:23:59.686 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:23:59.686 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=30 2026-02-18T08:23:59.687 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=411, tag_tid=29, entry_tid=30659' 2026-02-18T08:23:59.687 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:23:59.687 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:23:59.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=29 2026-02-18T08:23:59.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=425, tag_tid=30, entry_tid=1' 2026-02-18T08:23:59.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:23:59.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:23:59.691 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:23:59.692 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=411, tag_tid=29, entry_tid=30659' 2026-02-18T08:23:59.692 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:23:59.692 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:23:59.693 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=30659 2026-02-18T08:23:59.694 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 30 = 29 -a 1 -le 30659 2026-02-18T08:23:59.694 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:23:59.694 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:23:59.694 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:23:59.895 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:23:59.896 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:23:59.896 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:23:59.896 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:23:59.896 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:23:59.896 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:23:59.896 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:23:59.896 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:23:59.896 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:23:59.896 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:23:59.896 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:23:59.897 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:23:59.897 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:23:59.897 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:23:59.897 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:23:59.897 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:23:59.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:23:59.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:23:59.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:23:59.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:23:59.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:23:59.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:23:59.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:23:59.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:23:59.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:23:59.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:23:59.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:23:59.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:23:59.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:23:59.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:23:59.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:23:59.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:23:59.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:23:59.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:23:59.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:23:59.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:23:59.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:23:59.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:23:59.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:23:59.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:23:59.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:23:59.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:23:59.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:23:59.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:23:59.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:23:59.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:23:59.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:23:59.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:23:59.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:23:59.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:23:59.909 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:23:59.909 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:23:59.909 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:23:59.968 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:23:59.968 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:23:59.968 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:23:59.971 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:23:59.971 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:23:59.971 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:23:59.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:23:59.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:23:59.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:23:59.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:23:59.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:23:59.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:23:59.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:23:59.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:23:59.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:23:59.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:23:59.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:23:59.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:23:59.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:23:59.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:23:59.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:23:59.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:23:59.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:23:59.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:23:59.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:23:59.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:23:59.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:23:59.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:23:59.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:23:59.974 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:23:59.974 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:23:59.974 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:23:59.974 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:23:59.974 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:23:59.974 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:23:59.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:23:59.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:23:59.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:23:59.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:24:00.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:24:00.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:24:00.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:24:00.036 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:24:00.036 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:24:00.036 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:24:00.036 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:24:00.036 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:24:00.036 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:24:00.036 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:24:00.036 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:24:00.036 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:24:00.037 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:24:00.037 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:24:00.037 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:24:00.037 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:24:00.037 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:00.037 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:24:00.037 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:24:00.037 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:24:00.037 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:24:00.037 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:24:00.037 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:24:00.037 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:24:00.038 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:24:00.038 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:24:00.038 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:24:00.038 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:00.038 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:24:00.038 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:24:00.038 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:24:00.038 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:24:00.038 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:24:00.038 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:24:00.038 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:24:00.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:24:00.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:24:00.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:24:00.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:24:00.095 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:24:00.095 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:24:00.095 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:24:00.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:24:00.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:24:00.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:24:00.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:24:00.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:24:00.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:24:00.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:24:00.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:24:00.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:24:00.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:24:00.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:24:00.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:24:00.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:24:00.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:00.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:24:00.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:24:00.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:24:00.100 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:24:00.100 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:24:00.100 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:24:00.100 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:24:00.100 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:24:00.100 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:24:00.100 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:24:00.100 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:00.100 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:24:00.100 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:24:00.100 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:24:00.100 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:24:00.100 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:24:00.101 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:24:00.101 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:24:00.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:24:00.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:24:00.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:24:00.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:24:00.167 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:24:00.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:24:00.169 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:24:00.169 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:24:00.169 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:24:00.169 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:24:00.169 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:24:00.169 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:24:00.169 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:24:00.169 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:24:00.169 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:24:00.169 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:24:00.169 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:24:00.170 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:24:00.171 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:24:00.171 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:24:00.403 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 102 2026-02-18T08:24:00.403 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 106 2026-02-18T08:24:00.403 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:24:00.403 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=425, tag_tid=30, entry_tid=1], [object_number=424, tag_tid=30, entry_tid=0], [object_number=427, tag_tid=29, entry_tid=45931], [object_number=426, tag_tid=29, entry_tid=45930]]], state=connected] 2026-02-18T08:24:00.403 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=409, tag_tid=29, entry_tid=31017], [object_number=408, tag_tid=29, entry_tid=31016], [object_number=411, tag_tid=29, entry_tid=31015], [object_number=410, tag_tid=29, entry_tid=31014]]], state=connected] 2026-02-18T08:24:00.406 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:24:00.407 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=425, tag_tid=30, entry_tid=1' 2026-02-18T08:24:00.408 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:24:00.408 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:24:00.408 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:24:00.408 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:24:00.409 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:24:00.409 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:24:00.409 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:24:00.409 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:24:00.409 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:24:00.409 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:24:00.409 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:24:00.409 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:24:00.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:24:00.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:24:00.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:24:00.439 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 102 2026-02-18T08:24:00.439 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 106 2026-02-18T08:24:00.439 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:24:00.439 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=425, tag_tid=30, entry_tid=1], [object_number=424, tag_tid=30, entry_tid=0], [object_number=427, tag_tid=29, entry_tid=45931], [object_number=426, tag_tid=29, entry_tid=45930]]], state=connected] 2026-02-18T08:24:00.439 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=409, tag_tid=29, entry_tid=31017], [object_number=408, tag_tid=29, entry_tid=31016], [object_number=411, tag_tid=29, entry_tid=31015], [object_number=410, tag_tid=29, entry_tid=31014]]], state=connected] 2026-02-18T08:24:00.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:24:00.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=409, tag_tid=29, entry_tid=31017' 2026-02-18T08:24:00.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=425, tag_tid=30, entry_tid=1' -a 'object_number=425, tag_tid=30, entry_tid=1' = 'object_number=409, tag_tid=29, entry_tid=31017' 2026-02-18T08:24:00.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=409, tag_tid=29, entry_tid=31017' '!=' 'object_number=411, tag_tid=29, entry_tid=30659' 2026-02-18T08:24:00.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:24:00.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=409, tag_tid=29, entry_tid=31017' = 'object_number=411, tag_tid=29, entry_tid=30659' 2026-02-18T08:24:00.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=409, tag_tid=29, entry_tid=31017' 2026-02-18T08:24:00.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=425, tag_tid=30, entry_tid=1' 2026-02-18T08:24:00.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:24:00.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:24:00.446 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=30 2026-02-18T08:24:00.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=409, tag_tid=29, entry_tid=31017' 2026-02-18T08:24:00.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:24:00.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:24:00.448 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=29 2026-02-18T08:24:00.449 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=425, tag_tid=30, entry_tid=1' 2026-02-18T08:24:00.449 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:24:00.449 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:24:00.450 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:24:00.451 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=409, tag_tid=29, entry_tid=31017' 2026-02-18T08:24:00.452 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:24:00.452 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:24:00.453 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=31017 2026-02-18T08:24:00.453 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 30 = 29 -a 1 -le 31017 2026-02-18T08:24:00.454 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:24:00.454 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:24:00.454 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:24:00.655 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:24:00.655 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:24:00.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:24:00.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:24:00.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:24:00.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:24:00.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:24:00.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:24:00.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:24:00.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:24:00.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:24:00.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:24:00.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:24:00.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:24:00.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:24:00.660 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:24:00.660 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:24:00.660 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:24:00.660 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:24:00.660 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:24:00.661 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:24:00.661 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:24:00.661 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:24:00.661 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:24:00.661 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:24:00.661 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:24:00.661 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:24:00.661 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:24:00.661 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:24:00.661 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:24:00.661 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:00.661 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:24:00.662 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:24:00.662 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:24:00.662 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:24:00.662 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:24:00.662 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:24:00.662 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:24:00.662 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:24:00.662 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:24:00.662 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:24:00.662 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:00.662 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:24:00.662 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:24:00.663 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:24:00.663 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:24:00.663 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:24:00.663 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:24:00.663 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:24:00.671 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:24:00.672 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:24:00.672 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:24:00.672 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:24:00.722 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:24:00.722 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:24:00.722 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:24:00.725 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:24:00.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:24:00.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:24:00.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:24:00.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:24:00.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:24:00.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:24:00.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:24:00.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:24:00.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:24:00.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:24:00.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:24:00.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:24:00.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:00.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:24:00.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:24:00.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:24:00.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:24:00.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:24:00.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:24:00.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:24:00.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:24:00.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:24:00.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:24:00.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:00.728 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:24:00.728 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:24:00.728 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:24:00.728 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:24:00.728 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:24:00.728 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:24:00.728 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:24:00.736 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:24:00.737 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:24:00.737 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:24:00.737 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:24:00.799 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:24:00.799 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:24:00.799 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:24:00.802 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:24:00.802 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:24:00.802 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:24:00.802 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:24:00.802 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:24:00.802 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:24:00.802 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:24:00.802 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:24:00.802 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:24:00.803 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:24:00.803 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:24:00.803 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:24:00.803 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:24:00.803 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:00.803 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:24:00.803 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:24:00.803 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:24:00.803 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:24:00.803 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:24:00.803 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:24:00.803 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:24:00.804 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:24:00.804 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:24:00.804 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:24:00.804 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:00.804 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:24:00.804 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:24:00.804 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:24:00.804 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:24:00.804 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:24:00.804 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:24:00.804 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:24:00.812 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:24:00.812 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:24:00.813 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:24:00.813 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:24:00.862 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:24:00.862 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:24:00.862 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:24:00.865 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:24:00.865 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:24:00.865 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:24:00.865 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:24:00.865 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:24:00.865 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:24:00.865 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:24:00.865 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:24:00.865 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:24:00.866 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:24:00.866 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:24:00.866 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:24:00.866 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:24:00.866 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:00.866 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:24:00.866 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:24:00.866 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:24:00.866 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:24:00.866 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:24:00.866 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:24:00.866 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:24:00.866 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:24:00.867 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:24:00.867 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:24:00.867 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:00.867 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:24:00.867 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:24:00.867 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:24:00.867 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:24:00.867 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:24:00.867 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:24:00.867 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:24:00.875 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:24:00.875 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:24:00.875 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:24:00.875 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:24:00.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:24:00.936 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:24:00.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:24:00.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:24:00.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:24:00.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:24:00.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:24:00.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:24:00.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:24:00.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:24:00.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:24:00.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:24:00.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:24:00.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:24:00.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:24:00.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:24:00.967 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 102 2026-02-18T08:24:00.967 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 106 2026-02-18T08:24:00.967 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:24:00.968 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=425, tag_tid=30, entry_tid=1], [object_number=424, tag_tid=30, entry_tid=0], [object_number=427, tag_tid=29, entry_tid=45931], [object_number=426, tag_tid=29, entry_tid=45930]]], state=connected] 2026-02-18T08:24:00.968 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=409, tag_tid=29, entry_tid=31501], [object_number=408, tag_tid=29, entry_tid=31500], [object_number=411, tag_tid=29, entry_tid=31499], [object_number=410, tag_tid=29, entry_tid=31498]]], state=connected] 2026-02-18T08:24:00.970 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:24:00.971 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=425, tag_tid=30, entry_tid=1' 2026-02-18T08:24:00.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:24:00.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:24:00.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:24:00.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:24:00.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:24:00.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:24:00.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:24:00.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:24:00.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:24:00.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:24:00.974 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:24:00.974 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:24:00.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:24:00.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:24:00.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:24:01.003 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 102 2026-02-18T08:24:01.003 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 106 2026-02-18T08:24:01.003 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:24:01.003 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=425, tag_tid=30, entry_tid=1], [object_number=424, tag_tid=30, entry_tid=0], [object_number=427, tag_tid=29, entry_tid=45931], [object_number=426, tag_tid=29, entry_tid=45930]]], state=connected] 2026-02-18T08:24:01.004 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=409, tag_tid=29, entry_tid=31501], [object_number=408, tag_tid=29, entry_tid=31500], [object_number=411, tag_tid=29, entry_tid=31499], [object_number=410, tag_tid=29, entry_tid=31498]]], state=connected] 2026-02-18T08:24:01.006 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:24:01.007 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=409, tag_tid=29, entry_tid=31501' 2026-02-18T08:24:01.007 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=425, tag_tid=30, entry_tid=1' -a 'object_number=425, tag_tid=30, entry_tid=1' = 'object_number=409, tag_tid=29, entry_tid=31501' 2026-02-18T08:24:01.007 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=409, tag_tid=29, entry_tid=31501' '!=' 'object_number=409, tag_tid=29, entry_tid=31017' 2026-02-18T08:24:01.008 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:24:01.008 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=409, tag_tid=29, entry_tid=31501' = 'object_number=409, tag_tid=29, entry_tid=31017' 2026-02-18T08:24:01.008 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=409, tag_tid=29, entry_tid=31501' 2026-02-18T08:24:01.008 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=425, tag_tid=30, entry_tid=1' 2026-02-18T08:24:01.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:24:01.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:24:01.010 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=30 2026-02-18T08:24:01.010 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=409, tag_tid=29, entry_tid=31501' 2026-02-18T08:24:01.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:24:01.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:24:01.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=29 2026-02-18T08:24:01.012 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=425, tag_tid=30, entry_tid=1' 2026-02-18T08:24:01.012 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:24:01.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:24:01.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:24:01.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=409, tag_tid=29, entry_tid=31501' 2026-02-18T08:24:01.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:24:01.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:24:01.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=31501 2026-02-18T08:24:01.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 30 = 29 -a 1 -le 31501 2026-02-18T08:24:01.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:24:01.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:24:01.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:24:01.220 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:24:01.220 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:24:01.221 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:24:01.221 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:24:01.221 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:24:01.221 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:24:01.221 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:24:01.221 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:24:01.221 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:24:01.221 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:24:01.221 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:24:01.221 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:24:01.221 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:24:01.222 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:24:01.222 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:24:01.222 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:24:01.222 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:24:01.225 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:24:01.225 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:24:01.225 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:24:01.225 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:24:01.225 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:24:01.225 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:24:01.225 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:24:01.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:24:01.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:24:01.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:24:01.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:24:01.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:24:01.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:24:01.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:01.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:24:01.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:24:01.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:24:01.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:24:01.227 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:24:01.227 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:24:01.227 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:24:01.227 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:24:01.227 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:24:01.227 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:24:01.227 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:01.227 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:24:01.227 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:24:01.227 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:24:01.227 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:24:01.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:24:01.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:24:01.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:24:01.234 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:24:01.234 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:24:01.234 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:24:01.234 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:24:01.283 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:24:01.283 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:24:01.283 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:24:01.287 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:24:01.287 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:24:01.287 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:24:01.287 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:24:01.287 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:24:01.287 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:24:01.287 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:24:01.288 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:24:01.288 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:24:01.288 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:24:01.288 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:24:01.288 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:24:01.288 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:24:01.288 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:01.288 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:24:01.288 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:24:01.288 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:24:01.289 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:24:01.289 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:24:01.289 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:24:01.289 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:24:01.289 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:24:01.289 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:24:01.289 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:24:01.289 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:01.289 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:24:01.289 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:24:01.290 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:24:01.290 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:24:01.290 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:24:01.290 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:24:01.290 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:24:01.299 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:24:01.299 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:24:01.299 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:24:01.299 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:24:01.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:24:01.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:24:01.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:24:01.350 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:24:01.350 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:24:01.350 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:24:01.350 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:24:01.350 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:24:01.351 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:24:01.351 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:24:01.351 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:24:01.351 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:24:01.351 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:24:01.351 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:24:01.351 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:24:01.351 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:24:01.351 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:01.352 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:24:01.352 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:24:01.352 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:24:01.352 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:24:01.352 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:24:01.352 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:24:01.352 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:24:01.352 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:24:01.352 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:24:01.353 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:24:01.353 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:01.353 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:24:01.353 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:24:01.353 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:24:01.353 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:24:01.353 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:24:01.353 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:24:01.353 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:24:01.363 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:24:01.363 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:24:01.364 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:24:01.364 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:24:01.409 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:24:01.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:24:01.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:24:01.413 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:24:01.413 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:24:01.413 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:24:01.413 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:24:01.413 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:24:01.413 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:24:01.414 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:24:01.414 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:24:01.414 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:24:01.414 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:24:01.414 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:24:01.414 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:24:01.414 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:24:01.414 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:01.415 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:24:01.415 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:24:01.415 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:24:01.415 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:24:01.415 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:24:01.415 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:24:01.415 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:24:01.415 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:24:01.415 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:24:01.415 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:24:01.416 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:01.416 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:24:01.416 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:24:01.416 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:24:01.416 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:24:01.416 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:24:01.416 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:24:01.417 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:24:01.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:24:01.425 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:24:01.425 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:24:01.425 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:24:01.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:24:01.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:24:01.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:24:01.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:24:01.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:24:01.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:24:01.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:24:01.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:24:01.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:24:01.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:24:01.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:24:01.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:24:01.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:24:01.500 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:24:01.500 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:24:01.500 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:24:01.528 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 103 2026-02-18T08:24:01.528 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 106 2026-02-18T08:24:01.528 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:24:01.528 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=425, tag_tid=30, entry_tid=1], [object_number=424, tag_tid=30, entry_tid=0], [object_number=427, tag_tid=29, entry_tid=45931], [object_number=426, tag_tid=29, entry_tid=45930]]], state=connected] 2026-02-18T08:24:01.528 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=413, tag_tid=29, entry_tid=31757], [object_number=412, tag_tid=29, entry_tid=31756], [object_number=415, tag_tid=29, entry_tid=31755], [object_number=414, tag_tid=29, entry_tid=31754]]], state=connected] 2026-02-18T08:24:01.530 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:24:01.531 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=425, tag_tid=30, entry_tid=1' 2026-02-18T08:24:01.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:24:01.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:24:01.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:24:01.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:24:01.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:24:01.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:24:01.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:24:01.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:24:01.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:24:01.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:24:01.534 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:24:01.534 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:24:01.535 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:24:01.535 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:24:01.535 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:24:01.565 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 103 2026-02-18T08:24:01.565 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 106 2026-02-18T08:24:01.566 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:24:01.566 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=425, tag_tid=30, entry_tid=1], [object_number=424, tag_tid=30, entry_tid=0], [object_number=427, tag_tid=29, entry_tid=45931], [object_number=426, tag_tid=29, entry_tid=45930]]], state=connected] 2026-02-18T08:24:01.566 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=413, tag_tid=29, entry_tid=31757], [object_number=412, tag_tid=29, entry_tid=31756], [object_number=415, tag_tid=29, entry_tid=31755], [object_number=414, tag_tid=29, entry_tid=31754]]], state=connected] 2026-02-18T08:24:01.568 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:24:01.570 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=413, tag_tid=29, entry_tid=31757' 2026-02-18T08:24:01.570 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=425, tag_tid=30, entry_tid=1' -a 'object_number=425, tag_tid=30, entry_tid=1' = 'object_number=413, tag_tid=29, entry_tid=31757' 2026-02-18T08:24:01.570 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=413, tag_tid=29, entry_tid=31757' '!=' 'object_number=409, tag_tid=29, entry_tid=31501' 2026-02-18T08:24:01.570 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:24:01.570 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=413, tag_tid=29, entry_tid=31757' = 'object_number=409, tag_tid=29, entry_tid=31501' 2026-02-18T08:24:01.570 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=413, tag_tid=29, entry_tid=31757' 2026-02-18T08:24:01.571 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=425, tag_tid=30, entry_tid=1' 2026-02-18T08:24:01.571 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:24:01.571 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:24:01.572 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=30 2026-02-18T08:24:01.573 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=413, tag_tid=29, entry_tid=31757' 2026-02-18T08:24:01.573 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:24:01.573 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:24:01.574 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=29 2026-02-18T08:24:01.575 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=425, tag_tid=30, entry_tid=1' 2026-02-18T08:24:01.575 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:24:01.575 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:24:01.576 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:24:01.577 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=413, tag_tid=29, entry_tid=31757' 2026-02-18T08:24:01.577 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:24:01.577 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:24:01.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=31757 2026-02-18T08:24:01.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 30 = 29 -a 1 -le 31757 2026-02-18T08:24:01.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:24:01.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:24:01.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:24:01.781 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:24:01.781 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:24:01.781 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:24:01.781 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:24:01.781 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:24:01.781 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:24:01.781 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:24:01.782 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:24:01.782 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:24:01.782 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:24:01.782 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:24:01.782 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:24:01.782 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:24:01.782 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:24:01.782 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:24:01.782 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:24:01.785 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:24:01.785 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:24:01.785 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:24:01.785 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:24:01.785 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:24:01.785 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:24:01.785 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:24:01.785 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:24:01.785 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:24:01.785 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:24:01.786 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:24:01.786 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:24:01.786 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:24:01.786 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:24:01.786 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:01.786 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:24:01.786 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:24:01.786 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:24:01.786 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:24:01.786 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:24:01.786 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:24:01.786 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:24:01.787 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:24:01.787 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:24:01.787 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:24:01.787 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:01.787 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:24:01.787 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:24:01.787 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:24:01.787 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:24:01.787 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:24:01.787 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:24:01.787 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:24:01.796 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:24:01.796 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:24:01.796 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:24:01.796 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:24:01.844 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:24:01.845 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:24:01.845 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:24:01.847 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:24:01.848 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:24:01.848 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:24:01.848 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:24:01.848 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:24:01.848 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:24:01.848 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:24:01.848 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:24:01.848 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:24:01.848 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:24:01.848 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:24:01.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:24:01.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:24:01.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:01.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:24:01.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:24:01.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:24:01.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:24:01.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:24:01.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:24:01.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:24:01.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:24:01.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:24:01.850 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:24:01.850 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:01.850 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:24:01.850 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:24:01.850 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:24:01.850 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:24:01.850 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:24:01.850 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:24:01.850 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:24:01.859 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:24:01.860 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:24:01.860 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:24:01.860 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:24:01.916 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:24:01.916 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:24:01.916 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:24:01.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:24:01.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:24:01.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:24:01.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:24:01.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:24:01.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:24:01.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:24:01.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:24:01.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:24:01.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:24:01.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:24:01.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:24:01.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:24:01.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:01.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:24:01.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:24:01.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:24:01.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:24:01.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:24:01.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:24:01.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:24:01.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:24:01.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:24:01.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:24:01.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:01.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:24:01.922 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:24:01.922 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:24:01.922 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:24:01.922 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:24:01.922 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:24:01.922 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:24:01.931 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:24:01.931 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:24:01.931 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:24:01.931 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:24:01.988 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:24:01.988 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:24:01.988 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:24:01.990 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:24:01.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:24:01.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:24:01.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:24:01.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:24:01.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:24:01.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:24:01.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:24:01.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:24:01.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:24:01.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:24:01.992 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:24:01.992 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:24:01.992 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:01.992 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:24:01.992 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:24:01.992 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:24:01.992 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:24:01.992 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:24:01.992 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:24:01.992 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:24:01.992 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:24:01.992 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:24:01.993 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:24:01.993 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:01.993 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:24:01.993 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:24:01.993 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:24:01.993 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:24:01.993 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:24:01.993 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:24:01.993 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:24:02.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:24:02.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:24:02.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:24:02.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:24:02.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:24:02.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:24:02.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:24:02.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:24:02.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:24:02.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:24:02.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:24:02.062 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:24:02.062 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:24:02.062 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:24:02.062 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:24:02.062 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:24:02.062 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:24:02.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:24:02.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:24:02.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:24:02.092 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 103 2026-02-18T08:24:02.092 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 106 2026-02-18T08:24:02.093 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:24:02.093 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=425, tag_tid=30, entry_tid=1], [object_number=424, tag_tid=30, entry_tid=0], [object_number=427, tag_tid=29, entry_tid=45931], [object_number=426, tag_tid=29, entry_tid=45930]]], state=connected] 2026-02-18T08:24:02.093 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=412, tag_tid=29, entry_tid=32096], [object_number=415, tag_tid=29, entry_tid=32095], [object_number=414, tag_tid=29, entry_tid=32094], [object_number=413, tag_tid=29, entry_tid=32093]]], state=connected] 2026-02-18T08:24:02.095 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:24:02.097 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=425, tag_tid=30, entry_tid=1' 2026-02-18T08:24:02.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:24:02.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:24:02.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:24:02.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:24:02.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:24:02.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:24:02.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:24:02.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:24:02.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:24:02.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:24:02.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:24:02.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:24:02.100 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:24:02.100 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:24:02.100 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:24:02.138 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 103 2026-02-18T08:24:02.138 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 106 2026-02-18T08:24:02.139 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:24:02.139 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=425, tag_tid=30, entry_tid=1], [object_number=424, tag_tid=30, entry_tid=0], [object_number=427, tag_tid=29, entry_tid=45931], [object_number=426, tag_tid=29, entry_tid=45930]]], state=connected] 2026-02-18T08:24:02.139 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=412, tag_tid=29, entry_tid=32096], [object_number=415, tag_tid=29, entry_tid=32095], [object_number=414, tag_tid=29, entry_tid=32094], [object_number=413, tag_tid=29, entry_tid=32093]]], state=connected] 2026-02-18T08:24:02.142 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:24:02.145 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=412, tag_tid=29, entry_tid=32096' 2026-02-18T08:24:02.145 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=425, tag_tid=30, entry_tid=1' -a 'object_number=425, tag_tid=30, entry_tid=1' = 'object_number=412, tag_tid=29, entry_tid=32096' 2026-02-18T08:24:02.145 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=412, tag_tid=29, entry_tid=32096' '!=' 'object_number=413, tag_tid=29, entry_tid=31757' 2026-02-18T08:24:02.145 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:24:02.145 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=412, tag_tid=29, entry_tid=32096' = 'object_number=413, tag_tid=29, entry_tid=31757' 2026-02-18T08:24:02.145 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=412, tag_tid=29, entry_tid=32096' 2026-02-18T08:24:02.145 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=425, tag_tid=30, entry_tid=1' 2026-02-18T08:24:02.145 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:24:02.145 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:24:02.146 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=30 2026-02-18T08:24:02.147 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=412, tag_tid=29, entry_tid=32096' 2026-02-18T08:24:02.147 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:24:02.147 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:24:02.148 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=29 2026-02-18T08:24:02.149 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=425, tag_tid=30, entry_tid=1' 2026-02-18T08:24:02.149 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:24:02.150 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:24:02.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:24:02.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=412, tag_tid=29, entry_tid=32096' 2026-02-18T08:24:02.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:24:02.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:24:02.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=32096 2026-02-18T08:24:02.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 30 = 29 -a 1 -le 32096 2026-02-18T08:24:02.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:24:02.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:24:02.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:24:02.355 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:24:02.355 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:24:02.355 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:24:02.356 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:24:02.356 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:24:02.356 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:24:02.356 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:24:02.356 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:24:02.356 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:24:02.356 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:24:02.356 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:24:02.356 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:24:02.357 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:24:02.357 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:24:02.357 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:24:02.357 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:24:02.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:24:02.360 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:24:02.360 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:24:02.360 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:24:02.360 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:24:02.360 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:24:02.360 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:24:02.360 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:24:02.360 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:24:02.360 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:24:02.360 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:24:02.360 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:24:02.361 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:24:02.361 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:24:02.361 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:02.361 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:24:02.361 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:24:02.361 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:24:02.361 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:24:02.361 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:24:02.361 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:24:02.361 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:24:02.361 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:24:02.361 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:24:02.362 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:24:02.362 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:02.362 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:24:02.362 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:24:02.362 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:24:02.362 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:24:02.362 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:24:02.362 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:24:02.362 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:24:02.368 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:24:02.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:24:02.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:24:02.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:24:02.427 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:24:02.427 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:24:02.427 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:24:02.430 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:24:02.430 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:24:02.431 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:24:02.431 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:24:02.431 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:24:02.431 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:24:02.431 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:24:02.431 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:24:02.431 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:24:02.431 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:24:02.431 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:24:02.431 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:24:02.432 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:24:02.432 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:02.432 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:24:02.432 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:24:02.432 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:24:02.432 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:24:02.432 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:24:02.432 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:24:02.432 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:24:02.432 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:24:02.432 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:24:02.433 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:24:02.433 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:02.433 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:24:02.433 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:24:02.433 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:24:02.433 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:24:02.433 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:24:02.433 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:24:02.433 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:24:02.440 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:24:02.440 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:24:02.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:24:02.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:24:02.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:24:02.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:24:02.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:24:02.502 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:24:02.502 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:24:02.502 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:24:02.502 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:24:02.502 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:24:02.502 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:24:02.502 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:24:02.502 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:24:02.503 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:24:02.503 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:24:02.503 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:24:02.503 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:24:02.503 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:24:02.503 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:02.503 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:24:02.503 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:24:02.503 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:24:02.503 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:24:02.503 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:24:02.504 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:24:02.504 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:24:02.504 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:24:02.504 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:24:02.504 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:24:02.504 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:02.504 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:24:02.504 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:24:02.504 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:24:02.504 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:24:02.505 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:24:02.505 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:24:02.505 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:24:02.512 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:24:02.513 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:24:02.513 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:24:02.513 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:24:02.561 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:24:02.562 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:24:02.562 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:24:02.565 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:24:02.565 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:24:02.565 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:24:02.565 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:24:02.565 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:24:02.565 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:24:02.565 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:24:02.566 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:24:02.566 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:24:02.566 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:24:02.566 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:24:02.566 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:24:02.566 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:24:02.566 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:02.566 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:24:02.566 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:24:02.566 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:24:02.566 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:24:02.567 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:24:02.567 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:24:02.567 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:24:02.567 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:24:02.567 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:24:02.567 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:24:02.567 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:02.567 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:24:02.567 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:24:02.567 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:24:02.568 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:24:02.568 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:24:02.568 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:24:02.568 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:24:02.576 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:24:02.576 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:24:02.577 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:24:02.577 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:24:02.636 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:24:02.637 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:24:02.637 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:24:02.637 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:24:02.638 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:24:02.638 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:24:02.638 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:24:02.638 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:24:02.638 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:24:02.638 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:24:02.638 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:24:02.639 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:24:02.639 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:24:02.639 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:24:02.639 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:24:02.640 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:24:02.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 103 2026-02-18T08:24:02.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 106 2026-02-18T08:24:02.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:24:02.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=425, tag_tid=30, entry_tid=1], [object_number=424, tag_tid=30, entry_tid=0], [object_number=427, tag_tid=29, entry_tid=45931], [object_number=426, tag_tid=29, entry_tid=45930]]], state=connected] 2026-02-18T08:24:02.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=414, tag_tid=29, entry_tid=32378], [object_number=413, tag_tid=29, entry_tid=32377], [object_number=412, tag_tid=29, entry_tid=32376], [object_number=415, tag_tid=29, entry_tid=32375]]], state=connected] 2026-02-18T08:24:02.671 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:24:02.672 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=425, tag_tid=30, entry_tid=1' 2026-02-18T08:24:02.673 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:24:02.674 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:24:02.674 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:24:02.674 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:24:02.674 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:24:02.674 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:24:02.674 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:24:02.674 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:24:02.674 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:24:02.674 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:24:02.675 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:24:02.675 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:24:02.676 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:24:02.676 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:24:02.676 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:24:02.704 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 103 2026-02-18T08:24:02.704 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 106 2026-02-18T08:24:02.704 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:24:02.704 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=425, tag_tid=30, entry_tid=1], [object_number=424, tag_tid=30, entry_tid=0], [object_number=427, tag_tid=29, entry_tid=45931], [object_number=426, tag_tid=29, entry_tid=45930]]], state=connected] 2026-02-18T08:24:02.704 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=414, tag_tid=29, entry_tid=32378], [object_number=413, tag_tid=29, entry_tid=32377], [object_number=412, tag_tid=29, entry_tid=32376], [object_number=415, tag_tid=29, entry_tid=32375]]], state=connected] 2026-02-18T08:24:02.706 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:24:02.708 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=414, tag_tid=29, entry_tid=32378' 2026-02-18T08:24:02.708 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=425, tag_tid=30, entry_tid=1' -a 'object_number=425, tag_tid=30, entry_tid=1' = 'object_number=414, tag_tid=29, entry_tid=32378' 2026-02-18T08:24:02.708 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=414, tag_tid=29, entry_tid=32378' '!=' 'object_number=412, tag_tid=29, entry_tid=32096' 2026-02-18T08:24:02.709 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:24:02.709 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=414, tag_tid=29, entry_tid=32378' = 'object_number=412, tag_tid=29, entry_tid=32096' 2026-02-18T08:24:02.709 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=414, tag_tid=29, entry_tid=32378' 2026-02-18T08:24:02.709 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=425, tag_tid=30, entry_tid=1' 2026-02-18T08:24:02.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:24:02.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:24:02.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=30 2026-02-18T08:24:02.711 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=414, tag_tid=29, entry_tid=32378' 2026-02-18T08:24:02.711 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:24:02.711 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:24:02.713 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=29 2026-02-18T08:24:02.714 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=425, tag_tid=30, entry_tid=1' 2026-02-18T08:24:02.714 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:24:02.714 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:24:02.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:24:02.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=414, tag_tid=29, entry_tid=32378' 2026-02-18T08:24:02.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:24:02.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:24:02.717 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=32378 2026-02-18T08:24:02.718 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 30 = 29 -a 1 -le 32378 2026-02-18T08:24:02.718 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:24:02.718 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:24:02.718 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:24:02.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:24:02.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:24:02.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:24:02.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:24:02.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:24:02.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:24:02.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:24:02.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:24:02.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:24:02.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:24:02.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:24:02.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:24:02.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:24:02.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:24:02.922 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:24:02.922 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:24:02.924 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:24:02.924 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:24:02.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:24:02.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:24:02.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:24:02.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:24:02.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:24:02.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:24:02.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:24:02.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:24:02.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:24:02.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:24:02.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:24:02.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:24:02.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:02.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:24:02.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:24:02.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:24:02.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:24:02.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:24:02.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:24:02.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:24:02.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:24:02.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:24:02.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:24:02.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:02.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:24:02.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:24:02.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:24:02.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:24:02.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:24:02.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:24:02.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:24:02.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:24:02.936 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:24:02.936 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:24:02.936 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:24:02.986 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:24:02.986 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:24:02.986 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:24:02.989 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:24:02.989 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:24:02.989 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:24:02.989 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:24:02.989 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:24:02.990 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:24:02.990 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:24:02.990 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:24:02.990 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:24:02.990 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:24:02.990 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:24:02.990 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:24:02.990 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:24:02.990 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:02.990 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:24:02.990 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:24:02.990 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:24:02.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:24:02.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:24:02.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:24:02.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:24:02.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:24:02.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:24:02.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:24:02.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:02.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:24:02.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:24:02.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:24:02.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:24:02.992 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:24:02.992 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:24:02.992 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:24:03.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:24:03.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:24:03.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:24:03.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:24:03.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:24:03.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:24:03.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:24:03.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:24:03.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:24:03.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:24:03.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:24:03.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:24:03.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:24:03.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:24:03.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:24:03.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:24:03.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:24:03.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:24:03.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:24:03.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:24:03.055 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:03.055 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:24:03.055 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:24:03.055 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:24:03.055 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:24:03.055 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:24:03.055 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:24:03.055 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:24:03.055 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:24:03.055 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:24:03.055 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:24:03.055 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:03.055 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:24:03.056 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:24:03.056 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:24:03.056 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:24:03.056 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:24:03.056 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:24:03.056 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:24:03.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:24:03.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:24:03.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:24:03.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:24:03.112 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:24:03.112 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:24:03.112 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:24:03.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:24:03.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:24:03.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:24:03.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:24:03.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:24:03.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:24:03.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:24:03.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:24:03.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:24:03.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:24:03.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:24:03.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:24:03.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:24:03.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:03.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:24:03.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:24:03.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:24:03.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:24:03.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:24:03.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:24:03.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:24:03.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:24:03.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:24:03.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:24:03.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:03.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:24:03.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:24:03.118 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:24:03.118 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:24:03.118 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:24:03.118 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:24:03.118 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:24:03.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:24:03.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:24:03.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:24:03.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:24:03.188 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:24:03.189 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:24:03.189 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:24:03.189 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:24:03.189 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:24:03.189 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:24:03.189 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:24:03.189 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:24:03.189 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:24:03.190 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:24:03.190 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:24:03.190 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:24:03.190 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:24:03.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:24:03.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:24:03.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:24:03.220 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 103 2026-02-18T08:24:03.221 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 106 2026-02-18T08:24:03.221 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:24:03.221 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=425, tag_tid=30, entry_tid=1], [object_number=424, tag_tid=30, entry_tid=0], [object_number=427, tag_tid=29, entry_tid=45931], [object_number=426, tag_tid=29, entry_tid=45930]]], state=connected] 2026-02-18T08:24:03.221 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=413, tag_tid=29, entry_tid=32717], [object_number=412, tag_tid=29, entry_tid=32716], [object_number=415, tag_tid=29, entry_tid=32715], [object_number=414, tag_tid=29, entry_tid=32714]]], state=connected] 2026-02-18T08:24:03.223 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:24:03.225 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=425, tag_tid=30, entry_tid=1' 2026-02-18T08:24:03.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:24:03.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:24:03.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:24:03.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:24:03.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:24:03.227 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:24:03.227 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:24:03.227 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:24:03.227 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:24:03.227 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:24:03.227 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:24:03.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:24:03.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:24:03.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:24:03.229 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:24:03.256 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 103 2026-02-18T08:24:03.257 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 106 2026-02-18T08:24:03.257 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:24:03.257 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=425, tag_tid=30, entry_tid=1], [object_number=424, tag_tid=30, entry_tid=0], [object_number=427, tag_tid=29, entry_tid=45931], [object_number=426, tag_tid=29, entry_tid=45930]]], state=connected] 2026-02-18T08:24:03.257 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=413, tag_tid=29, entry_tid=32717], [object_number=412, tag_tid=29, entry_tid=32716], [object_number=415, tag_tid=29, entry_tid=32715], [object_number=414, tag_tid=29, entry_tid=32714]]], state=connected] 2026-02-18T08:24:03.260 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:24:03.261 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=413, tag_tid=29, entry_tid=32717' 2026-02-18T08:24:03.261 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=425, tag_tid=30, entry_tid=1' -a 'object_number=425, tag_tid=30, entry_tid=1' = 'object_number=413, tag_tid=29, entry_tid=32717' 2026-02-18T08:24:03.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=413, tag_tid=29, entry_tid=32717' '!=' 'object_number=414, tag_tid=29, entry_tid=32378' 2026-02-18T08:24:03.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:24:03.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=413, tag_tid=29, entry_tid=32717' = 'object_number=414, tag_tid=29, entry_tid=32378' 2026-02-18T08:24:03.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=413, tag_tid=29, entry_tid=32717' 2026-02-18T08:24:03.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=425, tag_tid=30, entry_tid=1' 2026-02-18T08:24:03.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:24:03.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:24:03.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=30 2026-02-18T08:24:03.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=413, tag_tid=29, entry_tid=32717' 2026-02-18T08:24:03.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:24:03.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:24:03.266 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=29 2026-02-18T08:24:03.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=425, tag_tid=30, entry_tid=1' 2026-02-18T08:24:03.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:24:03.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:24:03.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:24:03.269 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=413, tag_tid=29, entry_tid=32717' 2026-02-18T08:24:03.269 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:24:03.269 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:24:03.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=32717 2026-02-18T08:24:03.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 30 = 29 -a 1 -le 32717 2026-02-18T08:24:03.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:24:03.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:24:03.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:24:03.473 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:24:03.473 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:24:03.473 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:24:03.473 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:24:03.473 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:24:03.473 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:24:03.473 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:24:03.473 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:24:03.474 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:24:03.474 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:24:03.474 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:24:03.474 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:24:03.474 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:24:03.474 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:24:03.474 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:24:03.477 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:24:03.477 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:24:03.477 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:24:03.477 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:24:03.477 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:24:03.477 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:24:03.477 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:24:03.477 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:24:03.477 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:24:03.478 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:24:03.478 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:24:03.478 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:24:03.478 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:24:03.478 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:24:03.478 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:24:03.478 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:03.478 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:24:03.478 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:24:03.478 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:24:03.478 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:24:03.478 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:24:03.479 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:24:03.479 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:24:03.479 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:24:03.479 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:24:03.479 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:24:03.479 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:03.479 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:24:03.479 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:24:03.479 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:24:03.479 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:24:03.479 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:24:03.479 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:24:03.480 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:24:03.487 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:24:03.488 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:24:03.488 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:24:03.488 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:24:03.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:24:03.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:24:03.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:24:03.540 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:24:03.540 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:24:03.540 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:24:03.540 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:24:03.540 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:24:03.540 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:24:03.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:24:03.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:24:03.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:24:03.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:24:03.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:24:03.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:24:03.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:24:03.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:03.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:24:03.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:24:03.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:24:03.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:24:03.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:24:03.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:24:03.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:24:03.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:24:03.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:24:03.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:24:03.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:03.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:24:03.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:24:03.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:24:03.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:24:03.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:24:03.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:24:03.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:24:03.551 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:24:03.552 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:24:03.552 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:24:03.552 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:24:03.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:24:03.610 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:24:03.610 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:24:03.612 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:24:03.613 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:24:03.613 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:24:03.613 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:24:03.613 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:24:03.613 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:24:03.613 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:24:03.613 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:24:03.613 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:24:03.613 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:24:03.613 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:24:03.613 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:24:03.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:24:03.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:03.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:24:03.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:24:03.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:24:03.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:24:03.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:24:03.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:24:03.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:24:03.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:24:03.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:24:03.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:24:03.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:03.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:24:03.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:24:03.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:24:03.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:24:03.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:24:03.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:24:03.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:24:03.623 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:24:03.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:24:03.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:24:03.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:24:03.672 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:24:03.672 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:24:03.672 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:24:03.675 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:24:03.676 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:24:03.676 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:24:03.676 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:24:03.676 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:24:03.676 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:24:03.676 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:24:03.676 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:24:03.676 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:24:03.676 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:24:03.676 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:24:03.676 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:24:03.677 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:24:03.677 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:03.677 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:24:03.677 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:24:03.677 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:24:03.677 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:24:03.677 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:24:03.677 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:24:03.677 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:24:03.677 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:24:03.677 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:24:03.677 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:24:03.677 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:03.678 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:24:03.678 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:24:03.678 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:24:03.678 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:24:03.678 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:24:03.678 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:24:03.678 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:24:03.686 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:24:03.687 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:24:03.687 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:24:03.687 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:24:03.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:24:03.759 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:24:03.759 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:24:03.759 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:24:03.759 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:24:03.759 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:24:03.759 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:24:03.759 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:24:03.759 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:24:03.759 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:24:03.759 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:24:03.760 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:24:03.760 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:24:03.760 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:24:03.761 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:24:03.761 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:24:03.798 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 103 2026-02-18T08:24:03.798 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 106 2026-02-18T08:24:03.798 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:24:03.799 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=425, tag_tid=30, entry_tid=1], [object_number=424, tag_tid=30, entry_tid=0], [object_number=427, tag_tid=29, entry_tid=45931], [object_number=426, tag_tid=29, entry_tid=45930]]], state=connected] 2026-02-18T08:24:03.799 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=412, tag_tid=29, entry_tid=33064], [object_number=415, tag_tid=29, entry_tid=33063], [object_number=414, tag_tid=29, entry_tid=33062], [object_number=413, tag_tid=29, entry_tid=33061]]], state=connected] 2026-02-18T08:24:03.801 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:24:03.802 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=425, tag_tid=30, entry_tid=1' 2026-02-18T08:24:03.803 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:24:03.803 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:24:03.804 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:24:03.804 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:24:03.804 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:24:03.804 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:24:03.804 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:24:03.804 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:24:03.804 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:24:03.804 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:24:03.804 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:24:03.804 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:24:03.805 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:24:03.805 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:24:03.805 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:24:03.841 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 103 2026-02-18T08:24:03.841 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 106 2026-02-18T08:24:03.841 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:24:03.842 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=425, tag_tid=30, entry_tid=1], [object_number=424, tag_tid=30, entry_tid=0], [object_number=427, tag_tid=29, entry_tid=45931], [object_number=426, tag_tid=29, entry_tid=45930]]], state=connected] 2026-02-18T08:24:03.842 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=412, tag_tid=29, entry_tid=33064], [object_number=415, tag_tid=29, entry_tid=33063], [object_number=414, tag_tid=29, entry_tid=33062], [object_number=413, tag_tid=29, entry_tid=33061]]], state=connected] 2026-02-18T08:24:03.845 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:24:03.846 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=412, tag_tid=29, entry_tid=33064' 2026-02-18T08:24:03.847 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=425, tag_tid=30, entry_tid=1' -a 'object_number=425, tag_tid=30, entry_tid=1' = 'object_number=412, tag_tid=29, entry_tid=33064' 2026-02-18T08:24:03.847 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=412, tag_tid=29, entry_tid=33064' '!=' 'object_number=413, tag_tid=29, entry_tid=32717' 2026-02-18T08:24:03.847 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:24:03.847 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=412, tag_tid=29, entry_tid=33064' = 'object_number=413, tag_tid=29, entry_tid=32717' 2026-02-18T08:24:03.847 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=412, tag_tid=29, entry_tid=33064' 2026-02-18T08:24:03.848 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=425, tag_tid=30, entry_tid=1' 2026-02-18T08:24:03.848 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:24:03.848 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:24:03.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=30 2026-02-18T08:24:03.850 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=412, tag_tid=29, entry_tid=33064' 2026-02-18T08:24:03.850 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:24:03.850 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:24:03.851 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=29 2026-02-18T08:24:03.852 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=425, tag_tid=30, entry_tid=1' 2026-02-18T08:24:03.852 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:24:03.852 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:24:03.853 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:24:03.854 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=412, tag_tid=29, entry_tid=33064' 2026-02-18T08:24:03.854 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:24:03.854 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:24:03.856 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=33064 2026-02-18T08:24:03.856 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 30 = 29 -a 1 -le 33064 2026-02-18T08:24:03.856 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:24:03.856 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:24:03.856 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:24:04.058 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:24:04.058 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:24:04.058 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:24:04.058 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:24:04.059 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:24:04.059 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:24:04.059 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:24:04.059 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:24:04.059 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:24:04.059 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:24:04.059 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:24:04.059 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:24:04.059 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:24:04.059 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:24:04.060 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:24:04.060 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:24:04.062 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:24:04.062 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:24:04.062 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:24:04.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:24:04.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:24:04.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:24:04.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:24:04.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:24:04.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:24:04.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:24:04.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:24:04.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:24:04.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:24:04.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:24:04.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:04.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:24:04.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:24:04.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:24:04.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:24:04.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:24:04.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:24:04.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:24:04.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:24:04.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:24:04.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:24:04.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:04.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:24:04.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:24:04.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:24:04.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:24:04.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:24:04.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:24:04.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:24:04.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:24:04.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:24:04.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:24:04.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:24:04.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:24:04.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:24:04.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:24:04.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:24:04.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:24:04.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:24:04.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:24:04.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:24:04.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:24:04.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:24:04.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:24:04.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:24:04.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:24:04.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:24:04.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:24:04.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:24:04.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:04.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:24:04.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:24:04.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:24:04.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:24:04.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:24:04.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:24:04.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:24:04.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:24:04.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:24:04.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:24:04.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:04.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:24:04.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:24:04.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:24:04.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:24:04.129 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:24:04.129 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:24:04.129 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:24:04.137 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:24:04.137 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:24:04.138 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:24:04.138 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:24:04.187 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:24:04.187 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:24:04.188 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:24:04.190 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:24:04.190 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:24:04.190 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:24:04.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:24:04.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:24:04.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:24:04.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:24:04.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:24:04.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:24:04.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:24:04.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:24:04.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:24:04.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:24:04.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:04.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:24:04.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:24:04.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:24:04.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:24:04.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:24:04.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:24:04.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:24:04.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:24:04.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:24:04.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:24:04.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:04.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:24:04.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:24:04.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:24:04.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:24:04.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:24:04.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:24:04.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:24:04.201 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:24:04.201 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:24:04.201 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:24:04.202 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:24:04.250 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:24:04.250 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:24:04.250 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:24:04.253 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:24:04.253 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:24:04.253 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:24:04.253 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:24:04.254 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:24:04.254 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:24:04.254 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:24:04.254 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:24:04.254 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:24:04.254 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:24:04.254 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:24:04.254 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:24:04.254 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:24:04.254 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:04.254 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:24:04.255 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:24:04.255 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:24:04.255 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:24:04.255 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:24:04.255 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:24:04.255 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:24:04.255 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:24:04.255 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:24:04.255 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:24:04.255 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:04.255 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:24:04.255 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:24:04.256 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:24:04.256 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:24:04.256 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:24:04.256 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:24:04.256 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:24:04.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:24:04.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:24:04.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:24:04.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:24:04.323 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:24:04.324 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:24:04.325 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:24:04.325 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:24:04.325 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:24:04.325 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:24:04.325 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:24:04.325 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:24:04.325 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:24:04.325 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:24:04.325 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:24:04.325 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:24:04.325 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:24:04.326 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:24:04.327 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:24:04.327 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:24:04.355 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 103 2026-02-18T08:24:04.355 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 106 2026-02-18T08:24:04.355 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:24:04.355 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=425, tag_tid=30, entry_tid=1], [object_number=424, tag_tid=30, entry_tid=0], [object_number=427, tag_tid=29, entry_tid=45931], [object_number=426, tag_tid=29, entry_tid=45930]]], state=connected] 2026-02-18T08:24:04.356 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=415, tag_tid=29, entry_tid=33435], [object_number=414, tag_tid=29, entry_tid=33434], [object_number=413, tag_tid=29, entry_tid=33433], [object_number=412, tag_tid=29, entry_tid=33432]]], state=connected] 2026-02-18T08:24:04.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:24:04.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=425, tag_tid=30, entry_tid=1' 2026-02-18T08:24:04.360 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:24:04.361 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:24:04.361 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:24:04.361 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:24:04.361 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:24:04.361 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:24:04.361 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:24:04.361 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:24:04.361 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:24:04.361 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:24:04.362 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:24:04.362 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:24:04.362 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:24:04.363 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:24:04.363 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:24:04.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 103 2026-02-18T08:24:04.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 106 2026-02-18T08:24:04.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:24:04.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=425, tag_tid=30, entry_tid=1], [object_number=424, tag_tid=30, entry_tid=0], [object_number=427, tag_tid=29, entry_tid=45931], [object_number=426, tag_tid=29, entry_tid=45930]]], state=connected] 2026-02-18T08:24:04.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=415, tag_tid=29, entry_tid=33435], [object_number=414, tag_tid=29, entry_tid=33434], [object_number=413, tag_tid=29, entry_tid=33433], [object_number=412, tag_tid=29, entry_tid=33432]]], state=connected] 2026-02-18T08:24:04.395 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:24:04.397 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=415, tag_tid=29, entry_tid=33435' 2026-02-18T08:24:04.397 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=425, tag_tid=30, entry_tid=1' -a 'object_number=425, tag_tid=30, entry_tid=1' = 'object_number=415, tag_tid=29, entry_tid=33435' 2026-02-18T08:24:04.397 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=415, tag_tid=29, entry_tid=33435' '!=' 'object_number=412, tag_tid=29, entry_tid=33064' 2026-02-18T08:24:04.397 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:24:04.397 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=415, tag_tid=29, entry_tid=33435' = 'object_number=412, tag_tid=29, entry_tid=33064' 2026-02-18T08:24:04.397 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=415, tag_tid=29, entry_tid=33435' 2026-02-18T08:24:04.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=425, tag_tid=30, entry_tid=1' 2026-02-18T08:24:04.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:24:04.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:24:04.399 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=30 2026-02-18T08:24:04.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=415, tag_tid=29, entry_tid=33435' 2026-02-18T08:24:04.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:24:04.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:24:04.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=29 2026-02-18T08:24:04.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=425, tag_tid=30, entry_tid=1' 2026-02-18T08:24:04.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:24:04.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:24:04.403 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:24:04.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=415, tag_tid=29, entry_tid=33435' 2026-02-18T08:24:04.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:24:04.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:24:04.406 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=33435 2026-02-18T08:24:04.406 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 30 = 29 -a 1 -le 33435 2026-02-18T08:24:04.406 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:24:04.406 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:24:04.406 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:24:04.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:24:04.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:24:04.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:24:04.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:24:04.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:24:04.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:24:04.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:24:04.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:24:04.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:24:04.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:24:04.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:24:04.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:24:04.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:24:04.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:24:04.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:24:04.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:24:04.612 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:24:04.612 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:24:04.612 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:24:04.612 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:24:04.612 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:24:04.612 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:24:04.612 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:24:04.613 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:24:04.613 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:24:04.613 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:24:04.613 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:24:04.613 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:24:04.613 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:24:04.613 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:24:04.613 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:04.613 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:24:04.613 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:24:04.613 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:24:04.613 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:24:04.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:24:04.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:24:04.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:24:04.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:24:04.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:24:04.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:24:04.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:04.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:24:04.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:24:04.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:24:04.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:24:04.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:24:04.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:24:04.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:24:04.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:24:04.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:24:04.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:24:04.622 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:24:04.679 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:24:04.679 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:24:04.679 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:24:04.682 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:24:04.682 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:24:04.682 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:24:04.682 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:24:04.682 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:24:04.682 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:24:04.682 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:24:04.683 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:24:04.683 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:24:04.683 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:24:04.683 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:24:04.683 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:24:04.683 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:24:04.683 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:04.683 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:24:04.683 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:24:04.683 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:24:04.683 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:24:04.684 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:24:04.684 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:24:04.684 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:24:04.684 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:24:04.684 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:24:04.684 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:24:04.684 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:04.684 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:24:04.684 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:24:04.684 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:24:04.684 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:24:04.684 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:24:04.684 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:24:04.685 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:24:04.692 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:24:04.692 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:24:04.693 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:24:04.693 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:24:04.749 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:24:04.749 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:24:04.749 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:24:04.752 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:24:04.752 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:24:04.752 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:24:04.752 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:24:04.752 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:24:04.752 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:24:04.753 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:24:04.753 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:24:04.753 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:24:04.753 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:24:04.753 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:24:04.753 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:24:04.753 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:24:04.753 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:04.753 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:24:04.753 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:24:04.753 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:24:04.753 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:24:04.754 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:24:04.754 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:24:04.754 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:24:04.754 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:24:04.754 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:24:04.754 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:24:04.754 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:04.754 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:24:04.754 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:24:04.754 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:24:04.754 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:24:04.754 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:24:04.755 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:24:04.755 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:24:04.762 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:24:04.763 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:24:04.763 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:24:04.763 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:24:04.820 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:24:04.820 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:24:04.820 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:24:04.824 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:24:04.824 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:24:04.824 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:24:04.824 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:24:04.824 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:24:04.824 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:24:04.824 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:24:04.824 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:24:04.824 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:24:04.825 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:24:04.825 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:24:04.825 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:24:04.825 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:24:04.825 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:04.825 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:24:04.825 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:24:04.825 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:24:04.825 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:24:04.825 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:24:04.826 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:24:04.826 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:24:04.826 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:24:04.826 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:24:04.826 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:24:04.826 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:04.826 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:24:04.826 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:24:04.826 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:24:04.826 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:24:04.826 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:24:04.827 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:24:04.827 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:24:04.835 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:24:04.835 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:24:04.835 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:24:04.835 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:24:04.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:24:04.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:24:04.904 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:24:04.904 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:24:04.904 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:24:04.904 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:24:04.904 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:24:04.904 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:24:04.904 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:24:04.904 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:24:04.904 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:24:04.904 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:24:04.904 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:24:04.905 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:24:04.905 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:24:04.905 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:24:04.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 103 2026-02-18T08:24:04.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 106 2026-02-18T08:24:04.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:24:04.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=425, tag_tid=30, entry_tid=1], [object_number=424, tag_tid=30, entry_tid=0], [object_number=427, tag_tid=29, entry_tid=45931], [object_number=426, tag_tid=29, entry_tid=45930]]], state=connected] 2026-02-18T08:24:04.944 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=415, tag_tid=29, entry_tid=33783], [object_number=414, tag_tid=29, entry_tid=33782], [object_number=413, tag_tid=29, entry_tid=33781], [object_number=412, tag_tid=29, entry_tid=33780]]], state=connected] 2026-02-18T08:24:04.947 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:24:04.949 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=425, tag_tid=30, entry_tid=1' 2026-02-18T08:24:04.950 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:24:04.950 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:24:04.950 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:24:04.950 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:24:04.950 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:24:04.950 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:24:04.950 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:24:04.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:24:04.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:24:04.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:24:04.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:24:04.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:24:04.952 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:24:04.952 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:24:04.952 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:24:04.990 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 103 2026-02-18T08:24:04.990 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 106 2026-02-18T08:24:04.990 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:24:04.990 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=425, tag_tid=30, entry_tid=1], [object_number=424, tag_tid=30, entry_tid=0], [object_number=427, tag_tid=29, entry_tid=45931], [object_number=426, tag_tid=29, entry_tid=45930]]], state=connected] 2026-02-18T08:24:04.990 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=415, tag_tid=29, entry_tid=33783], [object_number=414, tag_tid=29, entry_tid=33782], [object_number=413, tag_tid=29, entry_tid=33781], [object_number=412, tag_tid=29, entry_tid=33780]]], state=connected] 2026-02-18T08:24:04.994 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:24:04.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=415, tag_tid=29, entry_tid=33783' 2026-02-18T08:24:04.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=425, tag_tid=30, entry_tid=1' -a 'object_number=425, tag_tid=30, entry_tid=1' = 'object_number=415, tag_tid=29, entry_tid=33783' 2026-02-18T08:24:04.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=415, tag_tid=29, entry_tid=33783' '!=' 'object_number=415, tag_tid=29, entry_tid=33435' 2026-02-18T08:24:04.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:24:04.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=415, tag_tid=29, entry_tid=33783' = 'object_number=415, tag_tid=29, entry_tid=33435' 2026-02-18T08:24:04.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=415, tag_tid=29, entry_tid=33783' 2026-02-18T08:24:04.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=425, tag_tid=30, entry_tid=1' 2026-02-18T08:24:04.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:24:04.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:24:04.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=30 2026-02-18T08:24:04.999 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=415, tag_tid=29, entry_tid=33783' 2026-02-18T08:24:04.999 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:24:04.999 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:24:05.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=29 2026-02-18T08:24:05.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=425, tag_tid=30, entry_tid=1' 2026-02-18T08:24:05.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:24:05.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:24:05.003 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:24:05.004 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=415, tag_tid=29, entry_tid=33783' 2026-02-18T08:24:05.004 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:24:05.004 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:24:05.006 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=33783 2026-02-18T08:24:05.006 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 30 = 29 -a 1 -le 33783 2026-02-18T08:24:05.006 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:24:05.006 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:24:05.006 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:24:05.207 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:24:05.207 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:24:05.207 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:24:05.208 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:24:05.208 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:24:05.208 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:24:05.208 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:24:05.208 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:24:05.208 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:24:05.208 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:24:05.208 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:24:05.208 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:24:05.208 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:24:05.208 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:24:05.209 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:24:05.211 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:24:05.212 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:24:05.212 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:24:05.212 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:24:05.212 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:24:05.212 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:24:05.212 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:24:05.212 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:24:05.212 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:24:05.212 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:24:05.212 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:24:05.212 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:24:05.213 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:24:05.213 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:24:05.213 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:24:05.213 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:05.213 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:24:05.213 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:24:05.213 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:24:05.213 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:24:05.213 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:24:05.213 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:24:05.213 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:24:05.213 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:24:05.213 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:24:05.214 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:24:05.214 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:05.214 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:24:05.214 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:24:05.214 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:24:05.214 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:24:05.214 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:24:05.214 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:24:05.214 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:24:05.221 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:24:05.221 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:24:05.222 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:24:05.222 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:24:05.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:24:05.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:24:05.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:24:05.274 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:24:05.274 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:24:05.275 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:24:05.275 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:24:05.275 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:24:05.275 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:24:05.275 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:24:05.275 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:24:05.275 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:24:05.275 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:24:05.275 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:24:05.275 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:24:05.276 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:24:05.276 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:05.276 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:24:05.276 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:24:05.276 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:24:05.276 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:24:05.276 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:24:05.276 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:24:05.276 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:24:05.276 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:24:05.276 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:24:05.276 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:24:05.277 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:05.277 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:24:05.277 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:24:05.277 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:24:05.277 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:24:05.277 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:24:05.277 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:24:05.277 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:24:05.284 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:24:05.285 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:24:05.285 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:24:05.285 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:24:05.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:24:05.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:24:05.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:24:05.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:24:05.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:24:05.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:24:05.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:24:05.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:24:05.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:24:05.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:24:05.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:24:05.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:24:05.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:24:05.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:24:05.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:24:05.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:24:05.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:05.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:24:05.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:24:05.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:24:05.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:24:05.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:24:05.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:24:05.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:24:05.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:24:05.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:24:05.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:24:05.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:05.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:24:05.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:24:05.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:24:05.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:24:05.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:24:05.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:24:05.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:24:05.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:24:05.348 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:24:05.348 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:24:05.348 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:24:05.397 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:24:05.397 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:24:05.397 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:24:05.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:24:05.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:24:05.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:24:05.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:24:05.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:24:05.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:24:05.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:24:05.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:24:05.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:24:05.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:24:05.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:24:05.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:24:05.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:24:05.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:05.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:24:05.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:24:05.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:24:05.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:24:05.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:24:05.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:24:05.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:24:05.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:24:05.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:24:05.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:24:05.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:05.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:24:05.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:24:05.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:24:05.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:24:05.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:24:05.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:24:05.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:24:05.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:24:05.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:24:05.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:24:05.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:24:05.473 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:24:05.474 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:24:05.474 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:24:05.474 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:24:05.474 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:24:05.474 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:24:05.474 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:24:05.474 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:24:05.474 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:24:05.474 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:24:05.474 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:24:05.474 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:24:05.474 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:24:05.476 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:24:05.476 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:24:05.476 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:24:05.504 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 103 2026-02-18T08:24:05.505 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 106 2026-02-18T08:24:05.505 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:24:05.505 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=425, tag_tid=30, entry_tid=1], [object_number=424, tag_tid=30, entry_tid=0], [object_number=427, tag_tid=29, entry_tid=45931], [object_number=426, tag_tid=29, entry_tid=45930]]], state=connected] 2026-02-18T08:24:05.505 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=414, tag_tid=29, entry_tid=34162], [object_number=413, tag_tid=29, entry_tid=34161], [object_number=412, tag_tid=29, entry_tid=34160], [object_number=415, tag_tid=29, entry_tid=34159]]], state=connected] 2026-02-18T08:24:05.507 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:24:05.508 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=425, tag_tid=30, entry_tid=1' 2026-02-18T08:24:05.509 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:24:05.509 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:24:05.509 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:24:05.509 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:24:05.509 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:24:05.509 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:24:05.509 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:24:05.509 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:24:05.509 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:24:05.509 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:24:05.510 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:24:05.510 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:24:05.511 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:24:05.512 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:24:05.512 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:24:05.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 103 2026-02-18T08:24:05.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 106 2026-02-18T08:24:05.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:24:05.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=425, tag_tid=30, entry_tid=1], [object_number=424, tag_tid=30, entry_tid=0], [object_number=427, tag_tid=29, entry_tid=45931], [object_number=426, tag_tid=29, entry_tid=45930]]], state=connected] 2026-02-18T08:24:05.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=414, tag_tid=29, entry_tid=34162], [object_number=413, tag_tid=29, entry_tid=34161], [object_number=412, tag_tid=29, entry_tid=34160], [object_number=415, tag_tid=29, entry_tid=34159]]], state=connected] 2026-02-18T08:24:05.545 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:24:05.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=414, tag_tid=29, entry_tid=34162' 2026-02-18T08:24:05.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=425, tag_tid=30, entry_tid=1' -a 'object_number=425, tag_tid=30, entry_tid=1' = 'object_number=414, tag_tid=29, entry_tid=34162' 2026-02-18T08:24:05.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=414, tag_tid=29, entry_tid=34162' '!=' 'object_number=415, tag_tid=29, entry_tid=33783' 2026-02-18T08:24:05.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:24:05.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=414, tag_tid=29, entry_tid=34162' = 'object_number=415, tag_tid=29, entry_tid=33783' 2026-02-18T08:24:05.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=414, tag_tid=29, entry_tid=34162' 2026-02-18T08:24:05.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=425, tag_tid=30, entry_tid=1' 2026-02-18T08:24:05.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:24:05.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:24:05.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=30 2026-02-18T08:24:05.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=414, tag_tid=29, entry_tid=34162' 2026-02-18T08:24:05.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:24:05.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:24:05.551 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=29 2026-02-18T08:24:05.551 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=425, tag_tid=30, entry_tid=1' 2026-02-18T08:24:05.552 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:24:05.552 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:24:05.553 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:24:05.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=414, tag_tid=29, entry_tid=34162' 2026-02-18T08:24:05.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:24:05.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:24:05.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=34162 2026-02-18T08:24:05.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 30 = 29 -a 1 -le 34162 2026-02-18T08:24:05.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:24:05.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:24:05.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:24:05.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:24:05.758 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:24:05.758 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:24:05.758 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:24:05.758 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:24:05.758 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:24:05.758 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:24:05.758 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:24:05.758 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:24:05.758 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:24:05.758 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:24:05.759 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:24:05.759 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:24:05.759 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:24:05.759 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:24:05.762 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:24:05.762 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:24:05.762 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:24:05.762 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:24:05.762 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:24:05.762 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:24:05.762 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:24:05.762 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:24:05.762 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:24:05.763 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:24:05.763 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:24:05.763 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:24:05.763 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:24:05.763 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:24:05.763 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:24:05.763 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:05.763 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:24:05.763 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:24:05.763 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:24:05.763 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:24:05.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:24:05.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:24:05.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:24:05.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:24:05.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:24:05.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:24:05.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:05.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:24:05.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:24:05.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:24:05.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:24:05.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:24:05.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:24:05.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:24:05.771 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:24:05.772 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:24:05.772 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:24:05.772 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:24:05.822 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:24:05.822 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:24:05.822 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:24:05.825 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:24:05.825 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:24:05.825 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:24:05.825 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:24:05.825 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:24:05.825 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:24:05.825 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:24:05.825 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:24:05.825 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:24:05.825 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:24:05.826 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:24:05.826 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:24:05.826 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:24:05.826 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:05.826 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:24:05.826 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:24:05.826 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:24:05.826 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:24:05.826 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:24:05.826 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:24:05.826 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:24:05.826 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:24:05.827 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:24:05.827 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:24:05.827 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:05.827 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:24:05.827 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:24:05.827 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:24:05.827 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:24:05.827 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:24:05.827 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:24:05.827 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:24:05.836 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:24:05.836 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:24:05.836 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:24:05.836 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:24:05.886 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:24:05.886 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:24:05.886 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:24:05.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:24:05.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:24:05.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:24:05.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:24:05.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:24:05.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:24:05.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:24:05.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:24:05.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:24:05.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:24:05.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:24:05.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:24:05.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:24:05.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:05.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:24:05.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:24:05.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:24:05.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:24:05.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:24:05.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:24:05.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:24:05.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:24:05.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:24:05.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:24:05.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:05.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:24:05.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:24:05.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:24:05.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:24:05.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:24:05.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:24:05.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:24:05.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:24:05.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:24:05.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:24:05.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:24:05.950 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:24:05.950 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:24:05.950 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:24:05.953 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:24:05.953 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:24:05.953 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:24:05.953 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:24:05.954 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:24:05.954 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:24:05.954 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:24:05.954 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:24:05.954 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:24:05.954 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:24:05.954 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:24:05.954 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:24:05.954 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:24:05.954 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:05.955 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:24:05.955 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:24:05.955 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:24:05.955 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:24:05.955 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:24:05.955 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:24:05.955 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:24:05.955 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:24:05.955 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:24:05.955 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:24:05.955 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:05.955 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:24:05.955 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:24:05.956 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:24:05.956 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:24:05.956 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:24:05.956 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:24:05.956 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:24:05.963 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:24:05.964 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:24:05.964 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:24:05.964 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:24:06.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:24:06.027 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:24:06.027 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:24:06.027 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:24:06.027 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:24:06.027 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:24:06.027 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:24:06.027 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:24:06.027 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:24:06.028 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:24:06.028 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:24:06.028 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:24:06.028 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:24:06.029 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:24:06.029 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:24:06.029 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:24:06.059 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 103 2026-02-18T08:24:06.059 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 106 2026-02-18T08:24:06.059 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:24:06.059 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=425, tag_tid=30, entry_tid=1], [object_number=424, tag_tid=30, entry_tid=0], [object_number=427, tag_tid=29, entry_tid=45931], [object_number=426, tag_tid=29, entry_tid=45930]]], state=connected] 2026-02-18T08:24:06.059 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=414, tag_tid=29, entry_tid=34506], [object_number=413, tag_tid=29, entry_tid=34505], [object_number=412, tag_tid=29, entry_tid=34504], [object_number=415, tag_tid=29, entry_tid=34503]]], state=connected] 2026-02-18T08:24:06.062 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:24:06.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=425, tag_tid=30, entry_tid=1' 2026-02-18T08:24:06.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:24:06.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:24:06.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:24:06.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:24:06.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:24:06.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:24:06.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:24:06.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:24:06.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:24:06.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:24:06.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:24:06.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:24:06.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:24:06.067 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:24:06.067 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:24:06.095 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 103 2026-02-18T08:24:06.095 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 106 2026-02-18T08:24:06.095 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:24:06.095 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=425, tag_tid=30, entry_tid=1], [object_number=424, tag_tid=30, entry_tid=0], [object_number=427, tag_tid=29, entry_tid=45931], [object_number=426, tag_tid=29, entry_tid=45930]]], state=connected] 2026-02-18T08:24:06.096 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=414, tag_tid=29, entry_tid=34506], [object_number=413, tag_tid=29, entry_tid=34505], [object_number=412, tag_tid=29, entry_tid=34504], [object_number=415, tag_tid=29, entry_tid=34503]]], state=connected] 2026-02-18T08:24:06.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:24:06.100 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=414, tag_tid=29, entry_tid=34506' 2026-02-18T08:24:06.100 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=425, tag_tid=30, entry_tid=1' -a 'object_number=425, tag_tid=30, entry_tid=1' = 'object_number=414, tag_tid=29, entry_tid=34506' 2026-02-18T08:24:06.100 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=414, tag_tid=29, entry_tid=34506' '!=' 'object_number=414, tag_tid=29, entry_tid=34162' 2026-02-18T08:24:06.100 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:24:06.100 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=414, tag_tid=29, entry_tid=34506' = 'object_number=414, tag_tid=29, entry_tid=34162' 2026-02-18T08:24:06.100 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=414, tag_tid=29, entry_tid=34506' 2026-02-18T08:24:06.100 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=425, tag_tid=30, entry_tid=1' 2026-02-18T08:24:06.101 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:24:06.101 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:24:06.102 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=30 2026-02-18T08:24:06.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=414, tag_tid=29, entry_tid=34506' 2026-02-18T08:24:06.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:24:06.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:24:06.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=29 2026-02-18T08:24:06.105 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=425, tag_tid=30, entry_tid=1' 2026-02-18T08:24:06.105 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:24:06.105 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:24:06.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:24:06.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=414, tag_tid=29, entry_tid=34506' 2026-02-18T08:24:06.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:24:06.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:24:06.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=34506 2026-02-18T08:24:06.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 30 = 29 -a 1 -le 34506 2026-02-18T08:24:06.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:24:06.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:24:06.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:24:06.310 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:24:06.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:24:06.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:24:06.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:24:06.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:24:06.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:24:06.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:24:06.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:24:06.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:24:06.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:24:06.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:24:06.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:24:06.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:24:06.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:24:06.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:24:06.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:24:06.312 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:24:06.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:24:06.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:24:06.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:24:06.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:24:06.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:24:06.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:24:06.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:24:06.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:24:06.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:24:06.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:24:06.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:24:06.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:24:06.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:24:06.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:06.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:24:06.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:24:06.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:24:06.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:24:06.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:24:06.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:24:06.314 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:24:06.314 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:24:06.314 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:24:06.314 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:24:06.314 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:06.314 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:24:06.314 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:24:06.314 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:24:06.314 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:24:06.314 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:24:06.314 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:24:06.314 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:24:06.326 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:24:06.326 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:24:06.326 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:24:06.326 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:24:06.376 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:24:06.376 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:24:06.376 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:24:06.377 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:24:06.377 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:24:06.377 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:24:06.377 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:24:06.377 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:24:06.377 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:24:06.377 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:24:06.377 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:24:06.377 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:24:06.377 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:24:06.377 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:24:06.377 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:24:06.377 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:24:06.377 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:06.377 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:24:06.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:24:06.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:24:06.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:24:06.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:24:06.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:24:06.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:24:06.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:24:06.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:24:06.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:24:06.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:06.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:24:06.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:24:06.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:24:06.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:24:06.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:24:06.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:24:06.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:24:06.390 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:24:06.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:24:06.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:24:06.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:24:06.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:24:06.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:24:06.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:24:06.448 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:24:06.448 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:24:06.449 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:24:06.449 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:24:06.449 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:24:06.449 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:24:06.449 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:24:06.449 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:24:06.449 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:24:06.449 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:24:06.449 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:24:06.449 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:24:06.449 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:24:06.449 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:06.449 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:24:06.449 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:24:06.449 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:24:06.449 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:24:06.449 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:24:06.449 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:24:06.449 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:24:06.449 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:24:06.449 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:24:06.449 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:24:06.450 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:06.450 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:24:06.450 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:24:06.450 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:24:06.450 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:24:06.450 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:24:06.450 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:24:06.450 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:24:06.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:24:06.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:24:06.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:24:06.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:24:06.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:24:06.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:24:06.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:24:06.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:24:06.523 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:24:06.523 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:24:06.523 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:24:06.523 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:24:06.523 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:24:06.523 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:24:06.523 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:24:06.523 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:24:06.523 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:24:06.523 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:24:06.523 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:24:06.523 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:24:06.524 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:06.524 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:24:06.524 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:24:06.524 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:24:06.524 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:24:06.524 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:24:06.524 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:24:06.524 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:24:06.524 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:24:06.524 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:24:06.524 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:24:06.525 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:06.525 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:24:06.525 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:24:06.525 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:24:06.525 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:24:06.525 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:24:06.525 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:24:06.525 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:24:06.534 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:24:06.534 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:24:06.534 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:24:06.534 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:24:06.596 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:24:06.597 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:24:06.597 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:24:06.597 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:24:06.597 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:24:06.597 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:24:06.597 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:24:06.597 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:24:06.598 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:24:06.598 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:24:06.598 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:24:06.598 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:24:06.598 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:24:06.599 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:24:06.599 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:24:06.599 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:24:06.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 103 2026-02-18T08:24:06.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 106 2026-02-18T08:24:06.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:24:06.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=425, tag_tid=30, entry_tid=1], [object_number=424, tag_tid=30, entry_tid=0], [object_number=427, tag_tid=29, entry_tid=45931], [object_number=426, tag_tid=29, entry_tid=45930]]], state=connected] 2026-02-18T08:24:06.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=415, tag_tid=29, entry_tid=34887], [object_number=414, tag_tid=29, entry_tid=34886], [object_number=413, tag_tid=29, entry_tid=34885], [object_number=412, tag_tid=29, entry_tid=34884]]], state=connected] 2026-02-18T08:24:06.629 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:24:06.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=425, tag_tid=30, entry_tid=1' 2026-02-18T08:24:06.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:24:06.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:24:06.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:24:06.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:24:06.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:24:06.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:24:06.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:24:06.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:24:06.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:24:06.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:24:06.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:24:06.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:24:06.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:24:06.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:24:06.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:24:06.662 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 103 2026-02-18T08:24:06.662 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 106 2026-02-18T08:24:06.662 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:24:06.662 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=425, tag_tid=30, entry_tid=1], [object_number=424, tag_tid=30, entry_tid=0], [object_number=427, tag_tid=29, entry_tid=45931], [object_number=426, tag_tid=29, entry_tid=45930]]], state=connected] 2026-02-18T08:24:06.662 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=415, tag_tid=29, entry_tid=34887], [object_number=414, tag_tid=29, entry_tid=34886], [object_number=413, tag_tid=29, entry_tid=34885], [object_number=412, tag_tid=29, entry_tid=34884]]], state=connected] 2026-02-18T08:24:06.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:24:06.667 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=415, tag_tid=29, entry_tid=34887' 2026-02-18T08:24:06.667 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=425, tag_tid=30, entry_tid=1' -a 'object_number=425, tag_tid=30, entry_tid=1' = 'object_number=415, tag_tid=29, entry_tid=34887' 2026-02-18T08:24:06.667 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=415, tag_tid=29, entry_tid=34887' '!=' 'object_number=414, tag_tid=29, entry_tid=34506' 2026-02-18T08:24:06.667 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:24:06.667 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=415, tag_tid=29, entry_tid=34887' = 'object_number=414, tag_tid=29, entry_tid=34506' 2026-02-18T08:24:06.667 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=415, tag_tid=29, entry_tid=34887' 2026-02-18T08:24:06.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=425, tag_tid=30, entry_tid=1' 2026-02-18T08:24:06.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:24:06.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:24:06.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=30 2026-02-18T08:24:06.670 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=415, tag_tid=29, entry_tid=34887' 2026-02-18T08:24:06.670 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:24:06.670 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:24:06.671 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=29 2026-02-18T08:24:06.672 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=425, tag_tid=30, entry_tid=1' 2026-02-18T08:24:06.672 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:24:06.672 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:24:06.674 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:24:06.674 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=415, tag_tid=29, entry_tid=34887' 2026-02-18T08:24:06.675 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:24:06.675 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:24:06.676 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=34887 2026-02-18T08:24:06.676 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 30 = 29 -a 1 -le 34887 2026-02-18T08:24:06.677 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:24:06.677 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:24:06.677 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:24:06.878 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:24:06.878 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:24:06.878 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:24:06.878 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:24:06.878 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:24:06.878 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:24:06.879 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:24:06.879 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:24:06.879 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:24:06.879 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:24:06.879 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:24:06.879 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:24:06.879 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:24:06.879 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:24:06.879 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:24:06.882 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:24:06.882 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:24:06.882 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:24:06.882 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:24:06.882 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:24:06.882 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:24:06.883 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:24:06.883 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:24:06.883 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:24:06.883 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:24:06.883 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:24:06.883 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:24:06.883 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:24:06.883 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:24:06.883 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:24:06.883 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:06.883 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:24:06.883 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:24:06.884 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:24:06.884 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:24:06.884 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:24:06.884 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:24:06.884 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:24:06.884 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:24:06.884 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:24:06.884 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:24:06.884 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:06.884 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:24:06.884 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:24:06.884 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:24:06.885 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:24:06.885 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:24:06.885 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:24:06.885 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:24:06.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:24:06.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:24:06.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:24:06.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:24:06.949 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:24:06.949 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:24:06.950 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:24:06.952 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:24:06.952 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:24:06.953 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:24:06.953 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:24:06.953 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:24:06.953 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:24:06.953 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:24:06.953 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:24:06.953 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:24:06.953 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:24:06.953 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:24:06.953 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:24:06.953 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:24:06.954 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:06.954 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:24:06.954 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:24:06.954 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:24:06.954 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:24:06.954 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:24:06.954 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:24:06.954 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:24:06.954 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:24:06.954 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:24:06.954 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:24:06.954 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:06.954 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:24:06.955 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:24:06.955 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:24:06.955 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:24:06.955 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:24:06.955 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:24:06.955 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:24:06.964 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:24:06.964 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:24:06.964 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:24:06.964 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:24:07.021 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:24:07.021 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:24:07.021 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:24:07.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:24:07.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:24:07.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:24:07.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:24:07.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:24:07.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:24:07.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:24:07.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:24:07.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:24:07.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:24:07.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:24:07.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:24:07.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:24:07.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:07.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:24:07.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:24:07.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:24:07.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:24:07.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:24:07.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:24:07.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:24:07.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:24:07.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:24:07.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:24:07.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:07.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:24:07.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:24:07.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:24:07.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:24:07.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:24:07.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:24:07.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:24:07.036 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:24:07.036 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:24:07.036 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:24:07.036 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:24:07.092 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:24:07.092 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:24:07.092 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:24:07.095 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:24:07.095 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:24:07.095 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:24:07.095 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:24:07.095 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:24:07.095 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:24:07.095 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:24:07.095 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:24:07.095 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:24:07.095 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:24:07.096 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:24:07.096 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:24:07.096 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:24:07.096 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:07.096 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:24:07.096 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:24:07.096 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:24:07.096 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:24:07.096 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:24:07.096 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:24:07.096 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:24:07.096 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:24:07.097 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:24:07.097 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:24:07.097 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:07.097 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:24:07.097 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:24:07.097 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:24:07.097 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:24:07.097 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:24:07.097 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:24:07.097 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:24:07.106 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:24:07.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:24:07.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:24:07.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:24:07.166 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:24:07.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:24:07.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:24:07.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:24:07.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:24:07.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:24:07.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:24:07.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:24:07.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:24:07.169 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:24:07.169 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:24:07.169 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:24:07.169 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:24:07.169 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:24:07.170 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:24:07.170 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:24:07.208 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 103 2026-02-18T08:24:07.209 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 106 2026-02-18T08:24:07.209 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:24:07.209 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=425, tag_tid=30, entry_tid=1], [object_number=424, tag_tid=30, entry_tid=0], [object_number=427, tag_tid=29, entry_tid=45931], [object_number=426, tag_tid=29, entry_tid=45930]]], state=connected] 2026-02-18T08:24:07.209 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=415, tag_tid=29, entry_tid=35251], [object_number=414, tag_tid=29, entry_tid=35250], [object_number=413, tag_tid=29, entry_tid=35249], [object_number=412, tag_tid=29, entry_tid=35248]]], state=connected] 2026-02-18T08:24:07.212 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:24:07.214 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=425, tag_tid=30, entry_tid=1' 2026-02-18T08:24:07.215 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:24:07.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:24:07.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:24:07.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:24:07.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:24:07.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:24:07.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:24:07.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:24:07.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:24:07.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:24:07.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:24:07.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:24:07.217 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:24:07.218 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:24:07.218 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:24:07.256 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 103 2026-02-18T08:24:07.256 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 106 2026-02-18T08:24:07.256 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:24:07.256 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=425, tag_tid=30, entry_tid=1], [object_number=424, tag_tid=30, entry_tid=0], [object_number=427, tag_tid=29, entry_tid=45931], [object_number=426, tag_tid=29, entry_tid=45930]]], state=connected] 2026-02-18T08:24:07.256 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=415, tag_tid=29, entry_tid=35251], [object_number=414, tag_tid=29, entry_tid=35250], [object_number=413, tag_tid=29, entry_tid=35249], [object_number=412, tag_tid=29, entry_tid=35248]]], state=connected] 2026-02-18T08:24:07.260 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:24:07.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=415, tag_tid=29, entry_tid=35251' 2026-02-18T08:24:07.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=425, tag_tid=30, entry_tid=1' -a 'object_number=425, tag_tid=30, entry_tid=1' = 'object_number=415, tag_tid=29, entry_tid=35251' 2026-02-18T08:24:07.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=415, tag_tid=29, entry_tid=35251' '!=' 'object_number=415, tag_tid=29, entry_tid=34887' 2026-02-18T08:24:07.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:24:07.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=415, tag_tid=29, entry_tid=35251' = 'object_number=415, tag_tid=29, entry_tid=34887' 2026-02-18T08:24:07.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=415, tag_tid=29, entry_tid=35251' 2026-02-18T08:24:07.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=425, tag_tid=30, entry_tid=1' 2026-02-18T08:24:07.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:24:07.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:24:07.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=30 2026-02-18T08:24:07.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=415, tag_tid=29, entry_tid=35251' 2026-02-18T08:24:07.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:24:07.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:24:07.266 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=29 2026-02-18T08:24:07.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=425, tag_tid=30, entry_tid=1' 2026-02-18T08:24:07.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:24:07.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:24:07.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:24:07.269 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=415, tag_tid=29, entry_tid=35251' 2026-02-18T08:24:07.269 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:24:07.269 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:24:07.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=35251 2026-02-18T08:24:07.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 30 = 29 -a 1 -le 35251 2026-02-18T08:24:07.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:24:07.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:24:07.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:24:07.473 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:24:07.473 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:24:07.473 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:24:07.473 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:24:07.473 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:24:07.473 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:24:07.474 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:24:07.474 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:24:07.474 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:24:07.474 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:24:07.474 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:24:07.474 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:24:07.474 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:24:07.474 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:24:07.474 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:24:07.474 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:24:07.477 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:24:07.477 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:24:07.477 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:24:07.477 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:24:07.477 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:24:07.478 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:24:07.478 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:24:07.478 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:24:07.478 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:24:07.478 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:24:07.478 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:24:07.478 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:24:07.478 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:24:07.478 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:24:07.478 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:07.478 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:24:07.478 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:24:07.479 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:24:07.479 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:24:07.479 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:24:07.479 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:24:07.479 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:24:07.479 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:24:07.479 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:24:07.479 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:24:07.479 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:07.479 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:24:07.479 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:24:07.479 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:24:07.480 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:24:07.480 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:24:07.480 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:24:07.480 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:24:07.488 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:24:07.488 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:24:07.488 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:24:07.488 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:24:07.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:24:07.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:24:07.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:24:07.540 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:24:07.540 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:24:07.540 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:24:07.540 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:24:07.540 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:24:07.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:24:07.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:24:07.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:24:07.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:24:07.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:24:07.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:24:07.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:24:07.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:24:07.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:07.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:24:07.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:24:07.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:24:07.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:24:07.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:24:07.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:24:07.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:24:07.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:24:07.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:24:07.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:24:07.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:07.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:24:07.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:24:07.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:24:07.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:24:07.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:24:07.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:24:07.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:24:07.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:24:07.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:24:07.551 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:24:07.551 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:24:07.602 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:24:07.602 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:24:07.602 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:24:07.605 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:24:07.605 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:24:07.605 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:24:07.606 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:24:07.606 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:24:07.606 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:24:07.606 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:24:07.606 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:24:07.606 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:24:07.606 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:24:07.606 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:24:07.606 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:24:07.606 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:24:07.606 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:07.607 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:24:07.607 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:24:07.607 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:24:07.607 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:24:07.607 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:24:07.607 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:24:07.607 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:24:07.607 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:24:07.607 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:24:07.607 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:24:07.607 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:07.607 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:24:07.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:24:07.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:24:07.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:24:07.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:24:07.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:24:07.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:24:07.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:24:07.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:24:07.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:24:07.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:24:07.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:24:07.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:24:07.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:24:07.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:24:07.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:24:07.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:24:07.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:24:07.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:24:07.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:24:07.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:24:07.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:24:07.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:24:07.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:24:07.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:24:07.670 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:24:07.670 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:24:07.670 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:07.670 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:24:07.670 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:24:07.670 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:24:07.670 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:24:07.670 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:24:07.670 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:24:07.670 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:24:07.670 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:24:07.670 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:24:07.671 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:24:07.671 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:07.671 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:24:07.671 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:24:07.671 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:24:07.671 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:24:07.671 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:24:07.671 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:24:07.671 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:24:07.679 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:24:07.679 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:24:07.680 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:24:07.680 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:24:07.741 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:24:07.742 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:24:07.743 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:24:07.743 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:24:07.743 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:24:07.743 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:24:07.743 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:24:07.743 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:24:07.743 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:24:07.743 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:24:07.743 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:24:07.744 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:24:07.744 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:24:07.744 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:24:07.745 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:24:07.745 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:24:07.782 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 103 2026-02-18T08:24:07.782 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 106 2026-02-18T08:24:07.782 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:24:07.782 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=425, tag_tid=30, entry_tid=1], [object_number=424, tag_tid=30, entry_tid=0], [object_number=427, tag_tid=29, entry_tid=45931], [object_number=426, tag_tid=29, entry_tid=45930]]], state=connected] 2026-02-18T08:24:07.782 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=415, tag_tid=29, entry_tid=35603], [object_number=414, tag_tid=29, entry_tid=35602], [object_number=413, tag_tid=29, entry_tid=35601], [object_number=412, tag_tid=29, entry_tid=35600]]], state=connected] 2026-02-18T08:24:07.786 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:24:07.787 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=425, tag_tid=30, entry_tid=1' 2026-02-18T08:24:07.788 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:24:07.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:24:07.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:24:07.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:24:07.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:24:07.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:24:07.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:24:07.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:24:07.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:24:07.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:24:07.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:24:07.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:24:07.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:24:07.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:24:07.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:24:07.826 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 103 2026-02-18T08:24:07.826 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 106 2026-02-18T08:24:07.826 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:24:07.826 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=425, tag_tid=30, entry_tid=1], [object_number=424, tag_tid=30, entry_tid=0], [object_number=427, tag_tid=29, entry_tid=45931], [object_number=426, tag_tid=29, entry_tid=45930]]], state=connected] 2026-02-18T08:24:07.826 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=415, tag_tid=29, entry_tid=35603], [object_number=414, tag_tid=29, entry_tid=35602], [object_number=413, tag_tid=29, entry_tid=35601], [object_number=412, tag_tid=29, entry_tid=35600]]], state=connected] 2026-02-18T08:24:07.830 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:24:07.832 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=415, tag_tid=29, entry_tid=35603' 2026-02-18T08:24:07.832 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=425, tag_tid=30, entry_tid=1' -a 'object_number=425, tag_tid=30, entry_tid=1' = 'object_number=415, tag_tid=29, entry_tid=35603' 2026-02-18T08:24:07.832 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=415, tag_tid=29, entry_tid=35603' '!=' 'object_number=415, tag_tid=29, entry_tid=35251' 2026-02-18T08:24:07.832 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:24:07.832 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=415, tag_tid=29, entry_tid=35603' = 'object_number=415, tag_tid=29, entry_tid=35251' 2026-02-18T08:24:07.832 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=415, tag_tid=29, entry_tid=35603' 2026-02-18T08:24:07.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=425, tag_tid=30, entry_tid=1' 2026-02-18T08:24:07.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:24:07.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:24:07.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=30 2026-02-18T08:24:07.835 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=415, tag_tid=29, entry_tid=35603' 2026-02-18T08:24:07.835 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:24:07.835 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:24:07.836 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=29 2026-02-18T08:24:07.837 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=425, tag_tid=30, entry_tid=1' 2026-02-18T08:24:07.838 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:24:07.838 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:24:07.838 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:24:07.839 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=415, tag_tid=29, entry_tid=35603' 2026-02-18T08:24:07.839 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:24:07.840 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:24:07.841 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=35603 2026-02-18T08:24:07.841 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 30 = 29 -a 1 -le 35603 2026-02-18T08:24:07.842 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:24:07.842 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:24:07.842 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:24:08.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:24:08.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:24:08.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:24:08.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:24:08.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:24:08.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:24:08.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:24:08.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:24:08.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:24:08.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:24:08.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:24:08.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:24:08.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:24:08.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:24:08.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:24:08.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:24:08.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:24:08.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:24:08.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:24:08.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:24:08.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:24:08.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:24:08.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:24:08.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:24:08.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:24:08.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:24:08.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:24:08.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:24:08.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:24:08.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:24:08.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:08.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:24:08.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:24:08.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:24:08.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:24:08.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:24:08.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:24:08.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:24:08.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:24:08.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:24:08.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:24:08.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:08.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:24:08.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:24:08.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:24:08.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:24:08.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:24:08.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:24:08.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:24:08.058 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:24:08.059 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:24:08.059 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:24:08.059 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:24:08.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:24:08.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:24:08.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:24:08.111 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:24:08.111 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:24:08.111 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:24:08.111 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:24:08.111 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:24:08.111 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:24:08.111 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:24:08.111 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:24:08.111 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:24:08.112 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:24:08.112 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:24:08.112 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:24:08.112 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:24:08.112 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:08.112 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:24:08.112 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:24:08.112 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:24:08.112 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:24:08.112 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:24:08.112 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:24:08.113 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:24:08.113 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:24:08.113 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:24:08.113 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:24:08.113 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:08.113 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:24:08.113 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:24:08.113 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:24:08.113 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:24:08.113 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:24:08.113 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:24:08.113 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:24:08.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:24:08.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:24:08.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:24:08.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:24:08.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:24:08.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:24:08.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:24:08.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:24:08.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:24:08.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:24:08.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:24:08.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:24:08.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:24:08.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:24:08.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:24:08.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:24:08.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:24:08.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:24:08.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:24:08.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:24:08.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:08.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:24:08.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:24:08.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:24:08.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:24:08.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:24:08.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:24:08.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:24:08.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:24:08.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:24:08.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:24:08.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:08.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:24:08.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:24:08.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:24:08.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:24:08.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:24:08.187 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:24:08.187 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:24:08.199 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:24:08.199 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:24:08.199 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:24:08.199 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:24:08.255 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:24:08.255 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:24:08.255 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:24:08.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:24:08.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:24:08.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:24:08.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:24:08.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:24:08.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:24:08.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:24:08.259 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:24:08.259 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:24:08.259 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:24:08.259 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:24:08.259 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:24:08.259 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:24:08.259 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:08.259 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:24:08.259 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:24:08.259 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:24:08.259 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:24:08.259 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:24:08.260 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:24:08.260 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:24:08.260 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:24:08.260 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:24:08.260 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:24:08.260 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:08.260 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:24:08.260 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:24:08.260 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:24:08.260 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:24:08.260 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:24:08.260 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:24:08.261 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:24:08.270 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:24:08.270 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:24:08.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:24:08.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:24:08.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:24:08.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:24:08.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:24:08.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:24:08.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:24:08.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:24:08.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:24:08.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:24:08.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:24:08.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:24:08.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:24:08.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:24:08.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:24:08.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:24:08.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:24:08.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:24:08.362 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 104 2026-02-18T08:24:08.362 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 106 2026-02-18T08:24:08.362 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:24:08.363 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=425, tag_tid=30, entry_tid=1], [object_number=424, tag_tid=30, entry_tid=0], [object_number=427, tag_tid=29, entry_tid=45931], [object_number=426, tag_tid=29, entry_tid=45930]]], state=connected] 2026-02-18T08:24:08.363 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=419, tag_tid=29, entry_tid=35859], [object_number=418, tag_tid=29, entry_tid=35858], [object_number=417, tag_tid=29, entry_tid=35857], [object_number=416, tag_tid=29, entry_tid=35856]]], state=connected] 2026-02-18T08:24:08.365 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:24:08.367 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=425, tag_tid=30, entry_tid=1' 2026-02-18T08:24:08.368 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:24:08.368 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:24:08.368 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:24:08.368 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:24:08.368 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:24:08.368 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:24:08.368 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:24:08.368 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:24:08.368 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:24:08.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:24:08.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:24:08.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:24:08.370 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:24:08.370 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:24:08.370 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:24:08.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 104 2026-02-18T08:24:08.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 106 2026-02-18T08:24:08.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:24:08.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=425, tag_tid=30, entry_tid=1], [object_number=424, tag_tid=30, entry_tid=0], [object_number=427, tag_tid=29, entry_tid=45931], [object_number=426, tag_tid=29, entry_tid=45930]]], state=connected] 2026-02-18T08:24:08.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=419, tag_tid=29, entry_tid=35859], [object_number=418, tag_tid=29, entry_tid=35858], [object_number=417, tag_tid=29, entry_tid=35857], [object_number=416, tag_tid=29, entry_tid=35856]]], state=connected] 2026-02-18T08:24:08.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:24:08.405 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=419, tag_tid=29, entry_tid=35859' 2026-02-18T08:24:08.405 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=425, tag_tid=30, entry_tid=1' -a 'object_number=425, tag_tid=30, entry_tid=1' = 'object_number=419, tag_tid=29, entry_tid=35859' 2026-02-18T08:24:08.405 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=419, tag_tid=29, entry_tid=35859' '!=' 'object_number=415, tag_tid=29, entry_tid=35603' 2026-02-18T08:24:08.405 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:24:08.405 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=419, tag_tid=29, entry_tid=35859' = 'object_number=415, tag_tid=29, entry_tid=35603' 2026-02-18T08:24:08.405 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=419, tag_tid=29, entry_tid=35859' 2026-02-18T08:24:08.406 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=425, tag_tid=30, entry_tid=1' 2026-02-18T08:24:08.406 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:24:08.406 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:24:08.407 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=30 2026-02-18T08:24:08.408 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=419, tag_tid=29, entry_tid=35859' 2026-02-18T08:24:08.408 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:24:08.408 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:24:08.409 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=29 2026-02-18T08:24:08.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=425, tag_tid=30, entry_tid=1' 2026-02-18T08:24:08.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:24:08.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:24:08.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:24:08.412 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=419, tag_tid=29, entry_tid=35859' 2026-02-18T08:24:08.412 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:24:08.412 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:24:08.413 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=35859 2026-02-18T08:24:08.413 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 30 = 29 -a 1 -le 35859 2026-02-18T08:24:08.414 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:24:08.414 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:24:08.414 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:24:08.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:24:08.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:24:08.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:24:08.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:24:08.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:24:08.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:24:08.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:24:08.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:24:08.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:24:08.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:24:08.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:24:08.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:24:08.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:24:08.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:24:08.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:24:08.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:24:08.619 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:24:08.620 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:24:08.620 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:24:08.620 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:24:08.620 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:24:08.620 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:24:08.620 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:24:08.620 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:24:08.620 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:24:08.620 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:24:08.620 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:24:08.620 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:24:08.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:24:08.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:24:08.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:08.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:24:08.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:24:08.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:24:08.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:24:08.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:24:08.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:24:08.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:24:08.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:24:08.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:24:08.622 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:24:08.622 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:08.622 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:24:08.622 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:24:08.622 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:24:08.622 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:24:08.622 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:24:08.622 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:24:08.622 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:24:08.629 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:24:08.629 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:24:08.629 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:24:08.629 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:24:08.678 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:24:08.678 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:24:08.678 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:24:08.680 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:24:08.681 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:24:08.681 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:24:08.681 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:24:08.681 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:24:08.681 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:24:08.681 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:24:08.681 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:24:08.681 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:24:08.681 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:24:08.681 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:24:08.681 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:24:08.682 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:24:08.682 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:08.682 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:24:08.682 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:24:08.682 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:24:08.682 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:24:08.682 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:24:08.682 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:24:08.682 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:24:08.682 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:24:08.682 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:24:08.682 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:24:08.683 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:08.683 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:24:08.683 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:24:08.683 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:24:08.683 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:24:08.683 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:24:08.683 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:24:08.683 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:24:08.691 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:24:08.692 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:24:08.692 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:24:08.692 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:24:08.749 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:24:08.749 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:24:08.749 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:24:08.752 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:24:08.752 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:24:08.752 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:24:08.752 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:24:08.752 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:24:08.752 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:24:08.752 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:24:08.753 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:24:08.753 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:24:08.753 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:24:08.753 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:24:08.753 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:24:08.753 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:24:08.753 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:08.753 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:24:08.753 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:24:08.753 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:24:08.753 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:24:08.753 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:24:08.753 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:24:08.754 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:24:08.754 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:24:08.754 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:24:08.754 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:24:08.754 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:08.754 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:24:08.754 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:24:08.754 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:24:08.754 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:24:08.754 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:24:08.754 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:24:08.754 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:24:08.762 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:24:08.763 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:24:08.763 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:24:08.763 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:24:08.812 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:24:08.812 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:24:08.812 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:24:08.815 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:24:08.815 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:24:08.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:24:08.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:24:08.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:24:08.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:24:08.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:24:08.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:24:08.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:24:08.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:24:08.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:24:08.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:24:08.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:24:08.817 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:08.817 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:24:08.817 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:24:08.817 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:24:08.817 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:24:08.817 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:24:08.817 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:24:08.817 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:24:08.817 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:24:08.817 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:24:08.817 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:24:08.817 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:08.817 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:24:08.818 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:24:08.818 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:24:08.818 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:24:08.818 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:24:08.818 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:24:08.818 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:24:08.826 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:24:08.827 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:24:08.827 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:24:08.827 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:24:08.887 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:24:08.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:24:08.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:24:08.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:24:08.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:24:08.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:24:08.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:24:08.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:24:08.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:24:08.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:24:08.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:24:08.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:24:08.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:24:08.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:24:08.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:24:08.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:24:08.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 104 2026-02-18T08:24:08.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 106 2026-02-18T08:24:08.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:24:08.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=425, tag_tid=30, entry_tid=1], [object_number=424, tag_tid=30, entry_tid=0], [object_number=427, tag_tid=29, entry_tid=45931], [object_number=426, tag_tid=29, entry_tid=45930]]], state=connected] 2026-02-18T08:24:08.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=418, tag_tid=29, entry_tid=36190], [object_number=417, tag_tid=29, entry_tid=36189], [object_number=416, tag_tid=29, entry_tid=36188], [object_number=419, tag_tid=29, entry_tid=36187]]], state=connected] 2026-02-18T08:24:08.923 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:24:08.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=425, tag_tid=30, entry_tid=1' 2026-02-18T08:24:08.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:24:08.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:24:08.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:24:08.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:24:08.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:24:08.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:24:08.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:24:08.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:24:08.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:24:08.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:24:08.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:24:08.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:24:08.928 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:24:08.928 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:24:08.928 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:24:08.956 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 104 2026-02-18T08:24:08.956 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 106 2026-02-18T08:24:08.956 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:24:08.956 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=425, tag_tid=30, entry_tid=1], [object_number=424, tag_tid=30, entry_tid=0], [object_number=427, tag_tid=29, entry_tid=45931], [object_number=426, tag_tid=29, entry_tid=45930]]], state=connected] 2026-02-18T08:24:08.956 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=418, tag_tid=29, entry_tid=36190], [object_number=417, tag_tid=29, entry_tid=36189], [object_number=416, tag_tid=29, entry_tid=36188], [object_number=419, tag_tid=29, entry_tid=36187]]], state=connected] 2026-02-18T08:24:08.959 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:24:08.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=418, tag_tid=29, entry_tid=36190' 2026-02-18T08:24:08.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=425, tag_tid=30, entry_tid=1' -a 'object_number=425, tag_tid=30, entry_tid=1' = 'object_number=418, tag_tid=29, entry_tid=36190' 2026-02-18T08:24:08.961 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=418, tag_tid=29, entry_tid=36190' '!=' 'object_number=419, tag_tid=29, entry_tid=35859' 2026-02-18T08:24:08.961 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:24:08.961 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=418, tag_tid=29, entry_tid=36190' = 'object_number=419, tag_tid=29, entry_tid=35859' 2026-02-18T08:24:08.961 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=418, tag_tid=29, entry_tid=36190' 2026-02-18T08:24:08.961 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=425, tag_tid=30, entry_tid=1' 2026-02-18T08:24:08.962 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:24:08.962 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:24:08.962 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=30 2026-02-18T08:24:08.963 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=418, tag_tid=29, entry_tid=36190' 2026-02-18T08:24:08.963 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:24:08.963 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:24:08.965 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=29 2026-02-18T08:24:08.965 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=425, tag_tid=30, entry_tid=1' 2026-02-18T08:24:08.966 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:24:08.966 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:24:08.967 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:24:08.968 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=418, tag_tid=29, entry_tid=36190' 2026-02-18T08:24:08.968 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:24:08.968 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:24:08.970 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=36190 2026-02-18T08:24:08.970 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 30 = 29 -a 1 -le 36190 2026-02-18T08:24:08.970 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:24:08.970 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:24:08.970 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:24:09.171 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:24:09.171 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:24:09.171 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:24:09.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:24:09.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:24:09.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:24:09.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:24:09.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:24:09.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:24:09.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:24:09.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:24:09.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:24:09.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:24:09.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:24:09.173 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:24:09.176 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:24:09.176 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:24:09.176 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:24:09.176 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:24:09.176 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:24:09.176 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:24:09.176 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:24:09.176 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:24:09.176 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:24:09.176 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:24:09.176 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:24:09.177 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:24:09.177 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:24:09.177 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:24:09.177 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:24:09.177 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:09.177 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:24:09.177 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:24:09.177 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:24:09.177 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:24:09.177 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:24:09.177 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:24:09.177 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:24:09.178 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:24:09.178 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:24:09.178 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:24:09.178 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:09.178 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:24:09.178 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:24:09.178 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:24:09.178 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:24:09.178 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:24:09.178 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:24:09.178 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:24:09.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:24:09.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:24:09.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:24:09.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:24:09.245 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:24:09.246 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:24:09.246 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:24:09.249 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:24:09.249 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:24:09.249 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:24:09.250 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:24:09.250 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:24:09.250 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:24:09.250 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:24:09.250 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:24:09.250 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:24:09.250 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:24:09.250 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:24:09.250 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:24:09.250 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:24:09.250 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:09.250 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:24:09.251 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:24:09.251 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:24:09.251 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:24:09.251 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:24:09.251 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:24:09.251 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:24:09.251 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:24:09.251 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:24:09.251 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:24:09.251 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:09.251 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:24:09.251 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:24:09.252 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:24:09.252 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:24:09.252 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:24:09.252 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:24:09.252 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:24:09.260 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:24:09.260 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:24:09.260 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:24:09.260 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:24:09.318 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:24:09.318 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:24:09.318 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:24:09.321 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:24:09.321 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:24:09.321 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:24:09.321 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:24:09.322 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:24:09.322 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:24:09.322 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:24:09.322 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:24:09.322 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:24:09.322 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:24:09.322 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:24:09.322 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:24:09.322 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:24:09.322 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:09.322 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:24:09.323 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:24:09.323 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:24:09.323 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:24:09.323 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:24:09.323 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:24:09.323 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:24:09.323 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:24:09.323 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:24:09.323 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:24:09.323 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:09.323 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:24:09.323 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:24:09.324 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:24:09.324 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:24:09.324 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:24:09.324 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:24:09.324 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:24:09.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:24:09.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:24:09.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:24:09.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:24:09.389 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:24:09.389 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:24:09.389 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:24:09.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:24:09.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:24:09.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:24:09.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:24:09.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:24:09.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:24:09.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:24:09.393 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:24:09.393 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:24:09.393 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:24:09.393 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:24:09.393 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:24:09.393 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:24:09.393 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:09.393 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:24:09.393 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:24:09.393 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:24:09.393 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:24:09.393 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:24:09.394 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:24:09.394 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:24:09.394 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:24:09.394 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:24:09.394 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:24:09.394 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:09.394 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:24:09.394 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:24:09.394 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:24:09.394 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:24:09.394 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:24:09.395 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:24:09.395 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:24:09.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:24:09.403 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:24:09.403 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:24:09.403 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:24:09.465 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:24:09.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:24:09.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:24:09.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:24:09.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:24:09.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:24:09.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:24:09.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:24:09.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:24:09.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:24:09.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:24:09.467 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:24:09.467 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:24:09.468 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:24:09.468 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:24:09.468 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:24:09.507 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 104 2026-02-18T08:24:09.507 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 106 2026-02-18T08:24:09.507 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:24:09.507 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=425, tag_tid=30, entry_tid=1], [object_number=424, tag_tid=30, entry_tid=0], [object_number=427, tag_tid=29, entry_tid=45931], [object_number=426, tag_tid=29, entry_tid=45930]]], state=connected] 2026-02-18T08:24:09.507 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=416, tag_tid=29, entry_tid=36520], [object_number=419, tag_tid=29, entry_tid=36519], [object_number=418, tag_tid=29, entry_tid=36518], [object_number=417, tag_tid=29, entry_tid=36517]]], state=connected] 2026-02-18T08:24:09.511 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:24:09.512 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=425, tag_tid=30, entry_tid=1' 2026-02-18T08:24:09.513 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:24:09.513 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:24:09.514 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:24:09.514 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:24:09.514 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:24:09.514 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:24:09.514 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:24:09.514 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:24:09.514 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:24:09.514 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:24:09.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:24:09.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:24:09.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:24:09.516 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:24:09.516 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:24:09.552 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 104 2026-02-18T08:24:09.552 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 106 2026-02-18T08:24:09.552 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:24:09.552 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=425, tag_tid=30, entry_tid=1], [object_number=424, tag_tid=30, entry_tid=0], [object_number=427, tag_tid=29, entry_tid=45931], [object_number=426, tag_tid=29, entry_tid=45930]]], state=connected] 2026-02-18T08:24:09.552 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=416, tag_tid=29, entry_tid=36520], [object_number=419, tag_tid=29, entry_tid=36519], [object_number=418, tag_tid=29, entry_tid=36518], [object_number=417, tag_tid=29, entry_tid=36517]]], state=connected] 2026-02-18T08:24:09.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:24:09.558 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=416, tag_tid=29, entry_tid=36520' 2026-02-18T08:24:09.558 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=425, tag_tid=30, entry_tid=1' -a 'object_number=425, tag_tid=30, entry_tid=1' = 'object_number=416, tag_tid=29, entry_tid=36520' 2026-02-18T08:24:09.558 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=416, tag_tid=29, entry_tid=36520' '!=' 'object_number=418, tag_tid=29, entry_tid=36190' 2026-02-18T08:24:09.558 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:24:09.558 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=416, tag_tid=29, entry_tid=36520' = 'object_number=418, tag_tid=29, entry_tid=36190' 2026-02-18T08:24:09.558 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=416, tag_tid=29, entry_tid=36520' 2026-02-18T08:24:09.559 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=425, tag_tid=30, entry_tid=1' 2026-02-18T08:24:09.559 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:24:09.559 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:24:09.560 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=30 2026-02-18T08:24:09.561 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=416, tag_tid=29, entry_tid=36520' 2026-02-18T08:24:09.561 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:24:09.561 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:24:09.562 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=29 2026-02-18T08:24:09.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=425, tag_tid=30, entry_tid=1' 2026-02-18T08:24:09.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:24:09.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:24:09.565 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:24:09.565 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=416, tag_tid=29, entry_tid=36520' 2026-02-18T08:24:09.566 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:24:09.566 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:24:09.567 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=36520 2026-02-18T08:24:09.567 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 30 = 29 -a 1 -le 36520 2026-02-18T08:24:09.567 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:24:09.567 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:24:09.567 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:24:09.769 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:24:09.769 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:24:09.769 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:24:09.770 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:24:09.770 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:24:09.770 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:24:09.770 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:24:09.770 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:24:09.770 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:24:09.770 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:24:09.770 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:24:09.771 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:24:09.771 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:24:09.771 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:24:09.771 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:24:09.771 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:24:09.774 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:24:09.774 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:24:09.774 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:24:09.774 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:24:09.774 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:24:09.774 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:24:09.774 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:24:09.774 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:24:09.774 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:24:09.774 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:24:09.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:24:09.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:24:09.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:24:09.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:24:09.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:09.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:24:09.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:24:09.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:24:09.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:24:09.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:24:09.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:24:09.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:24:09.776 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:24:09.776 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:24:09.776 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:24:09.776 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:09.776 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:24:09.776 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:24:09.776 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:24:09.776 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:24:09.776 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:24:09.776 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:24:09.776 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:24:09.783 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:24:09.784 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:24:09.784 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:24:09.784 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:24:09.842 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:24:09.842 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:24:09.842 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:24:09.846 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:24:09.846 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:24:09.846 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:24:09.846 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:24:09.846 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:24:09.846 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:24:09.846 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:24:09.846 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:24:09.846 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:24:09.846 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:24:09.846 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:24:09.847 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:24:09.847 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:24:09.847 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:09.847 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:24:09.847 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:24:09.847 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:24:09.847 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:24:09.847 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:24:09.847 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:24:09.847 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:24:09.847 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:24:09.847 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:24:09.848 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:24:09.848 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:09.848 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:24:09.848 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:24:09.848 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:24:09.848 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:24:09.848 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:24:09.848 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:24:09.848 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:24:09.856 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:24:09.857 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:24:09.857 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:24:09.857 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:24:09.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:24:09.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:24:09.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:24:09.910 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:24:09.910 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:24:09.910 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:24:09.911 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:24:09.911 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:24:09.911 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:24:09.911 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:24:09.911 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:24:09.911 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:24:09.911 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:24:09.911 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:24:09.911 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:24:09.911 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:24:09.911 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:09.911 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:24:09.912 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:24:09.912 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:24:09.912 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:24:09.912 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:24:09.912 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:24:09.912 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:24:09.912 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:24:09.912 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:24:09.912 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:24:09.912 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:09.912 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:24:09.913 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:24:09.913 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:24:09.913 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:24:09.913 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:24:09.913 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:24:09.913 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:24:09.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:24:09.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:24:09.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:24:09.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:24:09.969 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:24:09.970 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:24:09.970 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:24:09.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:24:09.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:24:09.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:24:09.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:24:09.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:24:09.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:24:09.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:24:09.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:24:09.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:24:09.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:24:09.974 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:24:09.974 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:24:09.974 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:24:09.974 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:09.974 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:24:09.974 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:24:09.974 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:24:09.974 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:24:09.974 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:24:09.974 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:24:09.974 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:24:09.974 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:24:09.974 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:24:09.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:24:09.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:09.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:24:09.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:24:09.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:24:09.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:24:09.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:24:09.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:24:09.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:24:09.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:24:09.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:24:09.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:24:09.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:24:10.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:24:10.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:24:10.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:24:10.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:24:10.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:24:10.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:24:10.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:24:10.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:24:10.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:24:10.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:24:10.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:24:10.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:24:10.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:24:10.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:24:10.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:24:10.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:24:10.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 104 2026-02-18T08:24:10.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 106 2026-02-18T08:24:10.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:24:10.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=425, tag_tid=30, entry_tid=1], [object_number=424, tag_tid=30, entry_tid=0], [object_number=427, tag_tid=29, entry_tid=45931], [object_number=426, tag_tid=29, entry_tid=45930]]], state=connected] 2026-02-18T08:24:10.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=416, tag_tid=29, entry_tid=36876], [object_number=419, tag_tid=29, entry_tid=36875], [object_number=418, tag_tid=29, entry_tid=36874], [object_number=417, tag_tid=29, entry_tid=36873]]], state=connected] 2026-02-18T08:24:10.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:24:10.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=425, tag_tid=30, entry_tid=1' 2026-02-18T08:24:10.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:24:10.092 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:24:10.092 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:24:10.092 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:24:10.092 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:24:10.092 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:24:10.092 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:24:10.092 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:24:10.092 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:24:10.092 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:24:10.093 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:24:10.093 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:24:10.093 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:24:10.094 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:24:10.094 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:24:10.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 104 2026-02-18T08:24:10.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 106 2026-02-18T08:24:10.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:24:10.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=425, tag_tid=30, entry_tid=1], [object_number=424, tag_tid=30, entry_tid=0], [object_number=427, tag_tid=29, entry_tid=45931], [object_number=426, tag_tid=29, entry_tid=45930]]], state=connected] 2026-02-18T08:24:10.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=416, tag_tid=29, entry_tid=36876], [object_number=419, tag_tid=29, entry_tid=36875], [object_number=418, tag_tid=29, entry_tid=36874], [object_number=417, tag_tid=29, entry_tid=36873]]], state=connected] 2026-02-18T08:24:10.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:24:10.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=416, tag_tid=29, entry_tid=36876' 2026-02-18T08:24:10.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=425, tag_tid=30, entry_tid=1' -a 'object_number=425, tag_tid=30, entry_tid=1' = 'object_number=416, tag_tid=29, entry_tid=36876' 2026-02-18T08:24:10.129 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=416, tag_tid=29, entry_tid=36876' '!=' 'object_number=416, tag_tid=29, entry_tid=36520' 2026-02-18T08:24:10.129 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:24:10.129 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=416, tag_tid=29, entry_tid=36876' = 'object_number=416, tag_tid=29, entry_tid=36520' 2026-02-18T08:24:10.129 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=416, tag_tid=29, entry_tid=36876' 2026-02-18T08:24:10.130 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=425, tag_tid=30, entry_tid=1' 2026-02-18T08:24:10.130 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:24:10.130 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:24:10.131 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=30 2026-02-18T08:24:10.132 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=416, tag_tid=29, entry_tid=36876' 2026-02-18T08:24:10.132 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:24:10.132 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:24:10.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=29 2026-02-18T08:24:10.134 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=425, tag_tid=30, entry_tid=1' 2026-02-18T08:24:10.134 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:24:10.134 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:24:10.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:24:10.136 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=416, tag_tid=29, entry_tid=36876' 2026-02-18T08:24:10.136 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:24:10.137 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:24:10.138 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=36876 2026-02-18T08:24:10.138 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 30 = 29 -a 1 -le 36876 2026-02-18T08:24:10.138 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:24:10.138 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:24:10.138 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:24:10.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:24:10.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:24:10.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:24:10.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:24:10.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:24:10.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:24:10.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:24:10.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:24:10.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:24:10.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:24:10.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:24:10.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:24:10.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:24:10.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:24:10.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:24:10.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:24:10.344 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:24:10.344 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:24:10.344 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:24:10.344 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:24:10.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:24:10.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:24:10.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:24:10.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:24:10.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:24:10.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:24:10.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:24:10.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:24:10.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:24:10.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:24:10.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:10.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:24:10.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:24:10.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:24:10.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:24:10.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:24:10.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:24:10.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:24:10.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:24:10.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:24:10.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:24:10.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:10.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:24:10.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:24:10.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:24:10.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:24:10.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:24:10.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:24:10.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:24:10.355 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:24:10.355 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:24:10.356 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:24:10.356 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:24:10.413 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:24:10.413 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:24:10.413 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:24:10.416 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:24:10.416 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:24:10.416 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:24:10.416 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:24:10.416 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:24:10.416 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:24:10.417 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:24:10.417 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:24:10.417 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:24:10.417 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:24:10.417 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:24:10.417 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:24:10.417 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:24:10.417 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:10.417 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:24:10.417 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:24:10.417 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:24:10.417 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:24:10.418 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:24:10.418 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:24:10.418 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:24:10.418 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:24:10.418 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:24:10.418 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:24:10.418 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:10.418 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:24:10.418 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:24:10.418 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:24:10.418 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:24:10.418 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:24:10.419 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:24:10.419 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:24:10.427 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:24:10.428 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:24:10.428 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:24:10.428 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:24:10.486 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:24:10.486 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:24:10.486 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:24:10.489 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:24:10.489 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:24:10.489 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:24:10.490 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:24:10.490 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:24:10.490 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:24:10.490 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:24:10.490 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:24:10.490 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:24:10.490 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:24:10.490 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:24:10.490 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:24:10.490 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:24:10.490 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:10.490 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:24:10.491 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:24:10.491 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:24:10.491 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:24:10.491 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:24:10.491 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:24:10.491 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:24:10.491 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:24:10.491 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:24:10.491 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:24:10.491 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:10.491 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:24:10.491 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:24:10.492 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:24:10.492 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:24:10.492 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:24:10.492 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:24:10.492 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:24:10.500 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:24:10.500 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:24:10.501 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:24:10.501 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:24:10.558 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:24:10.558 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:24:10.558 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:24:10.561 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:24:10.561 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:24:10.561 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:24:10.561 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:24:10.561 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:24:10.561 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:24:10.561 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:24:10.561 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:24:10.561 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:24:10.561 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:24:10.562 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:24:10.562 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:24:10.562 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:24:10.562 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:10.562 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:24:10.562 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:24:10.562 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:24:10.562 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:24:10.562 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:24:10.562 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:24:10.562 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:24:10.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:24:10.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:24:10.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:24:10.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:10.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:24:10.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:24:10.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:24:10.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:24:10.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:24:10.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:24:10.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:24:10.573 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:24:10.573 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:24:10.573 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:24:10.573 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:24:10.647 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:24:10.648 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:24:10.648 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:24:10.649 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:24:10.649 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:24:10.649 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:24:10.649 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:24:10.649 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:24:10.649 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:24:10.649 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:24:10.649 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:24:10.649 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:24:10.649 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:24:10.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:24:10.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:24:10.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:24:10.681 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 104 2026-02-18T08:24:10.681 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 106 2026-02-18T08:24:10.681 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:24:10.681 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=425, tag_tid=30, entry_tid=1], [object_number=424, tag_tid=30, entry_tid=0], [object_number=427, tag_tid=29, entry_tid=45931], [object_number=426, tag_tid=29, entry_tid=45930]]], state=connected] 2026-02-18T08:24:10.681 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=417, tag_tid=29, entry_tid=37249], [object_number=416, tag_tid=29, entry_tid=37248], [object_number=419, tag_tid=29, entry_tid=37247], [object_number=418, tag_tid=29, entry_tid=37246]]], state=connected] 2026-02-18T08:24:10.684 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:24:10.685 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=425, tag_tid=30, entry_tid=1' 2026-02-18T08:24:10.687 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:24:10.687 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:24:10.687 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:24:10.687 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:24:10.687 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:24:10.687 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:24:10.687 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:24:10.687 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:24:10.688 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:24:10.688 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:24:10.688 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:24:10.688 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:24:10.688 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:24:10.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:24:10.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:24:10.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 104 2026-02-18T08:24:10.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 106 2026-02-18T08:24:10.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:24:10.717 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=425, tag_tid=30, entry_tid=1], [object_number=424, tag_tid=30, entry_tid=0], [object_number=427, tag_tid=29, entry_tid=45931], [object_number=426, tag_tid=29, entry_tid=45930]]], state=connected] 2026-02-18T08:24:10.717 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=417, tag_tid=29, entry_tid=37249], [object_number=416, tag_tid=29, entry_tid=37248], [object_number=419, tag_tid=29, entry_tid=37247], [object_number=418, tag_tid=29, entry_tid=37246]]], state=connected] 2026-02-18T08:24:10.719 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:24:10.721 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=417, tag_tid=29, entry_tid=37249' 2026-02-18T08:24:10.721 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=425, tag_tid=30, entry_tid=1' -a 'object_number=425, tag_tid=30, entry_tid=1' = 'object_number=417, tag_tid=29, entry_tid=37249' 2026-02-18T08:24:10.721 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=417, tag_tid=29, entry_tid=37249' '!=' 'object_number=416, tag_tid=29, entry_tid=36876' 2026-02-18T08:24:10.721 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:24:10.721 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=417, tag_tid=29, entry_tid=37249' = 'object_number=416, tag_tid=29, entry_tid=36876' 2026-02-18T08:24:10.721 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=417, tag_tid=29, entry_tid=37249' 2026-02-18T08:24:10.722 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=425, tag_tid=30, entry_tid=1' 2026-02-18T08:24:10.722 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:24:10.722 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:24:10.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=30 2026-02-18T08:24:10.724 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=417, tag_tid=29, entry_tid=37249' 2026-02-18T08:24:10.724 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:24:10.724 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:24:10.725 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=29 2026-02-18T08:24:10.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=425, tag_tid=30, entry_tid=1' 2026-02-18T08:24:10.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:24:10.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:24:10.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:24:10.728 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=417, tag_tid=29, entry_tid=37249' 2026-02-18T08:24:10.728 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:24:10.728 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:24:10.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=37249 2026-02-18T08:24:10.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 30 = 29 -a 1 -le 37249 2026-02-18T08:24:10.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:24:10.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:24:10.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:24:10.932 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:24:10.932 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:24:10.932 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:24:10.932 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:24:10.932 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:24:10.933 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:24:10.933 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:24:10.933 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:24:10.933 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:24:10.933 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:24:10.933 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:24:10.933 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:24:10.933 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:24:10.933 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:24:10.933 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:24:10.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:24:10.936 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:24:10.936 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:24:10.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:24:10.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:24:10.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:24:10.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:24:10.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:24:10.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:24:10.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:24:10.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:24:10.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:24:10.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:24:10.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:24:10.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:24:10.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:10.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:24:10.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:24:10.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:24:10.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:24:10.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:24:10.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:24:10.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:24:10.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:24:10.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:24:10.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:24:10.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:10.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:24:10.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:24:10.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:24:10.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:24:10.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:24:10.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:24:10.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:24:10.946 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:24:10.947 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:24:10.947 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:24:10.947 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:24:10.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:24:10.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:24:10.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:24:11.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:24:11.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:24:11.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:24:11.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:24:11.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:24:11.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:24:11.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:24:11.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:24:11.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:24:11.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:24:11.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:24:11.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:24:11.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:24:11.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:11.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:24:11.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:24:11.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:24:11.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:24:11.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:24:11.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:24:11.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:24:11.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:24:11.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:24:11.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:24:11.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:11.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:24:11.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:24:11.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:24:11.003 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:24:11.003 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:24:11.003 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:24:11.003 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:24:11.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:24:11.012 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:24:11.012 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:24:11.012 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:24:11.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:24:11.062 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:24:11.062 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:24:11.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:24:11.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:24:11.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:24:11.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:24:11.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:24:11.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:24:11.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:24:11.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:24:11.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:24:11.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:24:11.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:24:11.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:24:11.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:24:11.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:11.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:24:11.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:24:11.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:24:11.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:24:11.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:24:11.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:24:11.067 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:24:11.067 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:24:11.067 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:24:11.067 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:24:11.067 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:11.067 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:24:11.067 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:24:11.067 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:24:11.067 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:24:11.067 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:24:11.067 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:24:11.067 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:24:11.076 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:24:11.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:24:11.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:24:11.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:24:11.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:24:11.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:24:11.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:24:11.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:24:11.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:24:11.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:24:11.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:24:11.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:24:11.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:24:11.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:24:11.129 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:24:11.129 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:24:11.129 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:24:11.129 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:24:11.129 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:24:11.129 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:24:11.129 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:11.129 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:24:11.129 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:24:11.129 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:24:11.129 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:24:11.129 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:24:11.130 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:24:11.130 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:24:11.130 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:24:11.130 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:24:11.130 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:24:11.130 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:11.130 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:24:11.130 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:24:11.130 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:24:11.130 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:24:11.130 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:24:11.130 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:24:11.131 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:24:11.140 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:24:11.140 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:24:11.140 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:24:11.141 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:24:11.200 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:24:11.201 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:24:11.201 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:24:11.201 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:24:11.201 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:24:11.202 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:24:11.202 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:24:11.202 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:24:11.202 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:24:11.202 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:24:11.202 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:24:11.202 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:24:11.202 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:24:11.203 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:24:11.203 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:24:11.203 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:24:11.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 104 2026-02-18T08:24:11.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 106 2026-02-18T08:24:11.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:24:11.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=425, tag_tid=30, entry_tid=1], [object_number=424, tag_tid=30, entry_tid=0], [object_number=427, tag_tid=29, entry_tid=45931], [object_number=426, tag_tid=29, entry_tid=45930]]], state=connected] 2026-02-18T08:24:11.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=416, tag_tid=29, entry_tid=37596], [object_number=419, tag_tid=29, entry_tid=37595], [object_number=418, tag_tid=29, entry_tid=37594], [object_number=417, tag_tid=29, entry_tid=37593]]], state=connected] 2026-02-18T08:24:11.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:24:11.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=425, tag_tid=30, entry_tid=1' 2026-02-18T08:24:11.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:24:11.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:24:11.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:24:11.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:24:11.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:24:11.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:24:11.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:24:11.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:24:11.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:24:11.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:24:11.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:24:11.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:24:11.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:24:11.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:24:11.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:24:11.277 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 104 2026-02-18T08:24:11.277 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 106 2026-02-18T08:24:11.277 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:24:11.277 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=425, tag_tid=30, entry_tid=1], [object_number=424, tag_tid=30, entry_tid=0], [object_number=427, tag_tid=29, entry_tid=45931], [object_number=426, tag_tid=29, entry_tid=45930]]], state=connected] 2026-02-18T08:24:11.277 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=416, tag_tid=29, entry_tid=37596], [object_number=419, tag_tid=29, entry_tid=37595], [object_number=418, tag_tid=29, entry_tid=37594], [object_number=417, tag_tid=29, entry_tid=37593]]], state=connected] 2026-02-18T08:24:11.281 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:24:11.282 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=416, tag_tid=29, entry_tid=37596' 2026-02-18T08:24:11.283 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=425, tag_tid=30, entry_tid=1' -a 'object_number=425, tag_tid=30, entry_tid=1' = 'object_number=416, tag_tid=29, entry_tid=37596' 2026-02-18T08:24:11.283 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=416, tag_tid=29, entry_tid=37596' '!=' 'object_number=417, tag_tid=29, entry_tid=37249' 2026-02-18T08:24:11.283 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:24:11.283 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=416, tag_tid=29, entry_tid=37596' = 'object_number=417, tag_tid=29, entry_tid=37249' 2026-02-18T08:24:11.283 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=416, tag_tid=29, entry_tid=37596' 2026-02-18T08:24:11.284 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=425, tag_tid=30, entry_tid=1' 2026-02-18T08:24:11.284 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:24:11.284 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:24:11.285 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=30 2026-02-18T08:24:11.286 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=416, tag_tid=29, entry_tid=37596' 2026-02-18T08:24:11.286 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:24:11.286 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:24:11.287 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=29 2026-02-18T08:24:11.288 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=425, tag_tid=30, entry_tid=1' 2026-02-18T08:24:11.288 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:24:11.288 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:24:11.289 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:24:11.290 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=416, tag_tid=29, entry_tid=37596' 2026-02-18T08:24:11.290 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:24:11.290 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:24:11.292 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=37596 2026-02-18T08:24:11.292 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 30 = 29 -a 1 -le 37596 2026-02-18T08:24:11.292 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:24:11.292 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:24:11.292 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:24:11.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:24:11.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:24:11.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:24:11.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:24:11.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:24:11.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:24:11.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:24:11.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:24:11.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:24:11.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:24:11.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:24:11.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:24:11.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:24:11.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:24:11.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:24:11.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:24:11.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:24:11.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:24:11.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:24:11.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:24:11.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:24:11.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:24:11.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:24:11.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:24:11.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:24:11.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:24:11.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:24:11.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:24:11.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:24:11.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:24:11.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:11.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:24:11.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:24:11.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:24:11.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:24:11.500 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:24:11.500 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:24:11.500 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:24:11.500 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:24:11.500 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:24:11.500 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:24:11.500 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:11.500 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:24:11.500 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:24:11.500 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:24:11.500 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:24:11.500 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:24:11.501 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:24:11.501 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:24:11.508 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:24:11.508 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:24:11.509 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:24:11.509 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:24:11.559 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:24:11.559 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:24:11.559 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:24:11.562 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:24:11.562 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:24:11.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:24:11.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:24:11.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:24:11.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:24:11.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:24:11.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:24:11.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:24:11.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:24:11.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:24:11.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:24:11.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:24:11.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:11.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:24:11.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:24:11.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:24:11.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:24:11.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:24:11.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:24:11.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:24:11.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:24:11.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:24:11.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:24:11.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:11.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:24:11.565 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:24:11.565 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:24:11.565 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:24:11.565 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:24:11.565 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:24:11.565 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:24:11.573 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:24:11.574 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:24:11.574 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:24:11.574 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:24:11.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:24:11.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:24:11.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:24:11.636 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:24:11.636 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:24:11.637 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:24:11.637 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:24:11.637 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:24:11.637 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:24:11.637 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:24:11.637 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:24:11.637 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:24:11.637 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:24:11.637 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:24:11.637 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:24:11.637 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:24:11.638 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:11.638 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:24:11.638 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:24:11.638 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:24:11.638 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:24:11.638 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:24:11.638 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:24:11.638 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:24:11.638 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:24:11.638 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:24:11.638 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:24:11.638 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:11.638 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:24:11.639 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:24:11.639 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:24:11.639 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:24:11.639 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:24:11.639 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:24:11.639 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:24:11.647 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:24:11.648 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:24:11.648 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:24:11.648 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:24:11.697 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:24:11.697 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:24:11.697 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:24:11.700 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:24:11.700 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:24:11.700 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:24:11.701 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:24:11.701 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:24:11.701 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:24:11.701 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:24:11.701 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:24:11.701 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:24:11.701 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:24:11.701 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:24:11.701 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:24:11.701 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:24:11.701 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:11.702 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:24:11.702 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:24:11.702 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:24:11.702 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:24:11.702 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:24:11.702 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:24:11.702 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:24:11.702 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:24:11.702 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:24:11.702 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:24:11.702 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:11.702 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:24:11.703 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:24:11.703 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:24:11.703 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:24:11.703 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:24:11.703 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:24:11.703 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:24:11.712 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:24:11.713 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:24:11.713 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:24:11.713 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:24:11.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:24:11.776 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:24:11.776 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:24:11.776 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:24:11.777 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:24:11.777 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:24:11.777 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:24:11.777 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:24:11.777 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:24:11.777 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:24:11.777 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:24:11.777 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:24:11.777 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:24:11.778 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:24:11.778 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:24:11.778 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:24:11.808 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 104 2026-02-18T08:24:11.808 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 106 2026-02-18T08:24:11.809 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:24:11.809 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=425, tag_tid=30, entry_tid=1], [object_number=424, tag_tid=30, entry_tid=0], [object_number=427, tag_tid=29, entry_tid=45931], [object_number=426, tag_tid=29, entry_tid=45930]]], state=connected] 2026-02-18T08:24:11.809 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=417, tag_tid=29, entry_tid=37925], [object_number=416, tag_tid=29, entry_tid=37924], [object_number=419, tag_tid=29, entry_tid=37923], [object_number=418, tag_tid=29, entry_tid=37922]]], state=connected] 2026-02-18T08:24:11.811 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:24:11.812 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=425, tag_tid=30, entry_tid=1' 2026-02-18T08:24:11.813 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:24:11.813 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:24:11.814 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:24:11.814 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:24:11.814 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:24:11.814 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:24:11.814 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:24:11.814 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:24:11.814 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:24:11.814 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:24:11.814 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:24:11.815 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:24:11.815 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:24:11.815 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:24:11.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:24:11.851 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 104 2026-02-18T08:24:11.851 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 106 2026-02-18T08:24:11.851 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:24:11.851 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=425, tag_tid=30, entry_tid=1], [object_number=424, tag_tid=30, entry_tid=0], [object_number=427, tag_tid=29, entry_tid=45931], [object_number=426, tag_tid=29, entry_tid=45930]]], state=connected] 2026-02-18T08:24:11.851 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=417, tag_tid=29, entry_tid=37925], [object_number=416, tag_tid=29, entry_tid=37924], [object_number=419, tag_tid=29, entry_tid=37923], [object_number=418, tag_tid=29, entry_tid=37922]]], state=connected] 2026-02-18T08:24:11.854 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:24:11.856 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=417, tag_tid=29, entry_tid=37925' 2026-02-18T08:24:11.856 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=425, tag_tid=30, entry_tid=1' -a 'object_number=425, tag_tid=30, entry_tid=1' = 'object_number=417, tag_tid=29, entry_tid=37925' 2026-02-18T08:24:11.856 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=417, tag_tid=29, entry_tid=37925' '!=' 'object_number=416, tag_tid=29, entry_tid=37596' 2026-02-18T08:24:11.856 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:24:11.856 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=417, tag_tid=29, entry_tid=37925' = 'object_number=416, tag_tid=29, entry_tid=37596' 2026-02-18T08:24:11.856 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=417, tag_tid=29, entry_tid=37925' 2026-02-18T08:24:11.857 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=425, tag_tid=30, entry_tid=1' 2026-02-18T08:24:11.857 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:24:11.857 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:24:11.858 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=30 2026-02-18T08:24:11.859 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=417, tag_tid=29, entry_tid=37925' 2026-02-18T08:24:11.859 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:24:11.859 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:24:11.860 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=29 2026-02-18T08:24:11.861 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=425, tag_tid=30, entry_tid=1' 2026-02-18T08:24:11.861 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:24:11.861 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:24:11.863 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:24:11.863 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=417, tag_tid=29, entry_tid=37925' 2026-02-18T08:24:11.864 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:24:11.864 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:24:11.865 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=37925 2026-02-18T08:24:11.865 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 30 = 29 -a 1 -le 37925 2026-02-18T08:24:11.865 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:24:11.865 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:24:11.865 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:24:12.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:24:12.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:24:12.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:24:12.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:24:12.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:24:12.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:24:12.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:24:12.067 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:24:12.067 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:24:12.067 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:24:12.067 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:24:12.067 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:24:12.067 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:24:12.067 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:24:12.067 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:24:12.067 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:24:12.067 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:24:12.068 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:24:12.068 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:24:12.068 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:24:12.068 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:24:12.068 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:24:12.068 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:24:12.068 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:24:12.069 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:24:12.069 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:24:12.069 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:24:12.069 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:24:12.069 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:24:12.069 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:24:12.069 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:12.069 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:24:12.069 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:24:12.069 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:24:12.069 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:24:12.069 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:24:12.069 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:24:12.069 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:24:12.069 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:24:12.069 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:24:12.069 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:24:12.069 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:12.069 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:24:12.069 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:24:12.069 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:24:12.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:24:12.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:24:12.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:24:12.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:24:12.080 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:24:12.080 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:24:12.080 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:24:12.080 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:24:12.131 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:24:12.131 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:24:12.131 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:24:12.132 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:24:12.132 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:24:12.132 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:24:12.132 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:24:12.132 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:24:12.132 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:24:12.132 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:24:12.132 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:24:12.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:24:12.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:24:12.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:24:12.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:24:12.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:24:12.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:12.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:24:12.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:24:12.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:24:12.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:24:12.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:24:12.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:24:12.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:24:12.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:24:12.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:24:12.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:24:12.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:12.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:24:12.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:24:12.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:24:12.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:24:12.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:24:12.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:24:12.134 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:24:12.144 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:24:12.144 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:24:12.144 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:24:12.144 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:24:12.194 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:24:12.194 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:24:12.194 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:24:12.196 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:24:12.196 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:24:12.196 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:24:12.196 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:24:12.196 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:24:12.196 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:24:12.196 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:24:12.196 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:24:12.196 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:24:12.196 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:24:12.196 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:24:12.196 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:24:12.196 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:24:12.196 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:12.196 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:24:12.196 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:24:12.197 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:24:12.197 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:24:12.197 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:24:12.197 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:24:12.197 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:24:12.197 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:24:12.197 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:24:12.197 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:24:12.197 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:12.197 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:24:12.197 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:24:12.197 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:24:12.197 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:24:12.197 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:24:12.197 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:24:12.197 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:24:12.208 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:24:12.209 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:24:12.209 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:24:12.209 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:24:12.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:24:12.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:24:12.259 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:24:12.261 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:24:12.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:24:12.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:24:12.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:24:12.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:24:12.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:24:12.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:24:12.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:24:12.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:24:12.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:24:12.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:24:12.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:24:12.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:24:12.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:12.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:24:12.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:24:12.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:24:12.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:24:12.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:24:12.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:24:12.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:24:12.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:24:12.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:24:12.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:24:12.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:12.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:24:12.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:24:12.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:24:12.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:24:12.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:24:12.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:24:12.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:24:12.273 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:24:12.273 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:24:12.273 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:24:12.273 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:24:12.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:24:12.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:24:12.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:24:12.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:24:12.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:24:12.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:24:12.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:24:12.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:24:12.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:24:12.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:24:12.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:24:12.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:24:12.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:24:12.348 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:24:12.348 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:24:12.348 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:24:12.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 104 2026-02-18T08:24:12.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 106 2026-02-18T08:24:12.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:24:12.379 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=425, tag_tid=30, entry_tid=1], [object_number=424, tag_tid=30, entry_tid=0], [object_number=427, tag_tid=29, entry_tid=45931], [object_number=426, tag_tid=29, entry_tid=45930]]], state=connected] 2026-02-18T08:24:12.379 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=418, tag_tid=29, entry_tid=38258], [object_number=417, tag_tid=29, entry_tid=38257], [object_number=416, tag_tid=29, entry_tid=38256], [object_number=419, tag_tid=29, entry_tid=38255]]], state=connected] 2026-02-18T08:24:12.381 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:24:12.382 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=425, tag_tid=30, entry_tid=1' 2026-02-18T08:24:12.383 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:24:12.383 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:24:12.384 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:24:12.384 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:24:12.384 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:24:12.384 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:24:12.384 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:24:12.384 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:24:12.384 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:24:12.384 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:24:12.385 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:24:12.385 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:24:12.385 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:24:12.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:24:12.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:24:12.413 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 104 2026-02-18T08:24:12.413 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 106 2026-02-18T08:24:12.413 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:24:12.413 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=425, tag_tid=30, entry_tid=1], [object_number=424, tag_tid=30, entry_tid=0], [object_number=427, tag_tid=29, entry_tid=45931], [object_number=426, tag_tid=29, entry_tid=45930]]], state=connected] 2026-02-18T08:24:12.413 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=418, tag_tid=29, entry_tid=38258], [object_number=417, tag_tid=29, entry_tid=38257], [object_number=416, tag_tid=29, entry_tid=38256], [object_number=419, tag_tid=29, entry_tid=38255]]], state=connected] 2026-02-18T08:24:12.416 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:24:12.418 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=418, tag_tid=29, entry_tid=38258' 2026-02-18T08:24:12.418 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=425, tag_tid=30, entry_tid=1' -a 'object_number=425, tag_tid=30, entry_tid=1' = 'object_number=418, tag_tid=29, entry_tid=38258' 2026-02-18T08:24:12.418 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=418, tag_tid=29, entry_tid=38258' '!=' 'object_number=417, tag_tid=29, entry_tid=37925' 2026-02-18T08:24:12.418 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:24:12.418 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=418, tag_tid=29, entry_tid=38258' = 'object_number=417, tag_tid=29, entry_tid=37925' 2026-02-18T08:24:12.418 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=418, tag_tid=29, entry_tid=38258' 2026-02-18T08:24:12.419 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=425, tag_tid=30, entry_tid=1' 2026-02-18T08:24:12.419 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:24:12.419 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:24:12.420 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=30 2026-02-18T08:24:12.421 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=418, tag_tid=29, entry_tid=38258' 2026-02-18T08:24:12.421 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:24:12.421 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:24:12.422 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=29 2026-02-18T08:24:12.423 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=425, tag_tid=30, entry_tid=1' 2026-02-18T08:24:12.423 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:24:12.423 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:24:12.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:24:12.425 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=418, tag_tid=29, entry_tid=38258' 2026-02-18T08:24:12.425 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:24:12.425 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:24:12.427 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=38258 2026-02-18T08:24:12.427 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 30 = 29 -a 1 -le 38258 2026-02-18T08:24:12.427 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:24:12.427 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:24:12.427 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:24:12.629 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:24:12.629 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:24:12.629 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:24:12.629 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:24:12.629 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:24:12.629 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:24:12.629 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:24:12.629 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:24:12.629 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:24:12.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:24:12.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:24:12.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:24:12.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:24:12.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:24:12.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:24:12.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:24:12.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:24:12.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:24:12.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:24:12.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:24:12.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:24:12.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:24:12.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:24:12.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:24:12.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:24:12.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:24:12.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:24:12.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:24:12.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:24:12.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:24:12.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:12.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:24:12.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:24:12.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:24:12.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:24:12.635 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:24:12.635 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:24:12.635 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:24:12.635 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:24:12.635 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:24:12.635 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:24:12.635 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:12.635 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:24:12.635 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:24:12.635 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:24:12.635 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:24:12.635 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:24:12.636 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:24:12.636 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:24:12.643 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:24:12.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:24:12.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:24:12.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:24:12.704 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:24:12.705 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:24:12.705 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:24:12.707 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:24:12.708 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:24:12.708 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:24:12.708 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:24:12.708 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:24:12.708 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:24:12.708 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:24:12.708 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:24:12.708 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:24:12.708 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:24:12.708 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:24:12.708 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:24:12.709 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:24:12.709 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:12.709 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:24:12.709 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:24:12.709 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:24:12.709 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:24:12.709 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:24:12.709 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:24:12.709 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:24:12.709 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:24:12.709 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:24:12.709 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:24:12.709 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:12.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:24:12.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:24:12.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:24:12.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:24:12.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:24:12.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:24:12.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:24:12.719 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:24:12.720 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:24:12.720 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:24:12.720 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:24:12.776 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:24:12.776 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:24:12.776 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:24:12.779 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:24:12.779 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:24:12.779 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:24:12.779 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:24:12.779 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:24:12.780 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:24:12.780 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:24:12.780 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:24:12.780 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:24:12.780 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:24:12.780 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:24:12.780 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:24:12.780 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:24:12.780 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:12.780 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:24:12.780 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:24:12.780 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:24:12.781 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:24:12.781 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:24:12.781 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:24:12.781 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:24:12.781 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:24:12.781 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:24:12.781 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:24:12.781 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:12.781 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:24:12.781 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:24:12.781 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:24:12.781 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:24:12.781 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:24:12.782 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:24:12.782 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:24:12.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:24:12.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:24:12.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:24:12.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:24:12.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:24:12.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:24:12.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:24:12.852 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:24:12.852 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:24:12.852 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:24:12.852 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:24:12.852 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:24:12.852 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:24:12.852 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:24:12.852 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:24:12.852 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:24:12.852 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:24:12.852 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:24:12.853 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:24:12.853 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:24:12.853 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:12.853 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:24:12.853 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:24:12.853 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:24:12.853 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:24:12.853 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:24:12.853 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:24:12.853 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:24:12.853 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:24:12.853 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:24:12.854 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:24:12.854 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:12.854 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:24:12.854 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:24:12.854 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:24:12.854 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:24:12.854 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:24:12.854 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:24:12.854 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:24:12.863 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:24:12.864 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:24:12.864 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:24:12.864 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:24:12.930 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:24:12.931 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:24:12.932 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:24:12.932 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:24:12.932 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:24:12.932 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:24:12.932 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:24:12.932 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:24:12.932 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:24:12.932 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:24:12.932 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:24:12.932 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:24:12.933 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:24:12.933 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:24:12.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:24:12.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:24:12.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 104 2026-02-18T08:24:12.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 106 2026-02-18T08:24:12.974 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:24:12.974 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=425, tag_tid=30, entry_tid=1], [object_number=424, tag_tid=30, entry_tid=0], [object_number=427, tag_tid=29, entry_tid=45931], [object_number=426, tag_tid=29, entry_tid=45930]]], state=connected] 2026-02-18T08:24:12.974 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=417, tag_tid=29, entry_tid=38601], [object_number=416, tag_tid=29, entry_tid=38600], [object_number=419, tag_tid=29, entry_tid=38599], [object_number=418, tag_tid=29, entry_tid=38598]]], state=connected] 2026-02-18T08:24:12.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:24:12.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=425, tag_tid=30, entry_tid=1' 2026-02-18T08:24:12.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:24:12.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:24:12.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:24:12.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:24:12.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:24:12.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:24:12.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:24:12.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:24:12.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:24:12.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:24:12.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:24:12.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:24:12.982 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:24:12.982 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:24:12.982 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:24:13.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 104 2026-02-18T08:24:13.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 106 2026-02-18T08:24:13.010 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:24:13.010 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=425, tag_tid=30, entry_tid=1], [object_number=424, tag_tid=30, entry_tid=0], [object_number=427, tag_tid=29, entry_tid=45931], [object_number=426, tag_tid=29, entry_tid=45930]]], state=connected] 2026-02-18T08:24:13.010 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=417, tag_tid=29, entry_tid=38601], [object_number=416, tag_tid=29, entry_tid=38600], [object_number=419, tag_tid=29, entry_tid=38599], [object_number=418, tag_tid=29, entry_tid=38598]]], state=connected] 2026-02-18T08:24:13.012 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:24:13.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=417, tag_tid=29, entry_tid=38601' 2026-02-18T08:24:13.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=425, tag_tid=30, entry_tid=1' -a 'object_number=425, tag_tid=30, entry_tid=1' = 'object_number=417, tag_tid=29, entry_tid=38601' 2026-02-18T08:24:13.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=417, tag_tid=29, entry_tid=38601' '!=' 'object_number=418, tag_tid=29, entry_tid=38258' 2026-02-18T08:24:13.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:24:13.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=417, tag_tid=29, entry_tid=38601' = 'object_number=418, tag_tid=29, entry_tid=38258' 2026-02-18T08:24:13.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=417, tag_tid=29, entry_tid=38601' 2026-02-18T08:24:13.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=425, tag_tid=30, entry_tid=1' 2026-02-18T08:24:13.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:24:13.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:24:13.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=30 2026-02-18T08:24:13.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=417, tag_tid=29, entry_tid=38601' 2026-02-18T08:24:13.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:24:13.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:24:13.018 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=29 2026-02-18T08:24:13.019 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=425, tag_tid=30, entry_tid=1' 2026-02-18T08:24:13.019 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:24:13.019 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:24:13.020 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:24:13.021 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=417, tag_tid=29, entry_tid=38601' 2026-02-18T08:24:13.021 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:24:13.021 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:24:13.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=38601 2026-02-18T08:24:13.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 30 = 29 -a 1 -le 38601 2026-02-18T08:24:13.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:24:13.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:24:13.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:24:13.224 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:24:13.225 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:24:13.225 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:24:13.225 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:24:13.225 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:24:13.225 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:24:13.225 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:24:13.225 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:24:13.225 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:24:13.225 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:24:13.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:24:13.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:24:13.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:24:13.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:24:13.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:24:13.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:24:13.229 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:24:13.229 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:24:13.229 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:24:13.229 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:24:13.229 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:24:13.229 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:24:13.229 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:24:13.229 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:24:13.229 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:24:13.230 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:24:13.230 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:24:13.230 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:24:13.230 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:24:13.230 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:24:13.230 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:13.230 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:24:13.230 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:24:13.230 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:24:13.230 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:24:13.230 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:24:13.230 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:24:13.231 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:24:13.231 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:24:13.231 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:24:13.231 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:24:13.231 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:13.231 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:24:13.231 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:24:13.231 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:24:13.231 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:24:13.231 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:24:13.231 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:24:13.231 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:24:13.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:24:13.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:24:13.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:24:13.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:24:13.290 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:24:13.290 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:24:13.290 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:24:13.293 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:24:13.293 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:24:13.293 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:24:13.293 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:24:13.293 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:24:13.293 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:24:13.293 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:24:13.293 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:24:13.293 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:24:13.294 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:24:13.294 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:24:13.294 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:24:13.294 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:24:13.294 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:13.294 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:24:13.294 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:24:13.294 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:24:13.294 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:24:13.294 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:24:13.294 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:24:13.294 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:24:13.294 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:24:13.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:24:13.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:24:13.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:13.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:24:13.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:24:13.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:24:13.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:24:13.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:24:13.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:24:13.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:24:13.304 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:24:13.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:24:13.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:24:13.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:24:13.354 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:24:13.354 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:24:13.354 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:24:13.357 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:24:13.357 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:24:13.357 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:24:13.357 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:24:13.357 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:24:13.357 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:24:13.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:24:13.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:24:13.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:24:13.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:24:13.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:24:13.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:24:13.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:24:13.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:13.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:24:13.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:24:13.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:24:13.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:24:13.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:24:13.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:24:13.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:24:13.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:24:13.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:24:13.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:24:13.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:13.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:24:13.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:24:13.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:24:13.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:24:13.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:24:13.360 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:24:13.360 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:24:13.367 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:24:13.368 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:24:13.368 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:24:13.368 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:24:13.418 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:24:13.418 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:24:13.418 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:24:13.421 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:24:13.421 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:24:13.421 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:24:13.421 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:24:13.422 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:24:13.422 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:24:13.422 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:24:13.422 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:24:13.422 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:24:13.422 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:24:13.422 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:24:13.422 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:24:13.422 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:24:13.422 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:13.422 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:24:13.422 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:24:13.423 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:24:13.423 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:24:13.423 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:24:13.423 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:24:13.423 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:24:13.423 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:24:13.423 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:24:13.423 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:24:13.423 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:13.423 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:24:13.423 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:24:13.423 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:24:13.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:24:13.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:24:13.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:24:13.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:24:13.431 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:24:13.432 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:24:13.432 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:24:13.432 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:24:13.492 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:24:13.493 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:24:13.493 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:24:13.493 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:24:13.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:24:13.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:24:13.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:24:13.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:24:13.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:24:13.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:24:13.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:24:13.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:24:13.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:24:13.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:24:13.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:24:13.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:24:13.532 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 104 2026-02-18T08:24:13.532 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 106 2026-02-18T08:24:13.532 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:24:13.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=425, tag_tid=30, entry_tid=1], [object_number=424, tag_tid=30, entry_tid=0], [object_number=427, tag_tid=29, entry_tid=45931], [object_number=426, tag_tid=29, entry_tid=45930]]], state=connected] 2026-02-18T08:24:13.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=416, tag_tid=29, entry_tid=38932], [object_number=419, tag_tid=29, entry_tid=38931], [object_number=418, tag_tid=29, entry_tid=38930], [object_number=417, tag_tid=29, entry_tid=38929]]], state=connected] 2026-02-18T08:24:13.536 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:24:13.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=425, tag_tid=30, entry_tid=1' 2026-02-18T08:24:13.539 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:24:13.539 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:24:13.539 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:24:13.539 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:24:13.539 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:24:13.539 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:24:13.539 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:24:13.539 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:24:13.540 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:24:13.540 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:24:13.540 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:24:13.540 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:24:13.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:24:13.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:24:13.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:24:13.570 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 104 2026-02-18T08:24:13.570 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 106 2026-02-18T08:24:13.570 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:24:13.570 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=425, tag_tid=30, entry_tid=1], [object_number=424, tag_tid=30, entry_tid=0], [object_number=427, tag_tid=29, entry_tid=45931], [object_number=426, tag_tid=29, entry_tid=45930]]], state=connected] 2026-02-18T08:24:13.570 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=416, tag_tid=29, entry_tid=38932], [object_number=419, tag_tid=29, entry_tid=38931], [object_number=418, tag_tid=29, entry_tid=38930], [object_number=417, tag_tid=29, entry_tid=38929]]], state=connected] 2026-02-18T08:24:13.573 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:24:13.574 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=416, tag_tid=29, entry_tid=38932' 2026-02-18T08:24:13.574 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=425, tag_tid=30, entry_tid=1' -a 'object_number=425, tag_tid=30, entry_tid=1' = 'object_number=416, tag_tid=29, entry_tid=38932' 2026-02-18T08:24:13.574 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=416, tag_tid=29, entry_tid=38932' '!=' 'object_number=417, tag_tid=29, entry_tid=38601' 2026-02-18T08:24:13.574 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:24:13.574 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=416, tag_tid=29, entry_tid=38932' = 'object_number=417, tag_tid=29, entry_tid=38601' 2026-02-18T08:24:13.575 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=416, tag_tid=29, entry_tid=38932' 2026-02-18T08:24:13.575 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=425, tag_tid=30, entry_tid=1' 2026-02-18T08:24:13.575 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:24:13.575 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:24:13.576 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=30 2026-02-18T08:24:13.577 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=416, tag_tid=29, entry_tid=38932' 2026-02-18T08:24:13.577 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:24:13.577 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:24:13.578 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=29 2026-02-18T08:24:13.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=425, tag_tid=30, entry_tid=1' 2026-02-18T08:24:13.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:24:13.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:24:13.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:24:13.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=416, tag_tid=29, entry_tid=38932' 2026-02-18T08:24:13.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:24:13.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:24:13.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=38932 2026-02-18T08:24:13.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 30 = 29 -a 1 -le 38932 2026-02-18T08:24:13.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:24:13.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:24:13.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:24:13.784 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:24:13.785 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:24:13.785 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:24:13.785 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:24:13.785 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:24:13.785 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:24:13.785 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:24:13.785 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:24:13.785 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:24:13.786 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:24:13.786 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:24:13.786 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:24:13.786 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:24:13.786 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:24:13.786 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:24:13.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:24:13.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:24:13.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:24:13.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:24:13.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:24:13.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:24:13.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:24:13.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:24:13.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:24:13.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:24:13.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:24:13.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:24:13.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:24:13.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:24:13.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:24:13.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:13.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:24:13.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:24:13.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:24:13.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:24:13.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:24:13.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:24:13.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:24:13.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:24:13.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:24:13.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:24:13.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:13.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:24:13.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:24:13.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:24:13.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:24:13.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:24:13.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:24:13.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:24:13.799 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:24:13.800 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:24:13.800 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:24:13.800 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:24:13.850 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:24:13.850 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:24:13.851 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:24:13.853 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:24:13.854 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:24:13.854 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:24:13.854 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:24:13.854 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:24:13.854 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:24:13.854 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:24:13.854 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:24:13.854 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:24:13.854 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:24:13.854 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:24:13.854 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:24:13.855 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:24:13.855 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:13.855 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:24:13.855 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:24:13.855 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:24:13.855 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:24:13.855 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:24:13.855 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:24:13.855 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:24:13.855 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:24:13.855 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:24:13.855 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:24:13.856 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:13.856 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:24:13.856 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:24:13.856 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:24:13.856 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:24:13.856 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:24:13.856 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:24:13.856 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:24:13.864 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:24:13.865 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:24:13.865 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:24:13.865 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:24:13.922 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:24:13.923 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:24:13.923 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:24:13.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:24:13.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:24:13.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:24:13.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:24:13.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:24:13.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:24:13.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:24:13.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:24:13.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:24:13.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:24:13.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:24:13.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:24:13.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:24:13.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:13.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:24:13.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:24:13.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:24:13.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:24:13.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:24:13.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:24:13.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:24:13.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:24:13.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:24:13.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:24:13.928 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:13.928 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:24:13.928 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:24:13.928 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:24:13.928 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:24:13.928 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:24:13.928 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:24:13.928 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:24:13.936 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:24:13.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:24:13.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:24:13.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:24:13.986 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:24:13.986 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:24:13.986 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:24:13.989 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:24:13.989 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:24:13.989 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:24:13.989 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:24:13.989 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:24:13.989 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:24:13.989 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:24:13.989 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:24:13.989 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:24:13.990 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:24:13.990 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:24:13.990 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:24:13.990 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:24:13.990 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:13.990 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:24:13.990 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:24:13.990 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:24:13.990 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:24:13.990 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:24:13.990 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:24:13.990 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:24:13.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:24:13.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:24:13.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:24:13.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:13.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:24:13.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:24:13.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:24:13.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:24:13.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:24:13.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:24:13.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:24:14.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:24:14.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:24:14.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:24:14.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:24:14.062 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:24:14.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:24:14.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:24:14.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:24:14.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:24:14.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:24:14.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:24:14.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:24:14.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:24:14.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:24:14.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:24:14.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:24:14.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:24:14.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:24:14.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:24:14.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:24:14.297 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 104 2026-02-18T08:24:14.297 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 106 2026-02-18T08:24:14.297 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:24:14.297 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=425, tag_tid=30, entry_tid=1], [object_number=424, tag_tid=30, entry_tid=0], [object_number=427, tag_tid=29, entry_tid=45931], [object_number=426, tag_tid=29, entry_tid=45930]]], state=connected] 2026-02-18T08:24:14.297 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=416, tag_tid=29, entry_tid=39288], [object_number=419, tag_tid=29, entry_tid=39287], [object_number=418, tag_tid=29, entry_tid=39286], [object_number=417, tag_tid=29, entry_tid=39285]]], state=connected] 2026-02-18T08:24:14.301 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:24:14.302 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=425, tag_tid=30, entry_tid=1' 2026-02-18T08:24:14.304 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:24:14.304 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:24:14.304 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:24:14.304 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:24:14.304 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:24:14.304 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:24:14.304 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:24:14.304 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:24:14.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:24:14.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:24:14.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:24:14.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:24:14.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:24:14.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:24:14.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:24:14.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 104 2026-02-18T08:24:14.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 106 2026-02-18T08:24:14.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:24:14.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=425, tag_tid=30, entry_tid=1], [object_number=424, tag_tid=30, entry_tid=0], [object_number=427, tag_tid=29, entry_tid=45931], [object_number=426, tag_tid=29, entry_tid=45930]]], state=connected] 2026-02-18T08:24:14.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=416, tag_tid=29, entry_tid=39288], [object_number=419, tag_tid=29, entry_tid=39287], [object_number=418, tag_tid=29, entry_tid=39286], [object_number=417, tag_tid=29, entry_tid=39285]]], state=connected] 2026-02-18T08:24:14.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:24:14.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=416, tag_tid=29, entry_tid=39288' 2026-02-18T08:24:14.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=425, tag_tid=30, entry_tid=1' -a 'object_number=425, tag_tid=30, entry_tid=1' = 'object_number=416, tag_tid=29, entry_tid=39288' 2026-02-18T08:24:14.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=416, tag_tid=29, entry_tid=39288' '!=' 'object_number=416, tag_tid=29, entry_tid=38932' 2026-02-18T08:24:14.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:24:14.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=416, tag_tid=29, entry_tid=39288' = 'object_number=416, tag_tid=29, entry_tid=38932' 2026-02-18T08:24:14.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=416, tag_tid=29, entry_tid=39288' 2026-02-18T08:24:14.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=425, tag_tid=30, entry_tid=1' 2026-02-18T08:24:14.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:24:14.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:24:14.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=30 2026-02-18T08:24:14.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=416, tag_tid=29, entry_tid=39288' 2026-02-18T08:24:14.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:24:14.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:24:14.344 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=29 2026-02-18T08:24:14.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=425, tag_tid=30, entry_tid=1' 2026-02-18T08:24:14.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:24:14.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:24:14.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:24:14.348 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=416, tag_tid=29, entry_tid=39288' 2026-02-18T08:24:14.348 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:24:14.348 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:24:14.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=39288 2026-02-18T08:24:14.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 30 = 29 -a 1 -le 39288 2026-02-18T08:24:14.350 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:24:14.350 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:24:14.350 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:24:14.551 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:24:14.551 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:24:14.551 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:24:14.552 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:24:14.552 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:24:14.552 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:24:14.552 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:24:14.552 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:24:14.552 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:24:14.552 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:24:14.552 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:24:14.552 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:24:14.552 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:24:14.553 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:24:14.553 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:24:14.553 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:24:14.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:24:14.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:24:14.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:24:14.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:24:14.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:24:14.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:24:14.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:24:14.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:24:14.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:24:14.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:24:14.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:24:14.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:24:14.557 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:24:14.557 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:24:14.557 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:14.557 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:24:14.557 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:24:14.557 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:24:14.557 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:24:14.557 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:24:14.557 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:24:14.557 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:24:14.557 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:24:14.557 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:24:14.557 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:24:14.558 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:14.558 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:24:14.558 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:24:14.558 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:24:14.558 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:24:14.558 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:24:14.558 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:24:14.558 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:24:14.565 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:24:14.566 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:24:14.566 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:24:14.566 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:24:14.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:24:14.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:24:14.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:24:14.619 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:24:14.619 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:24:14.619 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:24:14.619 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:24:14.619 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:24:14.619 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:24:14.619 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:24:14.619 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:24:14.619 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:24:14.620 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:24:14.620 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:24:14.620 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:24:14.620 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:24:14.620 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:14.620 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:24:14.620 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:24:14.620 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:24:14.620 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:24:14.620 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:24:14.620 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:24:14.620 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:24:14.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:24:14.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:24:14.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:24:14.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:14.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:24:14.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:24:14.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:24:14.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:24:14.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:24:14.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:24:14.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:24:14.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:24:14.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:24:14.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:24:14.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:24:14.687 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:24:14.688 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:24:14.688 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:24:14.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:24:14.691 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:24:14.691 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:24:14.691 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:24:14.691 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:24:14.691 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:24:14.691 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:24:14.691 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:24:14.691 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:24:14.691 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:24:14.691 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:24:14.692 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:24:14.692 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:24:14.692 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:14.692 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:24:14.692 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:24:14.692 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:24:14.692 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:24:14.692 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:24:14.692 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:24:14.692 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:24:14.692 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:24:14.692 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:24:14.692 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:24:14.693 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:14.693 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:24:14.693 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:24:14.693 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:24:14.693 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:24:14.693 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:24:14.693 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:24:14.693 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:24:14.701 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:24:14.702 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:24:14.702 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:24:14.702 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:24:14.750 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:24:14.750 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:24:14.750 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:24:14.753 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:24:14.753 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:24:14.753 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:24:14.753 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:24:14.753 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:24:14.753 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:24:14.754 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:24:14.754 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:24:14.754 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:24:14.754 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:24:14.754 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:24:14.754 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:24:14.754 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:24:14.754 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:14.754 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:24:14.754 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:24:14.754 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:24:14.754 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:24:14.755 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:24:14.755 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:24:14.755 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:24:14.755 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:24:14.755 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:24:14.755 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:24:14.755 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:14.755 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:24:14.755 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:24:14.755 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:24:14.755 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:24:14.755 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:24:14.756 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:24:14.756 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:24:14.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:24:14.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:24:14.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:24:14.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:24:14.832 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:24:14.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:24:14.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:24:14.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:24:14.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:24:14.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:24:14.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:24:14.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:24:14.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:24:14.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:24:14.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:24:14.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:24:14.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:24:14.835 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:24:14.835 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:24:14.835 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:24:14.865 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 104 2026-02-18T08:24:14.865 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 106 2026-02-18T08:24:14.865 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:24:14.865 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=425, tag_tid=30, entry_tid=1], [object_number=424, tag_tid=30, entry_tid=0], [object_number=427, tag_tid=29, entry_tid=45931], [object_number=426, tag_tid=29, entry_tid=45930]]], state=connected] 2026-02-18T08:24:14.865 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=417, tag_tid=29, entry_tid=39761], [object_number=416, tag_tid=29, entry_tid=39760], [object_number=419, tag_tid=29, entry_tid=39759], [object_number=418, tag_tid=29, entry_tid=39758]]], state=connected] 2026-02-18T08:24:14.868 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:24:14.869 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=425, tag_tid=30, entry_tid=1' 2026-02-18T08:24:14.870 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:24:14.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:24:14.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:24:14.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:24:14.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:24:14.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:24:14.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:24:14.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:24:14.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:24:14.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:24:14.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:24:14.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:24:14.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:24:14.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:24:14.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:24:14.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 104 2026-02-18T08:24:14.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 106 2026-02-18T08:24:14.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:24:14.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=425, tag_tid=30, entry_tid=1], [object_number=424, tag_tid=30, entry_tid=0], [object_number=427, tag_tid=29, entry_tid=45931], [object_number=426, tag_tid=29, entry_tid=45930]]], state=connected] 2026-02-18T08:24:14.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=417, tag_tid=29, entry_tid=39761], [object_number=416, tag_tid=29, entry_tid=39760], [object_number=419, tag_tid=29, entry_tid=39759], [object_number=418, tag_tid=29, entry_tid=39758]]], state=connected] 2026-02-18T08:24:14.905 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:24:14.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=417, tag_tid=29, entry_tid=39761' 2026-02-18T08:24:14.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=425, tag_tid=30, entry_tid=1' -a 'object_number=425, tag_tid=30, entry_tid=1' = 'object_number=417, tag_tid=29, entry_tid=39761' 2026-02-18T08:24:14.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=417, tag_tid=29, entry_tid=39761' '!=' 'object_number=416, tag_tid=29, entry_tid=39288' 2026-02-18T08:24:14.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:24:14.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=417, tag_tid=29, entry_tid=39761' = 'object_number=416, tag_tid=29, entry_tid=39288' 2026-02-18T08:24:14.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=417, tag_tid=29, entry_tid=39761' 2026-02-18T08:24:14.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=425, tag_tid=30, entry_tid=1' 2026-02-18T08:24:14.909 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:24:14.909 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:24:14.910 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=30 2026-02-18T08:24:14.910 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=417, tag_tid=29, entry_tid=39761' 2026-02-18T08:24:14.911 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:24:14.911 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:24:14.912 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=29 2026-02-18T08:24:14.913 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=425, tag_tid=30, entry_tid=1' 2026-02-18T08:24:14.913 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:24:14.913 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:24:14.914 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:24:14.915 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=417, tag_tid=29, entry_tid=39761' 2026-02-18T08:24:14.915 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:24:14.915 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:24:14.917 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=39761 2026-02-18T08:24:14.917 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 30 = 29 -a 1 -le 39761 2026-02-18T08:24:14.917 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:24:14.917 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:24:14.917 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:24:15.119 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:24:15.119 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:24:15.119 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:24:15.119 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:24:15.119 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:24:15.119 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:24:15.120 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:24:15.120 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:24:15.120 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:24:15.120 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:24:15.120 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:24:15.120 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:24:15.120 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:24:15.120 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:24:15.120 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:24:15.120 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:24:15.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:24:15.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:24:15.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:24:15.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:24:15.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:24:15.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:24:15.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:24:15.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:24:15.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:24:15.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:24:15.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:24:15.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:24:15.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:24:15.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:24:15.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:15.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:24:15.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:24:15.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:24:15.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:24:15.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:24:15.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:24:15.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:24:15.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:24:15.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:24:15.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:24:15.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:15.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:24:15.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:24:15.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:24:15.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:24:15.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:24:15.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:24:15.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:24:15.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:24:15.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:24:15.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:24:15.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:24:15.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:24:15.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:24:15.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:24:15.187 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:24:15.187 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:24:15.187 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:24:15.187 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:24:15.187 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:24:15.187 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:24:15.187 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:24:15.188 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:24:15.188 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:24:15.188 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:24:15.188 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:24:15.188 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:24:15.188 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:24:15.188 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:15.188 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:24:15.188 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:24:15.188 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:24:15.188 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:24:15.189 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:24:15.189 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:24:15.189 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:24:15.189 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:24:15.189 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:24:15.189 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:24:15.189 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:15.189 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:24:15.189 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:24:15.189 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:24:15.189 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:24:15.189 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:24:15.189 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:24:15.190 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:24:15.198 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:24:15.199 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:24:15.199 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:24:15.199 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:24:15.250 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:24:15.250 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:24:15.250 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:24:15.253 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:24:15.253 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:24:15.253 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:24:15.253 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:24:15.253 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:24:15.253 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:24:15.253 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:24:15.253 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:24:15.254 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:24:15.254 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:24:15.254 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:24:15.254 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:24:15.254 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:24:15.254 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:15.254 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:24:15.254 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:24:15.254 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:24:15.254 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:24:15.254 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:24:15.254 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:24:15.254 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:24:15.255 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:24:15.255 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:24:15.255 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:24:15.255 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:15.255 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:24:15.255 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:24:15.255 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:24:15.255 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:24:15.255 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:24:15.255 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:24:15.255 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:24:15.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:24:15.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:24:15.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:24:15.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:24:15.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:24:15.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:24:15.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:24:15.316 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:24:15.317 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:24:15.317 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:24:15.317 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:24:15.317 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:24:15.317 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:24:15.317 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:24:15.317 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:24:15.317 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:24:15.317 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:24:15.317 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:24:15.317 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:24:15.317 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:24:15.318 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:15.318 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:24:15.318 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:24:15.318 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:24:15.318 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:24:15.318 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:24:15.318 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:24:15.318 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:24:15.318 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:24:15.318 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:24:15.318 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:24:15.318 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:15.319 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:24:15.319 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:24:15.319 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:24:15.319 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:24:15.319 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:24:15.319 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:24:15.319 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:24:15.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:24:15.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:24:15.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:24:15.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:24:15.399 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:24:15.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:24:15.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:24:15.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:24:15.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:24:15.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:24:15.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:24:15.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:24:15.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:24:15.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:24:15.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:24:15.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:24:15.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:24:15.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:24:15.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:24:15.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:24:15.440 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 104 2026-02-18T08:24:15.440 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 106 2026-02-18T08:24:15.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:24:15.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=425, tag_tid=30, entry_tid=1], [object_number=424, tag_tid=30, entry_tid=0], [object_number=427, tag_tid=29, entry_tid=45931], [object_number=426, tag_tid=29, entry_tid=45930]]], state=connected] 2026-02-18T08:24:15.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=420, tag_tid=29, entry_tid=40036], [object_number=423, tag_tid=29, entry_tid=40035], [object_number=422, tag_tid=29, entry_tid=40034], [object_number=417, tag_tid=29, entry_tid=40033]]], state=connected] 2026-02-18T08:24:15.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:24:15.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=425, tag_tid=30, entry_tid=1' 2026-02-18T08:24:15.446 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:24:15.446 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:24:15.446 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:24:15.446 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:24:15.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:24:15.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:24:15.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:24:15.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:24:15.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:24:15.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:24:15.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:24:15.448 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:24:15.448 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:24:15.449 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:24:15.449 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:24:15.482 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 104 2026-02-18T08:24:15.482 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 106 2026-02-18T08:24:15.482 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:24:15.482 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=425, tag_tid=30, entry_tid=1], [object_number=424, tag_tid=30, entry_tid=0], [object_number=427, tag_tid=29, entry_tid=45931], [object_number=426, tag_tid=29, entry_tid=45930]]], state=connected] 2026-02-18T08:24:15.482 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=420, tag_tid=29, entry_tid=40036], [object_number=423, tag_tid=29, entry_tid=40035], [object_number=422, tag_tid=29, entry_tid=40034], [object_number=417, tag_tid=29, entry_tid=40033]]], state=connected] 2026-02-18T08:24:15.485 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:24:15.488 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=420, tag_tid=29, entry_tid=40036' 2026-02-18T08:24:15.488 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=425, tag_tid=30, entry_tid=1' -a 'object_number=425, tag_tid=30, entry_tid=1' = 'object_number=420, tag_tid=29, entry_tid=40036' 2026-02-18T08:24:15.488 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=420, tag_tid=29, entry_tid=40036' '!=' 'object_number=417, tag_tid=29, entry_tid=39761' 2026-02-18T08:24:15.488 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:24:15.488 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=420, tag_tid=29, entry_tid=40036' = 'object_number=417, tag_tid=29, entry_tid=39761' 2026-02-18T08:24:15.488 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=420, tag_tid=29, entry_tid=40036' 2026-02-18T08:24:15.489 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=425, tag_tid=30, entry_tid=1' 2026-02-18T08:24:15.489 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:24:15.489 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:24:15.490 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=30 2026-02-18T08:24:15.491 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=420, tag_tid=29, entry_tid=40036' 2026-02-18T08:24:15.491 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:24:15.491 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:24:15.492 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=29 2026-02-18T08:24:15.493 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=425, tag_tid=30, entry_tid=1' 2026-02-18T08:24:15.493 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:24:15.493 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:24:15.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:24:15.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=420, tag_tid=29, entry_tid=40036' 2026-02-18T08:24:15.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:24:15.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:24:15.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=40036 2026-02-18T08:24:15.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 30 = 29 -a 1 -le 40036 2026-02-18T08:24:15.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:24:15.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:24:15.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:24:15.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:24:15.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:24:15.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:24:15.700 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:24:15.700 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:24:15.700 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:24:15.700 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:24:15.700 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:24:15.700 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:24:15.700 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:24:15.700 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:24:15.700 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:24:15.700 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:24:15.700 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:24:15.701 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:24:15.701 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:24:15.703 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:24:15.703 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:24:15.703 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:24:15.703 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:24:15.704 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:24:15.704 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:24:15.704 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:24:15.704 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:24:15.704 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:24:15.704 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:24:15.704 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:24:15.704 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:24:15.704 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:24:15.704 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:24:15.704 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:15.704 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:24:15.705 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:24:15.705 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:24:15.705 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:24:15.705 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:24:15.705 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:24:15.705 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:24:15.705 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:24:15.705 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:24:15.705 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:24:15.705 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:15.705 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:24:15.705 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:24:15.706 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:24:15.706 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:24:15.706 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:24:15.706 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:24:15.706 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:24:15.712 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:24:15.713 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:24:15.713 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:24:15.713 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:24:15.762 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:24:15.762 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:24:15.762 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:24:15.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:24:15.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:24:15.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:24:15.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:24:15.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:24:15.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:24:15.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:24:15.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:24:15.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:24:15.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:24:15.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:24:15.766 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:24:15.766 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:24:15.766 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:15.766 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:24:15.766 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:24:15.766 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:24:15.766 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:24:15.766 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:24:15.766 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:24:15.766 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:24:15.766 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:24:15.766 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:24:15.766 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:24:15.767 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:15.767 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:24:15.767 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:24:15.767 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:24:15.767 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:24:15.767 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:24:15.767 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:24:15.767 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:24:15.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:24:15.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:24:15.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:24:15.776 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:24:15.824 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:24:15.824 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:24:15.824 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:24:15.827 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:24:15.828 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:24:15.828 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:24:15.828 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:24:15.828 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:24:15.828 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:24:15.828 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:24:15.828 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:24:15.828 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:24:15.828 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:24:15.828 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:24:15.829 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:24:15.829 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:24:15.829 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:15.829 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:24:15.829 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:24:15.829 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:24:15.829 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:24:15.829 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:24:15.829 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:24:15.829 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:24:15.829 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:24:15.830 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:24:15.830 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:24:15.830 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:15.830 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:24:15.830 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:24:15.830 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:24:15.830 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:24:15.830 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:24:15.830 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:24:15.830 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:24:15.838 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:24:15.838 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:24:15.838 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:24:15.839 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:24:15.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:24:15.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:24:15.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:24:15.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:24:15.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:24:15.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:24:15.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:24:15.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:24:15.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:24:15.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:24:15.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:24:15.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:24:15.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:24:15.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:24:15.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:24:15.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:24:15.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:15.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:24:15.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:24:15.894 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:24:15.894 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:24:15.894 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:24:15.894 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:24:15.894 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:24:15.894 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:24:15.894 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:24:15.894 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:24:15.894 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:15.894 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:24:15.894 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:24:15.894 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:24:15.895 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:24:15.895 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:24:15.895 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:24:15.895 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:24:15.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:24:15.904 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:24:15.904 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:24:15.904 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:24:15.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:24:15.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:24:15.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:24:15.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:24:15.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:24:15.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:24:15.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:24:15.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:24:15.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:24:15.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:24:15.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:24:15.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:24:15.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:24:15.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:24:15.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:24:15.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:24:16.006 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 105 2026-02-18T08:24:16.006 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 106 2026-02-18T08:24:16.006 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:24:16.006 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=425, tag_tid=30, entry_tid=1], [object_number=424, tag_tid=30, entry_tid=0], [object_number=427, tag_tid=29, entry_tid=45931], [object_number=426, tag_tid=29, entry_tid=45930]]], state=connected] 2026-02-18T08:24:16.007 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=421, tag_tid=29, entry_tid=40389], [object_number=420, tag_tid=29, entry_tid=40388], [object_number=423, tag_tid=29, entry_tid=40387], [object_number=422, tag_tid=29, entry_tid=40386]]], state=connected] 2026-02-18T08:24:16.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:24:16.010 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=425, tag_tid=30, entry_tid=1' 2026-02-18T08:24:16.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:24:16.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:24:16.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:24:16.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:24:16.012 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:24:16.012 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:24:16.012 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:24:16.012 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:24:16.012 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:24:16.012 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:24:16.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:24:16.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:24:16.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:24:16.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:24:16.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:24:16.245 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 105 2026-02-18T08:24:16.245 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 106 2026-02-18T08:24:16.245 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:24:16.245 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=425, tag_tid=30, entry_tid=1], [object_number=424, tag_tid=30, entry_tid=0], [object_number=427, tag_tid=29, entry_tid=45931], [object_number=426, tag_tid=29, entry_tid=45930]]], state=connected] 2026-02-18T08:24:16.245 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=421, tag_tid=29, entry_tid=40389], [object_number=420, tag_tid=29, entry_tid=40388], [object_number=423, tag_tid=29, entry_tid=40387], [object_number=422, tag_tid=29, entry_tid=40386]]], state=connected] 2026-02-18T08:24:16.249 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:24:16.251 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=421, tag_tid=29, entry_tid=40389' 2026-02-18T08:24:16.251 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=425, tag_tid=30, entry_tid=1' -a 'object_number=425, tag_tid=30, entry_tid=1' = 'object_number=421, tag_tid=29, entry_tid=40389' 2026-02-18T08:24:16.251 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=421, tag_tid=29, entry_tid=40389' '!=' 'object_number=420, tag_tid=29, entry_tid=40036' 2026-02-18T08:24:16.251 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:24:16.251 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=421, tag_tid=29, entry_tid=40389' = 'object_number=420, tag_tid=29, entry_tid=40036' 2026-02-18T08:24:16.252 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=421, tag_tid=29, entry_tid=40389' 2026-02-18T08:24:16.252 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=425, tag_tid=30, entry_tid=1' 2026-02-18T08:24:16.252 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:24:16.252 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:24:16.253 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=30 2026-02-18T08:24:16.254 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=421, tag_tid=29, entry_tid=40389' 2026-02-18T08:24:16.254 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:24:16.254 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:24:16.255 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=29 2026-02-18T08:24:16.256 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=425, tag_tid=30, entry_tid=1' 2026-02-18T08:24:16.256 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:24:16.256 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:24:16.257 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:24:16.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=421, tag_tid=29, entry_tid=40389' 2026-02-18T08:24:16.259 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:24:16.259 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:24:16.260 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=40389 2026-02-18T08:24:16.260 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 30 = 29 -a 1 -le 40389 2026-02-18T08:24:16.260 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:24:16.260 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:24:16.260 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:24:16.462 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:24:16.462 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:24:16.462 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:24:16.462 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:24:16.463 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:24:16.463 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:24:16.463 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:24:16.463 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:24:16.463 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:24:16.463 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:24:16.463 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:24:16.463 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:24:16.463 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:24:16.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:24:16.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:24:16.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:24:16.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:24:16.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:24:16.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:24:16.467 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:24:16.467 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:24:16.467 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:24:16.467 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:24:16.467 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:24:16.467 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:24:16.467 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:24:16.467 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:24:16.467 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:24:16.467 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:24:16.467 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:24:16.468 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:16.468 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:24:16.468 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:24:16.468 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:24:16.468 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:24:16.468 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:24:16.468 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:24:16.468 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:24:16.468 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:24:16.468 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:24:16.468 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:24:16.468 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:16.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:24:16.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:24:16.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:24:16.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:24:16.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:24:16.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:24:16.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:24:16.476 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:24:16.477 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:24:16.477 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:24:16.477 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:24:16.535 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:24:16.535 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:24:16.536 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:24:16.539 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:24:16.539 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:24:16.539 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:24:16.539 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:24:16.539 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:24:16.539 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:24:16.539 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:24:16.539 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:24:16.539 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:24:16.540 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:24:16.540 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:24:16.540 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:24:16.540 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:24:16.540 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:16.540 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:24:16.540 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:24:16.540 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:24:16.540 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:24:16.540 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:24:16.540 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:24:16.540 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:24:16.540 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:24:16.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:24:16.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:24:16.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:16.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:24:16.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:24:16.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:24:16.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:24:16.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:24:16.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:24:16.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:24:16.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:24:16.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:24:16.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:24:16.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:24:16.601 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:24:16.601 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:24:16.601 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:24:16.604 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:24:16.604 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:24:16.604 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:24:16.604 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:24:16.604 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:24:16.604 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:24:16.605 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:24:16.605 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:24:16.605 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:24:16.605 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:24:16.605 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:24:16.605 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:24:16.605 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:24:16.605 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:16.605 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:24:16.605 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:24:16.605 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:24:16.605 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:24:16.606 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:24:16.606 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:24:16.606 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:24:16.606 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:24:16.606 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:24:16.606 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:24:16.606 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:16.606 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:24:16.606 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:24:16.606 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:24:16.606 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:24:16.606 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:24:16.607 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:24:16.607 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:24:16.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:24:16.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:24:16.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:24:16.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:24:16.672 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:24:16.673 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:24:16.673 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:24:16.675 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:24:16.676 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:24:16.676 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:24:16.676 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:24:16.676 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:24:16.676 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:24:16.676 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:24:16.676 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:24:16.676 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:24:16.676 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:24:16.676 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:24:16.677 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:24:16.677 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:24:16.677 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:16.677 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:24:16.677 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:24:16.677 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:24:16.677 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:24:16.677 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:24:16.677 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:24:16.677 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:24:16.677 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:24:16.677 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:24:16.678 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:24:16.678 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:16.678 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:24:16.678 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:24:16.678 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:24:16.678 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:24:16.678 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:24:16.678 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:24:16.678 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:24:16.687 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:24:16.687 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:24:16.688 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:24:16.688 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:24:16.748 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:24:16.749 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:24:16.749 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:24:16.750 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:24:16.750 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:24:16.750 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:24:16.750 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:24:16.750 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:24:16.750 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:24:16.750 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:24:16.750 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:24:16.750 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:24:16.751 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:24:16.751 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:24:16.751 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:24:16.752 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:24:16.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 105 2026-02-18T08:24:16.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 106 2026-02-18T08:24:16.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:24:16.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=425, tag_tid=30, entry_tid=1], [object_number=424, tag_tid=30, entry_tid=0], [object_number=427, tag_tid=29, entry_tid=45931], [object_number=426, tag_tid=29, entry_tid=45930]]], state=connected] 2026-02-18T08:24:16.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=420, tag_tid=29, entry_tid=40868], [object_number=423, tag_tid=29, entry_tid=40867], [object_number=422, tag_tid=29, entry_tid=40866], [object_number=421, tag_tid=29, entry_tid=40865]]], state=connected] 2026-02-18T08:24:16.794 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:24:16.795 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=425, tag_tid=30, entry_tid=1' 2026-02-18T08:24:16.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:24:16.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:24:16.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:24:16.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:24:16.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:24:16.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:24:16.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:24:16.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:24:16.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:24:16.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:24:16.798 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:24:16.798 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:24:16.798 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:24:16.798 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:24:16.799 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:24:16.837 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 105 2026-02-18T08:24:16.837 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 106 2026-02-18T08:24:16.837 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:24:16.837 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=425, tag_tid=30, entry_tid=1], [object_number=424, tag_tid=30, entry_tid=0], [object_number=427, tag_tid=29, entry_tid=45931], [object_number=426, tag_tid=29, entry_tid=45930]]], state=connected] 2026-02-18T08:24:16.837 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=420, tag_tid=29, entry_tid=40868], [object_number=423, tag_tid=29, entry_tid=40867], [object_number=422, tag_tid=29, entry_tid=40866], [object_number=421, tag_tid=29, entry_tid=40865]]], state=connected] 2026-02-18T08:24:16.840 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:24:16.843 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=420, tag_tid=29, entry_tid=40868' 2026-02-18T08:24:16.843 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=425, tag_tid=30, entry_tid=1' -a 'object_number=425, tag_tid=30, entry_tid=1' = 'object_number=420, tag_tid=29, entry_tid=40868' 2026-02-18T08:24:16.843 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=420, tag_tid=29, entry_tid=40868' '!=' 'object_number=421, tag_tid=29, entry_tid=40389' 2026-02-18T08:24:16.843 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:24:16.843 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=420, tag_tid=29, entry_tid=40868' = 'object_number=421, tag_tid=29, entry_tid=40389' 2026-02-18T08:24:16.843 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=420, tag_tid=29, entry_tid=40868' 2026-02-18T08:24:16.844 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=425, tag_tid=30, entry_tid=1' 2026-02-18T08:24:16.844 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:24:16.844 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:24:16.845 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=30 2026-02-18T08:24:16.845 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=420, tag_tid=29, entry_tid=40868' 2026-02-18T08:24:16.845 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:24:16.846 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:24:16.847 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=29 2026-02-18T08:24:16.847 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=425, tag_tid=30, entry_tid=1' 2026-02-18T08:24:16.848 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:24:16.848 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:24:16.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:24:16.850 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=420, tag_tid=29, entry_tid=40868' 2026-02-18T08:24:16.850 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:24:16.850 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:24:16.852 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=40868 2026-02-18T08:24:16.852 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 30 = 29 -a 1 -le 40868 2026-02-18T08:24:16.852 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:24:16.852 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:24:16.852 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:24:17.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:24:17.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:24:17.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:24:17.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:24:17.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:24:17.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:24:17.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:24:17.055 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:24:17.055 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:24:17.055 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:24:17.055 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:24:17.055 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:24:17.055 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:24:17.055 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:24:17.055 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:24:17.055 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:24:17.058 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:24:17.058 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:24:17.058 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:24:17.058 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:24:17.058 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:24:17.058 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:24:17.059 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:24:17.059 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:24:17.059 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:24:17.059 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:24:17.059 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:24:17.059 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:24:17.059 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:24:17.059 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:24:17.059 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:17.059 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:24:17.059 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:24:17.060 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:24:17.060 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:24:17.060 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:24:17.060 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:24:17.060 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:24:17.060 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:24:17.060 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:24:17.060 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:24:17.060 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:17.060 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:24:17.060 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:24:17.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:24:17.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:24:17.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:24:17.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:24:17.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:24:17.068 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:24:17.069 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:24:17.069 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:24:17.069 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:24:17.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:24:17.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:24:17.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:24:17.129 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:24:17.129 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:24:17.129 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:24:17.129 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:24:17.130 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:24:17.130 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:24:17.130 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:24:17.130 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:24:17.130 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:24:17.130 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:24:17.130 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:24:17.130 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:24:17.130 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:24:17.130 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:17.131 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:24:17.131 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:24:17.131 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:24:17.131 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:24:17.131 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:24:17.131 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:24:17.131 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:24:17.131 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:24:17.131 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:24:17.131 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:24:17.131 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:17.131 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:24:17.132 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:24:17.132 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:24:17.132 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:24:17.132 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:24:17.132 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:24:17.132 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:24:17.140 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:24:17.141 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:24:17.141 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:24:17.141 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:24:17.198 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:24:17.198 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:24:17.199 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:24:17.201 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:24:17.201 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:24:17.202 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:24:17.202 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:24:17.202 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:24:17.202 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:24:17.202 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:24:17.202 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:24:17.202 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:24:17.202 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:24:17.202 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:24:17.202 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:24:17.202 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:24:17.202 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:17.202 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:24:17.203 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:24:17.203 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:24:17.203 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:24:17.203 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:24:17.203 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:24:17.203 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:24:17.203 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:24:17.203 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:24:17.203 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:24:17.203 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:17.204 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:24:17.204 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:24:17.204 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:24:17.204 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:24:17.204 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:24:17.204 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:24:17.204 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:24:17.212 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:24:17.213 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:24:17.213 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:24:17.213 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:24:17.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:24:17.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:24:17.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:24:17.266 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:24:17.266 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:24:17.266 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:24:17.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:24:17.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:24:17.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:24:17.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:24:17.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:24:17.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:24:17.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:24:17.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:24:17.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:24:17.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:24:17.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:17.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:24:17.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:24:17.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:24:17.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:24:17.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:24:17.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:24:17.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:24:17.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:24:17.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:24:17.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:24:17.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:17.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:24:17.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:24:17.269 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:24:17.269 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:24:17.269 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:24:17.269 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:24:17.269 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:24:17.276 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:24:17.277 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:24:17.277 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:24:17.277 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:24:17.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:24:17.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:24:17.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:24:17.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:24:17.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:24:17.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:24:17.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:24:17.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:24:17.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:24:17.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:24:17.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:24:17.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:24:17.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:24:17.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:24:17.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:24:17.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:24:17.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 105 2026-02-18T08:24:17.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 106 2026-02-18T08:24:17.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:24:17.370 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=425, tag_tid=30, entry_tid=1], [object_number=424, tag_tid=30, entry_tid=0], [object_number=427, tag_tid=29, entry_tid=45931], [object_number=426, tag_tid=29, entry_tid=45930]]], state=connected] 2026-02-18T08:24:17.370 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=422, tag_tid=29, entry_tid=41222], [object_number=421, tag_tid=29, entry_tid=41221], [object_number=420, tag_tid=29, entry_tid=41220], [object_number=423, tag_tid=29, entry_tid=41219]]], state=connected] 2026-02-18T08:24:17.372 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:24:17.373 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=425, tag_tid=30, entry_tid=1' 2026-02-18T08:24:17.375 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:24:17.375 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:24:17.376 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:24:17.376 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:24:17.376 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:24:17.376 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:24:17.376 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:24:17.376 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:24:17.376 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:24:17.376 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:24:17.376 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:24:17.376 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:24:17.377 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:24:17.377 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:24:17.377 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:24:17.405 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 105 2026-02-18T08:24:17.405 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 106 2026-02-18T08:24:17.406 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:24:17.406 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=425, tag_tid=30, entry_tid=1], [object_number=424, tag_tid=30, entry_tid=0], [object_number=427, tag_tid=29, entry_tid=45931], [object_number=426, tag_tid=29, entry_tid=45930]]], state=connected] 2026-02-18T08:24:17.406 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=422, tag_tid=29, entry_tid=41222], [object_number=421, tag_tid=29, entry_tid=41221], [object_number=420, tag_tid=29, entry_tid=41220], [object_number=423, tag_tid=29, entry_tid=41219]]], state=connected] 2026-02-18T08:24:17.408 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:24:17.409 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=422, tag_tid=29, entry_tid=41222' 2026-02-18T08:24:17.409 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=425, tag_tid=30, entry_tid=1' -a 'object_number=425, tag_tid=30, entry_tid=1' = 'object_number=422, tag_tid=29, entry_tid=41222' 2026-02-18T08:24:17.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=422, tag_tid=29, entry_tid=41222' '!=' 'object_number=420, tag_tid=29, entry_tid=40868' 2026-02-18T08:24:17.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:24:17.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=422, tag_tid=29, entry_tid=41222' = 'object_number=420, tag_tid=29, entry_tid=40868' 2026-02-18T08:24:17.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=422, tag_tid=29, entry_tid=41222' 2026-02-18T08:24:17.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=425, tag_tid=30, entry_tid=1' 2026-02-18T08:24:17.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:24:17.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:24:17.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=30 2026-02-18T08:24:17.412 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=422, tag_tid=29, entry_tid=41222' 2026-02-18T08:24:17.413 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:24:17.413 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:24:17.414 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=29 2026-02-18T08:24:17.415 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=425, tag_tid=30, entry_tid=1' 2026-02-18T08:24:17.415 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:24:17.415 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:24:17.416 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:24:17.417 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=422, tag_tid=29, entry_tid=41222' 2026-02-18T08:24:17.417 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:24:17.417 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:24:17.419 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=41222 2026-02-18T08:24:17.419 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 30 = 29 -a 1 -le 41222 2026-02-18T08:24:17.419 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:24:17.419 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:24:17.419 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:24:17.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:24:17.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:24:17.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:24:17.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:24:17.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:24:17.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:24:17.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:24:17.622 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:24:17.622 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:24:17.622 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:24:17.622 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:24:17.622 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:24:17.622 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:24:17.622 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:24:17.622 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:24:17.622 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:24:17.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:24:17.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:24:17.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:24:17.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:24:17.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:24:17.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:24:17.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:24:17.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:24:17.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:24:17.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:24:17.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:24:17.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:24:17.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:24:17.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:24:17.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:17.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:24:17.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:24:17.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:24:17.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:24:17.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:24:17.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:24:17.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:24:17.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:24:17.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:24:17.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:24:17.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:17.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:24:17.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:24:17.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:24:17.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:24:17.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:24:17.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:24:17.628 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:24:17.636 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:24:17.637 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:24:17.637 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:24:17.637 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:24:17.693 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:24:17.694 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:24:17.694 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:24:17.696 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:24:17.696 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:24:17.696 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:24:17.696 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:24:17.697 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:24:17.697 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:24:17.697 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:24:17.697 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:24:17.697 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:24:17.697 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:24:17.697 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:24:17.697 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:24:17.697 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:24:17.697 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:17.697 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:24:17.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:24:17.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:24:17.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:24:17.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:24:17.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:24:17.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:24:17.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:24:17.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:24:17.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:24:17.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:17.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:24:17.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:24:17.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:24:17.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:24:17.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:24:17.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:24:17.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:24:17.708 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:24:17.708 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:24:17.708 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:24:17.709 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:24:17.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:24:17.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:24:17.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:24:17.768 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:24:17.768 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:24:17.768 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:24:17.768 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:24:17.769 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:24:17.769 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:24:17.769 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:24:17.769 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:24:17.769 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:24:17.769 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:24:17.769 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:24:17.769 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:24:17.769 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:24:17.769 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:17.770 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:24:17.770 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:24:17.770 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:24:17.770 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:24:17.770 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:24:17.770 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:24:17.770 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:24:17.770 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:24:17.770 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:24:17.770 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:24:17.770 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:17.771 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:24:17.771 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:24:17.771 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:24:17.771 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:24:17.771 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:24:17.771 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:24:17.771 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:24:17.779 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:24:17.780 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:24:17.780 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:24:17.780 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:24:17.829 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:24:17.829 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:24:17.829 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:24:17.832 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:24:17.832 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:24:17.832 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:24:17.832 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:24:17.832 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:24:17.832 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:24:17.832 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:24:17.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:24:17.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:24:17.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:24:17.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:24:17.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:24:17.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:24:17.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:17.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:24:17.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:24:17.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:24:17.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:24:17.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:24:17.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:24:17.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:24:17.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:24:17.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:24:17.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:24:17.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:17.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:24:17.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:24:17.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:24:17.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:24:17.835 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:24:17.835 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:24:17.835 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:24:17.843 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:24:17.844 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:24:17.844 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:24:17.844 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:24:17.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:24:17.905 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:24:17.905 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:24:17.905 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:24:17.905 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:24:17.905 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:24:17.905 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:24:17.905 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:24:17.905 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:24:17.905 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:24:17.905 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:24:17.905 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:24:17.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:24:17.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:24:17.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:24:17.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:24:17.944 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 105 2026-02-18T08:24:17.944 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 106 2026-02-18T08:24:17.944 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:24:17.944 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=425, tag_tid=30, entry_tid=1], [object_number=424, tag_tid=30, entry_tid=0], [object_number=427, tag_tid=29, entry_tid=45931], [object_number=426, tag_tid=29, entry_tid=45930]]], state=connected] 2026-02-18T08:24:17.944 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=420, tag_tid=29, entry_tid=41556], [object_number=423, tag_tid=29, entry_tid=41555], [object_number=422, tag_tid=29, entry_tid=41554], [object_number=421, tag_tid=29, entry_tid=41553]]], state=connected] 2026-02-18T08:24:17.948 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:24:17.950 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=425, tag_tid=30, entry_tid=1' 2026-02-18T08:24:17.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:24:17.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:24:17.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:24:17.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:24:17.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:24:17.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:24:17.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:24:17.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:24:17.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:24:17.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:24:17.952 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:24:17.952 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:24:17.953 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:24:17.953 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:24:17.953 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:24:17.990 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 105 2026-02-18T08:24:17.990 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 106 2026-02-18T08:24:17.990 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:24:17.990 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=425, tag_tid=30, entry_tid=1], [object_number=424, tag_tid=30, entry_tid=0], [object_number=427, tag_tid=29, entry_tid=45931], [object_number=426, tag_tid=29, entry_tid=45930]]], state=connected] 2026-02-18T08:24:17.990 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=420, tag_tid=29, entry_tid=41556], [object_number=423, tag_tid=29, entry_tid=41555], [object_number=422, tag_tid=29, entry_tid=41554], [object_number=421, tag_tid=29, entry_tid=41553]]], state=connected] 2026-02-18T08:24:17.994 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:24:17.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=420, tag_tid=29, entry_tid=41556' 2026-02-18T08:24:17.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=425, tag_tid=30, entry_tid=1' -a 'object_number=425, tag_tid=30, entry_tid=1' = 'object_number=420, tag_tid=29, entry_tid=41556' 2026-02-18T08:24:17.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=420, tag_tid=29, entry_tid=41556' '!=' 'object_number=422, tag_tid=29, entry_tid=41222' 2026-02-18T08:24:17.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:24:17.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=420, tag_tid=29, entry_tid=41556' = 'object_number=422, tag_tid=29, entry_tid=41222' 2026-02-18T08:24:17.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=420, tag_tid=29, entry_tid=41556' 2026-02-18T08:24:17.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=425, tag_tid=30, entry_tid=1' 2026-02-18T08:24:17.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:24:17.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:24:17.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=30 2026-02-18T08:24:17.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=420, tag_tid=29, entry_tid=41556' 2026-02-18T08:24:17.999 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:24:17.999 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:24:18.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=29 2026-02-18T08:24:18.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=425, tag_tid=30, entry_tid=1' 2026-02-18T08:24:18.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:24:18.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:24:18.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:24:18.003 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=420, tag_tid=29, entry_tid=41556' 2026-02-18T08:24:18.003 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:24:18.003 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:24:18.005 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=41556 2026-02-18T08:24:18.005 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 30 = 29 -a 1 -le 41556 2026-02-18T08:24:18.005 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:24:18.005 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:24:18.005 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:24:18.207 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:24:18.207 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:24:18.207 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:24:18.207 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:24:18.207 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:24:18.207 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:24:18.207 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:24:18.207 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:24:18.208 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:24:18.208 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:24:18.208 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:24:18.208 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:24:18.208 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:24:18.208 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:24:18.208 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:24:18.208 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:24:18.211 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:24:18.211 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:24:18.211 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:24:18.211 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:24:18.211 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:24:18.211 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:24:18.211 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:24:18.211 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:24:18.212 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:24:18.212 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:24:18.212 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:24:18.212 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:24:18.212 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:24:18.212 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:24:18.212 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:18.212 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:24:18.212 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:24:18.212 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:24:18.212 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:24:18.212 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:24:18.213 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:24:18.213 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:24:18.213 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:24:18.213 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:24:18.213 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:24:18.213 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:18.213 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:24:18.213 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:24:18.213 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:24:18.213 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:24:18.213 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:24:18.213 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:24:18.214 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:24:18.222 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:24:18.222 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:24:18.222 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:24:18.222 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:24:18.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:24:18.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:24:18.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:24:18.274 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:24:18.274 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:24:18.274 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:24:18.274 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:24:18.274 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:24:18.274 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:24:18.275 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:24:18.275 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:24:18.275 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:24:18.275 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:24:18.275 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:24:18.275 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:24:18.275 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:24:18.275 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:18.275 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:24:18.275 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:24:18.275 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:24:18.276 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:24:18.276 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:24:18.276 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:24:18.276 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:24:18.276 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:24:18.276 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:24:18.276 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:24:18.276 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:18.276 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:24:18.276 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:24:18.277 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:24:18.277 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:24:18.277 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:24:18.277 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:24:18.277 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:24:18.285 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:24:18.286 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:24:18.286 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:24:18.286 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:24:18.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:24:18.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:24:18.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:24:18.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:24:18.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:24:18.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:24:18.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:24:18.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:24:18.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:24:18.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:24:18.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:24:18.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:24:18.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:24:18.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:24:18.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:24:18.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:24:18.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:18.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:24:18.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:24:18.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:24:18.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:24:18.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:24:18.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:24:18.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:24:18.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:24:18.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:24:18.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:24:18.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:18.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:24:18.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:24:18.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:24:18.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:24:18.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:24:18.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:24:18.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:24:18.350 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:24:18.351 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:24:18.351 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:24:18.351 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:24:18.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:24:18.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:24:18.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:24:18.405 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:24:18.405 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:24:18.405 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:24:18.405 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:24:18.406 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:24:18.406 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:24:18.406 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:24:18.406 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:24:18.406 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:24:18.406 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:24:18.406 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:24:18.406 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:24:18.406 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:24:18.406 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:18.406 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:24:18.407 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:24:18.407 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:24:18.407 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:24:18.407 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:24:18.407 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:24:18.407 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:24:18.407 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:24:18.407 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:24:18.407 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:24:18.407 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:18.407 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:24:18.407 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:24:18.407 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:24:18.408 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:24:18.408 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:24:18.408 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:24:18.408 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:24:18.416 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:24:18.417 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:24:18.417 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:24:18.417 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:24:18.478 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:24:18.479 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:24:18.479 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:24:18.479 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:24:18.479 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:24:18.480 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:24:18.480 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:24:18.480 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:24:18.480 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:24:18.480 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:24:18.480 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:24:18.480 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:24:18.480 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:24:18.481 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:24:18.481 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:24:18.482 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:24:18.511 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 105 2026-02-18T08:24:18.511 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 106 2026-02-18T08:24:18.512 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:24:18.512 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=425, tag_tid=30, entry_tid=1], [object_number=424, tag_tid=30, entry_tid=0], [object_number=427, tag_tid=29, entry_tid=45931], [object_number=426, tag_tid=29, entry_tid=45930]]], state=connected] 2026-02-18T08:24:18.512 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=423, tag_tid=29, entry_tid=41891], [object_number=422, tag_tid=29, entry_tid=41890], [object_number=421, tag_tid=29, entry_tid=41889], [object_number=420, tag_tid=29, entry_tid=41888]]], state=connected] 2026-02-18T08:24:18.514 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:24:18.516 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=425, tag_tid=30, entry_tid=1' 2026-02-18T08:24:18.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:24:18.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:24:18.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:24:18.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:24:18.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:24:18.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:24:18.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:24:18.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:24:18.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:24:18.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:24:18.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:24:18.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:24:18.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:24:18.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:24:18.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:24:18.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 105 2026-02-18T08:24:18.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 106 2026-02-18T08:24:18.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:24:18.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=425, tag_tid=30, entry_tid=1], [object_number=424, tag_tid=30, entry_tid=0], [object_number=427, tag_tid=29, entry_tid=45931], [object_number=426, tag_tid=29, entry_tid=45930]]], state=connected] 2026-02-18T08:24:18.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=423, tag_tid=29, entry_tid=41891], [object_number=422, tag_tid=29, entry_tid=41890], [object_number=421, tag_tid=29, entry_tid=41889], [object_number=420, tag_tid=29, entry_tid=41888]]], state=connected] 2026-02-18T08:24:18.552 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:24:18.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=423, tag_tid=29, entry_tid=41891' 2026-02-18T08:24:18.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=425, tag_tid=30, entry_tid=1' -a 'object_number=425, tag_tid=30, entry_tid=1' = 'object_number=423, tag_tid=29, entry_tid=41891' 2026-02-18T08:24:18.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=423, tag_tid=29, entry_tid=41891' '!=' 'object_number=420, tag_tid=29, entry_tid=41556' 2026-02-18T08:24:18.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:24:18.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=423, tag_tid=29, entry_tid=41891' = 'object_number=420, tag_tid=29, entry_tid=41556' 2026-02-18T08:24:18.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=423, tag_tid=29, entry_tid=41891' 2026-02-18T08:24:18.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=425, tag_tid=30, entry_tid=1' 2026-02-18T08:24:18.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:24:18.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:24:18.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=30 2026-02-18T08:24:18.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=423, tag_tid=29, entry_tid=41891' 2026-02-18T08:24:18.557 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:24:18.557 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:24:18.558 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=29 2026-02-18T08:24:18.558 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=425, tag_tid=30, entry_tid=1' 2026-02-18T08:24:18.559 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:24:18.559 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:24:18.560 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:24:18.561 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=423, tag_tid=29, entry_tid=41891' 2026-02-18T08:24:18.561 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:24:18.561 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:24:18.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=41891 2026-02-18T08:24:18.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 30 = 29 -a 1 -le 41891 2026-02-18T08:24:18.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:24:18.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:24:18.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:24:18.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:24:18.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:24:18.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:24:18.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:24:18.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:24:18.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:24:18.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:24:18.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:24:18.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:24:18.766 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:24:18.766 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:24:18.766 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:24:18.766 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:24:18.766 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:24:18.766 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:24:18.769 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:24:18.769 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:24:18.769 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:24:18.769 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:24:18.769 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:24:18.769 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:24:18.769 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:24:18.770 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:24:18.770 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:24:18.770 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:24:18.770 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:24:18.770 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:24:18.770 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:24:18.770 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:24:18.770 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:24:18.770 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:18.770 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:24:18.770 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:24:18.770 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:24:18.771 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:24:18.771 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:24:18.771 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:24:18.771 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:24:18.771 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:24:18.771 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:24:18.771 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:24:18.771 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:18.771 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:24:18.771 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:24:18.771 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:24:18.771 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:24:18.772 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:24:18.772 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:24:18.772 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:24:18.780 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:24:18.781 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:24:18.781 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:24:18.781 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:24:18.829 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:24:18.829 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:24:18.830 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:24:18.832 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:24:18.832 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:24:18.832 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:24:18.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:24:18.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:24:18.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:24:18.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:24:18.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:24:18.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:24:18.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:24:18.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:24:18.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:24:18.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:24:18.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:18.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:24:18.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:24:18.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:24:18.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:24:18.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:24:18.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:24:18.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:24:18.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:24:18.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:24:18.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:24:18.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:18.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:24:18.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:24:18.835 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:24:18.835 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:24:18.835 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:24:18.835 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:24:18.835 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:24:18.843 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:24:18.844 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:24:18.844 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:24:18.844 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:24:18.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:24:18.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:24:18.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:24:18.895 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:24:18.895 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:24:18.895 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:24:18.895 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:24:18.895 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:24:18.895 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:24:18.895 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:24:18.896 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:24:18.896 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:24:18.896 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:24:18.896 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:24:18.896 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:24:18.896 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:24:18.896 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:18.896 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:24:18.896 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:24:18.896 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:24:18.896 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:24:18.896 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:24:18.897 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:24:18.897 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:24:18.897 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:24:18.897 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:24:18.897 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:24:18.897 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:18.897 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:24:18.897 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:24:18.897 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:24:18.897 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:24:18.897 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:24:18.897 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:24:18.897 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:24:18.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:24:18.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:24:18.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:24:18.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:24:18.956 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:24:18.956 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:24:18.956 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:24:18.959 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:24:18.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:24:18.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:24:18.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:24:18.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:24:18.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:24:18.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:24:18.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:24:18.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:24:18.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:24:18.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:24:18.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:24:18.961 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:24:18.961 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:18.961 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:24:18.961 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:24:18.961 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:24:18.961 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:24:18.961 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:24:18.961 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:24:18.961 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:24:18.961 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:24:18.961 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:24:18.961 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:24:18.962 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:18.962 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:24:18.962 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:24:18.962 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:24:18.962 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:24:18.962 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:24:18.962 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:24:18.962 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:24:18.970 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:24:18.971 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:24:18.971 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:24:18.971 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:24:19.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:24:19.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:24:19.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:24:19.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:24:19.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:24:19.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:24:19.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:24:19.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:24:19.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:24:19.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:24:19.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:24:19.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:24:19.035 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:24:19.035 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:24:19.035 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:24:19.036 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:24:19.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 105 2026-02-18T08:24:19.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 106 2026-02-18T08:24:19.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:24:19.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=425, tag_tid=30, entry_tid=1], [object_number=424, tag_tid=30, entry_tid=0], [object_number=427, tag_tid=29, entry_tid=45931], [object_number=426, tag_tid=29, entry_tid=45930]]], state=connected] 2026-02-18T08:24:19.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=423, tag_tid=29, entry_tid=42223], [object_number=422, tag_tid=29, entry_tid=42222], [object_number=421, tag_tid=29, entry_tid=42221], [object_number=420, tag_tid=29, entry_tid=42220]]], state=connected] 2026-02-18T08:24:19.069 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:24:19.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=425, tag_tid=30, entry_tid=1' 2026-02-18T08:24:19.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:24:19.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:24:19.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:24:19.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:24:19.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:24:19.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:24:19.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:24:19.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:24:19.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:24:19.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:24:19.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:24:19.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:24:19.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:24:19.074 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:24:19.074 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:24:19.102 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 105 2026-02-18T08:24:19.102 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 106 2026-02-18T08:24:19.102 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:24:19.102 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=425, tag_tid=30, entry_tid=1], [object_number=424, tag_tid=30, entry_tid=0], [object_number=427, tag_tid=29, entry_tid=45931], [object_number=426, tag_tid=29, entry_tid=45930]]], state=connected] 2026-02-18T08:24:19.102 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=423, tag_tid=29, entry_tid=42223], [object_number=422, tag_tid=29, entry_tid=42222], [object_number=421, tag_tid=29, entry_tid=42221], [object_number=420, tag_tid=29, entry_tid=42220]]], state=connected] 2026-02-18T08:24:19.105 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:24:19.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=423, tag_tid=29, entry_tid=42223' 2026-02-18T08:24:19.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=425, tag_tid=30, entry_tid=1' -a 'object_number=425, tag_tid=30, entry_tid=1' = 'object_number=423, tag_tid=29, entry_tid=42223' 2026-02-18T08:24:19.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=423, tag_tid=29, entry_tid=42223' '!=' 'object_number=423, tag_tid=29, entry_tid=41891' 2026-02-18T08:24:19.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:24:19.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=423, tag_tid=29, entry_tid=42223' = 'object_number=423, tag_tid=29, entry_tid=41891' 2026-02-18T08:24:19.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=423, tag_tid=29, entry_tid=42223' 2026-02-18T08:24:19.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=425, tag_tid=30, entry_tid=1' 2026-02-18T08:24:19.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:24:19.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:24:19.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=30 2026-02-18T08:24:19.110 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=423, tag_tid=29, entry_tid=42223' 2026-02-18T08:24:19.110 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:24:19.110 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:24:19.111 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=29 2026-02-18T08:24:19.112 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=425, tag_tid=30, entry_tid=1' 2026-02-18T08:24:19.112 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:24:19.112 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:24:19.113 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:24:19.114 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=423, tag_tid=29, entry_tid=42223' 2026-02-18T08:24:19.114 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:24:19.114 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:24:19.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=42223 2026-02-18T08:24:19.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 30 = 29 -a 1 -le 42223 2026-02-18T08:24:19.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:24:19.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:24:19.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:24:19.318 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:24:19.318 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:24:19.318 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:24:19.318 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:24:19.318 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:24:19.318 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:24:19.318 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:24:19.318 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:24:19.319 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:24:19.319 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:24:19.319 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:24:19.319 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:24:19.319 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:24:19.319 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:24:19.319 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:24:19.322 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:24:19.322 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:24:19.322 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:24:19.322 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:24:19.322 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:24:19.322 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:24:19.322 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:24:19.322 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:24:19.322 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:24:19.323 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:24:19.323 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:24:19.323 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:24:19.323 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:24:19.323 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:24:19.323 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:24:19.323 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:19.323 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:24:19.323 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:24:19.323 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:24:19.323 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:24:19.323 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:24:19.323 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:24:19.324 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:24:19.324 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:24:19.324 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:24:19.324 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:24:19.324 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:19.324 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:24:19.324 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:24:19.324 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:24:19.324 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:24:19.324 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:24:19.324 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:24:19.324 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:24:19.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:24:19.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:24:19.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:24:19.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:24:19.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:24:19.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:24:19.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:24:19.395 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:24:19.395 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:24:19.395 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:24:19.395 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:24:19.395 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:24:19.396 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:24:19.396 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:24:19.396 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:24:19.396 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:24:19.396 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:24:19.396 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:24:19.396 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:24:19.396 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:24:19.396 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:19.396 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:24:19.396 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:24:19.396 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:24:19.397 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:24:19.397 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:24:19.397 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:24:19.397 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:24:19.397 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:24:19.397 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:24:19.397 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:24:19.397 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:19.397 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:24:19.397 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:24:19.397 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:24:19.397 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:24:19.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:24:19.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:24:19.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:24:19.406 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:24:19.407 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:24:19.407 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:24:19.407 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:24:19.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:24:19.458 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:24:19.458 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:24:19.460 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:24:19.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:24:19.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:24:19.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:24:19.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:24:19.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:24:19.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:24:19.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:24:19.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:24:19.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:24:19.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:24:19.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:24:19.462 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:24:19.462 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:19.462 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:24:19.462 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:24:19.462 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:24:19.462 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:24:19.462 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:24:19.462 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:24:19.462 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:24:19.462 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:24:19.462 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:24:19.462 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:24:19.463 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:19.463 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:24:19.463 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:24:19.463 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:24:19.463 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:24:19.463 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:24:19.463 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:24:19.463 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:24:19.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:24:19.472 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:24:19.472 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:24:19.472 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:24:19.529 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:24:19.530 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:24:19.530 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:24:19.532 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:24:19.532 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:24:19.532 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:24:19.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:24:19.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:24:19.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:24:19.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:24:19.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:24:19.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:24:19.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:24:19.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:24:19.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:24:19.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:24:19.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:19.534 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:24:19.534 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:24:19.534 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:24:19.534 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:24:19.534 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:24:19.534 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:24:19.534 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:24:19.534 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:24:19.534 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:24:19.534 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:24:19.534 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:19.534 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:24:19.534 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:24:19.535 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:24:19.535 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:24:19.535 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:24:19.535 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:24:19.535 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:24:19.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:24:19.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:24:19.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:24:19.545 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:24:19.606 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:24:19.607 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:24:19.607 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:24:19.607 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:24:19.607 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:24:19.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:24:19.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:24:19.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:24:19.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:24:19.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:24:19.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:24:19.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:24:19.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:24:19.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:24:19.610 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:24:19.610 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:24:19.637 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 105 2026-02-18T08:24:19.638 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 106 2026-02-18T08:24:19.638 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:24:19.638 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=425, tag_tid=30, entry_tid=1], [object_number=424, tag_tid=30, entry_tid=0], [object_number=427, tag_tid=29, entry_tid=45931], [object_number=426, tag_tid=29, entry_tid=45930]]], state=connected] 2026-02-18T08:24:19.638 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=422, tag_tid=29, entry_tid=42578], [object_number=421, tag_tid=29, entry_tid=42577], [object_number=420, tag_tid=29, entry_tid=42576], [object_number=423, tag_tid=29, entry_tid=42575]]], state=connected] 2026-02-18T08:24:19.640 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:24:19.642 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=425, tag_tid=30, entry_tid=1' 2026-02-18T08:24:19.643 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:24:19.643 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:24:19.643 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:24:19.643 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:24:19.643 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:24:19.643 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:24:19.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:24:19.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:24:19.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:24:19.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:24:19.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:24:19.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:24:19.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:24:19.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:24:19.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:24:19.674 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 105 2026-02-18T08:24:19.674 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 106 2026-02-18T08:24:19.674 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:24:19.674 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=425, tag_tid=30, entry_tid=1], [object_number=424, tag_tid=30, entry_tid=0], [object_number=427, tag_tid=29, entry_tid=45931], [object_number=426, tag_tid=29, entry_tid=45930]]], state=connected] 2026-02-18T08:24:19.674 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=422, tag_tid=29, entry_tid=42578], [object_number=421, tag_tid=29, entry_tid=42577], [object_number=420, tag_tid=29, entry_tid=42576], [object_number=423, tag_tid=29, entry_tid=42575]]], state=connected] 2026-02-18T08:24:19.677 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:24:19.678 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=422, tag_tid=29, entry_tid=42578' 2026-02-18T08:24:19.679 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=425, tag_tid=30, entry_tid=1' -a 'object_number=425, tag_tid=30, entry_tid=1' = 'object_number=422, tag_tid=29, entry_tid=42578' 2026-02-18T08:24:19.679 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=422, tag_tid=29, entry_tid=42578' '!=' 'object_number=423, tag_tid=29, entry_tid=42223' 2026-02-18T08:24:19.679 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:24:19.679 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=422, tag_tid=29, entry_tid=42578' = 'object_number=423, tag_tid=29, entry_tid=42223' 2026-02-18T08:24:19.679 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=422, tag_tid=29, entry_tid=42578' 2026-02-18T08:24:19.680 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=425, tag_tid=30, entry_tid=1' 2026-02-18T08:24:19.680 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:24:19.680 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:24:19.681 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=30 2026-02-18T08:24:19.681 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=422, tag_tid=29, entry_tid=42578' 2026-02-18T08:24:19.681 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:24:19.682 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:24:19.683 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=29 2026-02-18T08:24:19.683 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=425, tag_tid=30, entry_tid=1' 2026-02-18T08:24:19.684 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:24:19.684 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:24:19.685 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:24:19.686 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=422, tag_tid=29, entry_tid=42578' 2026-02-18T08:24:19.686 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:24:19.686 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:24:19.687 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=42578 2026-02-18T08:24:19.687 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 30 = 29 -a 1 -le 42578 2026-02-18T08:24:19.687 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:24:19.688 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:24:19.688 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:24:19.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:24:19.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:24:19.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:24:19.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:24:19.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:24:19.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:24:19.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:24:19.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:24:19.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:24:19.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:24:19.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:24:19.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:24:19.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:24:19.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:24:19.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:24:19.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:24:19.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:24:19.894 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:24:19.894 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:24:19.894 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:24:19.894 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:24:19.894 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:24:19.894 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:24:19.894 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:24:19.894 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:24:19.894 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:24:19.894 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:24:19.894 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:24:19.895 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:24:19.895 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:24:19.895 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:19.895 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:24:19.895 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:24:19.895 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:24:19.895 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:24:19.895 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:24:19.895 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:24:19.895 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:24:19.895 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:24:19.895 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:24:19.896 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:24:19.896 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:19.896 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:24:19.896 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:24:19.896 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:24:19.896 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:24:19.896 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:24:19.896 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:24:19.896 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:24:19.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:24:19.904 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:24:19.904 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:24:19.904 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:24:19.954 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:24:19.954 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:24:19.954 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:24:19.956 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:24:19.956 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:24:19.956 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:24:19.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:24:19.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:24:19.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:24:19.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:24:19.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:24:19.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:24:19.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:24:19.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:24:19.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:24:19.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:24:19.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:19.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:24:19.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:24:19.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:24:19.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:24:19.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:24:19.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:24:19.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:24:19.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:24:19.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:24:19.958 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:24:19.958 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:19.958 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:24:19.958 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:24:19.958 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:24:19.958 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:24:19.958 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:24:19.958 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:24:19.958 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:24:19.968 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:24:19.969 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:24:19.969 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:24:19.969 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:24:20.018 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:24:20.018 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:24:20.018 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:24:20.019 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:24:20.019 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:24:20.019 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:24:20.019 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:24:20.019 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:24:20.019 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:24:20.019 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:24:20.019 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:24:20.019 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:24:20.019 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:24:20.020 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:24:20.020 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:24:20.020 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:24:20.020 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:20.020 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:24:20.020 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:24:20.021 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:24:20.021 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:24:20.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:24:20.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:24:20.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:24:20.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:24:20.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:24:20.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:24:20.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:20.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:24:20.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:24:20.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:24:20.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:24:20.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:24:20.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:24:20.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:24:20.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:24:20.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:24:20.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:24:20.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:24:20.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:24:20.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:24:20.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:24:20.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:24:20.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:24:20.086 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:24:20.086 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:24:20.086 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:24:20.086 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:24:20.086 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:24:20.086 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:24:20.086 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:24:20.086 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:24:20.086 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:24:20.086 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:24:20.086 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:24:20.086 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:20.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:24:20.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:24:20.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:24:20.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:24:20.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:24:20.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:24:20.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:24:20.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:24:20.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:24:20.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:24:20.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:20.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:24:20.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:24:20.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:24:20.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:24:20.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:24:20.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:24:20.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:24:20.097 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:24:20.097 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:24:20.097 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:24:20.097 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:24:20.158 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:24:20.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:24:20.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:24:20.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:24:20.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:24:20.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:24:20.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:24:20.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:24:20.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:24:20.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:24:20.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:24:20.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:24:20.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:24:20.161 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:24:20.161 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:24:20.161 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:24:20.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 105 2026-02-18T08:24:20.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 106 2026-02-18T08:24:20.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:24:20.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=425, tag_tid=30, entry_tid=1], [object_number=424, tag_tid=30, entry_tid=0], [object_number=427, tag_tid=29, entry_tid=45931], [object_number=426, tag_tid=29, entry_tid=45930]]], state=connected] 2026-02-18T08:24:20.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=423, tag_tid=29, entry_tid=42939], [object_number=422, tag_tid=29, entry_tid=42938], [object_number=421, tag_tid=29, entry_tid=42937], [object_number=420, tag_tid=29, entry_tid=42936]]], state=connected] 2026-02-18T08:24:20.194 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:24:20.196 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=425, tag_tid=30, entry_tid=1' 2026-02-18T08:24:20.197 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:24:20.197 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:24:20.197 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:24:20.197 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:24:20.197 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:24:20.197 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:24:20.198 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:24:20.198 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:24:20.198 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:24:20.198 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:24:20.198 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:24:20.198 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:24:20.199 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:24:20.199 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:24:20.199 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:24:20.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 105 2026-02-18T08:24:20.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 106 2026-02-18T08:24:20.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:24:20.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=425, tag_tid=30, entry_tid=1], [object_number=424, tag_tid=30, entry_tid=0], [object_number=427, tag_tid=29, entry_tid=45931], [object_number=426, tag_tid=29, entry_tid=45930]]], state=connected] 2026-02-18T08:24:20.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=423, tag_tid=29, entry_tid=42939], [object_number=422, tag_tid=29, entry_tid=42938], [object_number=421, tag_tid=29, entry_tid=42937], [object_number=420, tag_tid=29, entry_tid=42936]]], state=connected] 2026-02-18T08:24:20.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:24:20.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=423, tag_tid=29, entry_tid=42939' 2026-02-18T08:24:20.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=425, tag_tid=30, entry_tid=1' -a 'object_number=425, tag_tid=30, entry_tid=1' = 'object_number=423, tag_tid=29, entry_tid=42939' 2026-02-18T08:24:20.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=423, tag_tid=29, entry_tid=42939' '!=' 'object_number=422, tag_tid=29, entry_tid=42578' 2026-02-18T08:24:20.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:24:20.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=423, tag_tid=29, entry_tid=42939' = 'object_number=422, tag_tid=29, entry_tid=42578' 2026-02-18T08:24:20.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=423, tag_tid=29, entry_tid=42939' 2026-02-18T08:24:20.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=425, tag_tid=30, entry_tid=1' 2026-02-18T08:24:20.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:24:20.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:24:20.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=30 2026-02-18T08:24:20.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=423, tag_tid=29, entry_tid=42939' 2026-02-18T08:24:20.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:24:20.244 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:24:20.245 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=29 2026-02-18T08:24:20.246 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=425, tag_tid=30, entry_tid=1' 2026-02-18T08:24:20.246 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:24:20.246 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:24:20.247 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:24:20.248 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=423, tag_tid=29, entry_tid=42939' 2026-02-18T08:24:20.248 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:24:20.248 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:24:20.250 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=42939 2026-02-18T08:24:20.250 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 30 = 29 -a 1 -le 42939 2026-02-18T08:24:20.250 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:24:20.250 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:24:20.250 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:24:20.451 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:24:20.452 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:24:20.452 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:24:20.452 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:24:20.452 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:24:20.452 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:24:20.452 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:24:20.452 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:24:20.452 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:24:20.452 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:24:20.453 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:24:20.453 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:24:20.453 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:24:20.453 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:24:20.453 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:24:20.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:24:20.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:24:20.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:24:20.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:24:20.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:24:20.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:24:20.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:24:20.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:24:20.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:24:20.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:24:20.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:24:20.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:24:20.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:24:20.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:24:20.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:24:20.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:20.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:24:20.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:24:20.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:24:20.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:24:20.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:24:20.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:24:20.458 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:24:20.458 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:24:20.458 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:24:20.458 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:24:20.458 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:20.458 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:24:20.458 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:24:20.458 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:24:20.458 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:24:20.458 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:24:20.458 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:24:20.458 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:24:20.467 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:24:20.467 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:24:20.467 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:24:20.467 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:24:20.516 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:24:20.516 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:24:20.516 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:24:20.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:24:20.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:24:20.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:24:20.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:24:20.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:24:20.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:24:20.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:24:20.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:24:20.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:24:20.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:24:20.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:24:20.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:24:20.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:24:20.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:20.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:24:20.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:24:20.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:24:20.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:24:20.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:24:20.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:24:20.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:24:20.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:24:20.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:24:20.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:24:20.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:20.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:24:20.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:24:20.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:24:20.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:24:20.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:24:20.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:24:20.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:24:20.532 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:24:20.532 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:24:20.532 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:24:20.532 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:24:20.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:24:20.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:24:20.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:24:20.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:24:20.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:24:20.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:24:20.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:24:20.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:24:20.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:24:20.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:24:20.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:24:20.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:24:20.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:24:20.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:24:20.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:24:20.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:24:20.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:20.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:24:20.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:24:20.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:24:20.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:24:20.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:24:20.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:24:20.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:24:20.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:24:20.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:24:20.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:24:20.594 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:20.594 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:24:20.594 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:24:20.594 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:24:20.594 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:24:20.594 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:24:20.594 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:24:20.594 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:24:20.603 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:24:20.604 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:24:20.604 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:24:20.604 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:24:20.653 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:24:20.653 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:24:20.653 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:24:20.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:24:20.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:24:20.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:24:20.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:24:20.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:24:20.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:24:20.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:24:20.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:24:20.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:24:20.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:24:20.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:24:20.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:24:20.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:24:20.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:20.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:24:20.658 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:24:20.658 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:24:20.658 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:24:20.658 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:24:20.658 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:24:20.658 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:24:20.658 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:24:20.658 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:24:20.658 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:24:20.658 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:20.658 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:24:20.658 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:24:20.658 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:24:20.659 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:24:20.659 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:24:20.659 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:24:20.659 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:24:20.667 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:24:20.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:24:20.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:24:20.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:24:20.729 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:24:20.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:24:20.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:24:20.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:24:20.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:24:20.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:24:20.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:24:20.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:24:20.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:24:20.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:24:20.731 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:24:20.731 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:24:20.731 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:24:20.731 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:24:20.731 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:24:20.732 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:24:20.766 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 105 2026-02-18T08:24:20.766 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 106 2026-02-18T08:24:20.766 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:24:20.766 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=425, tag_tid=30, entry_tid=1], [object_number=424, tag_tid=30, entry_tid=0], [object_number=427, tag_tid=29, entry_tid=45931], [object_number=426, tag_tid=29, entry_tid=45930]]], state=connected] 2026-02-18T08:24:20.767 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=422, tag_tid=29, entry_tid=43330], [object_number=421, tag_tid=29, entry_tid=43329], [object_number=420, tag_tid=29, entry_tid=43328], [object_number=423, tag_tid=29, entry_tid=43327]]], state=connected] 2026-02-18T08:24:20.770 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:24:20.772 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=425, tag_tid=30, entry_tid=1' 2026-02-18T08:24:20.773 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:24:20.773 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:24:20.773 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:24:20.773 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:24:20.773 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:24:20.773 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:24:20.773 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:24:20.773 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:24:20.773 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:24:20.774 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:24:20.774 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:24:20.774 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:24:20.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:24:20.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:24:20.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:24:20.809 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 105 2026-02-18T08:24:20.809 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 106 2026-02-18T08:24:20.809 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:24:20.809 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=425, tag_tid=30, entry_tid=1], [object_number=424, tag_tid=30, entry_tid=0], [object_number=427, tag_tid=29, entry_tid=45931], [object_number=426, tag_tid=29, entry_tid=45930]]], state=connected] 2026-02-18T08:24:20.810 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=422, tag_tid=29, entry_tid=43330], [object_number=421, tag_tid=29, entry_tid=43329], [object_number=420, tag_tid=29, entry_tid=43328], [object_number=423, tag_tid=29, entry_tid=43327]]], state=connected] 2026-02-18T08:24:20.813 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:24:20.815 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=422, tag_tid=29, entry_tid=43330' 2026-02-18T08:24:20.815 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=425, tag_tid=30, entry_tid=1' -a 'object_number=425, tag_tid=30, entry_tid=1' = 'object_number=422, tag_tid=29, entry_tid=43330' 2026-02-18T08:24:20.815 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=422, tag_tid=29, entry_tid=43330' '!=' 'object_number=423, tag_tid=29, entry_tid=42939' 2026-02-18T08:24:20.815 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:24:20.815 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=422, tag_tid=29, entry_tid=43330' = 'object_number=423, tag_tid=29, entry_tid=42939' 2026-02-18T08:24:20.815 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=422, tag_tid=29, entry_tid=43330' 2026-02-18T08:24:20.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=425, tag_tid=30, entry_tid=1' 2026-02-18T08:24:20.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:24:20.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:24:20.817 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=30 2026-02-18T08:24:20.818 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=422, tag_tid=29, entry_tid=43330' 2026-02-18T08:24:20.818 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:24:20.818 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:24:20.820 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=29 2026-02-18T08:24:20.820 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=425, tag_tid=30, entry_tid=1' 2026-02-18T08:24:20.821 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:24:20.821 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:24:20.822 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:24:20.822 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=422, tag_tid=29, entry_tid=43330' 2026-02-18T08:24:20.823 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:24:20.823 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:24:20.824 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=43330 2026-02-18T08:24:20.825 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 30 = 29 -a 1 -le 43330 2026-02-18T08:24:20.825 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:24:20.825 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:24:20.825 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:24:21.027 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:24:21.027 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:24:21.027 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:24:21.027 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:24:21.027 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:24:21.027 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:24:21.028 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:24:21.028 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:24:21.028 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:24:21.028 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:24:21.028 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:24:21.028 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:24:21.028 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:24:21.028 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:24:21.028 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:24:21.028 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:24:21.028 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:24:21.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:24:21.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:24:21.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:24:21.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:24:21.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:24:21.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:24:21.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:24:21.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:24:21.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:24:21.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:24:21.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:24:21.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:24:21.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:24:21.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:21.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:24:21.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:24:21.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:24:21.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:24:21.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:24:21.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:24:21.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:24:21.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:24:21.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:24:21.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:24:21.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:21.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:24:21.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:24:21.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:24:21.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:24:21.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:24:21.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:24:21.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:24:21.040 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:24:21.041 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:24:21.041 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:24:21.041 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:24:21.101 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:24:21.101 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:24:21.101 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:24:21.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:24:21.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:24:21.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:24:21.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:24:21.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:24:21.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:24:21.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:24:21.105 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:24:21.105 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:24:21.105 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:24:21.105 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:24:21.105 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:24:21.105 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:24:21.105 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:21.105 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:24:21.105 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:24:21.105 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:24:21.105 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:24:21.105 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:24:21.106 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:24:21.106 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:24:21.106 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:24:21.106 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:24:21.106 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:24:21.106 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:21.106 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:24:21.106 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:24:21.106 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:24:21.106 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:24:21.106 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:24:21.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:24:21.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:24:21.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:24:21.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:24:21.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:24:21.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:24:21.164 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:24:21.164 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:24:21.164 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:24:21.166 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:24:21.167 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:24:21.167 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:24:21.167 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:24:21.167 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:24:21.167 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:24:21.167 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:24:21.167 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:24:21.167 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:24:21.167 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:24:21.167 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:24:21.167 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:24:21.167 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:24:21.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:21.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:24:21.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:24:21.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:24:21.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:24:21.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:24:21.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:24:21.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:24:21.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:24:21.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:24:21.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:24:21.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:21.169 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:24:21.169 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:24:21.169 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:24:21.169 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:24:21.169 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:24:21.169 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:24:21.169 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:24:21.178 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:24:21.179 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:24:21.179 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:24:21.179 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:24:21.227 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:24:21.227 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:24:21.227 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:24:21.230 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:24:21.230 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:24:21.230 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:24:21.230 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:24:21.230 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:24:21.230 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:24:21.230 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:24:21.230 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:24:21.230 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:24:21.231 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:24:21.231 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:24:21.231 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:24:21.231 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:24:21.231 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:21.231 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:24:21.231 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:24:21.231 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:24:21.231 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:24:21.231 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:24:21.231 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:24:21.231 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:24:21.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:24:21.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:24:21.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:24:21.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:21.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:24:21.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:24:21.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:24:21.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:24:21.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:24:21.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:24:21.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:24:21.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:24:21.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:24:21.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:24:21.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:24:21.300 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:24:21.301 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:24:21.302 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:24:21.302 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:24:21.302 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:24:21.302 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:24:21.302 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:24:21.302 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:24:21.302 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:24:21.302 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:24:21.302 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:24:21.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:24:21.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:24:21.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:24:21.304 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:24:21.304 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:24:21.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 105 2026-02-18T08:24:21.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 106 2026-02-18T08:24:21.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:24:21.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=425, tag_tid=30, entry_tid=1], [object_number=424, tag_tid=30, entry_tid=0], [object_number=427, tag_tid=29, entry_tid=45931], [object_number=426, tag_tid=29, entry_tid=45930]]], state=connected] 2026-02-18T08:24:21.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=423, tag_tid=29, entry_tid=43695], [object_number=422, tag_tid=29, entry_tid=43694], [object_number=421, tag_tid=29, entry_tid=43693], [object_number=420, tag_tid=29, entry_tid=43692]]], state=connected] 2026-02-18T08:24:21.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:24:21.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=425, tag_tid=30, entry_tid=1' 2026-02-18T08:24:21.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:24:21.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:24:21.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:24:21.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:24:21.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:24:21.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:24:21.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:24:21.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:24:21.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:24:21.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:24:21.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:24:21.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:24:21.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:24:21.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:24:21.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:24:21.371 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 105 2026-02-18T08:24:21.372 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 106 2026-02-18T08:24:21.372 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:24:21.372 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=425, tag_tid=30, entry_tid=1], [object_number=424, tag_tid=30, entry_tid=0], [object_number=427, tag_tid=29, entry_tid=45931], [object_number=426, tag_tid=29, entry_tid=45930]]], state=connected] 2026-02-18T08:24:21.372 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=423, tag_tid=29, entry_tid=43695], [object_number=422, tag_tid=29, entry_tid=43694], [object_number=421, tag_tid=29, entry_tid=43693], [object_number=420, tag_tid=29, entry_tid=43692]]], state=connected] 2026-02-18T08:24:21.374 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:24:21.376 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=423, tag_tid=29, entry_tid=43695' 2026-02-18T08:24:21.376 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=425, tag_tid=30, entry_tid=1' -a 'object_number=425, tag_tid=30, entry_tid=1' = 'object_number=423, tag_tid=29, entry_tid=43695' 2026-02-18T08:24:21.377 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=423, tag_tid=29, entry_tid=43695' '!=' 'object_number=422, tag_tid=29, entry_tid=43330' 2026-02-18T08:24:21.377 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:24:21.377 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=423, tag_tid=29, entry_tid=43695' = 'object_number=422, tag_tid=29, entry_tid=43330' 2026-02-18T08:24:21.377 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=423, tag_tid=29, entry_tid=43695' 2026-02-18T08:24:21.377 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=425, tag_tid=30, entry_tid=1' 2026-02-18T08:24:21.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:24:21.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:24:21.379 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=30 2026-02-18T08:24:21.380 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=423, tag_tid=29, entry_tid=43695' 2026-02-18T08:24:21.380 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:24:21.380 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:24:21.381 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=29 2026-02-18T08:24:21.381 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=425, tag_tid=30, entry_tid=1' 2026-02-18T08:24:21.382 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:24:21.382 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:24:21.383 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:24:21.384 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=423, tag_tid=29, entry_tid=43695' 2026-02-18T08:24:21.384 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:24:21.384 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:24:21.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=43695 2026-02-18T08:24:21.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 30 = 29 -a 1 -le 43695 2026-02-18T08:24:21.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:24:21.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:24:21.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:24:21.587 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:24:21.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:24:21.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:24:21.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:24:21.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:24:21.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:24:21.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:24:21.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:24:21.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:24:21.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:24:21.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:24:21.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:24:21.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:24:21.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:24:21.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:24:21.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:24:21.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:24:21.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:24:21.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:24:21.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:24:21.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:24:21.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:24:21.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:24:21.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:24:21.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:24:21.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:24:21.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:24:21.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:24:21.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:24:21.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:24:21.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:21.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:24:21.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:24:21.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:24:21.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:24:21.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:24:21.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:24:21.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:24:21.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:24:21.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:24:21.594 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:24:21.594 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:21.594 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:24:21.594 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:24:21.594 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:24:21.594 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:24:21.594 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:24:21.594 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:24:21.594 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:24:21.602 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:24:21.602 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:24:21.602 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:24:21.602 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:24:21.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:24:21.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:24:21.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:24:21.653 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:24:21.653 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:24:21.653 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:24:21.653 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:24:21.654 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:24:21.654 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:24:21.654 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:24:21.654 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:24:21.654 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:24:21.654 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:24:21.654 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:24:21.654 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:24:21.654 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:24:21.654 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:21.654 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:24:21.654 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:24:21.655 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:24:21.655 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:24:21.655 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:24:21.655 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:24:21.655 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:24:21.655 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:24:21.655 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:24:21.655 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:24:21.655 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:21.655 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:24:21.655 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:24:21.655 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:24:21.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:24:21.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:24:21.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:24:21.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:24:21.664 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:24:21.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:24:21.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:24:21.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:24:21.713 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:24:21.713 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:24:21.713 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:24:21.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:24:21.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:24:21.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:24:21.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:24:21.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:24:21.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:24:21.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:24:21.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:24:21.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:24:21.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:24:21.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:24:21.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:24:21.717 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:24:21.717 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:21.717 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:24:21.717 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:24:21.717 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:24:21.717 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:24:21.717 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:24:21.717 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:24:21.717 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:24:21.717 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:24:21.717 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:24:21.717 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:24:21.718 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:21.718 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:24:21.718 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:24:21.718 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:24:21.718 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:24:21.718 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:24:21.718 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:24:21.718 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:24:21.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:24:21.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:24:21.728 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:24:21.728 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:24:21.785 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:24:21.785 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:24:21.785 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:24:21.788 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:24:21.788 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:24:21.788 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:24:21.788 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:24:21.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:24:21.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:24:21.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:24:21.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:24:21.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:24:21.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:24:21.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:24:21.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:24:21.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:24:21.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:21.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:24:21.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:24:21.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:24:21.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:24:21.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:24:21.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:24:21.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:24:21.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:24:21.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:24:21.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:24:21.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:21.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:24:21.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:24:21.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:24:21.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:24:21.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:24:21.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:24:21.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:24:21.799 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:24:21.800 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:24:21.800 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:24:21.800 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:24:21.861 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:24:21.862 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:24:21.862 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:24:21.862 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:24:21.862 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:24:21.862 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:24:21.863 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:24:21.863 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:24:21.863 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:24:21.863 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:24:21.863 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:24:21.863 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:24:21.864 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:24:21.864 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:24:21.864 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:24:21.865 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:24:21.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 105 2026-02-18T08:24:21.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 106 2026-02-18T08:24:21.894 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:24:21.894 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=425, tag_tid=30, entry_tid=1], [object_number=424, tag_tid=30, entry_tid=0], [object_number=427, tag_tid=29, entry_tid=45931], [object_number=426, tag_tid=29, entry_tid=45930]]], state=connected] 2026-02-18T08:24:21.894 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=421, tag_tid=29, entry_tid=44041], [object_number=420, tag_tid=29, entry_tid=44040], [object_number=423, tag_tid=29, entry_tid=44039], [object_number=422, tag_tid=29, entry_tid=44038]]], state=connected] 2026-02-18T08:24:21.896 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:24:21.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=425, tag_tid=30, entry_tid=1' 2026-02-18T08:24:21.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:24:21.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:24:21.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:24:21.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:24:21.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:24:21.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:24:21.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:24:21.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:24:21.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:24:21.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:24:21.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:24:21.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:24:21.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:24:21.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:24:21.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:24:21.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 105 2026-02-18T08:24:21.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 106 2026-02-18T08:24:21.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:24:21.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=425, tag_tid=30, entry_tid=1], [object_number=424, tag_tid=30, entry_tid=0], [object_number=427, tag_tid=29, entry_tid=45931], [object_number=426, tag_tid=29, entry_tid=45930]]], state=connected] 2026-02-18T08:24:21.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=421, tag_tid=29, entry_tid=44041], [object_number=420, tag_tid=29, entry_tid=44040], [object_number=423, tag_tid=29, entry_tid=44039], [object_number=422, tag_tid=29, entry_tid=44038]]], state=connected] 2026-02-18T08:24:21.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:24:21.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=421, tag_tid=29, entry_tid=44041' 2026-02-18T08:24:21.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=425, tag_tid=30, entry_tid=1' -a 'object_number=425, tag_tid=30, entry_tid=1' = 'object_number=421, tag_tid=29, entry_tid=44041' 2026-02-18T08:24:21.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=421, tag_tid=29, entry_tid=44041' '!=' 'object_number=423, tag_tid=29, entry_tid=43695' 2026-02-18T08:24:21.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:24:21.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=421, tag_tid=29, entry_tid=44041' = 'object_number=423, tag_tid=29, entry_tid=43695' 2026-02-18T08:24:21.946 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=421, tag_tid=29, entry_tid=44041' 2026-02-18T08:24:21.946 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=425, tag_tid=30, entry_tid=1' 2026-02-18T08:24:21.946 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:24:21.946 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:24:21.947 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=30 2026-02-18T08:24:21.948 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=421, tag_tid=29, entry_tid=44041' 2026-02-18T08:24:21.948 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:24:21.948 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:24:21.950 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=29 2026-02-18T08:24:21.950 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=425, tag_tid=30, entry_tid=1' 2026-02-18T08:24:21.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:24:21.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:24:21.952 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:24:21.953 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=421, tag_tid=29, entry_tid=44041' 2026-02-18T08:24:21.953 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:24:21.953 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:24:21.954 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=44041 2026-02-18T08:24:21.955 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 30 = 29 -a 1 -le 44041 2026-02-18T08:24:21.955 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:24:21.955 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:24:21.955 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:24:22.156 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:24:22.157 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:24:22.157 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:24:22.157 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:24:22.157 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:24:22.157 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:24:22.157 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:24:22.157 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:24:22.157 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:24:22.157 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:24:22.157 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:24:22.157 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:24:22.158 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:24:22.158 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:24:22.158 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:24:22.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:24:22.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:24:22.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:24:22.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:24:22.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:24:22.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:24:22.161 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:24:22.161 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:24:22.161 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:24:22.161 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:24:22.161 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:24:22.161 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:24:22.161 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:24:22.161 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:24:22.161 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:24:22.161 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:22.161 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:24:22.161 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:24:22.162 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:24:22.162 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:24:22.162 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:24:22.162 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:24:22.162 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:24:22.162 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:24:22.162 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:24:22.162 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:24:22.162 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:22.162 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:24:22.162 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:24:22.162 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:24:22.163 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:24:22.163 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:24:22.163 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:24:22.163 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:24:22.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:24:22.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:24:22.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:24:22.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:24:22.222 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:24:22.223 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:24:22.223 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:24:22.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:24:22.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:24:22.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:24:22.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:24:22.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:24:22.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:24:22.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:24:22.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:24:22.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:24:22.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:24:22.227 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:24:22.227 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:24:22.227 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:24:22.227 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:22.227 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:24:22.227 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:24:22.227 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:24:22.227 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:24:22.227 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:24:22.227 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:24:22.227 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:24:22.227 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:24:22.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:24:22.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:24:22.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:22.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:24:22.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:24:22.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:24:22.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:24:22.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:24:22.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:24:22.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:24:22.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:24:22.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:24:22.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:24:22.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:24:22.301 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:24:22.301 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:24:22.301 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:24:22.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:24:22.304 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:24:22.304 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:24:22.304 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:24:22.304 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:24:22.304 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:24:22.304 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:24:22.304 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:24:22.304 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:24:22.304 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:24:22.304 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:24:22.304 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:24:22.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:24:22.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:22.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:24:22.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:24:22.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:24:22.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:24:22.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:24:22.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:24:22.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:24:22.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:24:22.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:24:22.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:24:22.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:22.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:24:22.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:24:22.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:24:22.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:24:22.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:24:22.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:24:22.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:24:22.315 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:24:22.315 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:24:22.315 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:24:22.315 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:24:22.364 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:24:22.365 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:24:22.365 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:24:22.367 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:24:22.368 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:24:22.368 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:24:22.368 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:24:22.368 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:24:22.368 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:24:22.368 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:24:22.368 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:24:22.368 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:24:22.368 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:24:22.368 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:24:22.368 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:24:22.368 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:24:22.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:22.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:24:22.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:24:22.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:24:22.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:24:22.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:24:22.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:24:22.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:24:22.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:24:22.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:24:22.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:24:22.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:22.370 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:24:22.370 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:24:22.370 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:24:22.370 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:24:22.370 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:24:22.370 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:24:22.370 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:24:22.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:24:22.379 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:24:22.379 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:24:22.379 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:24:22.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:24:22.448 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:24:22.448 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:24:22.448 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:24:22.448 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:24:22.448 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:24:22.448 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:24:22.448 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:24:22.448 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:24:22.448 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:24:22.449 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:24:22.449 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:24:22.449 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:24:22.450 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:24:22.450 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:24:22.450 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:24:22.480 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 106 2026-02-18T08:24:22.480 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 106 2026-02-18T08:24:22.480 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:24:22.480 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=425, tag_tid=30, entry_tid=1], [object_number=424, tag_tid=30, entry_tid=0], [object_number=427, tag_tid=29, entry_tid=45931], [object_number=426, tag_tid=29, entry_tid=45930]]], state=connected] 2026-02-18T08:24:22.480 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=426, tag_tid=29, entry_tid=44322], [object_number=425, tag_tid=29, entry_tid=44321], [object_number=424, tag_tid=29, entry_tid=44320], [object_number=427, tag_tid=29, entry_tid=44319]]], state=connected] 2026-02-18T08:24:22.483 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:24:22.484 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=425, tag_tid=30, entry_tid=1' 2026-02-18T08:24:22.485 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:24:22.485 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:24:22.486 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:24:22.486 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:24:22.486 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:24:22.486 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:24:22.486 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:24:22.486 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:24:22.486 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:24:22.486 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:24:22.487 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:24:22.487 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:24:22.488 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:24:22.488 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:24:22.488 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:24:22.526 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 106 2026-02-18T08:24:22.526 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 106 2026-02-18T08:24:22.526 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:24:22.526 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=425, tag_tid=30, entry_tid=1], [object_number=424, tag_tid=30, entry_tid=0], [object_number=427, tag_tid=29, entry_tid=45931], [object_number=426, tag_tid=29, entry_tid=45930]]], state=connected] 2026-02-18T08:24:22.526 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=426, tag_tid=29, entry_tid=44322], [object_number=425, tag_tid=29, entry_tid=44321], [object_number=424, tag_tid=29, entry_tid=44320], [object_number=427, tag_tid=29, entry_tid=44319]]], state=connected] 2026-02-18T08:24:22.529 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:24:22.531 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=426, tag_tid=29, entry_tid=44322' 2026-02-18T08:24:22.531 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=425, tag_tid=30, entry_tid=1' -a 'object_number=425, tag_tid=30, entry_tid=1' = 'object_number=426, tag_tid=29, entry_tid=44322' 2026-02-18T08:24:22.531 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=426, tag_tid=29, entry_tid=44322' '!=' 'object_number=421, tag_tid=29, entry_tid=44041' 2026-02-18T08:24:22.531 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:24:22.532 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=426, tag_tid=29, entry_tid=44322' = 'object_number=421, tag_tid=29, entry_tid=44041' 2026-02-18T08:24:22.532 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=426, tag_tid=29, entry_tid=44322' 2026-02-18T08:24:22.532 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=425, tag_tid=30, entry_tid=1' 2026-02-18T08:24:22.532 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:24:22.532 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:24:22.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=30 2026-02-18T08:24:22.534 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=426, tag_tid=29, entry_tid=44322' 2026-02-18T08:24:22.534 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:24:22.534 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:24:22.535 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=29 2026-02-18T08:24:22.536 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=425, tag_tid=30, entry_tid=1' 2026-02-18T08:24:22.536 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:24:22.536 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:24:22.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:24:22.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=426, tag_tid=29, entry_tid=44322' 2026-02-18T08:24:22.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:24:22.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:24:22.540 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=44322 2026-02-18T08:24:22.540 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 30 = 29 -a 1 -le 44322 2026-02-18T08:24:22.540 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:24:22.540 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:24:22.540 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:24:22.742 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:24:22.742 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:24:22.742 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:24:22.742 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:24:22.742 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:24:22.742 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:24:22.742 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:24:22.742 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:24:22.743 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:24:22.743 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:24:22.743 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:24:22.743 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:24:22.743 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:24:22.743 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:24:22.743 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:24:22.745 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:24:22.746 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:24:22.746 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:24:22.746 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:24:22.746 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:24:22.746 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:24:22.746 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:24:22.746 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:24:22.746 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:24:22.746 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:24:22.746 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:24:22.747 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:24:22.747 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:24:22.747 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:24:22.747 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:24:22.747 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:22.747 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:24:22.747 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:24:22.747 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:24:22.747 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:24:22.747 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:24:22.747 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:24:22.747 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:24:22.747 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:24:22.748 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:24:22.748 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:24:22.748 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:22.748 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:24:22.748 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:24:22.748 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:24:22.748 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:24:22.748 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:24:22.748 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:24:22.748 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:24:22.756 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:24:22.756 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:24:22.756 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:24:22.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:24:22.807 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:24:22.807 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:24:22.807 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:24:22.810 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:24:22.810 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:24:22.810 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:24:22.810 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:24:22.810 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:24:22.810 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:24:22.810 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:24:22.810 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:24:22.811 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:24:22.811 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:24:22.811 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:24:22.811 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:24:22.811 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:24:22.811 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:22.811 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:24:22.811 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:24:22.811 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:24:22.811 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:24:22.811 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:24:22.811 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:24:22.811 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:24:22.812 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:24:22.812 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:24:22.812 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:24:22.812 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:22.812 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:24:22.812 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:24:22.812 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:24:22.812 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:24:22.812 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:24:22.812 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:24:22.812 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:24:22.822 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:24:22.823 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:24:22.823 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:24:22.823 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:24:22.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:24:22.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:24:22.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:24:22.876 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:24:22.876 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:24:22.876 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:24:22.876 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:24:22.877 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:24:22.877 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:24:22.877 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:24:22.877 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:24:22.877 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:24:22.877 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:24:22.877 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:24:22.877 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:24:22.877 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:24:22.877 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:22.877 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:24:22.877 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:24:22.878 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:24:22.878 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:24:22.878 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:24:22.878 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:24:22.878 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:24:22.878 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:24:22.878 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:24:22.878 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:24:22.878 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:22.878 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:24:22.878 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:24:22.878 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:24:22.878 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:24:22.879 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:24:22.879 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:24:22.879 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:24:22.887 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:24:22.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:24:22.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:24:22.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:24:22.936 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:24:22.936 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:24:22.936 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:24:22.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:24:22.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:24:22.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:24:22.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:24:22.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:24:22.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:24:22.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:24:22.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:24:22.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:24:22.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:24:22.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:24:22.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:24:22.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:24:22.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:22.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:24:22.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:24:22.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:24:22.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:24:22.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:24:22.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:24:22.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:24:22.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:24:22.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:24:22.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:24:22.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:22.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:24:22.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:24:22.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:24:22.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:24:22.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:24:22.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:24:22.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:24:22.950 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:24:22.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:24:22.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:24:22.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:24:23.012 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:24:23.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:24:23.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:24:23.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:24:23.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:24:23.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:24:23.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:24:23.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:24:23.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:24:23.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:24:23.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:24:23.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:24:23.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:24:23.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:24:23.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:24:23.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:24:23.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 106 2026-02-18T08:24:23.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 106 2026-02-18T08:24:23.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:24:23.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=425, tag_tid=30, entry_tid=1], [object_number=424, tag_tid=30, entry_tid=0], [object_number=427, tag_tid=29, entry_tid=45931], [object_number=426, tag_tid=29, entry_tid=45930]]], state=connected] 2026-02-18T08:24:23.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=426, tag_tid=29, entry_tid=44674], [object_number=425, tag_tid=29, entry_tid=44673], [object_number=424, tag_tid=29, entry_tid=44672], [object_number=427, tag_tid=29, entry_tid=44671]]], state=connected] 2026-02-18T08:24:23.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:24:23.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=425, tag_tid=30, entry_tid=1' 2026-02-18T08:24:23.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:24:23.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:24:23.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:24:23.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:24:23.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:24:23.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:24:23.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:24:23.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:24:23.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:24:23.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:24:23.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:24:23.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:24:23.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:24:23.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:24:23.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:24:23.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 106 2026-02-18T08:24:23.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 106 2026-02-18T08:24:23.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:24:23.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=425, tag_tid=30, entry_tid=1], [object_number=424, tag_tid=30, entry_tid=0], [object_number=427, tag_tid=29, entry_tid=45931], [object_number=426, tag_tid=29, entry_tid=45930]]], state=connected] 2026-02-18T08:24:23.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=426, tag_tid=29, entry_tid=44674], [object_number=425, tag_tid=29, entry_tid=44673], [object_number=424, tag_tid=29, entry_tid=44672], [object_number=427, tag_tid=29, entry_tid=44671]]], state=connected] 2026-02-18T08:24:23.094 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:24:23.095 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=426, tag_tid=29, entry_tid=44674' 2026-02-18T08:24:23.096 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=425, tag_tid=30, entry_tid=1' -a 'object_number=425, tag_tid=30, entry_tid=1' = 'object_number=426, tag_tid=29, entry_tid=44674' 2026-02-18T08:24:23.096 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=426, tag_tid=29, entry_tid=44674' '!=' 'object_number=426, tag_tid=29, entry_tid=44322' 2026-02-18T08:24:23.096 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:24:23.096 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=426, tag_tid=29, entry_tid=44674' = 'object_number=426, tag_tid=29, entry_tid=44322' 2026-02-18T08:24:23.096 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=426, tag_tid=29, entry_tid=44674' 2026-02-18T08:24:23.096 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=425, tag_tid=30, entry_tid=1' 2026-02-18T08:24:23.096 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:24:23.096 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:24:23.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=30 2026-02-18T08:24:23.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=426, tag_tid=29, entry_tid=44674' 2026-02-18T08:24:23.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:24:23.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:24:23.100 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=29 2026-02-18T08:24:23.101 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=425, tag_tid=30, entry_tid=1' 2026-02-18T08:24:23.101 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:24:23.101 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:24:23.102 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:24:23.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=426, tag_tid=29, entry_tid=44674' 2026-02-18T08:24:23.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:24:23.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:24:23.105 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=44674 2026-02-18T08:24:23.105 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 30 = 29 -a 1 -le 44674 2026-02-18T08:24:23.105 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:24:23.105 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:24:23.105 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:24:23.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:24:23.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:24:23.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:24:23.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:24:23.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:24:23.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:24:23.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:24:23.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:24:23.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:24:23.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:24:23.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:24:23.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:24:23.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:24:23.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:24:23.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:24:23.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:24:23.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:24:23.312 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:24:23.312 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:24:23.312 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:24:23.312 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:24:23.312 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:24:23.312 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:24:23.312 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:24:23.312 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:24:23.312 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:24:23.312 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:24:23.312 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:24:23.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:24:23.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:24:23.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:23.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:24:23.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:24:23.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:24:23.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:24:23.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:24:23.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:24:23.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:24:23.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:24:23.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:24:23.314 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:24:23.314 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:23.314 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:24:23.314 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:24:23.314 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:24:23.314 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:24:23.314 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:24:23.314 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:24:23.314 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:24:23.321 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:24:23.322 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:24:23.322 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:24:23.322 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:24:23.372 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:24:23.372 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:24:23.372 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:24:23.375 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:24:23.375 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:24:23.375 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:24:23.375 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:24:23.375 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:24:23.375 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:24:23.375 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:24:23.375 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:24:23.376 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:24:23.376 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:24:23.376 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:24:23.376 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:24:23.376 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:24:23.376 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:23.376 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:24:23.376 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:24:23.376 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:24:23.376 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:24:23.376 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:24:23.377 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:24:23.377 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:24:23.377 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:24:23.377 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:24:23.377 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:24:23.377 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:23.377 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:24:23.377 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:24:23.377 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:24:23.377 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:24:23.377 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:24:23.377 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:24:23.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:24:23.387 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:24:23.387 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:24:23.388 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:24:23.388 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:24:23.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:24:23.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:24:23.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:24:23.446 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:24:23.446 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:24:23.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:24:23.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:24:23.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:24:23.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:24:23.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:24:23.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:24:23.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:24:23.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:24:23.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:24:23.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:24:23.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:24:23.448 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:23.448 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:24:23.448 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:24:23.448 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:24:23.448 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:24:23.448 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:24:23.448 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:24:23.448 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:24:23.448 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:24:23.448 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:24:23.448 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:24:23.448 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:23.448 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:24:23.449 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:24:23.449 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:24:23.449 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:24:23.449 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:24:23.449 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:24:23.449 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:24:23.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:24:23.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:24:23.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:24:23.458 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:24:23.513 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:24:23.513 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:24:23.513 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:24:23.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:24:23.516 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:24:23.516 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:24:23.516 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:24:23.516 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:24:23.516 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:24:23.516 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:24:23.516 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:24:23.516 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:24:23.516 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:24:23.516 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:24:23.516 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:24:23.516 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:24:23.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:23.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:24:23.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:24:23.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:24:23.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:24:23.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:24:23.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:24:23.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:24:23.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:24:23.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:24:23.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:24:23.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:23.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:24:23.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:24:23.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:24:23.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:24:23.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:24:23.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:24:23.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:24:23.528 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:24:23.528 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:24:23.528 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:24:23.528 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:24:23.587 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:24:23.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:24:23.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:24:23.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:24:23.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:24:23.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:24:23.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:24:23.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:24:23.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:24:23.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:24:23.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:24:23.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:24:23.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:24:23.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:24:23.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:24:23.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:24:23.620 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 106 2026-02-18T08:24:23.620 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 106 2026-02-18T08:24:23.620 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:24:23.620 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=425, tag_tid=30, entry_tid=1], [object_number=424, tag_tid=30, entry_tid=0], [object_number=427, tag_tid=29, entry_tid=45931], [object_number=426, tag_tid=29, entry_tid=45930]]], state=connected] 2026-02-18T08:24:23.620 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=427, tag_tid=29, entry_tid=45019], [object_number=426, tag_tid=29, entry_tid=45018], [object_number=425, tag_tid=29, entry_tid=45017], [object_number=424, tag_tid=29, entry_tid=45016]]], state=connected] 2026-02-18T08:24:23.623 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:24:23.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=425, tag_tid=30, entry_tid=1' 2026-02-18T08:24:23.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:24:23.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:24:23.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:24:23.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:24:23.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:24:23.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:24:23.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:24:23.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:24:23.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:24:23.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:24:23.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:24:23.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:24:23.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:24:23.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:24:23.628 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:24:23.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 106 2026-02-18T08:24:23.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 106 2026-02-18T08:24:23.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:24:23.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=425, tag_tid=30, entry_tid=1], [object_number=424, tag_tid=30, entry_tid=0], [object_number=427, tag_tid=29, entry_tid=45931], [object_number=426, tag_tid=29, entry_tid=45930]]], state=connected] 2026-02-18T08:24:23.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=427, tag_tid=29, entry_tid=45019], [object_number=426, tag_tid=29, entry_tid=45018], [object_number=425, tag_tid=29, entry_tid=45017], [object_number=424, tag_tid=29, entry_tid=45016]]], state=connected] 2026-02-18T08:24:23.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:24:23.671 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=427, tag_tid=29, entry_tid=45019' 2026-02-18T08:24:23.671 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=425, tag_tid=30, entry_tid=1' -a 'object_number=425, tag_tid=30, entry_tid=1' = 'object_number=427, tag_tid=29, entry_tid=45019' 2026-02-18T08:24:23.671 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=427, tag_tid=29, entry_tid=45019' '!=' 'object_number=426, tag_tid=29, entry_tid=44674' 2026-02-18T08:24:23.671 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:24:23.671 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=427, tag_tid=29, entry_tid=45019' = 'object_number=426, tag_tid=29, entry_tid=44674' 2026-02-18T08:24:23.671 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=427, tag_tid=29, entry_tid=45019' 2026-02-18T08:24:23.672 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=425, tag_tid=30, entry_tid=1' 2026-02-18T08:24:23.672 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:24:23.672 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:24:23.673 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=30 2026-02-18T08:24:23.673 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=427, tag_tid=29, entry_tid=45019' 2026-02-18T08:24:23.674 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:24:23.674 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:24:23.675 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=29 2026-02-18T08:24:23.675 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=425, tag_tid=30, entry_tid=1' 2026-02-18T08:24:23.676 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:24:23.676 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:24:23.677 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:24:23.678 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=427, tag_tid=29, entry_tid=45019' 2026-02-18T08:24:23.678 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:24:23.678 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:24:23.679 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=45019 2026-02-18T08:24:23.679 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 30 = 29 -a 1 -le 45019 2026-02-18T08:24:23.679 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:24:23.680 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:24:23.680 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:24:23.881 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:24:23.882 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:24:23.882 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:24:23.882 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:24:23.882 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:24:23.882 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:24:23.882 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:24:23.882 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:24:23.883 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:24:23.883 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:24:23.883 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:24:23.883 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:24:23.883 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:24:23.883 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:24:23.883 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:24:23.883 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:24:23.883 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:24:23.886 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:24:23.886 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:24:23.886 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:24:23.886 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:24:23.886 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:24:23.886 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:24:23.886 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:24:23.886 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:24:23.887 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:24:23.887 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:24:23.887 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:24:23.887 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:24:23.887 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:24:23.887 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:23.887 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:24:23.887 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:24:23.887 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:24:23.887 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:24:23.887 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:24:23.887 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:24:23.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:24:23.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:24:23.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:24:23.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:24:23.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:23.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:24:23.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:24:23.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:24:23.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:24:23.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:24:23.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:24:23.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:24:23.895 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:24:23.896 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:24:23.896 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:24:23.896 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:24:23.955 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:24:23.955 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:24:23.955 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:24:23.958 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:24:23.958 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:24:23.958 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:24:23.958 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:24:23.958 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:24:23.959 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:24:23.959 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:24:23.959 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:24:23.959 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:24:23.959 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:24:23.959 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:24:23.959 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:24:23.959 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:24:23.959 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:23.959 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:24:23.959 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:24:23.959 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:24:23.959 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:24:23.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:24:23.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:24:23.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:24:23.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:24:23.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:24:23.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:24:23.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:23.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:24:23.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:24:23.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:24:23.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:24:23.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:24:23.961 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:24:23.961 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:24:23.969 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:24:23.969 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:24:23.969 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:24:23.970 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:24:24.018 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:24:24.018 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:24:24.018 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:24:24.021 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:24:24.021 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:24:24.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:24:24.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:24:24.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:24:24.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:24:24.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:24:24.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:24:24.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:24:24.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:24:24.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:24:24.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:24:24.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:24:24.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:24.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:24:24.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:24:24.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:24:24.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:24:24.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:24:24.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:24:24.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:24:24.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:24:24.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:24:24.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:24:24.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:24.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:24:24.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:24:24.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:24:24.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:24:24.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:24:24.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:24:24.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:24:24.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:24:24.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:24:24.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:24:24.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:24:24.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:24:24.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:24:24.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:24:24.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:24:24.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:24:24.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:24:24.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:24:24.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:24:24.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:24:24.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:24:24.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:24:24.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:24:24.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:24:24.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:24:24.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:24:24.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:24:24.086 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:24.086 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:24:24.086 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:24:24.086 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:24:24.086 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:24:24.086 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:24:24.086 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:24:24.086 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:24:24.086 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:24:24.086 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:24:24.086 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:24:24.086 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:24.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:24:24.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:24:24.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:24:24.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:24:24.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:24:24.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:24:24.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:24:24.094 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:24:24.095 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:24:24.095 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:24:24.095 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:24:24.157 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:24:24.158 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:24:24.158 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:24:24.158 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:24:24.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:24:24.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:24:24.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:24:24.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:24:24.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:24:24.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:24:24.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:24:24.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:24:24.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:24:24.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:24:24.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:24:24.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:24:24.198 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 106 2026-02-18T08:24:24.198 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 106 2026-02-18T08:24:24.198 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:24:24.198 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=425, tag_tid=30, entry_tid=1], [object_number=424, tag_tid=30, entry_tid=0], [object_number=427, tag_tid=29, entry_tid=45931], [object_number=426, tag_tid=29, entry_tid=45930]]], state=connected] 2026-02-18T08:24:24.198 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=426, tag_tid=29, entry_tid=45366], [object_number=425, tag_tid=29, entry_tid=45365], [object_number=424, tag_tid=29, entry_tid=45364], [object_number=427, tag_tid=29, entry_tid=45363]]], state=connected] 2026-02-18T08:24:24.202 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:24:24.203 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=425, tag_tid=30, entry_tid=1' 2026-02-18T08:24:24.204 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:24:24.204 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:24:24.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:24:24.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:24:24.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:24:24.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:24:24.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:24:24.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:24:24.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:24:24.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:24:24.206 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:24:24.206 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:24:24.206 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:24:24.207 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:24:24.207 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:24:24.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 106 2026-02-18T08:24:24.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 106 2026-02-18T08:24:24.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:24:24.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=425, tag_tid=30, entry_tid=1], [object_number=424, tag_tid=30, entry_tid=0], [object_number=427, tag_tid=29, entry_tid=45931], [object_number=426, tag_tid=29, entry_tid=45930]]], state=connected] 2026-02-18T08:24:24.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=426, tag_tid=29, entry_tid=45366], [object_number=425, tag_tid=29, entry_tid=45365], [object_number=424, tag_tid=29, entry_tid=45364], [object_number=427, tag_tid=29, entry_tid=45363]]], state=connected] 2026-02-18T08:24:24.238 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:24:24.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=426, tag_tid=29, entry_tid=45366' 2026-02-18T08:24:24.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=425, tag_tid=30, entry_tid=1' -a 'object_number=425, tag_tid=30, entry_tid=1' = 'object_number=426, tag_tid=29, entry_tid=45366' 2026-02-18T08:24:24.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=426, tag_tid=29, entry_tid=45366' '!=' 'object_number=427, tag_tid=29, entry_tid=45019' 2026-02-18T08:24:24.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:24:24.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=426, tag_tid=29, entry_tid=45366' = 'object_number=427, tag_tid=29, entry_tid=45019' 2026-02-18T08:24:24.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=426, tag_tid=29, entry_tid=45366' 2026-02-18T08:24:24.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=425, tag_tid=30, entry_tid=1' 2026-02-18T08:24:24.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:24:24.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:24:24.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=30 2026-02-18T08:24:24.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=426, tag_tid=29, entry_tid=45366' 2026-02-18T08:24:24.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:24:24.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:24:24.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=29 2026-02-18T08:24:24.244 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=425, tag_tid=30, entry_tid=1' 2026-02-18T08:24:24.244 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:24:24.245 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:24:24.246 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:24:24.247 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=426, tag_tid=29, entry_tid=45366' 2026-02-18T08:24:24.247 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:24:24.247 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:24:24.248 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=45366 2026-02-18T08:24:24.248 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 30 = 29 -a 1 -le 45366 2026-02-18T08:24:24.249 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:24:24.249 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:24:24.249 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:24:24.450 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:24:24.450 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:24:24.450 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:24:24.450 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:24:24.451 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:24:24.451 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:24:24.451 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:24:24.451 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:24:24.451 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:24:24.451 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:24:24.451 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:24:24.451 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:24:24.451 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:24:24.452 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:24:24.452 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:24:24.454 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:24:24.455 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:24:24.455 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:24:24.455 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:24:24.455 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:24:24.455 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:24:24.455 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:24:24.455 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:24:24.455 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:24:24.455 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:24:24.455 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:24:24.455 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:24:24.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:24:24.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:24:24.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:24:24.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:24.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:24:24.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:24:24.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:24:24.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:24:24.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:24:24.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:24:24.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:24:24.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:24:24.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:24:24.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:24:24.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:24.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:24:24.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:24:24.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:24:24.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:24:24.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:24:24.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:24:24.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:24:24.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:24:24.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:24:24.465 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:24:24.465 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:24:24.516 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:24:24.516 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:24:24.516 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:24:24.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:24:24.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:24:24.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:24:24.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:24:24.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:24:24.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:24:24.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:24:24.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:24:24.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:24:24.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:24:24.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:24:24.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:24:24.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:24:24.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:24.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:24:24.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:24:24.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:24:24.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:24:24.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:24:24.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:24:24.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:24:24.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:24:24.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:24:24.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:24:24.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:24.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:24:24.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:24:24.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:24:24.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:24:24.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:24:24.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:24:24.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:24:24.529 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:24:24.530 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:24:24.530 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:24:24.530 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:24:24.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:24:24.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:24:24.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:24:24.582 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:24:24.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:24:24.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:24:24.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:24:24.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:24:24.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:24:24.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:24:24.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:24:24.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:24:24.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:24:24.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:24:24.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:24:24.584 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:24:24.584 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:24.584 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:24:24.584 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:24:24.584 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:24:24.584 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:24:24.584 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:24:24.584 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:24:24.584 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:24:24.584 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:24:24.585 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:24:24.585 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:24:24.585 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:24.585 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:24:24.585 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:24:24.585 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:24:24.585 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:24:24.585 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:24:24.585 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:24:24.585 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:24:24.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:24:24.594 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:24:24.594 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:24:24.594 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:24:24.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:24:24.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:24:24.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:24:24.655 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:24:24.655 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:24:24.655 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:24:24.655 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:24:24.655 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:24:24.655 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:24:24.655 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:24:24.655 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:24:24.655 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:24:24.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:24:24.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:24:24.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:24:24.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:24:24.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:24.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:24:24.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:24:24.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:24:24.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:24:24.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:24:24.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:24:24.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:24:24.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:24:24.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:24:24.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:24:24.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:24.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:24:24.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:24:24.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:24:24.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:24:24.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:24:24.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:24:24.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:24:24.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:24:24.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:24:24.667 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:24:24.667 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:24:24.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:24:24.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:24:24.728 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:24:24.728 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:24:24.728 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:24:24.728 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:24:24.728 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:24:24.728 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:24:24.728 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:24:24.728 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:24:24.728 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:24:24.728 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:24:24.728 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:24:24.729 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:24:24.729 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:24:24.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:24:24.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 106 2026-02-18T08:24:24.758 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 106 2026-02-18T08:24:24.758 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:24:24.758 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=425, tag_tid=30, entry_tid=1], [object_number=424, tag_tid=30, entry_tid=0], [object_number=427, tag_tid=29, entry_tid=45931], [object_number=426, tag_tid=29, entry_tid=45930]]], state=connected] 2026-02-18T08:24:24.758 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=426, tag_tid=29, entry_tid=45746], [object_number=425, tag_tid=29, entry_tid=45745], [object_number=424, tag_tid=29, entry_tid=45744], [object_number=427, tag_tid=29, entry_tid=45743]]], state=connected] 2026-02-18T08:24:24.760 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:24:24.762 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=425, tag_tid=30, entry_tid=1' 2026-02-18T08:24:24.763 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:24:24.763 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:24:24.763 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:24:24.763 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:24:24.763 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:24:24.763 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:24:24.763 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:24:24.763 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:24:24.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:24:24.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:24:24.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:24:24.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:24:24.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:24:24.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:24:24.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:24:24.804 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 106 2026-02-18T08:24:24.804 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 106 2026-02-18T08:24:24.804 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:24:24.804 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=425, tag_tid=30, entry_tid=1], [object_number=424, tag_tid=30, entry_tid=0], [object_number=427, tag_tid=29, entry_tid=45931], [object_number=426, tag_tid=29, entry_tid=45930]]], state=connected] 2026-02-18T08:24:24.804 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=426, tag_tid=29, entry_tid=45746], [object_number=425, tag_tid=29, entry_tid=45745], [object_number=424, tag_tid=29, entry_tid=45744], [object_number=427, tag_tid=29, entry_tid=45743]]], state=connected] 2026-02-18T08:24:24.808 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:24:24.809 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=426, tag_tid=29, entry_tid=45746' 2026-02-18T08:24:24.809 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=425, tag_tid=30, entry_tid=1' -a 'object_number=425, tag_tid=30, entry_tid=1' = 'object_number=426, tag_tid=29, entry_tid=45746' 2026-02-18T08:24:24.810 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=426, tag_tid=29, entry_tid=45746' '!=' 'object_number=426, tag_tid=29, entry_tid=45366' 2026-02-18T08:24:24.810 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:24:24.810 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=426, tag_tid=29, entry_tid=45746' = 'object_number=426, tag_tid=29, entry_tid=45366' 2026-02-18T08:24:24.810 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=426, tag_tid=29, entry_tid=45746' 2026-02-18T08:24:24.810 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=425, tag_tid=30, entry_tid=1' 2026-02-18T08:24:24.811 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:24:24.811 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:24:24.811 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=30 2026-02-18T08:24:24.812 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=426, tag_tid=29, entry_tid=45746' 2026-02-18T08:24:24.812 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:24:24.812 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:24:24.813 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=29 2026-02-18T08:24:24.814 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=425, tag_tid=30, entry_tid=1' 2026-02-18T08:24:24.814 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:24:24.814 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:24:24.815 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:24:24.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=426, tag_tid=29, entry_tid=45746' 2026-02-18T08:24:24.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:24:24.817 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:24:24.818 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=45746 2026-02-18T08:24:24.818 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 30 = 29 -a 1 -le 45746 2026-02-18T08:24:24.818 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:24:24.818 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:24:24.818 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:24:25.020 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:24:25.020 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:24:25.020 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:24:25.020 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:24:25.020 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:24:25.020 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:24:25.021 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:24:25.021 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:24:25.021 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:24:25.021 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:24:25.021 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:24:25.021 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:24:25.021 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:24:25.021 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:24:25.021 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:24:25.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:24:25.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:24:25.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:24:25.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:24:25.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:24:25.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:24:25.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:24:25.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:24:25.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:24:25.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:24:25.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:24:25.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:24:25.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:24:25.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:24:25.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:24:25.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:25.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:24:25.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:24:25.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:24:25.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:24:25.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:24:25.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:24:25.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:24:25.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:24:25.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:24:25.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:24:25.027 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:25.027 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:24:25.027 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:24:25.027 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:24:25.027 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:24:25.027 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:24:25.027 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:24:25.027 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:24:25.035 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:24:25.036 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:24:25.036 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:24:25.036 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:24:25.093 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:24:25.093 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:24:25.093 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:24:25.096 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:24:25.096 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:24:25.096 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:24:25.096 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:24:25.096 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:24:25.096 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:24:25.096 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:24:25.097 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:24:25.097 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:24:25.097 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:24:25.097 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:24:25.097 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:24:25.097 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:24:25.097 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:25.097 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:24:25.097 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:24:25.097 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:24:25.097 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:24:25.097 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:24:25.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:24:25.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:24:25.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:24:25.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:24:25.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:24:25.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:25.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:24:25.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:24:25.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:24:25.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:24:25.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:24:25.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:24:25.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:24:25.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:24:25.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:24:25.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:24:25.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:24:25.165 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:24:25.165 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:24:25.165 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:24:25.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:24:25.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:24:25.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:24:25.169 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:24:25.169 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:24:25.169 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:24:25.169 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:24:25.169 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:24:25.169 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:24:25.169 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:24:25.169 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:24:25.169 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:24:25.169 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:24:25.169 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:25.169 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:24:25.170 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:24:25.170 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:24:25.170 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:24:25.170 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:24:25.170 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:24:25.170 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:24:25.170 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:24:25.170 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:24:25.170 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:24:25.170 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:25.170 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:24:25.170 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:24:25.171 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:24:25.171 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:24:25.171 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:24:25.171 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:24:25.171 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:24:25.179 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:24:25.180 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:24:25.180 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:24:25.180 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:24:25.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:24:25.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:24:25.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:24:25.238 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:24:25.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:24:25.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:24:25.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:24:25.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:24:25.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:24:25.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:24:25.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:24:25.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:24:25.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:24:25.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:24:25.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:24:25.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:24:25.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:25.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:24:25.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:24:25.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:24:25.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:24:25.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:24:25.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:24:25.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:24:25.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:24:25.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:24:25.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:24:25.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:25.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:24:25.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:24:25.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:24:25.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:24:25.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:24:25.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:24:25.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:24:25.248 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:24:25.249 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:24:25.249 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:24:25.249 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:24:25.315 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:24:25.316 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:24:25.316 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:24:25.316 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:24:25.316 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:24:25.316 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:24:25.316 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:24:25.316 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:24:25.316 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:24:25.316 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:24:25.317 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:24:25.317 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:24:25.317 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:24:25.317 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:24:25.318 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:24:25.318 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:24:25.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 106 2026-02-18T08:24:25.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 106 2026-02-18T08:24:25.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:24:25.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=425, tag_tid=30, entry_tid=1], [object_number=424, tag_tid=30, entry_tid=0], [object_number=427, tag_tid=29, entry_tid=45931], [object_number=426, tag_tid=29, entry_tid=45930]]], state=connected] 2026-02-18T08:24:25.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=425, tag_tid=29, entry_tid=45933], [object_number=424, tag_tid=29, entry_tid=45932], [object_number=427, tag_tid=29, entry_tid=45931], [object_number=426, tag_tid=29, entry_tid=45930]]], state=connected] 2026-02-18T08:24:25.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:24:25.350 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=425, tag_tid=30, entry_tid=1' 2026-02-18T08:24:25.352 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:24:25.352 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:24:25.352 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:24:25.352 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:24:25.352 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:24:25.352 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:24:25.352 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:24:25.352 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:24:25.352 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:24:25.353 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:24:25.353 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:24:25.353 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:24:25.353 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:24:25.354 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:24:25.354 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:24:25.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 106 2026-02-18T08:24:25.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 106 2026-02-18T08:24:25.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:24:25.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=425, tag_tid=30, entry_tid=1], [object_number=424, tag_tid=30, entry_tid=0], [object_number=427, tag_tid=29, entry_tid=45931], [object_number=426, tag_tid=29, entry_tid=45930]]], state=connected] 2026-02-18T08:24:25.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=425, tag_tid=29, entry_tid=45933], [object_number=424, tag_tid=29, entry_tid=45932], [object_number=427, tag_tid=29, entry_tid=45931], [object_number=426, tag_tid=29, entry_tid=45930]]], state=connected] 2026-02-18T08:24:25.396 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:24:25.397 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=425, tag_tid=29, entry_tid=45933' 2026-02-18T08:24:25.397 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=425, tag_tid=30, entry_tid=1' -a 'object_number=425, tag_tid=30, entry_tid=1' = 'object_number=425, tag_tid=29, entry_tid=45933' 2026-02-18T08:24:25.397 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=425, tag_tid=29, entry_tid=45933' '!=' 'object_number=426, tag_tid=29, entry_tid=45746' 2026-02-18T08:24:25.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:24:25.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=425, tag_tid=29, entry_tid=45933' = 'object_number=426, tag_tid=29, entry_tid=45746' 2026-02-18T08:24:25.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=425, tag_tid=29, entry_tid=45933' 2026-02-18T08:24:25.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=425, tag_tid=30, entry_tid=1' 2026-02-18T08:24:25.399 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:24:25.399 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:24:25.399 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=30 2026-02-18T08:24:25.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=425, tag_tid=29, entry_tid=45933' 2026-02-18T08:24:25.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:24:25.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:24:25.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=29 2026-02-18T08:24:25.403 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=425, tag_tid=30, entry_tid=1' 2026-02-18T08:24:25.403 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:24:25.403 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:24:25.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:24:25.405 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=425, tag_tid=29, entry_tid=45933' 2026-02-18T08:24:25.405 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:24:25.405 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:24:25.407 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=45933 2026-02-18T08:24:25.407 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 30 = 29 -a 1 -le 45933 2026-02-18T08:24:25.407 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:24:25.407 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:24:25.407 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:24:25.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:24:25.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:24:25.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:24:25.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:24:25.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:24:25.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:24:25.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:24:25.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:24:25.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:24:25.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:24:25.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:24:25.610 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:24:25.610 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:24:25.610 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:24:25.610 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:24:25.613 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:24:25.613 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:24:25.613 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:24:25.613 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:24:25.613 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:24:25.613 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:24:25.613 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:24:25.613 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:24:25.613 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:24:25.613 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:24:25.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:24:25.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:24:25.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:24:25.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:24:25.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:24:25.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:25.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:24:25.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:24:25.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:24:25.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:24:25.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:24:25.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:24:25.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:24:25.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:24:25.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:24:25.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:24:25.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:25.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:24:25.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:24:25.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:24:25.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:24:25.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:24:25.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:24:25.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:24:25.623 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:24:25.623 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:24:25.623 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:24:25.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:24:25.681 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:24:25.682 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:24:25.682 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:24:25.684 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:24:25.684 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:24:25.684 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:24:25.685 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:24:25.685 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:24:25.685 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:24:25.685 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:24:25.685 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:24:25.685 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:24:25.685 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:24:25.685 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:24:25.685 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:24:25.685 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:24:25.685 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:25.685 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:24:25.686 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:24:25.686 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:24:25.686 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:24:25.686 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:24:25.686 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:24:25.686 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:24:25.686 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:24:25.686 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:24:25.686 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:24:25.686 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:25.687 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:24:25.687 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:24:25.687 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:24:25.687 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:24:25.687 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:24:25.687 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:24:25.687 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:24:25.695 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:24:25.696 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:24:25.696 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:24:25.696 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:24:25.753 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:24:25.754 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:24:25.754 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:24:25.756 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:24:25.756 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:24:25.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:24:25.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:24:25.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:24:25.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:24:25.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:24:25.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:24:25.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:24:25.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:24:25.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:24:25.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:24:25.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:24:25.758 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:25.758 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:24:25.758 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:24:25.758 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:24:25.758 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:24:25.758 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:24:25.758 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:24:25.758 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:24:25.758 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:24:25.758 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:24:25.758 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:24:25.758 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:25.759 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:24:25.759 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:24:25.759 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:24:25.759 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:24:25.759 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:24:25.759 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:24:25.759 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:24:25.767 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:24:25.768 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:24:25.768 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:24:25.768 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:24:25.824 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:24:25.825 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:24:25.825 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:24:25.827 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:24:25.827 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:24:25.827 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:24:25.828 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:24:25.828 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:24:25.828 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:24:25.828 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:24:25.828 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:24:25.828 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:24:25.828 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:24:25.828 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:24:25.828 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:24:25.828 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:24:25.828 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:25.829 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:24:25.829 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:24:25.829 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:24:25.829 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:24:25.829 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:24:25.829 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:24:25.829 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:24:25.829 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:24:25.829 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:24:25.829 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:24:25.829 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:25.829 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:24:25.829 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:24:25.830 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:24:25.830 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:24:25.830 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:24:25.830 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:24:25.830 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:24:25.838 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:24:25.839 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:24:25.839 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:24:25.839 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:24:25.896 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:24:25.897 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:24:25.897 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:24:25.897 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:24:25.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:24:25.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:24:25.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:24:25.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:24:25.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:24:25.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:24:25.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:24:25.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:24:25.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:24:25.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:24:25.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:24:25.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:24:25.928 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 106 2026-02-18T08:24:25.928 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 106 2026-02-18T08:24:25.928 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:24:25.928 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=425, tag_tid=30, entry_tid=1], [object_number=424, tag_tid=30, entry_tid=0], [object_number=427, tag_tid=29, entry_tid=45931], [object_number=426, tag_tid=29, entry_tid=45930]]], state=connected] 2026-02-18T08:24:25.929 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=425, tag_tid=29, entry_tid=45933], [object_number=424, tag_tid=29, entry_tid=45932], [object_number=427, tag_tid=29, entry_tid=45931], [object_number=426, tag_tid=29, entry_tid=45930]]], state=connected] 2026-02-18T08:24:25.931 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:24:25.933 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=425, tag_tid=30, entry_tid=1' 2026-02-18T08:24:25.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:24:25.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:24:25.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:24:25.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:24:25.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:24:25.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:24:25.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:24:25.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:24:25.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:24:25.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:24:25.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:24:25.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:24:25.936 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:24:25.936 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:24:25.936 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:24:25.974 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 106 2026-02-18T08:24:25.974 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 106 2026-02-18T08:24:25.974 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:24:25.974 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=425, tag_tid=30, entry_tid=1], [object_number=424, tag_tid=30, entry_tid=0], [object_number=427, tag_tid=29, entry_tid=45931], [object_number=426, tag_tid=29, entry_tid=45930]]], state=connected] 2026-02-18T08:24:25.974 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=425, tag_tid=29, entry_tid=45933], [object_number=424, tag_tid=29, entry_tid=45932], [object_number=427, tag_tid=29, entry_tid=45931], [object_number=426, tag_tid=29, entry_tid=45930]]], state=connected] 2026-02-18T08:24:25.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:24:25.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=425, tag_tid=29, entry_tid=45933' 2026-02-18T08:24:25.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=425, tag_tid=30, entry_tid=1' -a 'object_number=425, tag_tid=30, entry_tid=1' = 'object_number=425, tag_tid=29, entry_tid=45933' 2026-02-18T08:24:25.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=425, tag_tid=29, entry_tid=45933' '!=' 'object_number=425, tag_tid=29, entry_tid=45933' 2026-02-18T08:24:25.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:24:25.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.4 2026-02-18T08:24:26.381 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:24:26.381 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:24:26.381 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:24:26.381 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:24:26.381 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:24:26.381 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:24:26.381 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:24:26.381 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:24:26.381 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:24:26.381 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:24:26.381 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:24:26.381 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:24:26.381 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:24:26.381 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:24:26.381 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:24:26.381 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:24:26.382 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:24:26.383 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:24:26.383 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:24:26.383 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:24:26.383 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:24:26.383 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:24:26.383 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:24:26.383 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:24:26.383 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:24:26.383 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:24:26.383 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:24:26.383 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:24:26.383 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:24:26.383 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:24:26.383 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:26.383 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:24:26.383 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:24:26.384 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:24:26.384 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:24:26.384 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:24:26.384 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:24:26.384 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:24:26.384 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:24:26.384 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:24:26.384 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:24:26.384 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:26.384 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:24:26.384 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:24:26.384 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:24:26.384 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:24:26.384 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:24:26.384 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:24:26.384 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:24:26.396 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:24:26.396 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:24:26.396 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:24:26.396 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:24:26.455 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:24:26.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:24:26.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:24:26.458 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:24:26.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:24:26.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:24:26.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:24:26.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:24:26.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:24:26.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:24:26.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:24:26.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:24:26.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:24:26.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:24:26.460 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:24:26.460 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:24:26.460 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:26.460 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:24:26.460 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:24:26.460 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:24:26.460 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:24:26.460 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:24:26.460 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:24:26.460 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:24:26.460 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:24:26.460 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:24:26.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:24:26.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:26.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:24:26.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:24:26.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:24:26.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:24:26.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:24:26.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:24:26.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:24:26.470 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:24:26.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:24:26.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:24:26.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:24:26.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:24:26.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:24:26.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:24:26.524 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:24:26.524 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:24:26.524 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:24:26.524 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:24:26.524 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:24:26.524 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:24:26.524 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:24:26.524 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:24:26.524 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:24:26.525 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:24:26.525 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:24:26.525 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:24:26.525 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:24:26.525 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:26.525 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:24:26.525 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:24:26.525 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:24:26.525 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:24:26.525 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:24:26.525 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:24:26.525 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:24:26.526 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:24:26.526 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:24:26.526 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:24:26.526 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:26.526 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:24:26.526 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:24:26.526 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:24:26.526 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:24:26.526 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:24:26.526 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:24:26.526 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:24:26.535 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:24:26.535 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:24:26.536 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:24:26.536 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:24:26.585 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:24:26.586 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:24:26.586 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:24:26.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:24:26.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:24:26.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:24:26.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:24:26.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:24:26.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:24:26.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:24:26.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:24:26.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:24:26.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:24:26.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:24:26.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:24:26.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:24:26.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:26.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:24:26.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:24:26.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:24:26.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:24:26.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:24:26.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:24:26.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:24:26.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:24:26.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:24:26.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:24:26.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:26.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:24:26.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:24:26.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:24:26.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:24:26.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:24:26.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:24:26.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:24:26.601 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:24:26.601 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:24:26.601 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:24:26.601 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:24:26.664 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:24:26.664 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:24:26.664 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:24:26.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:24:26.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:24:26.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:24:26.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:24:26.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:24:26.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:24:26.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:24:26.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:24:26.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:24:26.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:24:26.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:24:26.667 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:24:26.667 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:24:26.696 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 106 2026-02-18T08:24:26.696 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 106 2026-02-18T08:24:26.696 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:24:26.696 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=425, tag_tid=30, entry_tid=1], [object_number=424, tag_tid=30, entry_tid=0], [object_number=427, tag_tid=29, entry_tid=45931], [object_number=426, tag_tid=29, entry_tid=45930]]], state=connected] 2026-02-18T08:24:26.696 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=425, tag_tid=29, entry_tid=45933], [object_number=424, tag_tid=29, entry_tid=45932], [object_number=427, tag_tid=29, entry_tid=45931], [object_number=426, tag_tid=29, entry_tid=45930]]], state=connected] 2026-02-18T08:24:26.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:24:26.700 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=425, tag_tid=30, entry_tid=1' 2026-02-18T08:24:26.702 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:24:26.702 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:24:26.702 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:24:26.702 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:24:26.702 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:24:26.702 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:24:26.702 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:24:26.702 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:24:26.703 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:24:26.703 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:24:26.703 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:24:26.703 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:24:26.703 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:24:26.704 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:24:26.704 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:24:26.741 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 106 2026-02-18T08:24:26.742 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 106 2026-02-18T08:24:26.742 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:24:26.742 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=425, tag_tid=30, entry_tid=1], [object_number=424, tag_tid=30, entry_tid=0], [object_number=427, tag_tid=29, entry_tid=45931], [object_number=426, tag_tid=29, entry_tid=45930]]], state=connected] 2026-02-18T08:24:26.742 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=425, tag_tid=29, entry_tid=45933], [object_number=424, tag_tid=29, entry_tid=45932], [object_number=427, tag_tid=29, entry_tid=45931], [object_number=426, tag_tid=29, entry_tid=45930]]], state=connected] 2026-02-18T08:24:26.745 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:24:26.747 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=425, tag_tid=29, entry_tid=45933' 2026-02-18T08:24:26.748 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=425, tag_tid=30, entry_tid=1' -a 'object_number=425, tag_tid=30, entry_tid=1' = 'object_number=425, tag_tid=29, entry_tid=45933' 2026-02-18T08:24:26.748 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=425, tag_tid=29, entry_tid=45933' '!=' 'object_number=425, tag_tid=29, entry_tid=45933' 2026-02-18T08:24:26.748 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:24:26.748 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.8 2026-02-18T08:24:27.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:24:27.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:24:27.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:24:27.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:24:27.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:24:27.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:24:27.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:24:27.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:24:27.551 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:24:27.551 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:24:27.551 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:24:27.551 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:24:27.551 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:24:27.551 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:24:27.551 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:24:27.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:24:27.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:24:27.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:24:27.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:24:27.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:24:27.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:24:27.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:24:27.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:24:27.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:24:27.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:24:27.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:24:27.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:24:27.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:24:27.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:24:27.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:24:27.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:27.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:24:27.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:24:27.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:24:27.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:24:27.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:24:27.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:24:27.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:24:27.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:24:27.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:24:27.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:24:27.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:27.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:24:27.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:24:27.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:24:27.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:24:27.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:24:27.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:24:27.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:24:27.565 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:24:27.565 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:24:27.566 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:24:27.566 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:24:27.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:24:27.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:24:27.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:24:27.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:24:27.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:24:27.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:24:27.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:24:27.618 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:24:27.618 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:24:27.618 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:24:27.618 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:24:27.618 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:24:27.618 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:24:27.618 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:24:27.618 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:24:27.618 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:24:27.618 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:27.618 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:24:27.618 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:24:27.619 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:24:27.619 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:24:27.619 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:24:27.619 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:24:27.619 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:24:27.619 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:24:27.619 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:24:27.619 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:24:27.619 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:27.619 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:24:27.619 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:24:27.619 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:24:27.619 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:24:27.620 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:24:27.620 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:24:27.620 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:24:27.628 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:24:27.628 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:24:27.628 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:24:27.628 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:24:27.684 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:24:27.684 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:24:27.684 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:24:27.687 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:24:27.687 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:24:27.687 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:24:27.687 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:24:27.687 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:24:27.687 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:24:27.687 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:24:27.687 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:24:27.688 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:24:27.688 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:24:27.688 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:24:27.688 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:24:27.688 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:24:27.688 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:27.688 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:24:27.688 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:24:27.688 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:24:27.688 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:24:27.688 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:24:27.688 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:24:27.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:24:27.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:24:27.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:24:27.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:24:27.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:27.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:24:27.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:24:27.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:24:27.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:24:27.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:24:27.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:24:27.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:24:27.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:24:27.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:24:27.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:24:27.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:24:27.748 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:24:27.748 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:24:27.748 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:24:27.750 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:24:27.750 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:24:27.751 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:24:27.751 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:24:27.751 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:24:27.751 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:24:27.751 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:24:27.751 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:24:27.751 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:24:27.751 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:24:27.751 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:24:27.751 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:24:27.751 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:24:27.752 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:27.752 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:24:27.752 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:24:27.752 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:24:27.752 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:24:27.752 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:24:27.752 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:24:27.752 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:24:27.752 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:24:27.752 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:24:27.752 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:24:27.752 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:27.753 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:24:27.753 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:24:27.753 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:24:27.753 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:24:27.753 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:24:27.753 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:24:27.753 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:24:27.760 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:24:27.761 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:24:27.761 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:24:27.761 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:24:27.824 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:24:27.824 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:24:27.825 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:24:27.825 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:24:27.825 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:24:27.825 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:24:27.825 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:24:27.825 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:24:27.825 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:24:27.825 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:24:27.825 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:24:27.825 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:24:27.825 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:24:27.826 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:24:27.826 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:24:27.827 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:24:27.855 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 106 2026-02-18T08:24:27.855 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 106 2026-02-18T08:24:27.856 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:24:27.856 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=425, tag_tid=30, entry_tid=1], [object_number=424, tag_tid=30, entry_tid=0], [object_number=427, tag_tid=29, entry_tid=45931], [object_number=426, tag_tid=29, entry_tid=45930]]], state=connected] 2026-02-18T08:24:27.856 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=425, tag_tid=29, entry_tid=45933], [object_number=424, tag_tid=29, entry_tid=45932], [object_number=427, tag_tid=29, entry_tid=45931], [object_number=426, tag_tid=29, entry_tid=45930]]], state=connected] 2026-02-18T08:24:27.858 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:24:27.859 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=425, tag_tid=30, entry_tid=1' 2026-02-18T08:24:27.861 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:24:27.861 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:24:27.861 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:24:27.861 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:24:27.861 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:24:27.861 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:24:27.861 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:24:27.862 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:24:27.862 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:24:27.862 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:24:27.862 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:24:27.862 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:24:27.863 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:24:27.863 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:24:27.863 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:24:27.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 106 2026-02-18T08:24:27.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 106 2026-02-18T08:24:27.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:24:27.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=425, tag_tid=30, entry_tid=1], [object_number=424, tag_tid=30, entry_tid=0], [object_number=427, tag_tid=29, entry_tid=45931], [object_number=426, tag_tid=29, entry_tid=45930]]], state=connected] 2026-02-18T08:24:27.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=425, tag_tid=29, entry_tid=45933], [object_number=424, tag_tid=29, entry_tid=45932], [object_number=427, tag_tid=29, entry_tid=45931], [object_number=426, tag_tid=29, entry_tid=45930]]], state=connected] 2026-02-18T08:24:27.896 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:24:27.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=425, tag_tid=29, entry_tid=45933' 2026-02-18T08:24:27.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=425, tag_tid=30, entry_tid=1' -a 'object_number=425, tag_tid=30, entry_tid=1' = 'object_number=425, tag_tid=29, entry_tid=45933' 2026-02-18T08:24:27.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=425, tag_tid=29, entry_tid=45933' '!=' 'object_number=425, tag_tid=29, entry_tid=45933' 2026-02-18T08:24:27.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:24:27.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1.6 2026-02-18T08:24:29.337 INFO:tasks.rbd_mirror_thrash:kill cluster2.client.mirror.0 2026-02-18T08:24:29.337 INFO:tasks.rbd_mirror.cluster2.client.mirror.0:Sent signal 15 2026-02-18T08:24:29.337 INFO:tasks.rbd_mirror_thrash:waiting for 4.025637582527647 secs before reviving daemons 2026-02-18T08:24:29.338 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.trial163.stderr:2026-02-18T08:24:29.328+0000 7f41bf1f9640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.0 (PID: 75989) UID: 1000 2026-02-18T08:24:29.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:24:29.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:24:29.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:24:29.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:24:29.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:24:29.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:24:29.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:24:29.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:24:29.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:24:29.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:24:29.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:24:29.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:24:29.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:24:29.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:24:29.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:24:29.500 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:24:29.500 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:24:29.501 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:24:29.501 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:24:29.501 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:24:29.501 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:24:29.501 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:24:29.501 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:24:29.501 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:24:29.501 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:24:29.501 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:24:29.501 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:24:29.501 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:24:29.501 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:24:29.501 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:24:29.501 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:29.501 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:24:29.501 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:24:29.502 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:24:29.502 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:24:29.502 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:24:29.502 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:24:29.502 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:24:29.502 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:24:29.502 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:24:29.502 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:24:29.502 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:29.502 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:24:29.502 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:24:29.502 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:24:29.502 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:24:29.502 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:24:29.502 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:24:29.502 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:24:29.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:24:29.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:24:29.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:24:29.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:24:29.574 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:24:29.574 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:24:29.574 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:24:29.576 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:24:29.576 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:24:29.576 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:24:29.576 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:24:29.576 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:24:29.576 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:24:29.576 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:24:29.576 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:24:29.576 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:24:29.576 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:24:29.576 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:24:29.576 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:24:29.576 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:24:29.576 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:29.576 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:24:29.576 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:24:29.576 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:24:29.576 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:24:29.576 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:24:29.577 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:24:29.577 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:24:29.577 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:24:29.577 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:24:29.577 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:24:29.577 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:29.577 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:24:29.577 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:24:29.577 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:24:29.577 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:24:29.577 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:24:29.577 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:24:29.577 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:24:29.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:24:29.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:24:29.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:24:29.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:24:29.649 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:24:29.649 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:24:29.649 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:24:29.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:24:29.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:24:29.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:24:29.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:24:29.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:24:29.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:24:29.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:24:29.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:24:29.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:24:29.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:24:29.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:24:29.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:24:29.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:24:29.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:29.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:24:29.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:24:29.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:24:29.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:24:29.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:24:29.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:24:29.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:24:29.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:24:29.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:24:29.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:24:29.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:29.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:24:29.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:24:29.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:24:29.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:24:29.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:24:29.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:24:29.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:24:29.663 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:24:29.664 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:24:29.664 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:24:29.664 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:24:29.713 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:24:29.713 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:24:29.713 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:24:29.714 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:24:29.714 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:24:29.714 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:24:29.714 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:24:29.714 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:24:29.714 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:24:29.714 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:24:29.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:24:29.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:24:29.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:24:29.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:24:29.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:24:29.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:24:29.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:29.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:24:29.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:24:29.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:24:29.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:24:29.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:24:29.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:24:29.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:24:29.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:24:29.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:24:29.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:24:29.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:29.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:24:29.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:24:29.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:24:29.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:24:29.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:24:29.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:24:29.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:24:29.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:24:29.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:24:29.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:24:29.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:24:29.784 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:24:29.784 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:24:29.784 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:24:29.784 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:24:29.784 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:24:29.784 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:24:29.784 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:24:29.784 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:24:29.784 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:24:29.784 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:24:29.784 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:24:29.785 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:24:29.785 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:24:29.786 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:24:29.787 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:24:29.787 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:24:29.817 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 106 2026-02-18T08:24:29.817 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 106 2026-02-18T08:24:29.817 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:24:29.817 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=425, tag_tid=30, entry_tid=1], [object_number=424, tag_tid=30, entry_tid=0], [object_number=427, tag_tid=29, entry_tid=45931], [object_number=426, tag_tid=29, entry_tid=45930]]], state=connected] 2026-02-18T08:24:29.817 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=425, tag_tid=29, entry_tid=45933], [object_number=424, tag_tid=29, entry_tid=45932], [object_number=427, tag_tid=29, entry_tid=45931], [object_number=426, tag_tid=29, entry_tid=45930]]], state=connected] 2026-02-18T08:24:29.820 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:24:29.821 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=425, tag_tid=30, entry_tid=1' 2026-02-18T08:24:29.822 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:24:29.822 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:24:29.822 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:24:29.822 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:24:29.822 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:24:29.822 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:24:29.822 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:24:29.822 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:24:29.822 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:24:29.822 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:24:29.823 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:24:29.823 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:24:29.824 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:24:29.825 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:24:29.825 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:24:29.862 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 106 2026-02-18T08:24:29.862 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 106 2026-02-18T08:24:29.863 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:24:29.863 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=425, tag_tid=30, entry_tid=1], [object_number=424, tag_tid=30, entry_tid=0], [object_number=427, tag_tid=29, entry_tid=45931], [object_number=426, tag_tid=29, entry_tid=45930]]], state=connected] 2026-02-18T08:24:29.863 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=425, tag_tid=29, entry_tid=45933], [object_number=424, tag_tid=29, entry_tid=45932], [object_number=427, tag_tid=29, entry_tid=45931], [object_number=426, tag_tid=29, entry_tid=45930]]], state=connected] 2026-02-18T08:24:29.866 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:24:29.868 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=425, tag_tid=29, entry_tid=45933' 2026-02-18T08:24:29.868 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=425, tag_tid=30, entry_tid=1' -a 'object_number=425, tag_tid=30, entry_tid=1' = 'object_number=425, tag_tid=29, entry_tid=45933' 2026-02-18T08:24:29.868 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=425, tag_tid=29, entry_tid=45933' '!=' 'object_number=425, tag_tid=29, entry_tid=45933' 2026-02-18T08:24:29.868 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:24:29.868 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 2 2026-02-18T08:24:31.870 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:24:31.870 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:24:31.870 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:24:31.870 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:24:31.870 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:24:31.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:24:31.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:24:31.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:24:31.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:24:31.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:24:31.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:24:31.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:24:31.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:24:31.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:24:31.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:24:31.874 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:24:31.874 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:24:31.875 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:24:31.875 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:24:31.875 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:24:31.875 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:24:31.875 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:24:31.875 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:24:31.875 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:24:31.875 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:24:31.875 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:24:31.875 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:24:31.875 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:24:31.875 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:24:31.876 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:24:31.876 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:31.876 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:24:31.876 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:24:31.876 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:24:31.876 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:24:31.876 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:24:31.876 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:24:31.876 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:24:31.876 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:24:31.876 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:24:31.876 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:24:31.876 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:31.877 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:24:31.877 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:24:31.877 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:24:31.877 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:24:31.877 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:24:31.877 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:24:31.877 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:24:31.885 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:24:31.885 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:24:31.885 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:24:31.886 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:24:31.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:24:31.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:24:31.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:24:31.946 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:24:31.946 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:24:31.946 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:24:31.946 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:24:31.947 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:24:31.947 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:24:31.947 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:24:31.947 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:24:31.947 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:24:31.947 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:24:31.947 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:24:31.947 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:24:31.947 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:24:31.947 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:31.947 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:24:31.948 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:24:31.948 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:24:31.948 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:24:31.948 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:24:31.948 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:24:31.948 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:24:31.948 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:24:31.948 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:24:31.948 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:24:31.948 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:31.948 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:24:31.948 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:24:31.949 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:24:31.949 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:24:31.949 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:24:31.949 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:24:31.949 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:24:31.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:24:31.958 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:24:31.958 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:24:31.958 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:24:32.008 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:24:32.008 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:24:32.008 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:24:32.010 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:24:32.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:24:32.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:24:32.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:24:32.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:24:32.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:24:32.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:24:32.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:24:32.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:24:32.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:24:32.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:24:32.012 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:24:32.012 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:24:32.012 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:32.012 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:24:32.012 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:24:32.012 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:24:32.012 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:24:32.012 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:24:32.012 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:24:32.012 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:24:32.012 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:24:32.012 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:24:32.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:24:32.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:32.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:24:32.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:24:32.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:24:32.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:24:32.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:24:32.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:24:32.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:24:32.021 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:24:32.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:24:32.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:24:32.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:24:32.069 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:24:32.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:24:32.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:24:32.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:24:32.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:24:32.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:24:32.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:24:32.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:24:32.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:24:32.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:24:32.074 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:24:32.074 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:24:32.074 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:24:32.074 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:24:32.074 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:24:32.074 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:24:32.074 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:32.074 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:24:32.074 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:24:32.074 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:24:32.075 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:24:32.075 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:24:32.075 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:24:32.075 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:24:32.075 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:24:32.075 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:24:32.075 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:24:32.075 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:32.075 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:24:32.075 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:24:32.075 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:24:32.076 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:24:32.076 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:24:32.076 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:24:32.076 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:24:32.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:24:32.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:24:32.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:24:32.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:24:32.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:24:32.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:24:32.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:24:32.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:24:32.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:24:32.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:24:32.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:24:32.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:24:32.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:24:32.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:24:32.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:24:32.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:24:32.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:24:32.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:24:32.155 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:24:32.155 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:24:32.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 106 2026-02-18T08:24:32.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 106 2026-02-18T08:24:32.194 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:24:32.194 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=425, tag_tid=30, entry_tid=1], [object_number=424, tag_tid=30, entry_tid=0], [object_number=427, tag_tid=29, entry_tid=45931], [object_number=426, tag_tid=29, entry_tid=45930]]], state=connected] 2026-02-18T08:24:32.194 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=425, tag_tid=30, entry_tid=1], [object_number=424, tag_tid=30, entry_tid=0], [object_number=427, tag_tid=29, entry_tid=45931], [object_number=426, tag_tid=29, entry_tid=45930]]], state=connected] 2026-02-18T08:24:32.197 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:24:32.199 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=425, tag_tid=30, entry_tid=1' 2026-02-18T08:24:32.200 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:24:32.201 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:24:32.201 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:24:32.201 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:24:32.201 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:24:32.201 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:24:32.201 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:24:32.201 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:24:32.201 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:24:32.202 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:24:32.202 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:24:32.202 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:24:32.203 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:24:32.203 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:24:32.203 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:24:32.230 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 106 2026-02-18T08:24:32.230 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 106 2026-02-18T08:24:32.230 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:24:32.231 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=425, tag_tid=30, entry_tid=1], [object_number=424, tag_tid=30, entry_tid=0], [object_number=427, tag_tid=29, entry_tid=45931], [object_number=426, tag_tid=29, entry_tid=45930]]], state=connected] 2026-02-18T08:24:32.231 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=425, tag_tid=30, entry_tid=1], [object_number=424, tag_tid=30, entry_tid=0], [object_number=427, tag_tid=29, entry_tid=45931], [object_number=426, tag_tid=29, entry_tid=45930]]], state=connected] 2026-02-18T08:24:32.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:24:32.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=425, tag_tid=30, entry_tid=1' 2026-02-18T08:24:32.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=425, tag_tid=30, entry_tid=1' -a 'object_number=425, tag_tid=30, entry_tid=1' = 'object_number=425, tag_tid=30, entry_tid=1' 2026-02-18T08:24:32.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:24:32.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_snap_present cluster1 mirror test snap10 2026-02-18T08:24:32.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:24:32.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:24:32.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:24:32.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap10 2026-02-18T08:24:32.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-18T08:24:32.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-18T08:24:32.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1 2026-02-18T08:24:33.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster1 info mirror/test@snap10 2026-02-18T08:24:33.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:rbd image 'test': 2026-02-18T08:24:33.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: size 512 MiB in 128 objects 2026-02-18T08:24:33.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: order 22 (4 MiB objects) 2026-02-18T08:24:33.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: snapshot_count: 19 2026-02-18T08:24:33.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: id: 10a74aa8ebc0 2026-02-18T08:24:33.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: block_name_prefix: rbd_data.10a74aa8ebc0 2026-02-18T08:24:33.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: format: 2 2026-02-18T08:24:33.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: features: layering, exclusive-lock, journaling 2026-02-18T08:24:33.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: op_features: 2026-02-18T08:24:33.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: flags: 2026-02-18T08:24:33.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: create_timestamp: Wed Feb 18 08:10:48 2026 2026-02-18T08:24:33.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: access_timestamp: Wed Feb 18 08:23:00 2026 2026-02-18T08:24:33.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: modify_timestamp: Wed Feb 18 08:23:38 2026 2026-02-18T08:24:33.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: protected: False 2026-02-18T08:24:33.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: journal: 10a74aa8ebc0 2026-02-18T08:24:33.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring state: enabled 2026-02-18T08:24:33.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring mode: journal 2026-02-18T08:24:33.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring global id: 0b548823-d751-43df-a076-4efac2da4d3d 2026-02-18T08:24:33.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring primary: false 2026-02-18T08:24:33.270 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:24:33.270 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n snap9-clean ']' 2026-02-18T08:24:33.270 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ compare_image_snaps mirror test snap9-clean 2026-02-18T08:24:33.270 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:24:33.270 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:24:33.270 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap9-clean 2026-02-18T08:24:33.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local ret=0 2026-02-18T08:24:33.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local rmt_export=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.export 2026-02-18T08:24:33.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loc_export=/tmp/tmp.vybh1m9yJF/cluster1-mirror-test.export 2026-02-18T08:24:33.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.export /tmp/tmp.vybh1m9yJF/cluster1-mirror-test.export 2026-02-18T08:24:33.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 -p mirror export test@snap9-clean /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.export 2026-02-18T08:24:33.364 INFO:tasks.rbd_mirror_thrash:waiting for cluster2.client.mirror.0 2026-02-18T08:24:33.364 INFO:teuthology.orchestra.run:waiting for 600 2026-02-18T08:24:33.365 INFO:tasks.rbd_mirror_thrash:reviving cluster2.client.mirror.0 2026-02-18T08:24:33.365 INFO:tasks.rbd_mirror.cluster2.client.mirror.0:Restarting daemon 2026-02-18T08:24:33.365 DEBUG:teuthology.orchestra.run.trial163:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.0 2026-02-18T08:24:33.367 INFO:tasks.rbd_mirror.cluster2.client.mirror.0:Started 2026-02-18T08:24:33.367 INFO:tasks.rbd_mirror_thrash:waiting for 88.18396647455097 secs before thrashing 2026-02-18T08:24:33.763 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-18T08:24:33.779 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster1 -p mirror export test@snap9-clean /tmp/tmp.vybh1m9yJF/cluster1-mirror-test.export 2026-02-18T08:24:34.370 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-18T08:24:34.390 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmp /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.export /tmp/tmp.vybh1m9yJF/cluster1-mirror-test.export 2026-02-18T08:24:34.568 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.export /tmp/tmp.vybh1m9yJF/cluster1-mirror-test.export 2026-02-18T08:24:34.708 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:24:34.708 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ compare_image_snaps mirror test snap10 2026-02-18T08:24:34.708 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:24:34.708 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:24:34.708 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap10 2026-02-18T08:24:34.708 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local ret=0 2026-02-18T08:24:34.708 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local rmt_export=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.export 2026-02-18T08:24:34.708 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loc_export=/tmp/tmp.vybh1m9yJF/cluster1-mirror-test.export 2026-02-18T08:24:34.708 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.export /tmp/tmp.vybh1m9yJF/cluster1-mirror-test.export 2026-02-18T08:24:34.708 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 -p mirror export test@snap10 /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.export 2026-02-18T08:24:35.195 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-18T08:24:35.212 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster1 -p mirror export test@snap10 /tmp/tmp.vybh1m9yJF/cluster1-mirror-test.export 2026-02-18T08:24:35.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-18T08:24:35.818 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmp /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.export /tmp/tmp.vybh1m9yJF/cluster1-mirror-test.export 2026-02-18T08:24:35.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.export /tmp/tmp.vybh1m9yJF/cluster1-mirror-test.export 2026-02-18T08:24:36.139 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:24:36.139 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ clean_snap_name=snap10-clean 2026-02-18T08:24:36.139 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_snap cluster2 mirror test snap10-clean 2026-02-18T08:24:36.140 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:24:36.140 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:24:36.140 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:24:36.140 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap10-clean 2026-02-18T08:24:36.140 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 -p mirror snap create test@snap10-clean --debug-rbd=20 --debug-journaler=20 2026-02-18T08:24:36.830 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_replay_started cluster1 mirror test 2026-02-18T08:24:36.830 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:24:36.830 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:24:36.831 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:24:36.831 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_replay_state cluster1 mirror test started 2026-02-18T08:24:36.831 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:24:36.831 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:24:36.831 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:24:36.831 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local state=started 2026-02-18T08:24:36.831 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-18T08:24:36.831 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:24:36.831 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1 2026-02-18T08:24:37.831 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test_image_replay_state cluster1 mirror test started 2026-02-18T08:24:37.831 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:24:37.831 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:24:37.831 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:24:37.831 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local test_state=started 2026-02-18T08:24:37.831 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local status_result 2026-02-18T08:24:37.831 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_state=stopped 2026-02-18T08:24:37.831 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-18T08:24:37.831 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_instance=cluster1 2026-02-18T08:24:37.831 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -i state 2026-02-18T08:24:37.831 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-18T08:24:37.832 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:24:37.832 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance=cluster1 2026-02-18T08:24:37.832 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local loop_instance 2026-02-18T08:24:37.832 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:24:37.832 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sleep 0 2026-02-18T08:24:37.832 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:24:37.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ seq 0 3 2026-02-18T08:24:37.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:24:37.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-18T08:24:37.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:0 2026-02-18T08:24:37.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-18T08:24:37.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:24:37.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:24:37.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:24:37.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-18T08:24:37.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:24:37.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:37.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:24:37.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:24:37.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-18T08:24:37.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:24:37.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:24:37.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:24:37.835 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:24:37.835 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:0 2026-02-18T08:24:37.835 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-18T08:24:37.835 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-18T08:24:37.835 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:24:37.835 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:0 2026-02-18T08:24:37.835 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T08:24:37.835 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-18T08:24:37.835 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:24:37.835 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:24:37.835 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-18T08:24:37.835 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-18T08:24:37.835 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-18T08:24:37.835 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=0 2026-02-18T08:24:37.835 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=0 2026-02-18T08:24:37.835 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:24:37.846 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:24:37.846 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:24:37.846 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:24:37.846 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-18T08:24:37.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:24:37.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status : get status for rbd mirror 2026-02-18T08:24:37.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:24:37.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:24:37.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test 2026-02-18T08:24:37.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:1 2026-02-18T08:24:37.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-18T08:24:37.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:24:37.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:24:37.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:24:37.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-18T08:24:37.904 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:24:37.904 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:37.904 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:24:37.904 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:24:37.904 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-18T08:24:37.904 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:24:37.904 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:24:37.904 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:24:37.905 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:24:37.905 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:1 2026-02-18T08:24:37.905 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-18T08:24:37.905 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-18T08:24:37.905 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:24:37.905 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:1 2026-02-18T08:24:37.905 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T08:24:37.905 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-18T08:24:37.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:24:37.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:24:37.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-18T08:24:37.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-18T08:24:37.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-18T08:24:37.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=1 2026-02-18T08:24:37.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=1 2026-02-18T08:24:37.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:24:37.911 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:24:37.911 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:24:37.912 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:24:37.912 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test 2026-02-18T08:24:37.969 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:24:37.969 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status : get status for rbd mirror 2026-02-18T08:24:37.970 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:24:37.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:24:37.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/test 2026-02-18T08:24:37.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:2 2026-02-18T08:24:37.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-18T08:24:37.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:24:37.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:24:37.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:24:37.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-18T08:24:37.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:24:37.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:37.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:24:37.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:24:37.974 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-18T08:24:37.974 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:24:37.974 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:24:37.974 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:24:37.974 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:24:37.974 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:2 2026-02-18T08:24:37.974 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-18T08:24:37.974 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-18T08:24:37.974 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:24:37.974 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:2 2026-02-18T08:24:37.974 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T08:24:37.974 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-18T08:24:37.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:24:37.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:24:37.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-18T08:24:37.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-18T08:24:37.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-18T08:24:37.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=2 2026-02-18T08:24:37.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=2 2026-02-18T08:24:37.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:24:37.982 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:24:37.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:24:37.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:24:37.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/test 2026-02-18T08:24:38.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:24:38.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status : get status for rbd mirror 2026-02-18T08:24:38.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:24:38.035 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:24:38.036 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/test 2026-02-18T08:24:38.036 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:3 2026-02-18T08:24:38.036 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-18T08:24:38.036 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:24:38.036 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:24:38.036 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:24:38.036 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-18T08:24:38.036 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:24:38.036 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:38.036 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:24:38.036 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:24:38.037 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-18T08:24:38.037 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:24:38.037 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:24:38.037 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:24:38.037 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:24:38.037 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:3 2026-02-18T08:24:38.037 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-18T08:24:38.037 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-18T08:24:38.037 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:24:38.037 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:3 2026-02-18T08:24:38.037 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T08:24:38.037 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-18T08:24:38.038 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:24:38.038 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:24:38.038 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-18T08:24:38.038 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-18T08:24:38.038 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-18T08:24:38.038 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=3 2026-02-18T08:24:38.038 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=3 2026-02-18T08:24:38.038 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:24:38.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:24:38.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:24:38.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:24:38.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/test 2026-02-18T08:24:38.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ return 0 2026-02-18T08:24:38.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ status_result=' "state": "Replaying"' 2026-02-18T08:24:38.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ echo ' "state": "Replaying"' 2026-02-18T08:24:38.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ grep -i Replaying 2026-02-18T08:24:38.105 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: "state": "Replaying" 2026-02-18T08:24:38.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ current_state=started 2026-02-18T08:24:38.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test started = started 2026-02-18T08:24:38.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:24:38.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror test 2026-02-18T08:24:38.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_cluster=cluster1 2026-02-18T08:24:38.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:24:38.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_pool=mirror 2026-02-18T08:24:38.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local remote_pool=mirror 2026-02-18T08:24:38.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:24:38.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' journal = journal ']' 2026-02-18T08:24:38.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror mirror test 2026-02-18T08:24:38.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_cluster=cluster1 2026-02-18T08:24:38.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:24:38.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_pool=mirror 2026-02-18T08:24:38.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local remote_pool=mirror 2026-02-18T08:24:38.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:24:38.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-02-18T08:24:38.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-02-18T08:24:38.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:24:38.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:24:38.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:24:38.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:24:38.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:24:38.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:24:38.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:24:38.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:24:38.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:24:38.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:24:38.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:24:38.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:24:38.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:24:38.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:24:38.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:24:38.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:24:38.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:24:38.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:24:38.310 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:24:38.310 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:24:38.312 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:24:38.312 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:24:38.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:24:38.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:24:38.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:24:38.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:24:38.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:24:38.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:24:38.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:24:38.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:24:38.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:24:38.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:24:38.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:24:38.314 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:38.314 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:24:38.314 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:24:38.314 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:24:38.314 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:24:38.314 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:24:38.314 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:24:38.314 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:24:38.314 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:24:38.314 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:24:38.314 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:24:38.314 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:38.315 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:24:38.315 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:24:38.315 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:24:38.315 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:24:38.315 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:24:38.315 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:24:38.315 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:24:38.321 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:24:38.322 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:24:38.322 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:24:38.322 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:24:38.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:24:38.379 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:24:38.379 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:24:38.381 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:24:38.381 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:24:38.382 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:24:38.382 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:24:38.382 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:24:38.382 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:24:38.382 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:24:38.382 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:24:38.382 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:24:38.382 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:24:38.382 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:24:38.382 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:24:38.382 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:24:38.383 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:38.383 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:24:38.383 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:24:38.383 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:24:38.383 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:24:38.383 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:24:38.383 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:24:38.383 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:24:38.383 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:24:38.383 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:24:38.383 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:24:38.383 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:38.384 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:24:38.384 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:24:38.384 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:24:38.384 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:24:38.384 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:24:38.384 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:24:38.384 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:24:38.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:24:38.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:24:38.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:24:38.393 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:24:38.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:24:38.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:24:38.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:24:38.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:24:38.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:24:38.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:24:38.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:24:38.446 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:24:38.446 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:24:38.446 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:24:38.446 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:24:38.446 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:24:38.446 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:24:38.446 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:24:38.446 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:24:38.446 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:24:38.446 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:38.446 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:24:38.446 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:24:38.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:24:38.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:24:38.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:24:38.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:24:38.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:24:38.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:24:38.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:24:38.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:24:38.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:38.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:24:38.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:24:38.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:24:38.448 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:24:38.448 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:24:38.448 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:24:38.448 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:24:38.455 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:24:38.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:24:38.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:24:38.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:24:38.513 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:24:38.513 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:24:38.513 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:24:38.516 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:24:38.516 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:24:38.516 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:24:38.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:24:38.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:24:38.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:24:38.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:24:38.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:24:38.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:24:38.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:24:38.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:24:38.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:24:38.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:24:38.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:38.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:24:38.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:24:38.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:24:38.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:24:38.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:24:38.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:24:38.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:24:38.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:24:38.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:24:38.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:24:38.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:38.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:24:38.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:24:38.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:24:38.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:24:38.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:24:38.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:24:38.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:24:38.526 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:24:38.527 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:24:38.527 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:24:38.527 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:24:38.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:24:38.584 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:24:38.584 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:24:38.584 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:24:38.584 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:24:38.584 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:24:38.584 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:24:38.584 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:24:38.585 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:24:38.585 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:24:38.585 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:24:38.585 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:24:38.585 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:24:38.586 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:24:38.586 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:24:38.586 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:24:38.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 106 2026-02-18T08:24:38.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 106 2026-02-18T08:24:38.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:24:38.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=425, tag_tid=31, entry_tid=1], [object_number=424, tag_tid=31, entry_tid=0], [object_number=427, tag_tid=29, entry_tid=45931], [object_number=426, tag_tid=29, entry_tid=45930]]], state=connected] 2026-02-18T08:24:38.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=425, tag_tid=30, entry_tid=1], [object_number=424, tag_tid=30, entry_tid=0], [object_number=427, tag_tid=29, entry_tid=45931], [object_number=426, tag_tid=29, entry_tid=45930]]], state=connected] 2026-02-18T08:24:38.628 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:24:38.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=425, tag_tid=31, entry_tid=1' 2026-02-18T08:24:38.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:24:38.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:24:38.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:24:38.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:24:38.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:24:38.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:24:38.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:24:38.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:24:38.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:24:38.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:24:38.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:24:38.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:24:38.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:24:38.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:24:38.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:24:38.672 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 106 2026-02-18T08:24:38.672 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 106 2026-02-18T08:24:38.672 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:24:38.672 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=425, tag_tid=31, entry_tid=1], [object_number=424, tag_tid=31, entry_tid=0], [object_number=427, tag_tid=29, entry_tid=45931], [object_number=426, tag_tid=29, entry_tid=45930]]], state=connected] 2026-02-18T08:24:38.673 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=425, tag_tid=30, entry_tid=1], [object_number=424, tag_tid=30, entry_tid=0], [object_number=427, tag_tid=29, entry_tid=45931], [object_number=426, tag_tid=29, entry_tid=45930]]], state=connected] 2026-02-18T08:24:38.676 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:24:38.678 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=425, tag_tid=30, entry_tid=1' 2026-02-18T08:24:38.678 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=425, tag_tid=31, entry_tid=1' -a 'object_number=425, tag_tid=31, entry_tid=1' = 'object_number=425, tag_tid=30, entry_tid=1' 2026-02-18T08:24:38.678 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=425, tag_tid=30, entry_tid=1' '!=' '' 2026-02-18T08:24:38.678 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:24:38.678 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=425, tag_tid=30, entry_tid=1' = '' 2026-02-18T08:24:38.678 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=425, tag_tid=30, entry_tid=1' 2026-02-18T08:24:38.679 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=425, tag_tid=31, entry_tid=1' 2026-02-18T08:24:38.679 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:24:38.679 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:24:38.680 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=31 2026-02-18T08:24:38.681 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=425, tag_tid=30, entry_tid=1' 2026-02-18T08:24:38.681 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:24:38.681 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:24:38.682 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=30 2026-02-18T08:24:38.683 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=425, tag_tid=31, entry_tid=1' 2026-02-18T08:24:38.683 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:24:38.683 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:24:38.684 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:24:38.685 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=425, tag_tid=30, entry_tid=1' 2026-02-18T08:24:38.685 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:24:38.685 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:24:38.687 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=1 2026-02-18T08:24:38.687 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 31 = 30 -a 1 -le 1 2026-02-18T08:24:38.687 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:24:38.687 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:24:38.687 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:24:38.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:24:38.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:24:38.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:24:38.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:24:38.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:24:38.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:24:38.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:24:38.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:24:38.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:24:38.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:24:38.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:24:38.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:24:38.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:24:38.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:24:38.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:24:38.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:24:38.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:24:38.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:24:38.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:24:38.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:24:38.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:24:38.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:24:38.894 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:24:38.894 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:24:38.894 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:24:38.894 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:24:38.894 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:24:38.894 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:24:38.894 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:24:38.894 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:24:38.894 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:38.894 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:24:38.894 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:24:38.894 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:24:38.895 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:24:38.895 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:24:38.895 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:24:38.895 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:24:38.895 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:24:38.895 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:24:38.895 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:24:38.895 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:38.895 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:24:38.895 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:24:38.895 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:24:38.895 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:24:38.896 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:24:38.896 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:24:38.896 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:24:38.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:24:38.904 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:24:38.904 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:24:38.904 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:24:38.961 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:24:38.961 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:24:38.962 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:24:38.964 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:24:38.964 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:24:38.964 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:24:38.965 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:24:38.965 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:24:38.965 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:24:38.965 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:24:38.965 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:24:38.965 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:24:38.965 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:24:38.965 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:24:38.965 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:24:38.965 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:24:38.965 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:38.966 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:24:38.966 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:24:38.966 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:24:38.966 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:24:38.966 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:24:38.966 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:24:38.966 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:24:38.966 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:24:38.966 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:24:38.966 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:24:38.966 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:38.966 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:24:38.967 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:24:38.967 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:24:38.967 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:24:38.967 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:24:38.967 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:24:38.967 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:24:38.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:24:38.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:24:38.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:24:38.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:24:39.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:24:39.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:24:39.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:24:39.028 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:24:39.028 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:24:39.028 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:24:39.028 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:24:39.028 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:24:39.029 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:24:39.029 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:24:39.029 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:24:39.029 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:24:39.029 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:24:39.029 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:24:39.029 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:24:39.029 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:24:39.029 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:39.029 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:24:39.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:24:39.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:24:39.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:24:39.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:24:39.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:24:39.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:24:39.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:24:39.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:24:39.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:24:39.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:39.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:24:39.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:24:39.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:24:39.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:24:39.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:24:39.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:24:39.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:24:39.039 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:24:39.039 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:24:39.039 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:24:39.039 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:24:39.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:24:39.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:24:39.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:24:39.092 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:24:39.092 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:24:39.092 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:24:39.092 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:24:39.092 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:24:39.093 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:24:39.093 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:24:39.093 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:24:39.093 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:24:39.093 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:24:39.093 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:24:39.093 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:24:39.093 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:24:39.093 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:39.093 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:24:39.093 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:24:39.093 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:24:39.094 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:24:39.094 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:24:39.094 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:24:39.094 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:24:39.094 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:24:39.094 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:24:39.094 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:24:39.094 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:39.094 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:24:39.094 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:24:39.094 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:24:39.094 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:24:39.094 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:24:39.095 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:24:39.095 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:24:39.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:24:39.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:24:39.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:24:39.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:24:39.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:24:39.169 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:24:39.170 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:24:39.170 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:24:39.170 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:24:39.170 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:24:39.170 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:24:39.170 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:24:39.170 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:24:39.170 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:24:39.170 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:24:39.170 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:24:39.170 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:24:39.171 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:24:39.171 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:24:39.171 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:24:39.200 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 106 2026-02-18T08:24:39.200 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 106 2026-02-18T08:24:39.200 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:24:39.201 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=425, tag_tid=31, entry_tid=1], [object_number=424, tag_tid=31, entry_tid=0], [object_number=427, tag_tid=29, entry_tid=45931], [object_number=426, tag_tid=29, entry_tid=45930]]], state=connected] 2026-02-18T08:24:39.201 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=425, tag_tid=30, entry_tid=1], [object_number=424, tag_tid=30, entry_tid=0], [object_number=427, tag_tid=29, entry_tid=45931], [object_number=426, tag_tid=29, entry_tid=45930]]], state=connected] 2026-02-18T08:24:39.203 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:24:39.204 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=425, tag_tid=31, entry_tid=1' 2026-02-18T08:24:39.206 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:24:39.206 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:24:39.206 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:24:39.206 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:24:39.206 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:24:39.206 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:24:39.206 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:24:39.206 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:24:39.207 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:24:39.207 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:24:39.207 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:24:39.207 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:24:39.207 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:24:39.208 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:24:39.208 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:24:39.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 106 2026-02-18T08:24:39.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 106 2026-02-18T08:24:39.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:24:39.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=425, tag_tid=31, entry_tid=1], [object_number=424, tag_tid=31, entry_tid=0], [object_number=427, tag_tid=29, entry_tid=45931], [object_number=426, tag_tid=29, entry_tid=45930]]], state=connected] 2026-02-18T08:24:39.244 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=425, tag_tid=30, entry_tid=1], [object_number=424, tag_tid=30, entry_tid=0], [object_number=427, tag_tid=29, entry_tid=45931], [object_number=426, tag_tid=29, entry_tid=45930]]], state=connected] 2026-02-18T08:24:39.247 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:24:39.248 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=425, tag_tid=30, entry_tid=1' 2026-02-18T08:24:39.248 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=425, tag_tid=31, entry_tid=1' -a 'object_number=425, tag_tid=31, entry_tid=1' = 'object_number=425, tag_tid=30, entry_tid=1' 2026-02-18T08:24:39.249 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=425, tag_tid=30, entry_tid=1' '!=' 'object_number=425, tag_tid=30, entry_tid=1' 2026-02-18T08:24:39.249 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:24:39.249 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.4 2026-02-18T08:24:39.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:24:39.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:24:39.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:24:39.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:24:39.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:24:39.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:24:39.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:24:39.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:24:39.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:24:39.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:24:39.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:24:39.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:24:39.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:24:39.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:24:39.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:24:39.655 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:24:39.655 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:24:39.655 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:24:39.655 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:24:39.655 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:24:39.655 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:24:39.655 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:24:39.655 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:24:39.655 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:24:39.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:24:39.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:24:39.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:24:39.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:24:39.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:24:39.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:24:39.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:39.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:24:39.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:24:39.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:24:39.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:24:39.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:24:39.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:24:39.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:24:39.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:24:39.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:24:39.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:24:39.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:39.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:24:39.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:24:39.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:24:39.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:24:39.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:24:39.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:24:39.658 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:24:39.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:24:39.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:24:39.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:24:39.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:24:39.728 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:24:39.728 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:24:39.729 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:24:39.731 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:24:39.731 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:24:39.731 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:24:39.731 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:24:39.731 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:24:39.732 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:24:39.732 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:24:39.732 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:24:39.732 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:24:39.732 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:24:39.732 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:24:39.732 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:24:39.732 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:24:39.732 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:39.732 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:24:39.732 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:24:39.732 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:24:39.733 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:24:39.733 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:24:39.733 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:24:39.733 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:24:39.733 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:24:39.733 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:24:39.733 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:24:39.733 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:39.733 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:24:39.733 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:24:39.733 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:24:39.733 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:24:39.734 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:24:39.734 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:24:39.734 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:24:39.743 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:24:39.744 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:24:39.744 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:24:39.744 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:24:39.800 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:24:39.800 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:24:39.800 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:24:39.803 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:24:39.803 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:24:39.803 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:24:39.804 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:24:39.804 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:24:39.804 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:24:39.804 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:24:39.804 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:24:39.804 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:24:39.804 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:24:39.804 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:24:39.804 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:24:39.804 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:24:39.804 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:39.805 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:24:39.805 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:24:39.805 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:24:39.805 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:24:39.805 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:24:39.805 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:24:39.805 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:24:39.805 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:24:39.805 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:24:39.805 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:24:39.805 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:39.805 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:24:39.805 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:24:39.806 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:24:39.806 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:24:39.806 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:24:39.806 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:24:39.806 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:24:39.814 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:24:39.815 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:24:39.815 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:24:39.815 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:24:39.862 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:24:39.862 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:24:39.862 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:24:39.865 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:24:39.865 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:24:39.865 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:24:39.865 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:24:39.865 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:24:39.865 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:24:39.865 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:24:39.865 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:24:39.865 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:24:39.865 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:24:39.866 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:24:39.866 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:24:39.866 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:24:39.866 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:39.866 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:24:39.866 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:24:39.866 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:24:39.866 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:24:39.866 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:24:39.866 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:24:39.866 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:24:39.866 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:24:39.867 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:24:39.867 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:24:39.867 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:39.867 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:24:39.867 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:24:39.867 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:24:39.867 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:24:39.867 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:24:39.867 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:24:39.867 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:24:39.876 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:24:39.877 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:24:39.877 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:24:39.877 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:24:39.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:24:39.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:24:39.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:24:39.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:24:39.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:24:39.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:24:39.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:24:39.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:24:39.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:24:39.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:24:39.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:24:39.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:24:39.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:24:39.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:24:39.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:24:39.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:24:39.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 106 2026-02-18T08:24:39.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 106 2026-02-18T08:24:39.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:24:39.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=425, tag_tid=31, entry_tid=1], [object_number=424, tag_tid=31, entry_tid=0], [object_number=427, tag_tid=29, entry_tid=45931], [object_number=426, tag_tid=29, entry_tid=45930]]], state=connected] 2026-02-18T08:24:39.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=425, tag_tid=30, entry_tid=1], [object_number=424, tag_tid=30, entry_tid=0], [object_number=427, tag_tid=29, entry_tid=45931], [object_number=426, tag_tid=29, entry_tid=45930]]], state=connected] 2026-02-18T08:24:39.974 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:24:39.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=425, tag_tid=31, entry_tid=1' 2026-02-18T08:24:39.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:24:39.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:24:39.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:24:39.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:24:39.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:24:39.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:24:39.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:24:39.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:24:39.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:24:39.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:24:39.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:24:39.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:24:39.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:24:39.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:24:39.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:24:40.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 106 2026-02-18T08:24:40.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 106 2026-02-18T08:24:40.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:24:40.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=425, tag_tid=31, entry_tid=1], [object_number=424, tag_tid=31, entry_tid=0], [object_number=427, tag_tid=29, entry_tid=45931], [object_number=426, tag_tid=29, entry_tid=45930]]], state=connected] 2026-02-18T08:24:40.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=425, tag_tid=30, entry_tid=1], [object_number=424, tag_tid=30, entry_tid=0], [object_number=427, tag_tid=29, entry_tid=45931], [object_number=426, tag_tid=29, entry_tid=45930]]], state=connected] 2026-02-18T08:24:40.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:24:40.019 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=425, tag_tid=30, entry_tid=1' 2026-02-18T08:24:40.019 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=425, tag_tid=31, entry_tid=1' -a 'object_number=425, tag_tid=31, entry_tid=1' = 'object_number=425, tag_tid=30, entry_tid=1' 2026-02-18T08:24:40.019 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=425, tag_tid=30, entry_tid=1' '!=' 'object_number=425, tag_tid=30, entry_tid=1' 2026-02-18T08:24:40.019 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:24:40.019 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.8 2026-02-18T08:24:40.821 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:24:40.822 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:24:40.822 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:24:40.822 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:24:40.822 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:24:40.822 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:24:40.822 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:24:40.822 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:24:40.822 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:24:40.822 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:24:40.822 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:24:40.823 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:24:40.823 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:24:40.823 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:24:40.823 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:24:40.823 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:24:40.826 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:24:40.826 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:24:40.826 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:24:40.826 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:24:40.826 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:24:40.826 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:24:40.826 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:24:40.826 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:24:40.826 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:24:40.826 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:24:40.826 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:24:40.827 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:24:40.827 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:24:40.827 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:24:40.827 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:40.827 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:24:40.827 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:24:40.827 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:24:40.827 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:24:40.827 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:24:40.827 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:24:40.828 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:24:40.828 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:24:40.828 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:24:40.828 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:24:40.828 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:40.828 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:24:40.828 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:24:40.828 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:24:40.828 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:24:40.828 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:24:40.828 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:24:40.829 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:24:40.836 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:24:40.837 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:24:40.837 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:24:40.837 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:24:40.886 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:24:40.886 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:24:40.887 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:24:40.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:24:40.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:24:40.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:24:40.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:24:40.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:24:40.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:24:40.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:24:40.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:24:40.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:24:40.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:24:40.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:24:40.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:24:40.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:24:40.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:40.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:24:40.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:24:40.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:24:40.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:24:40.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:24:40.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:24:40.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:24:40.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:24:40.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:24:40.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:24:40.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:40.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:24:40.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:24:40.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:24:40.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:24:40.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:24:40.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:24:40.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:24:40.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:24:40.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:24:40.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:24:40.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:24:40.958 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:24:40.958 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:24:40.958 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:24:40.961 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:24:40.961 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:24:40.961 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:24:40.961 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:24:40.961 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:24:40.961 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:24:40.961 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:24:40.961 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:24:40.961 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:24:40.962 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:24:40.962 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:24:40.962 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:24:40.962 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:24:40.962 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:40.962 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:24:40.962 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:24:40.962 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:24:40.962 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:24:40.962 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:24:40.962 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:24:40.962 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:24:40.963 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:24:40.963 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:24:40.963 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:24:40.963 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:40.963 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:24:40.963 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:24:40.963 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:24:40.963 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:24:40.963 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:24:40.963 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:24:40.963 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:24:40.971 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:24:40.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:24:40.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:24:40.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:24:41.028 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:24:41.029 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:24:41.029 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:24:41.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:24:41.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:24:41.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:24:41.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:24:41.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:24:41.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:24:41.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:24:41.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:24:41.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:24:41.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:24:41.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:24:41.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:24:41.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:24:41.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:41.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:24:41.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:24:41.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:24:41.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:24:41.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:24:41.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:24:41.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:24:41.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:24:41.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:24:41.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:24:41.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:41.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:24:41.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:24:41.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:24:41.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:24:41.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:24:41.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:24:41.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:24:41.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:24:41.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:24:41.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:24:41.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:24:41.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:24:41.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:24:41.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:24:41.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:24:41.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:24:41.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:24:41.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:24:41.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:24:41.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:24:41.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:24:41.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:24:41.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:24:41.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:24:41.110 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:24:41.110 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:24:41.110 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:24:41.138 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 106 2026-02-18T08:24:41.138 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 106 2026-02-18T08:24:41.138 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:24:41.138 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=425, tag_tid=31, entry_tid=1], [object_number=424, tag_tid=31, entry_tid=0], [object_number=427, tag_tid=29, entry_tid=45931], [object_number=426, tag_tid=29, entry_tid=45930]]], state=connected] 2026-02-18T08:24:41.138 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=425, tag_tid=30, entry_tid=1], [object_number=424, tag_tid=30, entry_tid=0], [object_number=427, tag_tid=29, entry_tid=45931], [object_number=426, tag_tid=29, entry_tid=45930]]], state=connected] 2026-02-18T08:24:41.141 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:24:41.142 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=425, tag_tid=31, entry_tid=1' 2026-02-18T08:24:41.143 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:24:41.143 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:24:41.143 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:24:41.143 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:24:41.143 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:24:41.143 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:24:41.143 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:24:41.144 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:24:41.144 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:24:41.144 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:24:41.144 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:24:41.144 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:24:41.145 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:24:41.145 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:24:41.145 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:24:41.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 106 2026-02-18T08:24:41.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 106 2026-02-18T08:24:41.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:24:41.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=425, tag_tid=31, entry_tid=1], [object_number=424, tag_tid=31, entry_tid=0], [object_number=427, tag_tid=29, entry_tid=45931], [object_number=426, tag_tid=29, entry_tid=45930]]], state=connected] 2026-02-18T08:24:41.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=425, tag_tid=30, entry_tid=1], [object_number=424, tag_tid=30, entry_tid=0], [object_number=427, tag_tid=29, entry_tid=45931], [object_number=426, tag_tid=29, entry_tid=45930]]], state=connected] 2026-02-18T08:24:41.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:24:41.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=425, tag_tid=30, entry_tid=1' 2026-02-18T08:24:41.187 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=425, tag_tid=31, entry_tid=1' -a 'object_number=425, tag_tid=31, entry_tid=1' = 'object_number=425, tag_tid=30, entry_tid=1' 2026-02-18T08:24:41.187 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=425, tag_tid=30, entry_tid=1' '!=' 'object_number=425, tag_tid=30, entry_tid=1' 2026-02-18T08:24:41.187 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:24:41.187 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1.6 2026-02-18T08:24:42.788 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:24:42.788 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:24:42.788 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:24:42.788 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:24:42.788 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:24:42.788 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:24:42.788 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:24:42.788 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:24:42.788 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:24:42.788 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:24:42.788 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:24:42.788 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:24:42.788 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:24:42.788 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:24:42.788 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:24:42.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:24:42.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:24:42.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:24:42.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:24:42.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:24:42.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:24:42.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:24:42.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:24:42.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:24:42.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:24:42.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:24:42.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:24:42.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:24:42.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:24:42.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:24:42.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:42.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:24:42.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:24:42.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:24:42.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:24:42.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:24:42.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:24:42.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:24:42.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:24:42.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:24:42.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:24:42.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:42.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:24:42.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:24:42.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:24:42.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:24:42.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:24:42.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:24:42.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:24:42.804 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:24:42.804 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:24:42.804 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:24:42.805 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:24:42.863 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:24:42.864 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:24:42.864 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:24:42.865 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:24:42.865 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:24:42.865 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:24:42.865 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:24:42.865 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:24:42.865 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:24:42.865 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:24:42.865 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:24:42.865 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:24:42.865 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:24:42.865 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:24:42.865 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:24:42.865 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:24:42.865 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:42.865 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:24:42.866 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:24:42.866 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:24:42.866 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:24:42.866 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:24:42.866 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:24:42.866 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:24:42.866 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:24:42.866 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:24:42.866 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:24:42.866 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:42.866 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:24:42.866 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:24:42.866 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:24:42.867 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:24:42.867 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:24:42.867 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:24:42.867 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:24:42.877 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:24:42.877 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:24:42.877 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:24:42.877 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:24:42.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:24:42.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:24:42.936 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:24:42.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:24:42.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:24:42.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:24:42.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:24:42.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:24:42.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:24:42.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:24:42.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:24:42.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:24:42.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:24:42.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:24:42.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:24:42.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:24:42.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:42.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:24:42.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:24:42.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:24:42.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:24:42.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:24:42.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:24:42.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:24:42.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:24:42.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:24:42.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:24:42.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:42.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:24:42.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:24:42.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:24:42.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:24:42.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:24:42.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:24:42.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:24:42.949 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:24:42.949 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:24:42.949 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:24:42.949 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:24:43.007 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:24:43.007 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:24:43.007 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:24:43.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:24:43.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:24:43.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:24:43.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:24:43.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:24:43.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:24:43.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:24:43.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:24:43.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:24:43.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:24:43.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:24:43.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:24:43.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:24:43.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:43.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:24:43.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:24:43.010 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:24:43.010 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:24:43.010 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:24:43.010 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:24:43.010 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:24:43.010 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:24:43.010 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:24:43.010 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:24:43.010 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:43.010 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:24:43.010 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:24:43.010 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:24:43.010 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:24:43.010 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:24:43.010 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:24:43.010 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:24:43.021 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:24:43.021 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:24:43.021 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:24:43.021 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:24:43.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:24:43.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:24:43.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:24:43.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:24:43.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:24:43.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:24:43.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:24:43.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:24:43.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:24:43.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:24:43.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:24:43.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:24:43.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:24:43.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:24:43.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:24:43.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:24:43.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 106 2026-02-18T08:24:43.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 106 2026-02-18T08:24:43.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:24:43.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=425, tag_tid=31, entry_tid=1], [object_number=424, tag_tid=31, entry_tid=0], [object_number=427, tag_tid=29, entry_tid=45931], [object_number=426, tag_tid=29, entry_tid=45930]]], state=connected] 2026-02-18T08:24:43.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=425, tag_tid=30, entry_tid=1], [object_number=424, tag_tid=30, entry_tid=0], [object_number=427, tag_tid=29, entry_tid=45931], [object_number=426, tag_tid=29, entry_tid=45930]]], state=connected] 2026-02-18T08:24:43.132 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:24:43.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=425, tag_tid=31, entry_tid=1' 2026-02-18T08:24:43.134 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:24:43.134 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:24:43.134 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:24:43.134 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:24:43.134 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:24:43.134 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:24:43.134 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:24:43.134 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:24:43.134 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:24:43.134 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:24:43.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:24:43.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:24:43.136 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:24:43.136 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:24:43.137 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:24:43.171 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 106 2026-02-18T08:24:43.171 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 106 2026-02-18T08:24:43.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:24:43.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=425, tag_tid=31, entry_tid=1], [object_number=424, tag_tid=31, entry_tid=0], [object_number=427, tag_tid=29, entry_tid=45931], [object_number=426, tag_tid=29, entry_tid=45930]]], state=connected] 2026-02-18T08:24:43.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=425, tag_tid=30, entry_tid=1], [object_number=424, tag_tid=30, entry_tid=0], [object_number=427, tag_tid=29, entry_tid=45931], [object_number=426, tag_tid=29, entry_tid=45930]]], state=connected] 2026-02-18T08:24:43.174 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:24:43.175 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=425, tag_tid=30, entry_tid=1' 2026-02-18T08:24:43.176 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=425, tag_tid=31, entry_tid=1' -a 'object_number=425, tag_tid=31, entry_tid=1' = 'object_number=425, tag_tid=30, entry_tid=1' 2026-02-18T08:24:43.176 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=425, tag_tid=30, entry_tid=1' '!=' 'object_number=425, tag_tid=30, entry_tid=1' 2026-02-18T08:24:43.176 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:24:43.176 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 2 2026-02-18T08:24:45.178 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:24:45.178 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:24:45.178 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:24:45.178 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:24:45.179 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:24:45.179 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:24:45.179 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:24:45.179 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:24:45.179 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:24:45.179 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:24:45.179 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:24:45.179 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:24:45.179 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:24:45.179 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:24:45.179 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:24:45.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:24:45.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:24:45.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:24:45.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:24:45.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:24:45.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:24:45.183 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:24:45.183 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:24:45.183 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:24:45.183 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:24:45.183 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:24:45.183 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:24:45.183 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:24:45.183 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:24:45.183 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:24:45.183 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:45.183 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:24:45.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:24:45.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:24:45.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:24:45.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:24:45.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:24:45.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:24:45.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:24:45.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:24:45.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:24:45.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:45.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:24:45.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:24:45.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:24:45.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:24:45.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:24:45.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:24:45.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:24:45.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:24:45.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:24:45.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:24:45.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:24:45.251 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:24:45.251 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:24:45.252 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:24:45.254 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:24:45.254 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:24:45.254 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:24:45.254 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:24:45.255 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:24:45.255 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:24:45.255 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:24:45.255 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:24:45.255 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:24:45.255 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:24:45.255 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:24:45.255 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:24:45.255 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:24:45.255 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:45.255 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:24:45.255 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:24:45.256 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:24:45.256 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:24:45.256 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:24:45.256 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:24:45.256 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:24:45.256 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:24:45.256 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:24:45.256 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:24:45.256 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:45.256 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:24:45.256 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:24:45.256 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:24:45.257 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:24:45.257 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:24:45.257 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:24:45.257 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:24:45.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:24:45.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:24:45.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:24:45.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:24:45.314 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:24:45.314 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:24:45.314 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:24:45.317 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:24:45.317 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:24:45.317 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:24:45.317 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:24:45.317 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:24:45.318 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:24:45.318 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:24:45.318 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:24:45.318 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:24:45.318 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:24:45.318 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:24:45.318 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:24:45.318 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:24:45.318 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:45.318 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:24:45.318 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:24:45.318 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:24:45.318 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:24:45.319 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:24:45.319 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:24:45.319 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:24:45.319 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:24:45.319 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:24:45.319 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:24:45.319 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:45.319 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:24:45.319 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:24:45.319 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:24:45.319 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:24:45.319 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:24:45.320 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:24:45.320 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:24:45.327 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:24:45.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:24:45.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:24:45.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:24:45.385 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:24:45.385 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:24:45.385 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:24:45.388 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:24:45.388 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:24:45.388 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:24:45.388 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:24:45.388 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:24:45.388 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:24:45.388 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:24:45.388 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:24:45.389 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:24:45.389 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:24:45.389 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:24:45.389 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:24:45.389 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:24:45.389 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:45.389 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:24:45.389 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:24:45.389 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:24:45.389 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:24:45.389 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:24:45.390 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:24:45.390 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:24:45.390 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:24:45.390 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:24:45.390 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:24:45.390 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:45.390 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:24:45.390 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:24:45.390 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:24:45.390 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:24:45.390 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:24:45.390 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:24:45.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:24:45.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:24:45.399 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:24:45.399 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:24:45.399 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:24:45.455 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:24:45.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:24:45.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:24:45.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:24:45.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:24:45.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:24:45.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:24:45.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:24:45.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:24:45.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:24:45.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:24:45.458 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:24:45.458 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:24:45.458 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:24:45.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:24:45.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:24:45.489 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 106 2026-02-18T08:24:45.489 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 106 2026-02-18T08:24:45.489 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:24:45.489 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=425, tag_tid=31, entry_tid=1], [object_number=424, tag_tid=31, entry_tid=0], [object_number=427, tag_tid=29, entry_tid=45931], [object_number=426, tag_tid=29, entry_tid=45930]]], state=connected] 2026-02-18T08:24:45.489 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=425, tag_tid=30, entry_tid=1], [object_number=424, tag_tid=30, entry_tid=0], [object_number=427, tag_tid=29, entry_tid=45931], [object_number=426, tag_tid=29, entry_tid=45930]]], state=connected] 2026-02-18T08:24:45.491 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:24:45.493 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=425, tag_tid=31, entry_tid=1' 2026-02-18T08:24:45.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:24:45.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:24:45.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:24:45.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:24:45.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:24:45.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:24:45.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:24:45.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:24:45.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:24:45.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:24:45.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:24:45.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:24:45.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:24:45.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:24:45.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:24:45.524 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 106 2026-02-18T08:24:45.524 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 106 2026-02-18T08:24:45.524 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:24:45.524 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=425, tag_tid=31, entry_tid=1], [object_number=424, tag_tid=31, entry_tid=0], [object_number=427, tag_tid=29, entry_tid=45931], [object_number=426, tag_tid=29, entry_tid=45930]]], state=connected] 2026-02-18T08:24:45.525 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=425, tag_tid=30, entry_tid=1], [object_number=424, tag_tid=30, entry_tid=0], [object_number=427, tag_tid=29, entry_tid=45931], [object_number=426, tag_tid=29, entry_tid=45930]]], state=connected] 2026-02-18T08:24:45.527 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:24:45.528 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=425, tag_tid=30, entry_tid=1' 2026-02-18T08:24:45.528 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=425, tag_tid=31, entry_tid=1' -a 'object_number=425, tag_tid=31, entry_tid=1' = 'object_number=425, tag_tid=30, entry_tid=1' 2026-02-18T08:24:45.529 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=425, tag_tid=30, entry_tid=1' '!=' 'object_number=425, tag_tid=30, entry_tid=1' 2026-02-18T08:24:45.529 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:24:45.529 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 2 2026-02-18T08:24:47.530 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-18T08:24:47.530 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:24:47.530 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:24:47.531 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:24:47.531 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:24:47.531 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-18T08:24:47.531 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-18T08:24:47.531 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-18T08:24:47.531 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:24:47.531 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:24:47.531 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:24:47.531 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:24:47.531 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:24:47.532 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:24:47.532 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:24:47.535 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:24:47.535 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:24:47.535 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:24:47.535 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-18T08:24:47.535 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:24:47.535 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:24:47.535 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:24:47.535 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:24:47.535 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:24:47.536 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:24:47.536 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:24:47.536 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:24:47.536 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:24:47.536 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:24:47.536 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:24:47.536 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:47.536 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:24:47.536 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:24:47.536 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:24:47.536 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:24:47.536 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:24:47.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:24:47.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:24:47.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:24:47.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:24:47.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:24:47.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:47.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:24:47.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:24:47.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:24:47.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:24:47.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:24:47.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:24:47.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:24:47.545 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:24:47.545 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:24:47.545 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:24:47.545 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-18T08:24:47.602 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:24:47.602 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:24:47.602 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:24:47.605 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:24:47.605 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-18T08:24:47.605 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:24:47.605 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:24:47.605 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:24:47.605 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:24:47.605 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:24:47.605 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:24:47.605 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:24:47.605 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:24:47.606 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:24:47.606 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:24:47.606 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:24:47.606 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:47.606 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:24:47.606 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:24:47.606 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:24:47.606 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:24:47.606 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:24:47.606 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:24:47.606 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:24:47.606 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:24:47.606 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:24:47.607 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:24:47.607 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:47.607 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:24:47.607 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:24:47.607 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:24:47.607 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:24:47.607 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:24:47.607 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:24:47.607 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:24:47.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:24:47.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:24:47.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:24:47.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-18T08:24:47.673 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:24:47.673 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:24:47.673 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:24:47.676 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:24:47.676 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-18T08:24:47.676 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:24:47.676 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:24:47.676 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:24:47.676 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:24:47.676 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:24:47.676 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:24:47.676 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:24:47.676 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:24:47.677 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:24:47.677 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:24:47.677 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:24:47.677 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:47.677 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:24:47.677 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:24:47.677 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:24:47.677 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:24:47.677 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:24:47.677 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:24:47.677 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:24:47.678 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:24:47.678 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:24:47.678 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:24:47.678 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:47.678 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:24:47.678 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:24:47.678 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:24:47.678 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:24:47.678 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:24:47.678 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:24:47.678 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:24:47.687 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:24:47.687 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:24:47.687 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:24:47.687 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-18T08:24:47.743 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T08:24:47.743 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:24:47.743 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:24:47.746 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:24:47.746 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-18T08:24:47.746 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:24:47.746 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:24:47.746 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:24:47.746 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:24:47.746 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:24:47.746 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:24:47.747 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:24:47.747 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:24:47.747 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:24:47.747 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:24:47.747 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:24:47.747 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:47.747 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:24:47.747 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:24:47.747 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:24:47.747 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:24:47.747 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:24:47.747 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:24:47.748 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:24:47.748 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:24:47.748 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:24:47.748 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:24:47.748 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:24:47.748 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:24:47.748 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:24:47.748 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:24:47.748 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:24:47.748 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:24:47.748 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:24:47.748 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:24:47.756 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:24:47.756 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:24:47.756 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:24:47.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-18T08:24:47.815 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:24:47.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-18T08:24:47.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:24:47.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:24:47.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:24:47.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-18T08:24:47.817 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:24:47.817 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:24:47.817 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:24:47.817 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:24:47.817 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:24:47.817 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:24:47.817 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:24:47.817 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:24:47.818 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:24:47.818 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:24:47.847 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 106 2026-02-18T08:24:47.847 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 106 2026-02-18T08:24:47.847 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:24:47.847 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=425, tag_tid=31, entry_tid=1], [object_number=424, tag_tid=31, entry_tid=0], [object_number=427, tag_tid=29, entry_tid=45931], [object_number=426, tag_tid=29, entry_tid=45930]]], state=connected] 2026-02-18T08:24:47.847 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=425, tag_tid=31, entry_tid=1], [object_number=424, tag_tid=31, entry_tid=0], [object_number=427, tag_tid=29, entry_tid=45931], [object_number=426, tag_tid=29, entry_tid=45930]]], state=connected] 2026-02-18T08:24:47.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:24:47.851 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=425, tag_tid=31, entry_tid=1' 2026-02-18T08:24:47.852 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-18T08:24:47.852 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:24:47.852 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:24:47.852 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:24:47.852 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-18T08:24:47.852 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:24:47.853 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:24:47.853 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:24:47.853 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:24:47.853 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-18T08:24:47.853 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-18T08:24:47.854 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:24:47.854 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:24:47.855 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-18T08:24:47.855 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:24:47.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 106 2026-02-18T08:24:47.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 106 2026-02-18T08:24:47.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:24:47.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=425, tag_tid=31, entry_tid=1], [object_number=424, tag_tid=31, entry_tid=0], [object_number=427, tag_tid=29, entry_tid=45931], [object_number=426, tag_tid=29, entry_tid=45930]]], state=connected] 2026-02-18T08:24:47.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=425, tag_tid=31, entry_tid=1], [object_number=424, tag_tid=31, entry_tid=0], [object_number=427, tag_tid=29, entry_tid=45931], [object_number=426, tag_tid=29, entry_tid=45930]]], state=connected] 2026-02-18T08:24:47.895 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.status 2026-02-18T08:24:47.897 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=425, tag_tid=31, entry_tid=1' 2026-02-18T08:24:47.897 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=425, tag_tid=31, entry_tid=1' -a 'object_number=425, tag_tid=31, entry_tid=1' = 'object_number=425, tag_tid=31, entry_tid=1' 2026-02-18T08:24:47.897 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:24:47.897 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_snap_present cluster1 mirror test snap10-clean 2026-02-18T08:24:47.897 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:24:47.897 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:24:47.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:24:47.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap10-clean 2026-02-18T08:24:47.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-18T08:24:47.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-18T08:24:47.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1 2026-02-18T08:24:48.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster1 info mirror/test@snap10-clean 2026-02-18T08:24:48.928 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:rbd image 'test': 2026-02-18T08:24:48.928 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: size 512 MiB in 128 objects 2026-02-18T08:24:48.928 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: order 22 (4 MiB objects) 2026-02-18T08:24:48.928 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: snapshot_count: 20 2026-02-18T08:24:48.928 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: id: 10a74aa8ebc0 2026-02-18T08:24:48.929 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: block_name_prefix: rbd_data.10a74aa8ebc0 2026-02-18T08:24:48.929 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: format: 2 2026-02-18T08:24:48.929 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: features: layering, exclusive-lock, journaling 2026-02-18T08:24:48.929 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: op_features: 2026-02-18T08:24:48.929 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: flags: 2026-02-18T08:24:48.929 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: create_timestamp: Wed Feb 18 08:10:48 2026 2026-02-18T08:24:48.929 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: access_timestamp: Wed Feb 18 08:24:33 2026 2026-02-18T08:24:48.929 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: modify_timestamp: Wed Feb 18 08:23:38 2026 2026-02-18T08:24:48.929 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: protected: False 2026-02-18T08:24:48.929 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: journal: 10a74aa8ebc0 2026-02-18T08:24:48.929 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring state: enabled 2026-02-18T08:24:48.930 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring mode: journal 2026-02-18T08:24:48.930 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring global id: 0b548823-d751-43df-a076-4efac2da4d3d 2026-02-18T08:24:48.930 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring primary: false 2026-02-18T08:24:48.931 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:24:48.931 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 1 10 2026-02-18T08:24:48.932 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 10` 2026-02-18T08:24:48.932 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ snap_name=snap1 2026-02-18T08:24:48.933 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ compare_image_snaps mirror test snap1 2026-02-18T08:24:48.933 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:24:48.933 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:24:48.933 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap1 2026-02-18T08:24:48.933 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local ret=0 2026-02-18T08:24:48.933 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local rmt_export=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.export 2026-02-18T08:24:48.933 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loc_export=/tmp/tmp.vybh1m9yJF/cluster1-mirror-test.export 2026-02-18T08:24:48.933 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.export /tmp/tmp.vybh1m9yJF/cluster1-mirror-test.export 2026-02-18T08:24:48.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 -p mirror export test@snap1 /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.export 2026-02-18T08:24:49.408 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-18T08:24:49.418 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster1 -p mirror export test@snap1 /tmp/tmp.vybh1m9yJF/cluster1-mirror-test.export 2026-02-18T08:24:49.959 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-18T08:24:49.970 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmp /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.export /tmp/tmp.vybh1m9yJF/cluster1-mirror-test.export 2026-02-18T08:24:50.157 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.export /tmp/tmp.vybh1m9yJF/cluster1-mirror-test.export 2026-02-18T08:24:50.299 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:24:50.300 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ snap_name=snap1-clean 2026-02-18T08:24:50.300 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ compare_image_snaps mirror test snap1-clean 2026-02-18T08:24:50.300 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:24:50.300 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:24:50.300 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap1-clean 2026-02-18T08:24:50.300 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local ret=0 2026-02-18T08:24:50.300 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local rmt_export=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.export 2026-02-18T08:24:50.300 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loc_export=/tmp/tmp.vybh1m9yJF/cluster1-mirror-test.export 2026-02-18T08:24:50.301 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.export /tmp/tmp.vybh1m9yJF/cluster1-mirror-test.export 2026-02-18T08:24:50.301 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 -p mirror export test@snap1-clean /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.export 2026-02-18T08:24:50.733 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-18T08:24:50.743 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster1 -p mirror export test@snap1-clean /tmp/tmp.vybh1m9yJF/cluster1-mirror-test.export 2026-02-18T08:24:51.302 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-18T08:24:51.316 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmp /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.export /tmp/tmp.vybh1m9yJF/cluster1-mirror-test.export 2026-02-18T08:24:51.508 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.export /tmp/tmp.vybh1m9yJF/cluster1-mirror-test.export 2026-02-18T08:24:51.648 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:24:51.649 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 10` 2026-02-18T08:24:51.649 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ snap_name=snap2 2026-02-18T08:24:51.649 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ compare_image_snaps mirror test snap2 2026-02-18T08:24:51.649 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:24:51.649 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:24:51.649 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap2 2026-02-18T08:24:51.649 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local ret=0 2026-02-18T08:24:51.649 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local rmt_export=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.export 2026-02-18T08:24:51.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loc_export=/tmp/tmp.vybh1m9yJF/cluster1-mirror-test.export 2026-02-18T08:24:51.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.export /tmp/tmp.vybh1m9yJF/cluster1-mirror-test.export 2026-02-18T08:24:51.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 -p mirror export test@snap2 /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.export 2026-02-18T08:24:52.144 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-18T08:24:52.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster1 -p mirror export test@snap2 /tmp/tmp.vybh1m9yJF/cluster1-mirror-test.export 2026-02-18T08:24:52.800 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-18T08:24:52.811 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmp /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.export /tmp/tmp.vybh1m9yJF/cluster1-mirror-test.export 2026-02-18T08:24:53.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.export /tmp/tmp.vybh1m9yJF/cluster1-mirror-test.export 2026-02-18T08:24:53.142 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:24:53.143 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ snap_name=snap2-clean 2026-02-18T08:24:53.143 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ compare_image_snaps mirror test snap2-clean 2026-02-18T08:24:53.143 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:24:53.143 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:24:53.143 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap2-clean 2026-02-18T08:24:53.143 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local ret=0 2026-02-18T08:24:53.143 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local rmt_export=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.export 2026-02-18T08:24:53.143 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loc_export=/tmp/tmp.vybh1m9yJF/cluster1-mirror-test.export 2026-02-18T08:24:53.143 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.export /tmp/tmp.vybh1m9yJF/cluster1-mirror-test.export 2026-02-18T08:24:53.143 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 -p mirror export test@snap2-clean /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.export 2026-02-18T08:24:53.623 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-18T08:24:53.637 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster1 -p mirror export test@snap2-clean /tmp/tmp.vybh1m9yJF/cluster1-mirror-test.export 2026-02-18T08:24:54.269 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-18T08:24:54.282 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmp /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.export /tmp/tmp.vybh1m9yJF/cluster1-mirror-test.export 2026-02-18T08:24:54.473 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.export /tmp/tmp.vybh1m9yJF/cluster1-mirror-test.export 2026-02-18T08:24:54.606 INFO:tasks.rbd_mirror_thrash:kill cluster1.client.mirror.0 2026-02-18T08:24:54.606 INFO:tasks.rbd_mirror.cluster1.client.mirror.0:Sent signal 15 2026-02-18T08:24:54.606 INFO:tasks.rbd_mirror_thrash:waiting for 0.559206336439898 secs before reviving daemons 2026-02-18T08:24:54.607 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.trial163.stderr:2026-02-18T08:24:54.599+0000 7f04d0ac5640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.0 (PID: 58731) UID: 1000 2026-02-18T08:24:54.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:24:54.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 10` 2026-02-18T08:24:54.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ snap_name=snap3 2026-02-18T08:24:54.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ compare_image_snaps mirror test snap3 2026-02-18T08:24:54.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:24:54.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:24:54.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap3 2026-02-18T08:24:54.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local ret=0 2026-02-18T08:24:54.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local rmt_export=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.export 2026-02-18T08:24:54.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loc_export=/tmp/tmp.vybh1m9yJF/cluster1-mirror-test.export 2026-02-18T08:24:54.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.export /tmp/tmp.vybh1m9yJF/cluster1-mirror-test.export 2026-02-18T08:24:54.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 -p mirror export test@snap3 /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.export 2026-02-18T08:24:55.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-18T08:24:55.120 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster1 -p mirror export test@snap3 /tmp/tmp.vybh1m9yJF/cluster1-mirror-test.export 2026-02-18T08:24:55.166 INFO:tasks.rbd_mirror_thrash:waiting for cluster1.client.mirror.0 2026-02-18T08:24:55.166 INFO:teuthology.orchestra.run:waiting for 600 2026-02-18T08:24:55.166 INFO:tasks.rbd_mirror_thrash:reviving cluster1.client.mirror.0 2026-02-18T08:24:55.167 INFO:tasks.rbd_mirror.cluster1.client.mirror.0:Restarting daemon 2026-02-18T08:24:55.167 DEBUG:teuthology.orchestra.run.trial163:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.0 2026-02-18T08:24:55.169 INFO:tasks.rbd_mirror.cluster1.client.mirror.0:Started 2026-02-18T08:24:55.169 INFO:tasks.rbd_mirror_thrash:waiting for 117.77494157406406 secs before thrashing 2026-02-18T08:24:55.724 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-18T08:24:55.738 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmp /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.export /tmp/tmp.vybh1m9yJF/cluster1-mirror-test.export 2026-02-18T08:24:55.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.export /tmp/tmp.vybh1m9yJF/cluster1-mirror-test.export 2026-02-18T08:24:56.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:24:56.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ snap_name=snap3-clean 2026-02-18T08:24:56.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ compare_image_snaps mirror test snap3-clean 2026-02-18T08:24:56.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:24:56.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:24:56.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap3-clean 2026-02-18T08:24:56.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local ret=0 2026-02-18T08:24:56.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local rmt_export=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.export 2026-02-18T08:24:56.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loc_export=/tmp/tmp.vybh1m9yJF/cluster1-mirror-test.export 2026-02-18T08:24:56.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.export /tmp/tmp.vybh1m9yJF/cluster1-mirror-test.export 2026-02-18T08:24:56.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 -p mirror export test@snap3-clean /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.export 2026-02-18T08:24:56.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-18T08:24:56.562 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster1 -p mirror export test@snap3-clean /tmp/tmp.vybh1m9yJF/cluster1-mirror-test.export 2026-02-18T08:24:57.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-18T08:24:57.121 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmp /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.export /tmp/tmp.vybh1m9yJF/cluster1-mirror-test.export 2026-02-18T08:24:57.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.export /tmp/tmp.vybh1m9yJF/cluster1-mirror-test.export 2026-02-18T08:24:57.454 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:24:57.454 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 10` 2026-02-18T08:24:57.454 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ snap_name=snap4 2026-02-18T08:24:57.454 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ compare_image_snaps mirror test snap4 2026-02-18T08:24:57.454 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:24:57.454 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:24:57.455 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap4 2026-02-18T08:24:57.455 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local ret=0 2026-02-18T08:24:57.455 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local rmt_export=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.export 2026-02-18T08:24:57.455 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loc_export=/tmp/tmp.vybh1m9yJF/cluster1-mirror-test.export 2026-02-18T08:24:57.455 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.export /tmp/tmp.vybh1m9yJF/cluster1-mirror-test.export 2026-02-18T08:24:57.455 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 -p mirror export test@snap4 /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.export 2026-02-18T08:24:57.955 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-18T08:24:57.965 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster1 -p mirror export test@snap4 /tmp/tmp.vybh1m9yJF/cluster1-mirror-test.export 2026-02-18T08:24:58.571 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-18T08:24:58.586 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmp /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.export /tmp/tmp.vybh1m9yJF/cluster1-mirror-test.export 2026-02-18T08:24:58.777 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.export /tmp/tmp.vybh1m9yJF/cluster1-mirror-test.export 2026-02-18T08:24:58.918 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:24:58.918 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ snap_name=snap4-clean 2026-02-18T08:24:58.918 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ compare_image_snaps mirror test snap4-clean 2026-02-18T08:24:58.918 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:24:58.918 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:24:58.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap4-clean 2026-02-18T08:24:58.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local ret=0 2026-02-18T08:24:58.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local rmt_export=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.export 2026-02-18T08:24:58.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loc_export=/tmp/tmp.vybh1m9yJF/cluster1-mirror-test.export 2026-02-18T08:24:58.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.export /tmp/tmp.vybh1m9yJF/cluster1-mirror-test.export 2026-02-18T08:24:58.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 -p mirror export test@snap4-clean /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.export 2026-02-18T08:24:59.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-18T08:24:59.415 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster1 -p mirror export test@snap4-clean /tmp/tmp.vybh1m9yJF/cluster1-mirror-test.export 2026-02-18T08:24:59.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-18T08:25:00.012 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmp /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.export /tmp/tmp.vybh1m9yJF/cluster1-mirror-test.export 2026-02-18T08:25:00.201 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.export /tmp/tmp.vybh1m9yJF/cluster1-mirror-test.export 2026-02-18T08:25:00.344 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:25:00.344 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 10` 2026-02-18T08:25:00.344 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ snap_name=snap5 2026-02-18T08:25:00.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ compare_image_snaps mirror test snap5 2026-02-18T08:25:00.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:25:00.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:25:00.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap5 2026-02-18T08:25:00.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local ret=0 2026-02-18T08:25:00.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local rmt_export=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.export 2026-02-18T08:25:00.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loc_export=/tmp/tmp.vybh1m9yJF/cluster1-mirror-test.export 2026-02-18T08:25:00.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.export /tmp/tmp.vybh1m9yJF/cluster1-mirror-test.export 2026-02-18T08:25:00.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 -p mirror export test@snap5 /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.export 2026-02-18T08:25:00.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-18T08:25:00.828 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster1 -p mirror export test@snap5 /tmp/tmp.vybh1m9yJF/cluster1-mirror-test.export 2026-02-18T08:25:01.443 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-18T08:25:01.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmp /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.export /tmp/tmp.vybh1m9yJF/cluster1-mirror-test.export 2026-02-18T08:25:01.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.export /tmp/tmp.vybh1m9yJF/cluster1-mirror-test.export 2026-02-18T08:25:01.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:25:01.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ snap_name=snap5-clean 2026-02-18T08:25:01.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ compare_image_snaps mirror test snap5-clean 2026-02-18T08:25:01.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:25:01.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:25:01.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap5-clean 2026-02-18T08:25:01.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local ret=0 2026-02-18T08:25:01.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local rmt_export=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.export 2026-02-18T08:25:01.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loc_export=/tmp/tmp.vybh1m9yJF/cluster1-mirror-test.export 2026-02-18T08:25:01.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.export /tmp/tmp.vybh1m9yJF/cluster1-mirror-test.export 2026-02-18T08:25:01.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 -p mirror export test@snap5-clean /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.export 2026-02-18T08:25:02.274 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-18T08:25:02.285 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster1 -p mirror export test@snap5-clean /tmp/tmp.vybh1m9yJF/cluster1-mirror-test.export 2026-02-18T08:25:02.909 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-18T08:25:02.928 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmp /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.export /tmp/tmp.vybh1m9yJF/cluster1-mirror-test.export 2026-02-18T08:25:03.119 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.export /tmp/tmp.vybh1m9yJF/cluster1-mirror-test.export 2026-02-18T08:25:03.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:25:03.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 10` 2026-02-18T08:25:03.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ snap_name=snap6 2026-02-18T08:25:03.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ compare_image_snaps mirror test snap6 2026-02-18T08:25:03.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:25:03.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:25:03.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap6 2026-02-18T08:25:03.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local ret=0 2026-02-18T08:25:03.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local rmt_export=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.export 2026-02-18T08:25:03.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loc_export=/tmp/tmp.vybh1m9yJF/cluster1-mirror-test.export 2026-02-18T08:25:03.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.export /tmp/tmp.vybh1m9yJF/cluster1-mirror-test.export 2026-02-18T08:25:03.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 -p mirror export test@snap6 /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.export 2026-02-18T08:25:03.760 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-18T08:25:03.774 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster1 -p mirror export test@snap6 /tmp/tmp.vybh1m9yJF/cluster1-mirror-test.export 2026-02-18T08:25:04.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-18T08:25:04.403 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmp /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.export /tmp/tmp.vybh1m9yJF/cluster1-mirror-test.export 2026-02-18T08:25:04.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.export /tmp/tmp.vybh1m9yJF/cluster1-mirror-test.export 2026-02-18T08:25:04.736 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:25:04.736 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ snap_name=snap6-clean 2026-02-18T08:25:04.736 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ compare_image_snaps mirror test snap6-clean 2026-02-18T08:25:04.736 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:25:04.736 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:25:04.736 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap6-clean 2026-02-18T08:25:04.736 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local ret=0 2026-02-18T08:25:04.736 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local rmt_export=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.export 2026-02-18T08:25:04.737 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loc_export=/tmp/tmp.vybh1m9yJF/cluster1-mirror-test.export 2026-02-18T08:25:04.737 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.export /tmp/tmp.vybh1m9yJF/cluster1-mirror-test.export 2026-02-18T08:25:04.737 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 -p mirror export test@snap6-clean /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.export 2026-02-18T08:25:05.224 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-18T08:25:05.234 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster1 -p mirror export test@snap6-clean /tmp/tmp.vybh1m9yJF/cluster1-mirror-test.export 2026-02-18T08:25:05.840 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-18T08:25:05.855 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmp /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.export /tmp/tmp.vybh1m9yJF/cluster1-mirror-test.export 2026-02-18T08:25:06.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.export /tmp/tmp.vybh1m9yJF/cluster1-mirror-test.export 2026-02-18T08:25:06.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:25:06.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 10` 2026-02-18T08:25:06.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ snap_name=snap7 2026-02-18T08:25:06.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ compare_image_snaps mirror test snap7 2026-02-18T08:25:06.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:25:06.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:25:06.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap7 2026-02-18T08:25:06.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local ret=0 2026-02-18T08:25:06.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local rmt_export=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.export 2026-02-18T08:25:06.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loc_export=/tmp/tmp.vybh1m9yJF/cluster1-mirror-test.export 2026-02-18T08:25:06.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.export /tmp/tmp.vybh1m9yJF/cluster1-mirror-test.export 2026-02-18T08:25:06.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 -p mirror export test@snap7 /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.export 2026-02-18T08:25:06.662 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-18T08:25:06.674 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster1 -p mirror export test@snap7 /tmp/tmp.vybh1m9yJF/cluster1-mirror-test.export 2026-02-18T08:25:07.278 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-18T08:25:07.293 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmp /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.export /tmp/tmp.vybh1m9yJF/cluster1-mirror-test.export 2026-02-18T08:25:07.486 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.export /tmp/tmp.vybh1m9yJF/cluster1-mirror-test.export 2026-02-18T08:25:07.628 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:25:07.629 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ snap_name=snap7-clean 2026-02-18T08:25:07.629 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ compare_image_snaps mirror test snap7-clean 2026-02-18T08:25:07.629 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:25:07.629 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:25:07.629 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap7-clean 2026-02-18T08:25:07.629 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local ret=0 2026-02-18T08:25:07.629 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local rmt_export=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.export 2026-02-18T08:25:07.629 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loc_export=/tmp/tmp.vybh1m9yJF/cluster1-mirror-test.export 2026-02-18T08:25:07.629 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.export /tmp/tmp.vybh1m9yJF/cluster1-mirror-test.export 2026-02-18T08:25:07.629 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 -p mirror export test@snap7-clean /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.export 2026-02-18T08:25:08.093 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-18T08:25:08.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster1 -p mirror export test@snap7-clean /tmp/tmp.vybh1m9yJF/cluster1-mirror-test.export 2026-02-18T08:25:08.729 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-18T08:25:08.742 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmp /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.export /tmp/tmp.vybh1m9yJF/cluster1-mirror-test.export 2026-02-18T08:25:08.929 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.export /tmp/tmp.vybh1m9yJF/cluster1-mirror-test.export 2026-02-18T08:25:09.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:25:09.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 10` 2026-02-18T08:25:09.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ snap_name=snap8 2026-02-18T08:25:09.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ compare_image_snaps mirror test snap8 2026-02-18T08:25:09.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:25:09.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:25:09.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap8 2026-02-18T08:25:09.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local ret=0 2026-02-18T08:25:09.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local rmt_export=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.export 2026-02-18T08:25:09.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loc_export=/tmp/tmp.vybh1m9yJF/cluster1-mirror-test.export 2026-02-18T08:25:09.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.export /tmp/tmp.vybh1m9yJF/cluster1-mirror-test.export 2026-02-18T08:25:09.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 -p mirror export test@snap8 /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.export 2026-02-18T08:25:09.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-18T08:25:09.567 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster1 -p mirror export test@snap8 /tmp/tmp.vybh1m9yJF/cluster1-mirror-test.export 2026-02-18T08:25:10.211 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-18T08:25:10.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmp /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.export /tmp/tmp.vybh1m9yJF/cluster1-mirror-test.export 2026-02-18T08:25:10.416 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.export /tmp/tmp.vybh1m9yJF/cluster1-mirror-test.export 2026-02-18T08:25:10.558 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:25:10.558 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ snap_name=snap8-clean 2026-02-18T08:25:10.558 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ compare_image_snaps mirror test snap8-clean 2026-02-18T08:25:10.558 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:25:10.558 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:25:10.558 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap8-clean 2026-02-18T08:25:10.558 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local ret=0 2026-02-18T08:25:10.559 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local rmt_export=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.export 2026-02-18T08:25:10.559 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loc_export=/tmp/tmp.vybh1m9yJF/cluster1-mirror-test.export 2026-02-18T08:25:10.559 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.export /tmp/tmp.vybh1m9yJF/cluster1-mirror-test.export 2026-02-18T08:25:10.559 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 -p mirror export test@snap8-clean /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.export 2026-02-18T08:25:11.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-18T08:25:11.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster1 -p mirror export test@snap8-clean /tmp/tmp.vybh1m9yJF/cluster1-mirror-test.export 2026-02-18T08:25:11.441 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:25:11.431+0000 7fafee558640 -1 bdev(0x5644a1b41400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 1 2026-02-18T08:25:11.442 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:25:11.431+0000 7fafef55a640 -1 bdev(0x5644a1b41400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 1 2026-02-18T08:25:11.442 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:25:11.431+0000 7fafedd57640 -1 bdev(0x5644a1b41400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 1 2026-02-18T08:25:11.638 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-18T08:25:11.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmp /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.export /tmp/tmp.vybh1m9yJF/cluster1-mirror-test.export 2026-02-18T08:25:11.843 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.export /tmp/tmp.vybh1m9yJF/cluster1-mirror-test.export 2026-02-18T08:25:11.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:25:11.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 10` 2026-02-18T08:25:11.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ snap_name=snap9 2026-02-18T08:25:11.985 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ compare_image_snaps mirror test snap9 2026-02-18T08:25:11.985 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:25:11.985 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:25:11.985 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap9 2026-02-18T08:25:11.985 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local ret=0 2026-02-18T08:25:11.985 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local rmt_export=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.export 2026-02-18T08:25:11.985 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loc_export=/tmp/tmp.vybh1m9yJF/cluster1-mirror-test.export 2026-02-18T08:25:11.985 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.export /tmp/tmp.vybh1m9yJF/cluster1-mirror-test.export 2026-02-18T08:25:11.985 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 -p mirror export test@snap9 /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.export 2026-02-18T08:25:12.462 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-18T08:25:12.473 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster1 -p mirror export test@snap9 /tmp/tmp.vybh1m9yJF/cluster1-mirror-test.export 2026-02-18T08:25:13.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-18T08:25:13.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmp /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.export /tmp/tmp.vybh1m9yJF/cluster1-mirror-test.export 2026-02-18T08:25:13.281 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.export /tmp/tmp.vybh1m9yJF/cluster1-mirror-test.export 2026-02-18T08:25:13.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:25:13.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ snap_name=snap9-clean 2026-02-18T08:25:13.425 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ compare_image_snaps mirror test snap9-clean 2026-02-18T08:25:13.425 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:25:13.425 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:25:13.425 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap9-clean 2026-02-18T08:25:13.425 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local ret=0 2026-02-18T08:25:13.425 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local rmt_export=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.export 2026-02-18T08:25:13.425 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loc_export=/tmp/tmp.vybh1m9yJF/cluster1-mirror-test.export 2026-02-18T08:25:13.425 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.export /tmp/tmp.vybh1m9yJF/cluster1-mirror-test.export 2026-02-18T08:25:13.425 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 -p mirror export test@snap9-clean /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.export 2026-02-18T08:25:14.122 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-18T08:25:14.140 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster1 -p mirror export test@snap9-clean /tmp/tmp.vybh1m9yJF/cluster1-mirror-test.export 2026-02-18T08:25:14.766 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-18T08:25:14.782 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmp /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.export /tmp/tmp.vybh1m9yJF/cluster1-mirror-test.export 2026-02-18T08:25:14.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.export /tmp/tmp.vybh1m9yJF/cluster1-mirror-test.export 2026-02-18T08:25:15.119 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:25:15.119 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 10` 2026-02-18T08:25:15.119 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ snap_name=snap10 2026-02-18T08:25:15.119 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ compare_image_snaps mirror test snap10 2026-02-18T08:25:15.120 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:25:15.120 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:25:15.120 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap10 2026-02-18T08:25:15.120 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local ret=0 2026-02-18T08:25:15.120 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local rmt_export=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.export 2026-02-18T08:25:15.120 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loc_export=/tmp/tmp.vybh1m9yJF/cluster1-mirror-test.export 2026-02-18T08:25:15.120 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.export /tmp/tmp.vybh1m9yJF/cluster1-mirror-test.export 2026-02-18T08:25:15.120 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 -p mirror export test@snap10 /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.export 2026-02-18T08:25:15.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-18T08:25:15.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster1 -p mirror export test@snap10 /tmp/tmp.vybh1m9yJF/cluster1-mirror-test.export 2026-02-18T08:25:16.234 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-18T08:25:16.249 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmp /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.export /tmp/tmp.vybh1m9yJF/cluster1-mirror-test.export 2026-02-18T08:25:16.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.export /tmp/tmp.vybh1m9yJF/cluster1-mirror-test.export 2026-02-18T08:25:16.585 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:25:16.585 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ snap_name=snap10-clean 2026-02-18T08:25:16.585 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ compare_image_snaps mirror test snap10-clean 2026-02-18T08:25:16.585 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:25:16.585 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:25:16.585 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap10-clean 2026-02-18T08:25:16.586 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local ret=0 2026-02-18T08:25:16.586 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local rmt_export=/tmp/tmp.vybh1m9yJF/cluster2-mirror-test.export 2026-02-18T08:25:16.586 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loc_export=/tmp/tmp.vybh1m9yJF/cluster1-mirror-test.export 2026-02-18T08:25:16.586 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.export /tmp/tmp.vybh1m9yJF/cluster1-mirror-test.export 2026-02-18T08:25:16.586 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 -p mirror export test@snap10-clean /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.export 2026-02-18T08:25:17.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-18T08:25:17.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster1 -p mirror export test@snap10-clean /tmp/tmp.vybh1m9yJF/cluster1-mirror-test.export 2026-02-18T08:25:17.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-18T08:25:17.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmp /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.export /tmp/tmp.vybh1m9yJF/cluster1-mirror-test.export 2026-02-18T08:25:17.860 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.vybh1m9yJF/cluster2-mirror-test.export /tmp/tmp.vybh1m9yJF/cluster1-mirror-test.export 2026-02-18T08:25:18.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:25:18.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 1 10 2026-02-18T08:25:18.003 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 10` 2026-02-18T08:25:18.003 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ snap_name=snap1 2026-02-18T08:25:18.004 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_snapshot cluster2 mirror test snap1 2026-02-18T08:25:18.004 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:25:18.004 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:25:18.004 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:25:18.004 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap=snap1 2026-02-18T08:25:18.004 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 snap rm mirror/test@snap1 2026-02-18T08:25:18.855 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Removing snap: 100% complete...done. 2026-02-18T08:25:18.870 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ snap_name=snap1-clean 2026-02-18T08:25:18.870 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_snapshot cluster2 mirror test snap1-clean 2026-02-18T08:25:18.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:25:18.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:25:18.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:25:18.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap=snap1-clean 2026-02-18T08:25:18.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 snap rm mirror/test@snap1-clean 2026-02-18T08:25:19.861 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Removing snap: 100% complete...done. 2026-02-18T08:25:19.881 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 10` 2026-02-18T08:25:19.881 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ snap_name=snap2 2026-02-18T08:25:19.881 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_snapshot cluster2 mirror test snap2 2026-02-18T08:25:19.881 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:25:19.882 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:25:19.882 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:25:19.882 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap=snap2 2026-02-18T08:25:19.882 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 snap rm mirror/test@snap2 2026-02-18T08:25:20.865 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Removing snap: 100% complete...done. 2026-02-18T08:25:20.883 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ snap_name=snap2-clean 2026-02-18T08:25:20.883 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_snapshot cluster2 mirror test snap2-clean 2026-02-18T08:25:20.883 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:25:20.883 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:25:20.883 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:25:20.883 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap=snap2-clean 2026-02-18T08:25:20.883 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 snap rm mirror/test@snap2-clean 2026-02-18T08:25:21.867 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Removing snap: 100% complete...done. 2026-02-18T08:25:21.881 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 10` 2026-02-18T08:25:21.882 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ snap_name=snap3 2026-02-18T08:25:21.882 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_snapshot cluster2 mirror test snap3 2026-02-18T08:25:21.882 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:25:21.882 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:25:21.882 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:25:21.882 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap=snap3 2026-02-18T08:25:21.882 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 snap rm mirror/test@snap3 2026-02-18T08:25:22.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Removing snap: 100% complete...done. 2026-02-18T08:25:22.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ snap_name=snap3-clean 2026-02-18T08:25:22.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_snapshot cluster2 mirror test snap3-clean 2026-02-18T08:25:22.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:25:22.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:25:22.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:25:22.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap=snap3-clean 2026-02-18T08:25:22.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 snap rm mirror/test@snap3-clean 2026-02-18T08:25:23.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Removing snap: 100% complete...done. 2026-02-18T08:25:23.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 10` 2026-02-18T08:25:23.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ snap_name=snap4 2026-02-18T08:25:23.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_snapshot cluster2 mirror test snap4 2026-02-18T08:25:23.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:25:23.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:25:23.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:25:23.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap=snap4 2026-02-18T08:25:23.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 snap rm mirror/test@snap4 2026-02-18T08:25:24.923 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Removing snap: 100% complete...done. 2026-02-18T08:25:24.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ snap_name=snap4-clean 2026-02-18T08:25:24.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_snapshot cluster2 mirror test snap4-clean 2026-02-18T08:25:24.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:25:24.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:25:24.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:25:24.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap=snap4-clean 2026-02-18T08:25:24.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 snap rm mirror/test@snap4-clean 2026-02-18T08:25:25.929 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Removing snap: 100% complete...done. 2026-02-18T08:25:25.944 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 10` 2026-02-18T08:25:25.944 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ snap_name=snap5 2026-02-18T08:25:25.944 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_snapshot cluster2 mirror test snap5 2026-02-18T08:25:25.944 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:25:25.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:25:25.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:25:25.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap=snap5 2026-02-18T08:25:25.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 snap rm mirror/test@snap5 2026-02-18T08:25:26.930 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Removing snap: 100% complete...done. 2026-02-18T08:25:26.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ snap_name=snap5-clean 2026-02-18T08:25:26.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_snapshot cluster2 mirror test snap5-clean 2026-02-18T08:25:26.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:25:26.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:25:26.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:25:26.946 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap=snap5-clean 2026-02-18T08:25:26.946 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 snap rm mirror/test@snap5-clean 2026-02-18T08:25:27.929 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Removing snap: 100% complete...done. 2026-02-18T08:25:27.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 10` 2026-02-18T08:25:27.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ snap_name=snap6 2026-02-18T08:25:27.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_snapshot cluster2 mirror test snap6 2026-02-18T08:25:27.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:25:27.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:25:27.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:25:27.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap=snap6 2026-02-18T08:25:27.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 snap rm mirror/test@snap6 2026-02-18T08:25:28.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Removing snap: 100% complete...done. 2026-02-18T08:25:28.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ snap_name=snap6-clean 2026-02-18T08:25:28.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_snapshot cluster2 mirror test snap6-clean 2026-02-18T08:25:28.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:25:28.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:25:28.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:25:28.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap=snap6-clean 2026-02-18T08:25:28.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 snap rm mirror/test@snap6-clean 2026-02-18T08:25:29.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Removing snap: 100% complete...done. 2026-02-18T08:25:29.950 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 10` 2026-02-18T08:25:29.950 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ snap_name=snap7 2026-02-18T08:25:29.950 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_snapshot cluster2 mirror test snap7 2026-02-18T08:25:29.950 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:25:29.950 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:25:29.950 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:25:29.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap=snap7 2026-02-18T08:25:29.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 snap rm mirror/test@snap7 2026-02-18T08:25:30.944 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Removing snap: 100% complete...done. 2026-02-18T08:25:30.959 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ snap_name=snap7-clean 2026-02-18T08:25:30.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_snapshot cluster2 mirror test snap7-clean 2026-02-18T08:25:30.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:25:30.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:25:30.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:25:30.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap=snap7-clean 2026-02-18T08:25:30.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 snap rm mirror/test@snap7-clean 2026-02-18T08:25:31.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Removing snap: 100% complete...done. 2026-02-18T08:25:31.955 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 10` 2026-02-18T08:25:31.956 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ snap_name=snap8 2026-02-18T08:25:31.956 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_snapshot cluster2 mirror test snap8 2026-02-18T08:25:31.956 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:25:31.956 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:25:31.956 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:25:31.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap=snap8 2026-02-18T08:25:31.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 snap rm mirror/test@snap8 2026-02-18T08:25:32.947 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Removing snap: 100% complete...done. 2026-02-18T08:25:32.963 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ snap_name=snap8-clean 2026-02-18T08:25:32.963 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_snapshot cluster2 mirror test snap8-clean 2026-02-18T08:25:32.963 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:25:32.964 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:25:32.964 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:25:32.964 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap=snap8-clean 2026-02-18T08:25:32.964 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 snap rm mirror/test@snap8-clean 2026-02-18T08:25:33.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Removing snap: 100% complete...done. 2026-02-18T08:25:33.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 10` 2026-02-18T08:25:33.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ snap_name=snap9 2026-02-18T08:25:33.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_snapshot cluster2 mirror test snap9 2026-02-18T08:25:33.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:25:33.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:25:33.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:25:33.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap=snap9 2026-02-18T08:25:33.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 snap rm mirror/test@snap9 2026-02-18T08:25:34.949 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Removing snap: 100% complete...done. 2026-02-18T08:25:34.965 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ snap_name=snap9-clean 2026-02-18T08:25:34.965 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_snapshot cluster2 mirror test snap9-clean 2026-02-18T08:25:34.965 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:25:34.965 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:25:34.965 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:25:34.966 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap=snap9-clean 2026-02-18T08:25:34.966 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 snap rm mirror/test@snap9-clean 2026-02-18T08:25:35.954 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Removing snap: 100% complete...done. 2026-02-18T08:25:35.967 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 10` 2026-02-18T08:25:35.967 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ snap_name=snap10 2026-02-18T08:25:35.967 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_snapshot cluster2 mirror test snap10 2026-02-18T08:25:35.967 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:25:35.967 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:25:35.967 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:25:35.968 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap=snap10 2026-02-18T08:25:35.968 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 snap rm mirror/test@snap10 2026-02-18T08:25:36.956 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Removing snap: 100% complete...done. 2026-02-18T08:25:36.966 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ snap_name=snap10-clean 2026-02-18T08:25:36.966 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_snapshot cluster2 mirror test snap10-clean 2026-02-18T08:25:36.966 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:25:36.966 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:25:36.966 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:25:36.966 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap=snap10-clean 2026-02-18T08:25:36.966 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 snap rm mirror/test@snap10-clean 2026-02-18T08:25:37.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Removing snap: 100% complete...done. 2026-02-18T08:25:37.969 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_image_retry cluster2 mirror test 2026-02-18T08:25:37.970 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:25:37.970 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:25:37.970 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:25:37.970 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-18T08:25:37.970 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:25:37.971 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_image cluster2 mirror test 2026-02-18T08:25:37.971 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:25:37.971 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:25:37.971 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:25:37.971 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster=cluster2 snap purge mirror/test 2026-02-18T08:25:38.005 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster=cluster2 rm mirror/test 2026-02-18T08:25:38.177 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Removing image: 1% complete... Removing image: 2% complete... Removing image: 3% complete... Removing image: 4% complete... Removing image: 5% complete... Removing image: 6% complete... Removing image: 7% complete... Removing image: 8% complete... Removing image: 9% complete... Removing image: 10% complete... Removing image: 11% complete... Removing image: 12% complete... Removing image: 13% complete... Removing image: 14% complete... Removing image: 15% complete... Removing image: 16% complete... Removing image: 17% complete... Removing image: 18% complete... Removing image: 19% complete... Removing image: 20% complete... Removing image: 21% complete... Removing image: 22% complete... Removing image: 23% complete... Removing image: 24% complete... Removing image: 25% complete... Removing image: 26% complete... Removing image: 27% complete... Removing image: 28% complete... Removing image: 29% complete... Removing image: 30% complete... Removing image: 31% complete... Removing image: 32% complete... Removing image: 33% complete... Removing image: 34% complete... Removing image: 35% complete... Removing image: 36% complete... Removing image: 37% complete... Removing image: 38% complete... Removing image: 39% complete... Removing image: 40% complete... Removing image: 41% complete... Removing image: 42% complete... Removing image: 43% complete... Removing image: 44% complete... Removing image: 45% complete... Removing image: 46% complete... Removing image: 47% complete... Removing image: 48% complete... Removing image: 49% complete... Removing image: 50% complete... Removing image: 51% complete... Removing image: 52% complete... Removing image: 53% complete... Removing image: 54% complete... Removing image: 55% complete... Removing image: 56% complete... Removing image: 57% complete... Removing image: 58% complete... Removing image: 59% complete... Removing image: 60% complete... Removing image: 61% complete... Removing image: 62% complete... Removing image: 63% complete... Removing image: 64% complete... Removing image: 65% complete... Removing image: 66% complete... Removing image: 67% complete... Removing image: 68% complete... Removing image: 69% complete... Removing image: 70% complete... Removing image: 71% complete... Removing image: 72% complete... Removing image: 73% complete... Removing image: 74% complete... Removing image: 75% complete... Removing image: 76% complete... Removing image: 77% complete... Removing image: 78% complete... Removing image: 79% complete... Removing image: 80% complete... Removing image: 81% complete... Removing image: 82% complete... Removing image: 83% complete... Removing image: 84% complete... Removing image: 85% complete... Removing image: 86% complete... Removing image: 87% complete... Removing image: 88% complete... Removing image: 89% complete... Removing image: 90% complete... Removing image: 91% complete... Removing image: 92% complete... Removing image: 93% complete... Removing image: 94% complete... Removing image: 95% complete... Removing image: 96% complete... Removing image: 97% complete... Removing image: 98% complete... Removing image: 99% complete... Removing image: 100% complete...done. 2026-02-18T08:25:38.183 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:25:38.183 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_present cluster1 mirror test deleted 2026-02-18T08:25:38.183 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:25:38.183 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:25:38.183 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:25:38.183 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local state=deleted 2026-02-18T08:25:38.183 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image_id= 2026-02-18T08:25:38.183 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-18T08:25:38.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n '' 2026-02-18T08:25:38.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_image_id cluster1 mirror test 2026-02-18T08:25:38.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:25:38.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:25:38.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:25:38.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster=cluster1 info mirror/test 2026-02-18T08:25:38.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-18T08:25:38.207 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd: error opening image test: (2) No such file or directory 2026-02-18T08:25:38.209 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image_id= 2026-02-18T08:25:38.210 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-18T08:25:38.210 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.1 2026-02-18T08:25:38.312 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test_image_present cluster1 mirror test deleted '' 2026-02-18T08:25:38.312 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:25:38.312 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:25:38.312 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-18T08:25:38.312 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local test_state=deleted 2026-02-18T08:25:38.312 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image_id= 2026-02-18T08:25:38.312 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_state=deleted 2026-02-18T08:25:38.312 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_image_id 2026-02-18T08:25:38.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_image_id cluster1 mirror test 2026-02-18T08:25:38.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:25:38.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:25:38.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-18T08:25:38.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster=cluster1 info mirror/test 2026-02-18T08:25:38.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-18T08:25:38.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd: error opening image test: (2) No such file or directory 2026-02-18T08:25:38.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ current_image_id= 2026-02-18T08:25:38.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n '' 2026-02-18T08:25:38.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test deleted = deleted 2026-02-18T08:25:38.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:25:38.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ testlog 'TEST: create many images' 2026-02-18T08:25:38.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ tee -a /tmp/tmp.vybh1m9yJF/rbd-mirror.test.log 2026-02-18T08:25:38.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ date '+%F %T' 2026-02-18T08:25:38.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ echo 2026-02-18 08:25:38 TEST: create many images 2026-02-18T08:25:38.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:2026-02-18 08:25:38 TEST: create many images 2026-02-18T08:25:38.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ snap_name=snap 2026-02-18T08:25:38.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 1 50 2026-02-18T08:25:38.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T08:25:38.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_1 2026-02-18T08:25:38.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_image_and_enable_mirror cluster2 mirror image_1 journal 128M 2026-02-18T08:25:38.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:25:38.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:25:38.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:25:38.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:25:38.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_1 2026-02-18T08:25:38.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:25:38.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local mode=journal 2026-02-18T08:25:38.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n journal ']' 2026-02-18T08:25:38.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:25:38.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_image cluster2 mirror image_1 128M 2026-02-18T08:25:38.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:25:38.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:25:38.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:25:38.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:25:38.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_1 2026-02-18T08:25:38.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:25:38.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local size=128 2026-02-18T08:25:38.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n 128M ']' 2026-02-18T08:25:38.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ size=128M 2026-02-18T08:25:38.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:25:38.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,journaling mirror/image_1 2026-02-18T08:25:38.385 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ is_pool_mirror_mode_image mirror 2026-02-18T08:25:38.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:25:38.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' pool = image ']' 2026-02-18T08:25:38.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ case "${pool}" in 2026-02-18T08:25:38.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 1 2026-02-18T08:25:38.387 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n '' ']' 2026-02-18T08:25:38.387 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ stress_write_image cluster2 mirror image_1 2026-02-18T08:25:38.387 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:25:38.387 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:25:38.387 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_1 2026-02-18T08:25:38.387 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-18T08:25:38.388 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local duration=13 2026-02-18T08:25:38.388 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set +e 2026-02-18T08:25:38.388 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ timeout 13s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_1 --debug-rbd=20 --debug-journaler=20 2026-02-18T08:25:39.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:100 IOs, 61911040 bytes 2026-02-18T08:25:39.785 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:200 IOs, 86356992 bytes 2026-02-18T08:25:39.884 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:300 IOs, 121197568 bytes 2026-02-18T08:25:41.297 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:400 IOs, 159426560 bytes 2026-02-18T08:25:42.619 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:500 IOs, 231420928 bytes 2026-02-18T08:25:43.218 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:600 IOs, 265966592 bytes 2026-02-18T08:25:43.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:700 IOs, 331490304 bytes 2026-02-18T08:25:45.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:800 IOs, 369660928 bytes 2026-02-18T08:25:45.604 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:900 IOs, 444883968 bytes 2026-02-18T08:25:47.276 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1000 IOs, 513104896 bytes 2026-02-18T08:25:47.802 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1100 IOs, 535257088 bytes 2026-02-18T08:25:48.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1200 IOs, 561620992 bytes 2026-02-18T08:25:49.971 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1300 IOs, 686963712 bytes 2026-02-18T08:25:50.492 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1400 IOs, 708870144 bytes 2026-02-18T08:25:50.946 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1500 IOs, 732938240 bytes 2026-02-18T08:25:51.394 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ error_code=124 2026-02-18T08:25:51.395 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set -e 2026-02-18T08:25:51.395 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 124 -eq 124 ']' 2026-02-18T08:25:51.395 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:25:51.395 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T08:25:51.395 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_2 2026-02-18T08:25:51.395 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_image_and_enable_mirror cluster2 mirror image_2 journal 128M 2026-02-18T08:25:51.395 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:25:51.395 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:25:51.395 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:25:51.395 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:25:51.395 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_2 2026-02-18T08:25:51.396 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:25:51.396 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local mode=journal 2026-02-18T08:25:51.396 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n journal ']' 2026-02-18T08:25:51.396 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:25:51.396 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_image cluster2 mirror image_2 128M 2026-02-18T08:25:51.396 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:25:51.396 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:25:51.396 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:25:51.396 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:25:51.396 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_2 2026-02-18T08:25:51.396 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:25:51.396 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local size=128 2026-02-18T08:25:51.397 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n 128M ']' 2026-02-18T08:25:51.397 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ size=128M 2026-02-18T08:25:51.397 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:25:51.397 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,journaling mirror/image_2 2026-02-18T08:25:51.439 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ is_pool_mirror_mode_image mirror 2026-02-18T08:25:51.439 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:25:51.439 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' pool = image ']' 2026-02-18T08:25:51.439 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ case "${pool}" in 2026-02-18T08:25:51.440 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 1 2026-02-18T08:25:51.440 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n '' ']' 2026-02-18T08:25:51.440 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ stress_write_image cluster2 mirror image_2 2026-02-18T08:25:51.440 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:25:51.440 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:25:51.440 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_2 2026-02-18T08:25:51.440 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-18T08:25:51.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local duration=10 2026-02-18T08:25:51.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set +e 2026-02-18T08:25:51.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ timeout 10s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_2 --debug-rbd=20 --debug-journaler=20 2026-02-18T08:25:51.680 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:100 IOs, 7164928 bytes 2026-02-18T08:25:52.211 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:200 IOs, 27440128 bytes 2026-02-18T08:25:52.735 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:300 IOs, 47362048 bytes 2026-02-18T08:25:53.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:400 IOs, 66846720 bytes 2026-02-18T08:25:54.667 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:500 IOs, 144663552 bytes 2026-02-18T08:25:55.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:600 IOs, 170257408 bytes 2026-02-18T08:25:56.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:700 IOs, 244653056 bytes 2026-02-18T08:25:56.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:800 IOs, 302810112 bytes 2026-02-18T08:25:57.434 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:900 IOs, 327054336 bytes 2026-02-18T08:25:58.119 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1000 IOs, 378429440 bytes 2026-02-18T08:25:58.567 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1100 IOs, 379052032 bytes 2026-02-18T08:25:59.437 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1200 IOs, 422353920 bytes 2026-02-18T08:26:00.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1300 IOs, 450857984 bytes 2026-02-18T08:26:00.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1400 IOs, 491888640 bytes 2026-02-18T08:26:01.449 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ error_code=124 2026-02-18T08:26:01.449 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set -e 2026-02-18T08:26:01.449 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 124 -eq 124 ']' 2026-02-18T08:26:01.450 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:26:01.450 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T08:26:01.450 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_3 2026-02-18T08:26:01.450 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_image_and_enable_mirror cluster2 mirror image_3 journal 128M 2026-02-18T08:26:01.450 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:26:01.450 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:26:01.450 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:26:01.450 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:26:01.450 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_3 2026-02-18T08:26:01.450 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:26:01.450 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local mode=journal 2026-02-18T08:26:01.450 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n journal ']' 2026-02-18T08:26:01.450 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:26:01.450 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_image cluster2 mirror image_3 128M 2026-02-18T08:26:01.450 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:26:01.450 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:26:01.450 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:26:01.450 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:26:01.450 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_3 2026-02-18T08:26:01.451 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:26:01.451 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local size=128 2026-02-18T08:26:01.451 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n 128M ']' 2026-02-18T08:26:01.451 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ size=128M 2026-02-18T08:26:01.451 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:26:01.451 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,journaling mirror/image_3 2026-02-18T08:26:01.489 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ is_pool_mirror_mode_image mirror 2026-02-18T08:26:01.489 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:26:01.489 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' pool = image ']' 2026-02-18T08:26:01.489 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ case "${pool}" in 2026-02-18T08:26:01.489 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 1 2026-02-18T08:26:01.489 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n '' ']' 2026-02-18T08:26:01.489 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ stress_write_image cluster2 mirror image_3 2026-02-18T08:26:01.489 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:26:01.490 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:26:01.490 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_3 2026-02-18T08:26:01.490 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-18T08:26:01.491 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local duration=11 2026-02-18T08:26:01.491 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set +e 2026-02-18T08:26:01.491 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ timeout 11s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_3 --debug-rbd=20 --debug-journaler=20 2026-02-18T08:26:01.552 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.0 with skip (0.47493737402995095) > weight (0.25) 2026-02-18T08:26:01.552 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.1 with skip (0.7750358782085617) > weight (0.25) 2026-02-18T08:26:01.552 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.2 with skip (0.694739510612011) > weight (0.25) 2026-02-18T08:26:01.552 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.3 with skip (0.6142254881189391) > weight (0.25) 2026-02-18T08:26:01.552 INFO:tasks.rbd_mirror_thrash:waiting for 67.57714474883589 secs before thrashing 2026-02-18T08:26:02.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:100 IOs, 46214144 bytes 2026-02-18T08:26:03.105 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:200 IOs, 87157760 bytes 2026-02-18T08:26:03.594 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:300 IOs, 118233088 bytes 2026-02-18T08:26:04.251 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:400 IOs, 136213504 bytes 2026-02-18T08:26:05.179 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:500 IOs, 204141568 bytes 2026-02-18T08:26:05.836 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:600 IOs, 239209472 bytes 2026-02-18T08:26:06.514 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:700 IOs, 309809152 bytes 2026-02-18T08:26:06.725 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:26:06.722+0000 7fb27e7f3640 -1 reset not still connected to 0x55e1703c8b60 2026-02-18T08:26:06.726 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:26:06.722+0000 7fb27e7f3640 -1 reset not still connected to 0x55e17071e820 2026-02-18T08:26:07.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:800 IOs, 370757632 bytes 2026-02-18T08:26:07.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:900 IOs, 417882112 bytes 2026-02-18T08:26:07.855 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1000 IOs, 432795648 bytes 2026-02-18T08:26:08.029 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1100 IOs, 454749184 bytes 2026-02-18T08:26:09.619 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1200 IOs, 549888000 bytes 2026-02-18T08:26:09.915 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1300 IOs, 579842048 bytes 2026-02-18T08:26:10.122 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1400 IOs, 589798400 bytes 2026-02-18T08:26:10.746 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1500 IOs, 647363584 bytes 2026-02-18T08:26:10.947 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1600 IOs, 657082368 bytes 2026-02-18T08:26:11.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1700 IOs, 684797952 bytes 2026-02-18T08:26:11.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1800 IOs, 709470208 bytes 2026-02-18T08:26:12.501 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ error_code=124 2026-02-18T08:26:12.501 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set -e 2026-02-18T08:26:12.501 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 124 -eq 124 ']' 2026-02-18T08:26:12.502 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:26:12.502 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T08:26:12.502 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_4 2026-02-18T08:26:12.502 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_image_and_enable_mirror cluster2 mirror image_4 journal 128M 2026-02-18T08:26:12.502 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:26:12.502 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:26:12.502 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:26:12.502 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:26:12.502 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_4 2026-02-18T08:26:12.502 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:26:12.502 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local mode=journal 2026-02-18T08:26:12.503 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n journal ']' 2026-02-18T08:26:12.503 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:26:12.503 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_image cluster2 mirror image_4 128M 2026-02-18T08:26:12.503 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:26:12.503 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:26:12.503 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:26:12.503 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:26:12.503 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_4 2026-02-18T08:26:12.503 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:26:12.503 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local size=128 2026-02-18T08:26:12.503 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n 128M ']' 2026-02-18T08:26:12.503 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ size=128M 2026-02-18T08:26:12.504 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:26:12.504 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,journaling mirror/image_4 2026-02-18T08:26:12.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ is_pool_mirror_mode_image mirror 2026-02-18T08:26:12.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:26:12.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' pool = image ']' 2026-02-18T08:26:12.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ case "${pool}" in 2026-02-18T08:26:12.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 1 2026-02-18T08:26:12.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n '' ']' 2026-02-18T08:26:12.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ stress_write_image cluster2 mirror image_4 2026-02-18T08:26:12.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:26:12.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:26:12.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_4 2026-02-18T08:26:12.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-18T08:26:12.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local duration=8 2026-02-18T08:26:12.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set +e 2026-02-18T08:26:12.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ timeout 8s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_4 --debug-rbd=20 --debug-journaler=20 2026-02-18T08:26:13.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:100 IOs, 32423936 bytes 2026-02-18T08:26:13.601 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:200 IOs, 59812864 bytes 2026-02-18T08:26:14.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:300 IOs, 101530624 bytes 2026-02-18T08:26:14.525 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:400 IOs, 149043200 bytes 2026-02-18T08:26:17.352 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:500 IOs, 259853312 bytes 2026-02-18T08:26:18.389 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:600 IOs, 370173952 bytes 2026-02-18T08:26:18.780 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:700 IOs, 392670208 bytes 2026-02-18T08:26:19.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:800 IOs, 428028928 bytes 2026-02-18T08:26:19.730 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:26:19.725+0000 7fb27e7f3640 -1 reset not still connected to 0x55e183c11860 2026-02-18T08:26:19.730 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:26:19.725+0000 7fb27e7f3640 -1 reset not still connected to 0x55e18895cea0 2026-02-18T08:26:19.787 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:26:19.781+0000 7fb27e7f3640 -1 reset not still connected to 0x55e183c11860 2026-02-18T08:26:19.787 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:26:19.781+0000 7fb27e7f3640 -1 reset not still connected to 0x55e18895cea0 2026-02-18T08:26:19.974 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:900 IOs, 481608704 bytes 2026-02-18T08:26:20.212 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1000 IOs, 492598272 bytes 2026-02-18T08:26:20.553 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ error_code=124 2026-02-18T08:26:20.553 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set -e 2026-02-18T08:26:20.553 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 124 -eq 124 ']' 2026-02-18T08:26:20.553 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:26:20.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T08:26:20.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_5 2026-02-18T08:26:20.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_image_and_enable_mirror cluster2 mirror image_5 journal 128M 2026-02-18T08:26:20.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:26:20.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:26:20.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:26:20.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:26:20.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_5 2026-02-18T08:26:20.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:26:20.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local mode=journal 2026-02-18T08:26:20.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n journal ']' 2026-02-18T08:26:20.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:26:20.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_image cluster2 mirror image_5 128M 2026-02-18T08:26:20.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:26:20.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:26:20.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:26:20.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:26:20.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_5 2026-02-18T08:26:20.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:26:20.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local size=128 2026-02-18T08:26:20.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n 128M ']' 2026-02-18T08:26:20.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ size=128M 2026-02-18T08:26:20.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:26:20.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,journaling mirror/image_5 2026-02-18T08:26:20.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ is_pool_mirror_mode_image mirror 2026-02-18T08:26:20.594 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:26:20.594 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' pool = image ']' 2026-02-18T08:26:20.594 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ case "${pool}" in 2026-02-18T08:26:20.594 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 1 2026-02-18T08:26:20.594 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n '' ']' 2026-02-18T08:26:20.594 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ stress_write_image cluster2 mirror image_5 2026-02-18T08:26:20.594 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:26:20.594 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:26:20.594 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_5 2026-02-18T08:26:20.594 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-18T08:26:20.595 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local duration=13 2026-02-18T08:26:20.595 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set +e 2026-02-18T08:26:20.595 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ timeout 13s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_5 --debug-rbd=20 --debug-journaler=20 2026-02-18T08:26:20.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:100 IOs, 15856640 bytes 2026-02-18T08:26:21.620 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:200 IOs, 53526528 bytes 2026-02-18T08:26:21.895 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:300 IOs, 66519040 bytes 2026-02-18T08:26:23.714 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:400 IOs, 140402688 bytes 2026-02-18T08:26:24.165 INFO:tasks.ceph.cluster2.osd.2.trial163.stderr:2026-02-18T08:26:24.153+0000 7f2b60c87640 -1 reset not still connected to 0x55fbf91dfee0 2026-02-18T08:26:24.486 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:500 IOs, 173273088 bytes 2026-02-18T08:26:25.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:600 IOs, 211869696 bytes 2026-02-18T08:26:25.831 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:700 IOs, 276060160 bytes 2026-02-18T08:26:26.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:800 IOs, 301697024 bytes 2026-02-18T08:26:26.801 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:900 IOs, 336653312 bytes 2026-02-18T08:26:27.114 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1000 IOs, 337272832 bytes 2026-02-18T08:26:27.279 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1100 IOs, 344516608 bytes 2026-02-18T08:26:27.987 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1200 IOs, 383694848 bytes 2026-02-18T08:26:28.065 INFO:tasks.ceph.cluster1.osd.1.trial034.stderr:2026-02-18T08:26:28.061+0000 7f20d7266640 -1 reset not still connected to 0x5563fcd7b790 2026-02-18T08:26:28.066 INFO:tasks.ceph.cluster1.osd.1.trial034.stderr:2026-02-18T08:26:28.061+0000 7f20d7266640 -1 reset not still connected to 0x5563fcd7bee0 2026-02-18T08:26:28.066 INFO:tasks.ceph.cluster1.osd.1.trial034.stderr:2026-02-18T08:26:28.061+0000 7f20d7266640 -1 reset not still connected to 0x5563fd730a90 2026-02-18T08:26:28.066 INFO:tasks.ceph.cluster1.osd.1.trial034.stderr:2026-02-18T08:26:28.061+0000 7f20d7266640 -1 reset not still connected to 0x5563fd7312b0 2026-02-18T08:26:28.366 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1300 IOs, 420745216 bytes 2026-02-18T08:26:28.523 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1400 IOs, 427123712 bytes 2026-02-18T08:26:29.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1500 IOs, 484567040 bytes 2026-02-18T08:26:29.759 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1600 IOs, 491824128 bytes 2026-02-18T08:26:30.385 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1700 IOs, 530605056 bytes 2026-02-18T08:26:31.559 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1800 IOs, 593251328 bytes 2026-02-18T08:26:32.602 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1900 IOs, 662914048 bytes 2026-02-18T08:26:33.559 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:2000 IOs, 738518016 bytes 2026-02-18T08:26:33.605 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ error_code=124 2026-02-18T08:26:33.605 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set -e 2026-02-18T08:26:33.605 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 124 -eq 124 ']' 2026-02-18T08:26:33.605 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:26:33.605 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T08:26:33.605 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_6 2026-02-18T08:26:33.605 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_image_and_enable_mirror cluster2 mirror image_6 journal 128M 2026-02-18T08:26:33.605 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:26:33.605 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:26:33.605 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:26:33.605 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:26:33.605 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_6 2026-02-18T08:26:33.605 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:26:33.605 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local mode=journal 2026-02-18T08:26:33.605 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n journal ']' 2026-02-18T08:26:33.605 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:26:33.605 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_image cluster2 mirror image_6 128M 2026-02-18T08:26:33.605 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:26:33.605 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:26:33.606 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:26:33.606 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:26:33.606 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_6 2026-02-18T08:26:33.606 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:26:33.606 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local size=128 2026-02-18T08:26:33.606 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n 128M ']' 2026-02-18T08:26:33.606 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ size=128M 2026-02-18T08:26:33.606 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:26:33.606 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,journaling mirror/image_6 2026-02-18T08:26:33.647 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ is_pool_mirror_mode_image mirror 2026-02-18T08:26:33.647 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:26:33.647 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' pool = image ']' 2026-02-18T08:26:33.647 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ case "${pool}" in 2026-02-18T08:26:33.647 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 1 2026-02-18T08:26:33.647 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n '' ']' 2026-02-18T08:26:33.647 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ stress_write_image cluster2 mirror image_6 2026-02-18T08:26:33.647 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:26:33.647 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:26:33.647 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_6 2026-02-18T08:26:33.647 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-18T08:26:33.648 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local duration=6 2026-02-18T08:26:33.648 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set +e 2026-02-18T08:26:33.648 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ timeout 6s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_6 --debug-rbd=20 --debug-journaler=20 2026-02-18T08:26:34.994 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:100 IOs, 64719872 bytes 2026-02-18T08:26:35.293 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:200 IOs, 108443648 bytes 2026-02-18T08:26:35.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:300 IOs, 122759168 bytes 2026-02-18T08:26:35.721 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:26:35.717+0000 7fb27e7f3640 -1 reset not still connected to 0x55e183c11860 2026-02-18T08:26:35.721 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:26:35.717+0000 7fb27e7f3640 -1 reset not still connected to 0x55e18895cea0 2026-02-18T08:26:37.150 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:400 IOs, 206641152 bytes 2026-02-18T08:26:37.779 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:500 IOs, 230922240 bytes 2026-02-18T08:26:38.121 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:26:38.117+0000 7faffc95c640 -1 reset not still connected to 0x5644a5073110 2026-02-18T08:26:38.357 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:600 IOs, 257931264 bytes 2026-02-18T08:26:38.501 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:700 IOs, 262835200 bytes 2026-02-18T08:26:38.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:800 IOs, 268881920 bytes 2026-02-18T08:26:39.008 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:900 IOs, 293259264 bytes 2026-02-18T08:26:39.462 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1000 IOs, 309083136 bytes 2026-02-18T08:26:39.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ error_code=124 2026-02-18T08:26:39.658 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set -e 2026-02-18T08:26:39.658 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 124 -eq 124 ']' 2026-02-18T08:26:39.658 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:26:39.658 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T08:26:39.658 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_7 2026-02-18T08:26:39.658 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_image_and_enable_mirror cluster2 mirror image_7 journal 128M 2026-02-18T08:26:39.658 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:26:39.658 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:26:39.658 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:26:39.658 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:26:39.658 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_7 2026-02-18T08:26:39.658 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:26:39.658 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local mode=journal 2026-02-18T08:26:39.659 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n journal ']' 2026-02-18T08:26:39.659 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:26:39.659 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_image cluster2 mirror image_7 128M 2026-02-18T08:26:39.659 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:26:39.659 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:26:39.659 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:26:39.659 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:26:39.659 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_7 2026-02-18T08:26:39.659 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:26:39.659 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local size=128 2026-02-18T08:26:39.659 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n 128M ']' 2026-02-18T08:26:39.659 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ size=128M 2026-02-18T08:26:39.659 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:26:39.659 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,journaling mirror/image_7 2026-02-18T08:26:39.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ is_pool_mirror_mode_image mirror 2026-02-18T08:26:39.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:26:39.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' pool = image ']' 2026-02-18T08:26:39.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ case "${pool}" in 2026-02-18T08:26:39.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 1 2026-02-18T08:26:39.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n '' ']' 2026-02-18T08:26:39.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ stress_write_image cluster2 mirror image_7 2026-02-18T08:26:39.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:26:39.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:26:39.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_7 2026-02-18T08:26:39.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-18T08:26:39.700 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local duration=8 2026-02-18T08:26:39.700 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set +e 2026-02-18T08:26:39.700 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ timeout 8s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_7 --debug-rbd=20 --debug-journaler=20 2026-02-18T08:26:40.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:100 IOs, 44069888 bytes 2026-02-18T08:26:40.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:200 IOs, 56446976 bytes 2026-02-18T08:26:42.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:300 IOs, 99366912 bytes 2026-02-18T08:26:43.566 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:400 IOs, 144626688 bytes 2026-02-18T08:26:43.662 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:500 IOs, 147399680 bytes 2026-02-18T08:26:44.161 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:600 IOs, 171873280 bytes 2026-02-18T08:26:44.524 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:700 IOs, 189299712 bytes 2026-02-18T08:26:44.779 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:26:44.773+0000 7fb27e7f3640 -1 reset not still connected to 0x55e1703c8b60 2026-02-18T08:26:44.780 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:26:44.773+0000 7fb27e7f3640 -1 reset not still connected to 0x55e17071e820 2026-02-18T08:26:44.780 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:26:44.773+0000 7fb27e7f3640 -1 reset not still connected to 0x55e178bde8f0 2026-02-18T08:26:45.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:800 IOs, 240486400 bytes 2026-02-18T08:26:45.482 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:900 IOs, 251565056 bytes 2026-02-18T08:26:45.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1000 IOs, 265798656 bytes 2026-02-18T08:26:46.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1100 IOs, 266413056 bytes 2026-02-18T08:26:46.486 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1200 IOs, 307244032 bytes 2026-02-18T08:26:46.868 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1300 IOs, 328951808 bytes 2026-02-18T08:26:47.465 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1400 IOs, 384037888 bytes 2026-02-18T08:26:47.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ error_code=124 2026-02-18T08:26:47.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set -e 2026-02-18T08:26:47.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 124 -eq 124 ']' 2026-02-18T08:26:47.711 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:26:47.711 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T08:26:47.711 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_8 2026-02-18T08:26:47.711 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_image_and_enable_mirror cluster2 mirror image_8 journal 128M 2026-02-18T08:26:47.711 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:26:47.711 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:26:47.711 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:26:47.711 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:26:47.711 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_8 2026-02-18T08:26:47.711 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:26:47.712 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local mode=journal 2026-02-18T08:26:47.712 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n journal ']' 2026-02-18T08:26:47.712 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:26:47.712 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_image cluster2 mirror image_8 128M 2026-02-18T08:26:47.712 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:26:47.712 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:26:47.712 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:26:47.712 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:26:47.712 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_8 2026-02-18T08:26:47.712 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:26:47.712 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local size=128 2026-02-18T08:26:47.713 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n 128M ']' 2026-02-18T08:26:47.713 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ size=128M 2026-02-18T08:26:47.713 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:26:47.713 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,journaling mirror/image_8 2026-02-18T08:26:47.750 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ is_pool_mirror_mode_image mirror 2026-02-18T08:26:47.750 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:26:47.750 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' pool = image ']' 2026-02-18T08:26:47.750 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ case "${pool}" in 2026-02-18T08:26:47.751 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 1 2026-02-18T08:26:47.751 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n '' ']' 2026-02-18T08:26:47.751 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ stress_write_image cluster2 mirror image_8 2026-02-18T08:26:47.751 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:26:47.751 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:26:47.751 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_8 2026-02-18T08:26:47.751 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-18T08:26:47.752 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local duration=14 2026-02-18T08:26:47.752 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set +e 2026-02-18T08:26:47.752 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ timeout 14s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_8 --debug-rbd=20 --debug-journaler=20 2026-02-18T08:26:47.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:100 IOs, 17688576 bytes 2026-02-18T08:26:48.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:200 IOs, 58502144 bytes 2026-02-18T08:26:51.573 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:300 IOs, 191039488 bytes 2026-02-18T08:26:52.204 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:400 IOs, 227835904 bytes 2026-02-18T08:26:52.379 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:26:52.372+0000 7fb27e7f3640 -1 reset not still connected to 0x55e1705e44e0 2026-02-18T08:26:52.380 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:26:52.372+0000 7fb27e7f3640 -1 reset not still connected to 0x55e170742000 2026-02-18T08:26:52.381 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:26:52.372+0000 7fb27e7f3640 -1 reset not still connected to 0x55e170c6e000 2026-02-18T08:26:52.381 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:26:52.372+0000 7fb27e7f3640 -1 reset not still connected to 0x55e171e68000 2026-02-18T08:26:52.381 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:26:52.376+0000 7fb27e7f3640 -1 reset not still connected to 0x55e171e69380 2026-02-18T08:26:52.381 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:26:52.376+0000 7fb27e7f3640 -1 reset not still connected to 0x55e18817ba00 2026-02-18T08:26:52.423 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:500 IOs, 263698432 bytes 2026-02-18T08:26:52.945 INFO:tasks.rbd_mirror_thrash:kill cluster1.client.mirror.0 2026-02-18T08:26:52.946 INFO:tasks.rbd_mirror.cluster1.client.mirror.0:Sent signal 15 2026-02-18T08:26:52.946 INFO:tasks.rbd_mirror_thrash:waiting for 2.227471809859937 secs before reviving daemons 2026-02-18T08:26:52.947 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.trial163.stderr:2026-02-18T08:26:52.937+0000 7f622d54a640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.0 (PID: 84397) UID: 1000 2026-02-18T08:26:52.947 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.trial163.stderr:2026-02-18T08:26:52.937+0000 7f6225d3b640 -1 rbd::mirror::image_replayer::journal::Replayer: 0x55c450def500 handle_process_entry_safe: failed to commit journal event: (85) Interrupted system call should be restarted 2026-02-18T08:26:52.947 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.trial163.stderr:2026-02-18T08:26:52.937+0000 7f6225d3b640 -1 rbd::mirror::image_replayer::journal::Replayer: 0x55c450def500 handle_replay_complete: replay encountered an error: (85) Interrupted system call should be restarted 2026-02-18T08:26:53.199 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:600 IOs, 309020672 bytes 2026-02-18T08:26:53.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:700 IOs, 324060160 bytes 2026-02-18T08:26:54.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:800 IOs, 345375744 bytes 2026-02-18T08:26:54.388 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:900 IOs, 361220096 bytes 2026-02-18T08:26:54.539 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1000 IOs, 388254720 bytes 2026-02-18T08:26:55.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1100 IOs, 399316992 bytes 2026-02-18T08:26:55.174 INFO:tasks.rbd_mirror_thrash:waiting for cluster1.client.mirror.0 2026-02-18T08:26:55.175 INFO:teuthology.orchestra.run:waiting for 600 2026-02-18T08:26:55.175 INFO:tasks.rbd_mirror_thrash:reviving cluster1.client.mirror.0 2026-02-18T08:26:55.175 INFO:tasks.rbd_mirror.cluster1.client.mirror.0:Restarting daemon 2026-02-18T08:26:55.175 DEBUG:teuthology.orchestra.run.trial163:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.0 2026-02-18T08:26:55.178 INFO:tasks.rbd_mirror.cluster1.client.mirror.0:Started 2026-02-18T08:26:55.178 INFO:tasks.rbd_mirror_thrash:waiting for 104.58137917197467 secs before thrashing 2026-02-18T08:26:55.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1200 IOs, 477091840 bytes 2026-02-18T08:26:56.963 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1300 IOs, 563600384 bytes 2026-02-18T08:26:57.187 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1400 IOs, 572998656 bytes 2026-02-18T08:26:57.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1500 IOs, 637669376 bytes 2026-02-18T08:26:58.413 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1600 IOs, 659421184 bytes 2026-02-18T08:26:58.513 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:26:58.508+0000 7fb27e7f3640 -1 reset not still connected to 0x55e18895cea0 2026-02-18T08:26:59.019 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1700 IOs, 719479808 bytes 2026-02-18T08:26:59.811 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1800 IOs, 774530048 bytes 2026-02-18T08:27:00.164 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1900 IOs, 790784000 bytes 2026-02-18T08:27:00.675 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:2000 IOs, 825351168 bytes 2026-02-18T08:27:00.855 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:2100 IOs, 833269760 bytes 2026-02-18T08:27:01.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:2200 IOs, 882213888 bytes 2026-02-18T08:27:01.762 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ error_code=124 2026-02-18T08:27:01.762 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set -e 2026-02-18T08:27:01.762 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 124 -eq 124 ']' 2026-02-18T08:27:01.763 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:27:01.763 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T08:27:01.763 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_9 2026-02-18T08:27:01.763 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_image_and_enable_mirror cluster2 mirror image_9 journal 128M 2026-02-18T08:27:01.763 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:27:01.763 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:27:01.763 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:27:01.763 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:27:01.763 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_9 2026-02-18T08:27:01.763 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:27:01.763 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local mode=journal 2026-02-18T08:27:01.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n journal ']' 2026-02-18T08:27:01.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:27:01.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_image cluster2 mirror image_9 128M 2026-02-18T08:27:01.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:27:01.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:27:01.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:27:01.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:27:01.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_9 2026-02-18T08:27:01.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:27:01.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local size=128 2026-02-18T08:27:01.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n 128M ']' 2026-02-18T08:27:01.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ size=128M 2026-02-18T08:27:01.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:27:01.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,journaling mirror/image_9 2026-02-18T08:27:01.812 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ is_pool_mirror_mode_image mirror 2026-02-18T08:27:01.812 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:27:01.812 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' pool = image ']' 2026-02-18T08:27:01.812 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ case "${pool}" in 2026-02-18T08:27:01.812 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 1 2026-02-18T08:27:01.812 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n '' ']' 2026-02-18T08:27:01.813 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ stress_write_image cluster2 mirror image_9 2026-02-18T08:27:01.813 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:27:01.813 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:27:01.813 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_9 2026-02-18T08:27:01.813 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-18T08:27:01.814 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local duration=14 2026-02-18T08:27:01.814 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set +e 2026-02-18T08:27:01.814 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ timeout 14s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_9 --debug-rbd=20 --debug-journaler=20 2026-02-18T08:27:02.540 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:100 IOs, 44206080 bytes 2026-02-18T08:27:03.212 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:200 IOs, 75418624 bytes 2026-02-18T08:27:03.597 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:300 IOs, 98141184 bytes 2026-02-18T08:27:03.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:400 IOs, 115719168 bytes 2026-02-18T08:27:05.094 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:500 IOs, 139515904 bytes 2026-02-18T08:27:05.770 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:600 IOs, 176336896 bytes 2026-02-18T08:27:06.208 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:700 IOs, 204163072 bytes 2026-02-18T08:27:07.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:800 IOs, 290280448 bytes 2026-02-18T08:27:08.285 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:900 IOs, 364262400 bytes 2026-02-18T08:27:09.130 INFO:tasks.rbd_mirror_thrash:kill cluster2.client.mirror.0 2026-02-18T08:27:09.130 INFO:tasks.rbd_mirror.cluster2.client.mirror.0:Sent signal 15 2026-02-18T08:27:09.130 INFO:tasks.rbd_mirror_thrash:waiting for 7.9569991528000426 secs before reviving daemons 2026-02-18T08:27:09.131 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.trial163.stderr:2026-02-18T08:27:09.121+0000 7f65536d2640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.0 (PID: 83314) UID: 1000 2026-02-18T08:27:09.948 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1000 IOs, 431906816 bytes 2026-02-18T08:27:12.486 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1100 IOs, 542659584 bytes 2026-02-18T08:27:13.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1200 IOs, 608083968 bytes 2026-02-18T08:27:14.511 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1300 IOs, 703552512 bytes 2026-02-18T08:27:14.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1400 IOs, 712270848 bytes 2026-02-18T08:27:15.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1500 IOs, 747987968 bytes 2026-02-18T08:27:15.826 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ error_code=124 2026-02-18T08:27:15.826 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set -e 2026-02-18T08:27:15.826 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 124 -eq 124 ']' 2026-02-18T08:27:15.826 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:27:15.826 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T08:27:15.826 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_10 2026-02-18T08:27:15.826 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_image_and_enable_mirror cluster2 mirror image_10 journal 128M 2026-02-18T08:27:15.827 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:27:15.827 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:27:15.827 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:27:15.827 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:27:15.827 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_10 2026-02-18T08:27:15.827 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:27:15.827 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local mode=journal 2026-02-18T08:27:15.827 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n journal ']' 2026-02-18T08:27:15.827 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:27:15.827 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_image cluster2 mirror image_10 128M 2026-02-18T08:27:15.827 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:27:15.827 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:27:15.828 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:27:15.828 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:27:15.828 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_10 2026-02-18T08:27:15.828 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:27:15.828 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local size=128 2026-02-18T08:27:15.828 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n 128M ']' 2026-02-18T08:27:15.828 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ size=128M 2026-02-18T08:27:15.828 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:27:15.828 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,journaling mirror/image_10 2026-02-18T08:27:15.865 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ is_pool_mirror_mode_image mirror 2026-02-18T08:27:15.865 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:27:15.865 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' pool = image ']' 2026-02-18T08:27:15.865 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ case "${pool}" in 2026-02-18T08:27:15.866 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 1 2026-02-18T08:27:15.866 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n '' ']' 2026-02-18T08:27:15.866 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ stress_write_image cluster2 mirror image_10 2026-02-18T08:27:15.866 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:27:15.866 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:27:15.866 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_10 2026-02-18T08:27:15.866 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-18T08:27:15.867 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local duration=6 2026-02-18T08:27:15.867 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set +e 2026-02-18T08:27:15.867 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ timeout 6s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_10 --debug-rbd=20 --debug-journaler=20 2026-02-18T08:27:16.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:100 IOs, 10050560 bytes 2026-02-18T08:27:16.430 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:200 IOs, 20752384 bytes 2026-02-18T08:27:17.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:300 IOs, 49782784 bytes 2026-02-18T08:27:17.088 INFO:tasks.rbd_mirror_thrash:waiting for cluster2.client.mirror.0 2026-02-18T08:27:17.088 INFO:teuthology.orchestra.run:waiting for 600 2026-02-18T08:27:17.089 INFO:tasks.rbd_mirror_thrash:reviving cluster2.client.mirror.0 2026-02-18T08:27:17.089 INFO:tasks.rbd_mirror.cluster2.client.mirror.0:Restarting daemon 2026-02-18T08:27:17.089 DEBUG:teuthology.orchestra.run.trial163:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.0 2026-02-18T08:27:17.091 INFO:tasks.rbd_mirror.cluster2.client.mirror.0:Started 2026-02-18T08:27:17.091 INFO:tasks.rbd_mirror_thrash:waiting for 60.44888131253592 secs before thrashing 2026-02-18T08:27:17.448 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:400 IOs, 67164160 bytes 2026-02-18T08:27:17.786 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:500 IOs, 87957504 bytes 2026-02-18T08:27:19.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:600 IOs, 150316032 bytes 2026-02-18T08:27:19.835 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:700 IOs, 177771520 bytes 2026-02-18T08:27:20.293 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:800 IOs, 214456320 bytes 2026-02-18T08:27:20.962 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:900 IOs, 280344576 bytes 2026-02-18T08:27:21.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1000 IOs, 292890624 bytes 2026-02-18T08:27:21.570 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1100 IOs, 305863680 bytes 2026-02-18T08:27:21.876 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ error_code=124 2026-02-18T08:27:21.877 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set -e 2026-02-18T08:27:21.877 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 124 -eq 124 ']' 2026-02-18T08:27:21.877 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:27:21.877 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T08:27:21.877 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_11 2026-02-18T08:27:21.877 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_image_and_enable_mirror cluster2 mirror image_11 journal 128M 2026-02-18T08:27:21.877 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:27:21.877 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:27:21.877 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:27:21.878 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:27:21.878 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_11 2026-02-18T08:27:21.878 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:27:21.878 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local mode=journal 2026-02-18T08:27:21.878 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n journal ']' 2026-02-18T08:27:21.878 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:27:21.878 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_image cluster2 mirror image_11 128M 2026-02-18T08:27:21.878 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:27:21.878 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:27:21.878 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:27:21.878 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:27:21.878 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_11 2026-02-18T08:27:21.879 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:27:21.879 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local size=128 2026-02-18T08:27:21.879 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n 128M ']' 2026-02-18T08:27:21.879 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ size=128M 2026-02-18T08:27:21.879 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:27:21.879 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,journaling mirror/image_11 2026-02-18T08:27:21.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ is_pool_mirror_mode_image mirror 2026-02-18T08:27:21.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:27:21.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' pool = image ']' 2026-02-18T08:27:21.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ case "${pool}" in 2026-02-18T08:27:21.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 1 2026-02-18T08:27:21.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n '' ']' 2026-02-18T08:27:21.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ stress_write_image cluster2 mirror image_11 2026-02-18T08:27:21.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:27:21.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:27:21.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_11 2026-02-18T08:27:21.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-18T08:27:21.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local duration=12 2026-02-18T08:27:21.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set +e 2026-02-18T08:27:21.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ timeout 12s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_11 --debug-rbd=20 --debug-journaler=20 2026-02-18T08:27:22.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:100 IOs, 4190208 bytes 2026-02-18T08:27:23.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:200 IOs, 100779008 bytes 2026-02-18T08:27:23.969 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:300 IOs, 121027584 bytes 2026-02-18T08:27:24.297 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:400 IOs, 161720320 bytes 2026-02-18T08:27:24.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:500 IOs, 175990784 bytes 2026-02-18T08:27:25.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:600 IOs, 226301952 bytes 2026-02-18T08:27:26.259 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:27:26.252+0000 7faffc95c640 -1 reset not still connected to 0x5644a7b78d00 2026-02-18T08:27:26.523 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:700 IOs, 276809728 bytes 2026-02-18T08:27:27.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:800 IOs, 299162624 bytes 2026-02-18T08:27:28.018 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:900 IOs, 374383616 bytes 2026-02-18T08:27:28.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1000 IOs, 399110144 bytes 2026-02-18T08:27:28.703 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1100 IOs, 415962112 bytes 2026-02-18T08:27:29.720 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1200 IOs, 495320064 bytes 2026-02-18T08:27:31.165 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1300 IOs, 566006784 bytes 2026-02-18T08:27:32.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1400 IOs, 618714112 bytes 2026-02-18T08:27:32.748 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:27:32.743+0000 7faffc95c640 -1 reset not still connected to 0x5644a5073860 2026-02-18T08:27:32.749 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:27:32.747+0000 7faffc95c640 -1 reset not still connected to 0x5644a84a6750 2026-02-18T08:27:32.749 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:27:32.747+0000 7faffc95c640 -1 reset not still connected to 0x5644c1b595f0 2026-02-18T08:27:32.809 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1500 IOs, 650257408 bytes 2026-02-18T08:27:33.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1600 IOs, 688898048 bytes 2026-02-18T08:27:33.193 INFO:tasks.ceph.cluster2.osd.0.trial163.stderr:2026-02-18T08:27:33.180+0000 7f2129686640 -1 reset not still connected to 0x561767dda9c0 2026-02-18T08:27:33.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ error_code=124 2026-02-18T08:27:33.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set -e 2026-02-18T08:27:33.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 124 -eq 124 ']' 2026-02-18T08:27:33.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:27:33.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T08:27:33.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_12 2026-02-18T08:27:33.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_image_and_enable_mirror cluster2 mirror image_12 journal 128M 2026-02-18T08:27:33.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:27:33.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:27:33.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:27:33.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:27:33.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_12 2026-02-18T08:27:33.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:27:33.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local mode=journal 2026-02-18T08:27:33.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n journal ']' 2026-02-18T08:27:33.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:27:33.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_image cluster2 mirror image_12 128M 2026-02-18T08:27:33.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:27:33.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:27:33.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:27:33.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:27:33.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_12 2026-02-18T08:27:33.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:27:33.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local size=128 2026-02-18T08:27:33.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n 128M ']' 2026-02-18T08:27:33.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ size=128M 2026-02-18T08:27:33.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:27:33.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,journaling mirror/image_12 2026-02-18T08:27:33.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ is_pool_mirror_mode_image mirror 2026-02-18T08:27:33.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:27:33.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' pool = image ']' 2026-02-18T08:27:33.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ case "${pool}" in 2026-02-18T08:27:33.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 1 2026-02-18T08:27:33.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n '' ']' 2026-02-18T08:27:33.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ stress_write_image cluster2 mirror image_12 2026-02-18T08:27:33.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:27:33.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:27:33.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_12 2026-02-18T08:27:33.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-18T08:27:33.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local duration=14 2026-02-18T08:27:33.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set +e 2026-02-18T08:27:33.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ timeout 14s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_12 --debug-rbd=20 --debug-journaler=20 2026-02-18T08:27:34.805 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:100 IOs, 40783872 bytes 2026-02-18T08:27:35.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:200 IOs, 62442496 bytes 2026-02-18T08:27:35.950 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:300 IOs, 122817536 bytes 2026-02-18T08:27:36.849 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:27:36.847+0000 7fb27e7f3640 -1 reset not still connected to 0x55e170c6e000 2026-02-18T08:27:36.849 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:27:36.847+0000 7fb27e7f3640 -1 reset not still connected to 0x55e18817ba00 2026-02-18T08:27:37.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:400 IOs, 169589760 bytes 2026-02-18T08:27:37.475 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:500 IOs, 187931648 bytes 2026-02-18T08:27:37.552 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:600 IOs, 188546048 bytes 2026-02-18T08:27:39.552 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:700 IOs, 323894272 bytes 2026-02-18T08:27:39.561 INFO:tasks.ceph.cluster1.osd.1.trial034.stderr:2026-02-18T08:27:39.559+0000 7f20d7266640 -1 reset not still connected to 0x5563fcd7bee0 2026-02-18T08:27:39.562 INFO:tasks.ceph.cluster1.osd.1.trial034.stderr:2026-02-18T08:27:39.559+0000 7f20d7266640 -1 reset not still connected to 0x5563fd14cf70 2026-02-18T08:27:39.562 INFO:tasks.ceph.cluster1.osd.1.trial034.stderr:2026-02-18T08:27:39.559+0000 7f20d7266640 -1 reset not still connected to 0x5563fd730a90 2026-02-18T08:27:39.563 INFO:tasks.ceph.cluster1.osd.1.trial034.stderr:2026-02-18T08:27:39.559+0000 7f20d7266640 -1 reset not still connected to 0x5563fd7312b0 2026-02-18T08:27:39.563 INFO:tasks.ceph.cluster1.osd.1.trial034.stderr:2026-02-18T08:27:39.559+0000 7f20d7266640 -1 reset not still connected to 0x5563fe8d60d0 2026-02-18T08:27:41.018 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:800 IOs, 439334912 bytes 2026-02-18T08:27:41.406 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:900 IOs, 457220096 bytes 2026-02-18T08:27:41.571 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1000 IOs, 463931392 bytes 2026-02-18T08:27:41.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1100 IOs, 470679552 bytes 2026-02-18T08:27:42.161 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1200 IOs, 499708928 bytes 2026-02-18T08:27:42.782 INFO:tasks.ceph.cluster1.osd.1.trial034.stderr:2026-02-18T08:27:42.779+0000 7f20d7266640 -1 reset not still connected to 0x5563fd7308f0 2026-02-18T08:27:42.782 INFO:tasks.ceph.cluster1.osd.1.trial034.stderr:2026-02-18T08:27:42.779+0000 7f20d7266640 -1 reset not still connected to 0x5563fd731450 2026-02-18T08:27:42.782 INFO:tasks.ceph.cluster1.osd.1.trial034.stderr:2026-02-18T08:27:42.779+0000 7f20d7266640 -1 reset not still connected to 0x5563fe2316c0 2026-02-18T08:27:42.782 INFO:tasks.ceph.cluster1.osd.1.trial034.stderr:2026-02-18T08:27:42.779+0000 7f20d7266640 -1 reset not still connected to 0x5564228bfd40 2026-02-18T08:27:43.228 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:27:43.223+0000 7faffc95c640 -1 reset not still connected to 0x5644a84a6750 2026-02-18T08:27:43.228 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:27:43.223+0000 7faffc95c640 -1 reset not still connected to 0x5644c1b595f0 2026-02-18T08:27:43.228 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:27:43.223+0000 7faffc95c640 -1 reset not still connected to 0x5644d9b0e410 2026-02-18T08:27:43.405 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1300 IOs, 562575360 bytes 2026-02-18T08:27:43.505 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1400 IOs, 563179520 bytes 2026-02-18T08:27:43.551 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:27:43.547+0000 7faffc95c640 -1 reset not still connected to 0x5644a84a6750 2026-02-18T08:27:43.551 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:27:43.547+0000 7faffc95c640 -1 reset not still connected to 0x5644c1b595f0 2026-02-18T08:27:43.551 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:27:43.547+0000 7faffc95c640 -1 reset not still connected to 0x5644d9b0e410 2026-02-18T08:27:43.824 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:27:43.819+0000 7fb27e7f3640 -1 reset not still connected to 0x55e183c11860 2026-02-18T08:27:43.824 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:27:43.819+0000 7fb27e7f3640 -1 reset not still connected to 0x55e18895cea0 2026-02-18T08:27:44.483 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1500 IOs, 643581952 bytes 2026-02-18T08:27:44.585 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1600 IOs, 646699008 bytes 2026-02-18T08:27:46.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1700 IOs, 750993408 bytes 2026-02-18T08:27:47.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1800 IOs, 802957312 bytes 2026-02-18T08:27:47.988 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ error_code=124 2026-02-18T08:27:47.989 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set -e 2026-02-18T08:27:47.989 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 124 -eq 124 ']' 2026-02-18T08:27:47.989 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:27:47.989 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T08:27:47.989 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_13 2026-02-18T08:27:47.989 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_image_and_enable_mirror cluster2 mirror image_13 journal 128M 2026-02-18T08:27:47.989 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:27:47.989 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:27:47.989 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:27:47.989 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:27:47.990 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_13 2026-02-18T08:27:47.990 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:27:47.990 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local mode=journal 2026-02-18T08:27:47.990 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n journal ']' 2026-02-18T08:27:47.990 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:27:47.990 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_image cluster2 mirror image_13 128M 2026-02-18T08:27:47.990 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:27:47.990 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:27:47.990 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:27:47.990 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:27:47.990 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_13 2026-02-18T08:27:47.990 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:27:47.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local size=128 2026-02-18T08:27:47.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n 128M ']' 2026-02-18T08:27:47.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ size=128M 2026-02-18T08:27:47.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:27:47.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,journaling mirror/image_13 2026-02-18T08:27:48.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ is_pool_mirror_mode_image mirror 2026-02-18T08:27:48.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:27:48.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' pool = image ']' 2026-02-18T08:27:48.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ case "${pool}" in 2026-02-18T08:27:48.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 1 2026-02-18T08:27:48.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n '' ']' 2026-02-18T08:27:48.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ stress_write_image cluster2 mirror image_13 2026-02-18T08:27:48.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:27:48.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:27:48.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_13 2026-02-18T08:27:48.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-18T08:27:48.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local duration=14 2026-02-18T08:27:48.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set +e 2026-02-18T08:27:48.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ timeout 14s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_13 --debug-rbd=20 --debug-journaler=20 2026-02-18T08:27:49.675 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:100 IOs, 94118912 bytes 2026-02-18T08:27:49.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:200 IOs, 108926976 bytes 2026-02-18T08:27:50.138 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:300 IOs, 118879232 bytes 2026-02-18T08:27:50.762 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:27:50.759+0000 7fb27e7f3640 -1 reset not still connected to 0x55e1703c8b60 2026-02-18T08:27:50.762 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:27:50.759+0000 7fb27e7f3640 -1 reset not still connected to 0x55e17071e820 2026-02-18T08:27:50.763 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:27:50.759+0000 7fb27e7f3640 -1 reset not still connected to 0x55e178bde8f0 2026-02-18T08:27:51.097 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:400 IOs, 162306048 bytes 2026-02-18T08:27:51.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:500 IOs, 192513024 bytes 2026-02-18T08:27:52.487 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:600 IOs, 231072768 bytes 2026-02-18T08:27:53.453 INFO:tasks.ceph.cluster1.osd.1.trial034.stderr:2026-02-18T08:27:53.451+0000 7f20d7266640 -1 reset not still connected to 0x5563fe8d60d0 2026-02-18T08:27:53.569 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:700 IOs, 295124992 bytes 2026-02-18T08:27:53.853 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:800 IOs, 303860736 bytes 2026-02-18T08:27:54.148 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:900 IOs, 304990208 bytes 2026-02-18T08:27:55.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1000 IOs, 347814912 bytes 2026-02-18T08:27:56.244 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:27:56.239+0000 7fb27e7f3640 -1 reset not still connected to 0x55e17071ec30 2026-02-18T08:27:56.245 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:27:56.239+0000 7fb27e7f3640 -1 reset not still connected to 0x55e170c6e8f0 2026-02-18T08:27:56.245 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:27:56.243+0000 7fb27e7f3640 -1 reset not still connected to 0x55e178bde340 2026-02-18T08:27:56.245 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:27:56.243+0000 7fb27e7f3640 -1 reset not still connected to 0x55e183c11860 2026-02-18T08:27:56.245 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:27:56.243+0000 7fb27e7f3640 -1 reset not still connected to 0x55e18895cea0 2026-02-18T08:27:56.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1100 IOs, 436196352 bytes 2026-02-18T08:27:57.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1200 IOs, 436799488 bytes 2026-02-18T08:27:57.791 INFO:tasks.ceph.cluster1.osd.1.trial034.stderr:2026-02-18T08:27:57.787+0000 7f20d7266640 -1 reset not still connected to 0x556422af5ad0 2026-02-18T08:27:57.817 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1300 IOs, 485112832 bytes 2026-02-18T08:27:58.075 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1400 IOs, 496327680 bytes 2026-02-18T08:27:58.800 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1500 IOs, 557852672 bytes 2026-02-18T08:27:59.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1600 IOs, 609062912 bytes 2026-02-18T08:27:59.862 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1700 IOs, 611215360 bytes 2026-02-18T08:27:59.924 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1800 IOs, 611848192 bytes 2026-02-18T08:28:00.317 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1900 IOs, 651246592 bytes 2026-02-18T08:28:00.324 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:28:00.319+0000 7faffc95c640 -1 reset not still connected to 0x5644a84a6750 2026-02-18T08:28:00.325 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:28:00.319+0000 7faffc95c640 -1 reset not still connected to 0x5644c1b595f0 2026-02-18T08:28:00.325 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:28:00.319+0000 7faffc95c640 -1 reset not still connected to 0x5644d9b0e410 2026-02-18T08:28:00.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:2000 IOs, 682203136 bytes 2026-02-18T08:28:01.574 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:2100 IOs, 723302400 bytes 2026-02-18T08:28:02.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ error_code=124 2026-02-18T08:28:02.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set -e 2026-02-18T08:28:02.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 124 -eq 124 ']' 2026-02-18T08:28:02.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:28:02.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T08:28:02.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_14 2026-02-18T08:28:02.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_image_and_enable_mirror cluster2 mirror image_14 journal 128M 2026-02-18T08:28:02.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:28:02.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:28:02.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:28:02.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:28:02.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_14 2026-02-18T08:28:02.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:28:02.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local mode=journal 2026-02-18T08:28:02.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n journal ']' 2026-02-18T08:28:02.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:28:02.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_image cluster2 mirror image_14 128M 2026-02-18T08:28:02.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:28:02.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:28:02.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:28:02.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:28:02.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_14 2026-02-18T08:28:02.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:28:02.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local size=128 2026-02-18T08:28:02.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n 128M ']' 2026-02-18T08:28:02.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ size=128M 2026-02-18T08:28:02.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:28:02.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,journaling mirror/image_14 2026-02-18T08:28:02.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ is_pool_mirror_mode_image mirror 2026-02-18T08:28:02.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:28:02.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' pool = image ']' 2026-02-18T08:28:02.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ case "${pool}" in 2026-02-18T08:28:02.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 1 2026-02-18T08:28:02.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n '' ']' 2026-02-18T08:28:02.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ stress_write_image cluster2 mirror image_14 2026-02-18T08:28:02.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:28:02.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:28:02.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_14 2026-02-18T08:28:02.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-18T08:28:02.086 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local duration=8 2026-02-18T08:28:02.086 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set +e 2026-02-18T08:28:02.086 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ timeout 8s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_14 --debug-rbd=20 --debug-journaler=20 2026-02-18T08:28:02.799 INFO:tasks.ceph.cluster1.osd.1.trial034.stderr:2026-02-18T08:28:02.795+0000 7f20d7266640 -1 reset not still connected to 0x5563fcd7b790 2026-02-18T08:28:02.799 INFO:tasks.ceph.cluster1.osd.1.trial034.stderr:2026-02-18T08:28:02.795+0000 7f20d7266640 -1 reset not still connected to 0x5563fcd7bee0 2026-02-18T08:28:02.800 INFO:tasks.ceph.cluster1.osd.1.trial034.stderr:2026-02-18T08:28:02.795+0000 7f20d7266640 -1 reset not still connected to 0x5563fd14cf70 2026-02-18T08:28:02.800 INFO:tasks.ceph.cluster1.osd.1.trial034.stderr:2026-02-18T08:28:02.795+0000 7f20d7266640 -1 reset not still connected to 0x5563fd730a90 2026-02-18T08:28:02.800 INFO:tasks.ceph.cluster1.osd.1.trial034.stderr:2026-02-18T08:28:02.795+0000 7f20d7266640 -1 reset not still connected to 0x5563fd7312b0 2026-02-18T08:28:02.800 INFO:tasks.ceph.cluster1.osd.1.trial034.stderr:2026-02-18T08:28:02.795+0000 7f20d7266640 -1 reset not still connected to 0x5563fe8d60d0 2026-02-18T08:28:03.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:100 IOs, 92103680 bytes 2026-02-18T08:28:03.839 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:200 IOs, 126929920 bytes 2026-02-18T08:28:06.021 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:300 IOs, 207261696 bytes 2026-02-18T08:28:08.084 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:28:08.079+0000 7faffc95c640 -1 reset not still connected to 0x5644c0efc5b0 2026-02-18T08:28:08.095 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:400 IOs, 289998848 bytes 2026-02-18T08:28:08.452 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:500 IOs, 298394624 bytes 2026-02-18T08:28:09.254 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:600 IOs, 358352896 bytes 2026-02-18T08:28:09.778 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:700 IOs, 386023424 bytes 2026-02-18T08:28:10.096 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ error_code=124 2026-02-18T08:28:10.097 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set -e 2026-02-18T08:28:10.097 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 124 -eq 124 ']' 2026-02-18T08:28:10.097 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:28:10.097 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T08:28:10.097 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_15 2026-02-18T08:28:10.097 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_image_and_enable_mirror cluster2 mirror image_15 journal 128M 2026-02-18T08:28:10.097 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:28:10.097 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:28:10.097 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:28:10.097 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:28:10.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_15 2026-02-18T08:28:10.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:28:10.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local mode=journal 2026-02-18T08:28:10.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n journal ']' 2026-02-18T08:28:10.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:28:10.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_image cluster2 mirror image_15 128M 2026-02-18T08:28:10.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:28:10.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:28:10.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:28:10.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:28:10.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_15 2026-02-18T08:28:10.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:28:10.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local size=128 2026-02-18T08:28:10.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n 128M ']' 2026-02-18T08:28:10.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ size=128M 2026-02-18T08:28:10.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:28:10.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,journaling mirror/image_15 2026-02-18T08:28:10.138 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ is_pool_mirror_mode_image mirror 2026-02-18T08:28:10.138 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:28:10.138 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' pool = image ']' 2026-02-18T08:28:10.138 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ case "${pool}" in 2026-02-18T08:28:10.138 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 1 2026-02-18T08:28:10.138 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n '' ']' 2026-02-18T08:28:10.138 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ stress_write_image cluster2 mirror image_15 2026-02-18T08:28:10.139 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:28:10.139 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:28:10.139 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_15 2026-02-18T08:28:10.139 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-18T08:28:10.139 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local duration=13 2026-02-18T08:28:10.140 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set +e 2026-02-18T08:28:10.140 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ timeout 13s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_15 --debug-rbd=20 --debug-journaler=20 2026-02-18T08:28:10.678 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:100 IOs, 16413696 bytes 2026-02-18T08:28:11.443 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:200 IOs, 59572224 bytes 2026-02-18T08:28:11.641 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:300 IOs, 95798272 bytes 2026-02-18T08:28:12.970 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:400 IOs, 124541952 bytes 2026-02-18T08:28:13.676 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:500 IOs, 188177408 bytes 2026-02-18T08:28:14.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:600 IOs, 216249344 bytes 2026-02-18T08:28:15.190 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:700 IOs, 276703232 bytes 2026-02-18T08:28:15.847 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:800 IOs, 312175616 bytes 2026-02-18T08:28:15.927 INFO:tasks.ceph.cluster1.osd.1.trial034.stderr:2026-02-18T08:28:15.922+0000 7f20d7266640 -1 reset not still connected to 0x5563fe2316c0 2026-02-18T08:28:15.927 INFO:tasks.ceph.cluster1.osd.1.trial034.stderr:2026-02-18T08:28:15.922+0000 7f20d7266640 -1 reset not still connected to 0x55640bcd1110 2026-02-18T08:28:15.927 INFO:tasks.ceph.cluster1.osd.1.trial034.stderr:2026-02-18T08:28:15.922+0000 7f20d7266640 -1 reset not still connected to 0x5564155c3d40 2026-02-18T08:28:15.927 INFO:tasks.ceph.cluster1.osd.1.trial034.stderr:2026-02-18T08:28:15.922+0000 7f20d7266640 -1 reset not still connected to 0x5564228bfd40 2026-02-18T08:28:16.489 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:900 IOs, 343237632 bytes 2026-02-18T08:28:16.837 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1000 IOs, 364215296 bytes 2026-02-18T08:28:17.524 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1100 IOs, 418564096 bytes 2026-02-18T08:28:17.541 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.0 with skip (0.8001697992777208) > weight (0.25) 2026-02-18T08:28:17.541 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.1 with skip (0.4212302865611395) > weight (0.25) 2026-02-18T08:28:17.541 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.2 with skip (0.6170958360445024) > weight (0.25) 2026-02-18T08:28:17.541 INFO:tasks.rbd_mirror_thrash:kill cluster2.client.mirror.3 2026-02-18T08:28:17.541 INFO:tasks.rbd_mirror.cluster2.client.mirror.3:Sent signal 15 2026-02-18T08:28:17.541 INFO:tasks.rbd_mirror_thrash:waiting for 0.4163697481442563 secs before reviving daemons 2026-02-18T08:28:17.542 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.trial163.stderr:2026-02-18T08:28:17.532+0000 7f7064bb5640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.3 (PID: 55452) UID: 1000 2026-02-18T08:28:17.958 INFO:tasks.rbd_mirror_thrash:waiting for cluster2.client.mirror.3 2026-02-18T08:28:17.959 INFO:teuthology.orchestra.run:waiting for 600 2026-02-18T08:28:17.959 INFO:tasks.rbd_mirror_thrash:reviving cluster2.client.mirror.3 2026-02-18T08:28:17.959 INFO:tasks.rbd_mirror.cluster2.client.mirror.3:Restarting daemon 2026-02-18T08:28:17.959 DEBUG:teuthology.orchestra.run.trial163:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.3 2026-02-18T08:28:17.995 INFO:tasks.rbd_mirror.cluster2.client.mirror.3:Started 2026-02-18T08:28:17.995 INFO:tasks.rbd_mirror_thrash:waiting for 101.11604417594492 secs before thrashing 2026-02-18T08:28:20.106 INFO:tasks.ceph.cluster1.osd.1.trial034.stderr:2026-02-18T08:28:20.102+0000 7f20d7266640 -1 reset not still connected to 0x556422af5ad0 2026-02-18T08:28:20.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1200 IOs, 546656256 bytes 2026-02-18T08:28:21.007 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1300 IOs, 606251008 bytes 2026-02-18T08:28:21.187 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1400 IOs, 614707200 bytes 2026-02-18T08:28:21.574 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1500 IOs, 636200960 bytes 2026-02-18T08:28:22.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1600 IOs, 719039488 bytes 2026-02-18T08:28:23.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ error_code=124 2026-02-18T08:28:23.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set -e 2026-02-18T08:28:23.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 124 -eq 124 ']' 2026-02-18T08:28:23.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:28:23.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T08:28:23.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_16 2026-02-18T08:28:23.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_image_and_enable_mirror cluster2 mirror image_16 journal 128M 2026-02-18T08:28:23.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:28:23.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:28:23.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:28:23.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:28:23.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_16 2026-02-18T08:28:23.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:28:23.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local mode=journal 2026-02-18T08:28:23.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n journal ']' 2026-02-18T08:28:23.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:28:23.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_image cluster2 mirror image_16 128M 2026-02-18T08:28:23.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:28:23.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:28:23.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:28:23.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:28:23.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_16 2026-02-18T08:28:23.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:28:23.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local size=128 2026-02-18T08:28:23.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n 128M ']' 2026-02-18T08:28:23.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ size=128M 2026-02-18T08:28:23.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:28:23.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,journaling mirror/image_16 2026-02-18T08:28:23.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ is_pool_mirror_mode_image mirror 2026-02-18T08:28:23.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:28:23.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' pool = image ']' 2026-02-18T08:28:23.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ case "${pool}" in 2026-02-18T08:28:23.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 1 2026-02-18T08:28:23.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n '' ']' 2026-02-18T08:28:23.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ stress_write_image cluster2 mirror image_16 2026-02-18T08:28:23.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:28:23.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:28:23.194 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_16 2026-02-18T08:28:23.194 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-18T08:28:23.194 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local duration=12 2026-02-18T08:28:23.194 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set +e 2026-02-18T08:28:23.194 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ timeout 12s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_16 --debug-rbd=20 --debug-journaler=20 2026-02-18T08:28:23.557 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:100 IOs, 17755136 bytes 2026-02-18T08:28:24.463 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:200 IOs, 93685760 bytes 2026-02-18T08:28:24.761 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:28:24.758+0000 7fb27e7f3640 -1 reset not still connected to 0x55e183c11860 2026-02-18T08:28:24.761 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:28:24.758+0000 7fb27e7f3640 -1 reset not still connected to 0x55e18895cea0 2026-02-18T08:28:24.947 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:28:24.942+0000 7fb27e7f3640 -1 reset not still connected to 0x55e178bde340 2026-02-18T08:28:24.948 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:28:24.942+0000 7fb27e7f3640 -1 reset not still connected to 0x55e183c11860 2026-02-18T08:28:24.948 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:28:24.942+0000 7fb27e7f3640 -1 reset not still connected to 0x55e18895cea0 2026-02-18T08:28:26.491 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:300 IOs, 178922496 bytes 2026-02-18T08:28:26.785 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:400 IOs, 195328000 bytes 2026-02-18T08:28:27.302 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:500 IOs, 259390464 bytes 2026-02-18T08:28:27.559 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:28:27.554+0000 7faffc95c640 -1 reset not still connected to 0x5644a6840410 2026-02-18T08:28:27.559 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:28:27.554+0000 7faffc95c640 -1 reset not still connected to 0x5644a84a6750 2026-02-18T08:28:27.560 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:28:27.554+0000 7faffc95c640 -1 reset not still connected to 0x5644a84d5930 2026-02-18T08:28:27.560 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:28:27.554+0000 7faffc95c640 -1 reset not still connected to 0x5644c1b595f0 2026-02-18T08:28:27.560 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:28:27.554+0000 7faffc95c640 -1 reset not still connected to 0x5644d9b0e410 2026-02-18T08:28:28.224 INFO:tasks.ceph.cluster1.osd.1.trial034.stderr:2026-02-18T08:28:28.218+0000 7f20d7266640 -1 reset not still connected to 0x5564228bf520 2026-02-18T08:28:28.224 INFO:tasks.ceph.cluster1.osd.1.trial034.stderr:2026-02-18T08:28:28.218+0000 7f20d7266640 -1 reset not still connected to 0x556422af5ad0 2026-02-18T08:28:28.390 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:600 IOs, 327535616 bytes 2026-02-18T08:28:28.566 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:700 IOs, 340705280 bytes 2026-02-18T08:28:28.693 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:800 IOs, 343557120 bytes 2026-02-18T08:28:29.321 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:28:29.318+0000 7faffc95c640 -1 reset not still connected to 0x5644a5073110 2026-02-18T08:28:29.351 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:900 IOs, 369252352 bytes 2026-02-18T08:28:29.691 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:28:29.686+0000 7faffc95c640 -1 reset not still connected to 0x5644c1b595f0 2026-02-18T08:28:29.692 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:28:29.686+0000 7faffc95c640 -1 reset not still connected to 0x5644d9b0e410 2026-02-18T08:28:30.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1000 IOs, 397714432 bytes 2026-02-18T08:28:30.249 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1100 IOs, 412291072 bytes 2026-02-18T08:28:31.209 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1200 IOs, 489846784 bytes 2026-02-18T08:28:32.056 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1300 IOs, 537590784 bytes 2026-02-18T08:28:32.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1400 IOs, 563277824 bytes 2026-02-18T08:28:32.845 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1500 IOs, 569003008 bytes 2026-02-18T08:28:32.914 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1600 IOs, 569606144 bytes 2026-02-18T08:28:33.493 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:28:33.490+0000 7fb27e7f3640 -1 reset not still connected to 0x55e170742000 2026-02-18T08:28:33.493 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:28:33.490+0000 7fb27e7f3640 -1 reset not still connected to 0x55e170c6e000 2026-02-18T08:28:33.493 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:28:33.490+0000 7fb27e7f3640 -1 reset not still connected to 0x55e171e68000 2026-02-18T08:28:33.494 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:28:33.490+0000 7fb27e7f3640 -1 reset not still connected to 0x55e171e69380 2026-02-18T08:28:33.494 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:28:33.490+0000 7fb27e7f3640 -1 reset not still connected to 0x55e183c10750 2026-02-18T08:28:33.494 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:28:33.490+0000 7fb27e7f3640 -1 reset not still connected to 0x55e18817ba00 2026-02-18T08:28:33.618 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:28:33.614+0000 7fb27e7f3640 -1 reset not still connected to 0x55e18817ba00 2026-02-18T08:28:33.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1700 IOs, 635425792 bytes 2026-02-18T08:28:34.287 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1800 IOs, 657281024 bytes 2026-02-18T08:28:34.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1900 IOs, 692603904 bytes 2026-02-18T08:28:35.204 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ error_code=124 2026-02-18T08:28:35.204 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set -e 2026-02-18T08:28:35.204 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 124 -eq 124 ']' 2026-02-18T08:28:35.204 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:28:35.204 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T08:28:35.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_17 2026-02-18T08:28:35.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_image_and_enable_mirror cluster2 mirror image_17 journal 128M 2026-02-18T08:28:35.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:28:35.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:28:35.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:28:35.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:28:35.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_17 2026-02-18T08:28:35.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:28:35.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local mode=journal 2026-02-18T08:28:35.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n journal ']' 2026-02-18T08:28:35.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:28:35.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_image cluster2 mirror image_17 128M 2026-02-18T08:28:35.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:28:35.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:28:35.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:28:35.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:28:35.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_17 2026-02-18T08:28:35.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:28:35.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local size=128 2026-02-18T08:28:35.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n 128M ']' 2026-02-18T08:28:35.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ size=128M 2026-02-18T08:28:35.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:28:35.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,journaling mirror/image_17 2026-02-18T08:28:35.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ is_pool_mirror_mode_image mirror 2026-02-18T08:28:35.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:28:35.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' pool = image ']' 2026-02-18T08:28:35.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ case "${pool}" in 2026-02-18T08:28:35.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 1 2026-02-18T08:28:35.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n '' ']' 2026-02-18T08:28:35.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ stress_write_image cluster2 mirror image_17 2026-02-18T08:28:35.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:28:35.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:28:35.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_17 2026-02-18T08:28:35.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-18T08:28:35.245 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local duration=14 2026-02-18T08:28:35.245 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set +e 2026-02-18T08:28:35.245 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ timeout 14s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_17 --debug-rbd=20 --debug-journaler=20 2026-02-18T08:28:35.266 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:28:35.262+0000 7faffc95c640 -1 reset not still connected to 0x5644a5073860 2026-02-18T08:28:35.266 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:28:35.262+0000 7faffc95c640 -1 reset not still connected to 0x5644a6840410 2026-02-18T08:28:35.267 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:28:35.262+0000 7faffc95c640 -1 reset not still connected to 0x5644a84a6750 2026-02-18T08:28:35.267 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:28:35.262+0000 7faffc95c640 -1 reset not still connected to 0x5644a84d5930 2026-02-18T08:28:35.268 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:28:35.262+0000 7faffc95c640 -1 reset not still connected to 0x5644c1b595f0 2026-02-18T08:28:35.268 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:28:35.262+0000 7faffc95c640 -1 reset not still connected to 0x5644d9b0e410 2026-02-18T08:28:35.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:100 IOs, 607232 bytes 2026-02-18T08:28:35.932 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:200 IOs, 36123648 bytes 2026-02-18T08:28:36.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:300 IOs, 77667328 bytes 2026-02-18T08:28:36.756 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:400 IOs, 96560128 bytes 2026-02-18T08:28:37.315 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:28:37.310+0000 7faffc95c640 -1 reset not still connected to 0x5644a5073110 2026-02-18T08:28:37.316 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:28:37.310+0000 7faffc95c640 -1 reset not still connected to 0x5644a84a61a0 2026-02-18T08:28:38.248 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:500 IOs, 141969408 bytes 2026-02-18T08:28:39.149 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:600 IOs, 203052032 bytes 2026-02-18T08:28:39.760 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.0 with skip (0.4129324970426994) > weight (0.25) 2026-02-18T08:28:39.760 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.1 with skip (0.7447076121014715) > weight (0.25) 2026-02-18T08:28:39.760 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.2 with skip (0.9338602713567169) > weight (0.25) 2026-02-18T08:28:39.760 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.3 with skip (0.9447798940337506) > weight (0.25) 2026-02-18T08:28:39.760 INFO:tasks.rbd_mirror_thrash:waiting for 72.99421895983514 secs before thrashing 2026-02-18T08:28:40.227 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:700 IOs, 268122112 bytes 2026-02-18T08:28:40.759 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:800 IOs, 292023296 bytes 2026-02-18T08:28:41.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:900 IOs, 305320960 bytes 2026-02-18T08:28:41.230 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:28:41.226+0000 7faffc95c640 -1 reset not still connected to 0x5644a6840410 2026-02-18T08:28:41.232 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:28:41.226+0000 7faffc95c640 -1 reset not still connected to 0x5644a84a6750 2026-02-18T08:28:41.232 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:28:41.230+0000 7faffc95c640 -1 reset not still connected to 0x5644a84d5930 2026-02-18T08:28:41.233 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:28:41.230+0000 7faffc95c640 -1 reset not still connected to 0x5644c1b595f0 2026-02-18T08:28:41.233 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:28:41.230+0000 7faffc95c640 -1 reset not still connected to 0x5644d9b0e410 2026-02-18T08:28:41.570 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1000 IOs, 328787968 bytes 2026-02-18T08:28:41.950 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1100 IOs, 347158528 bytes 2026-02-18T08:28:42.223 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1200 IOs, 347780096 bytes 2026-02-18T08:28:42.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1300 IOs, 382625792 bytes 2026-02-18T08:28:43.576 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1400 IOs, 427488256 bytes 2026-02-18T08:28:43.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1500 IOs, 454189056 bytes 2026-02-18T08:28:44.291 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1600 IOs, 463581184 bytes 2026-02-18T08:28:44.427 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1700 IOs, 474753024 bytes 2026-02-18T08:28:44.959 INFO:tasks.ceph.cluster1.osd.1.trial034.stderr:2026-02-18T08:28:44.954+0000 7f20d7266640 -1 reset not still connected to 0x5563fcd7b790 2026-02-18T08:28:44.960 INFO:tasks.ceph.cluster1.osd.1.trial034.stderr:2026-02-18T08:28:44.954+0000 7f20d7266640 -1 reset not still connected to 0x5563fcd7bee0 2026-02-18T08:28:44.960 INFO:tasks.ceph.cluster1.osd.1.trial034.stderr:2026-02-18T08:28:44.954+0000 7f20d7266640 -1 reset not still connected to 0x5563fd14cf70 2026-02-18T08:28:44.960 INFO:tasks.ceph.cluster1.osd.1.trial034.stderr:2026-02-18T08:28:44.954+0000 7f20d7266640 -1 reset not still connected to 0x5563fd730a90 2026-02-18T08:28:44.960 INFO:tasks.ceph.cluster1.osd.1.trial034.stderr:2026-02-18T08:28:44.954+0000 7f20d7266640 -1 reset not still connected to 0x5563fd7312b0 2026-02-18T08:28:44.960 INFO:tasks.ceph.cluster1.osd.1.trial034.stderr:2026-02-18T08:28:44.954+0000 7f20d7266640 -1 reset not still connected to 0x5563fe8d60d0 2026-02-18T08:28:44.960 INFO:tasks.ceph.cluster1.osd.1.trial034.stderr:2026-02-18T08:28:44.954+0000 7f20d7266640 -1 reset not still connected to 0x55641ff40dd0 2026-02-18T08:28:45.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1800 IOs, 516578304 bytes 2026-02-18T08:28:45.146 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:28:45.142+0000 7faffc95c640 -1 reset not still connected to 0x5644a84a6750 2026-02-18T08:28:45.147 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:28:45.142+0000 7faffc95c640 -1 reset not still connected to 0x5644c1b595f0 2026-02-18T08:28:45.147 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:28:45.142+0000 7faffc95c640 -1 reset not still connected to 0x5644d9b0e410 2026-02-18T08:28:46.208 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1900 IOs, 575569920 bytes 2026-02-18T08:28:46.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:2000 IOs, 591404032 bytes 2026-02-18T08:28:46.809 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:28:46.802+0000 7faffc95c640 -1 reset not still connected to 0x5644a5073860 2026-02-18T08:28:46.809 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:28:46.806+0000 7faffc95c640 -1 reset not still connected to 0x5644a6840410 2026-02-18T08:28:46.810 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:28:46.806+0000 7faffc95c640 -1 reset not still connected to 0x5644a84a6750 2026-02-18T08:28:46.810 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:28:46.806+0000 7faffc95c640 -1 reset not still connected to 0x5644a84d5930 2026-02-18T08:28:46.810 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:28:46.806+0000 7faffc95c640 -1 reset not still connected to 0x5644c1b595f0 2026-02-18T08:28:46.810 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:28:46.806+0000 7faffc95c640 -1 reset not still connected to 0x5644d9b0e410 2026-02-18T08:28:46.879 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:2100 IOs, 610120704 bytes 2026-02-18T08:28:47.795 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:2200 IOs, 661700608 bytes 2026-02-18T08:28:48.161 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:2300 IOs, 674925568 bytes 2026-02-18T08:28:48.907 INFO:tasks.ceph.cluster1.osd.1.trial034.stderr:2026-02-18T08:28:48.902+0000 7f20d7266640 -1 reset not still connected to 0x5563fe2316c0 2026-02-18T08:28:48.907 INFO:tasks.ceph.cluster1.osd.1.trial034.stderr:2026-02-18T08:28:48.902+0000 7f20d7266640 -1 reset not still connected to 0x55640bcd1110 2026-02-18T08:28:48.907 INFO:tasks.ceph.cluster1.osd.1.trial034.stderr:2026-02-18T08:28:48.902+0000 7f20d7266640 -1 reset not still connected to 0x5564155c3d40 2026-02-18T08:28:48.907 INFO:tasks.ceph.cluster1.osd.1.trial034.stderr:2026-02-18T08:28:48.902+0000 7f20d7266640 -1 reset not still connected to 0x5564228bfd40 2026-02-18T08:28:49.255 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ error_code=124 2026-02-18T08:28:49.255 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set -e 2026-02-18T08:28:49.256 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 124 -eq 124 ']' 2026-02-18T08:28:49.256 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:28:49.256 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T08:28:49.256 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_18 2026-02-18T08:28:49.256 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_image_and_enable_mirror cluster2 mirror image_18 journal 128M 2026-02-18T08:28:49.256 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:28:49.256 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:28:49.256 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:28:49.256 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:28:49.256 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_18 2026-02-18T08:28:49.256 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:28:49.256 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local mode=journal 2026-02-18T08:28:49.256 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n journal ']' 2026-02-18T08:28:49.256 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:28:49.256 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_image cluster2 mirror image_18 128M 2026-02-18T08:28:49.256 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:28:49.257 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:28:49.257 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:28:49.257 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:28:49.257 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_18 2026-02-18T08:28:49.257 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:28:49.257 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local size=128 2026-02-18T08:28:49.257 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n 128M ']' 2026-02-18T08:28:49.257 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ size=128M 2026-02-18T08:28:49.257 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:28:49.257 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,journaling mirror/image_18 2026-02-18T08:28:49.294 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ is_pool_mirror_mode_image mirror 2026-02-18T08:28:49.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:28:49.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' pool = image ']' 2026-02-18T08:28:49.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ case "${pool}" in 2026-02-18T08:28:49.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 1 2026-02-18T08:28:49.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n '' ']' 2026-02-18T08:28:49.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ stress_write_image cluster2 mirror image_18 2026-02-18T08:28:49.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:28:49.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:28:49.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_18 2026-02-18T08:28:49.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-18T08:28:49.297 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local duration=8 2026-02-18T08:28:49.297 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set +e 2026-02-18T08:28:49.297 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ timeout 8s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_18 --debug-rbd=20 --debug-journaler=20 2026-02-18T08:28:50.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:100 IOs, 38005760 bytes 2026-02-18T08:28:50.240 INFO:tasks.ceph.cluster1.osd.1.trial034.stderr:2026-02-18T08:28:50.234+0000 7f20d7266640 -1 reset not still connected to 0x5563fcd7b790 2026-02-18T08:28:50.240 INFO:tasks.ceph.cluster1.osd.1.trial034.stderr:2026-02-18T08:28:50.234+0000 7f20d7266640 -1 reset not still connected to 0x5563fcd7bee0 2026-02-18T08:28:50.241 INFO:tasks.ceph.cluster1.osd.1.trial034.stderr:2026-02-18T08:28:50.234+0000 7f20d7266640 -1 reset not still connected to 0x5563fd14cf70 2026-02-18T08:28:50.241 INFO:tasks.ceph.cluster1.osd.1.trial034.stderr:2026-02-18T08:28:50.234+0000 7f20d7266640 -1 reset not still connected to 0x5563fd730a90 2026-02-18T08:28:50.241 INFO:tasks.ceph.cluster1.osd.1.trial034.stderr:2026-02-18T08:28:50.234+0000 7f20d7266640 -1 reset not still connected to 0x5563fd7312b0 2026-02-18T08:28:50.241 INFO:tasks.ceph.cluster1.osd.1.trial034.stderr:2026-02-18T08:28:50.234+0000 7f20d7266640 -1 reset not still connected to 0x5563fe8d60d0 2026-02-18T08:28:50.241 INFO:tasks.ceph.cluster1.osd.1.trial034.stderr:2026-02-18T08:28:50.234+0000 7f20d7266640 -1 reset not still connected to 0x55641ff40dd0 2026-02-18T08:28:50.607 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:200 IOs, 79419392 bytes 2026-02-18T08:28:51.390 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:300 IOs, 120095744 bytes 2026-02-18T08:28:52.287 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:400 IOs, 146152448 bytes 2026-02-18T08:28:53.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:500 IOs, 174603264 bytes 2026-02-18T08:28:53.540 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:600 IOs, 199257088 bytes 2026-02-18T08:28:53.968 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:700 IOs, 223794176 bytes 2026-02-18T08:28:54.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:800 IOs, 238971904 bytes 2026-02-18T08:28:54.753 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:900 IOs, 284563456 bytes 2026-02-18T08:28:55.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1000 IOs, 301358080 bytes 2026-02-18T08:28:55.430 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1100 IOs, 321215488 bytes 2026-02-18T08:28:55.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1200 IOs, 359821312 bytes 2026-02-18T08:28:56.855 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:28:56.849+0000 7faffc95c640 -1 reset not still connected to 0x5644a6840410 2026-02-18T08:28:56.856 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:28:56.849+0000 7faffc95c640 -1 reset not still connected to 0x5644a84a6750 2026-02-18T08:28:56.857 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:28:56.853+0000 7faffc95c640 -1 reset not still connected to 0x5644a84d5930 2026-02-18T08:28:56.857 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:28:56.853+0000 7faffc95c640 -1 reset not still connected to 0x5644c1b595f0 2026-02-18T08:28:56.857 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:28:56.853+0000 7faffc95c640 -1 reset not still connected to 0x5644d9b0e410 2026-02-18T08:28:57.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ error_code=124 2026-02-18T08:28:57.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set -e 2026-02-18T08:28:57.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 124 -eq 124 ']' 2026-02-18T08:28:57.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:28:57.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T08:28:57.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_19 2026-02-18T08:28:57.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_image_and_enable_mirror cluster2 mirror image_19 journal 128M 2026-02-18T08:28:57.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:28:57.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:28:57.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:28:57.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:28:57.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_19 2026-02-18T08:28:57.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:28:57.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local mode=journal 2026-02-18T08:28:57.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n journal ']' 2026-02-18T08:28:57.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:28:57.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_image cluster2 mirror image_19 128M 2026-02-18T08:28:57.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:28:57.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:28:57.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:28:57.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:28:57.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_19 2026-02-18T08:28:57.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:28:57.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local size=128 2026-02-18T08:28:57.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n 128M ']' 2026-02-18T08:28:57.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ size=128M 2026-02-18T08:28:57.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:28:57.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,journaling mirror/image_19 2026-02-18T08:28:57.357 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ is_pool_mirror_mode_image mirror 2026-02-18T08:28:57.357 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:28:57.357 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' pool = image ']' 2026-02-18T08:28:57.357 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ case "${pool}" in 2026-02-18T08:28:57.357 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 1 2026-02-18T08:28:57.357 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n '' ']' 2026-02-18T08:28:57.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ stress_write_image cluster2 mirror image_19 2026-02-18T08:28:57.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:28:57.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:28:57.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_19 2026-02-18T08:28:57.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-18T08:28:57.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local duration=14 2026-02-18T08:28:57.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set +e 2026-02-18T08:28:57.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ timeout 14s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_19 --debug-rbd=20 --debug-journaler=20 2026-02-18T08:28:58.414 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:100 IOs, 54824960 bytes 2026-02-18T08:28:58.857 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:200 IOs, 91074560 bytes 2026-02-18T08:28:59.713 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:300 IOs, 157933568 bytes 2026-02-18T08:29:00.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:400 IOs, 219294720 bytes 2026-02-18T08:29:00.956 INFO:tasks.ceph.cluster1.osd.1.trial034.stderr:2026-02-18T08:29:00.949+0000 7f20d7266640 -1 reset not still connected to 0x5563fcd7b790 2026-02-18T08:29:00.956 INFO:tasks.ceph.cluster1.osd.1.trial034.stderr:2026-02-18T08:29:00.949+0000 7f20d7266640 -1 reset not still connected to 0x5563fcd7bee0 2026-02-18T08:29:00.957 INFO:tasks.ceph.cluster1.osd.1.trial034.stderr:2026-02-18T08:29:00.953+0000 7f20d7266640 -1 reset not still connected to 0x5563fd14cf70 2026-02-18T08:29:00.957 INFO:tasks.ceph.cluster1.osd.1.trial034.stderr:2026-02-18T08:29:00.953+0000 7f20d7266640 -1 reset not still connected to 0x5563fd730a90 2026-02-18T08:29:00.958 INFO:tasks.ceph.cluster1.osd.1.trial034.stderr:2026-02-18T08:29:00.953+0000 7f20d7266640 -1 reset not still connected to 0x5563fd7312b0 2026-02-18T08:29:00.958 INFO:tasks.ceph.cluster1.osd.1.trial034.stderr:2026-02-18T08:29:00.953+0000 7f20d7266640 -1 reset not still connected to 0x5563fe8d60d0 2026-02-18T08:29:00.958 INFO:tasks.ceph.cluster1.osd.1.trial034.stderr:2026-02-18T08:29:00.953+0000 7f20d7266640 -1 reset not still connected to 0x55641ff40dd0 2026-02-18T08:29:01.681 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:500 IOs, 256710656 bytes 2026-02-18T08:29:03.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:600 IOs, 363063296 bytes 2026-02-18T08:29:04.016 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:29:04.009+0000 7fb27e7f3640 -1 reset not still connected to 0x55e170742000 2026-02-18T08:29:04.018 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:29:04.013+0000 7fb27e7f3640 -1 reset not still connected to 0x55e170c6e000 2026-02-18T08:29:04.018 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:29:04.013+0000 7fb27e7f3640 -1 reset not still connected to 0x55e171e68000 2026-02-18T08:29:04.018 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:29:04.013+0000 7fb27e7f3640 -1 reset not still connected to 0x55e171e69380 2026-02-18T08:29:04.018 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:29:04.013+0000 7fb27e7f3640 -1 reset not still connected to 0x55e183c10750 2026-02-18T08:29:04.018 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:29:04.013+0000 7fb27e7f3640 -1 reset not still connected to 0x55e18817ba00 2026-02-18T08:29:04.019 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:29:04.013+0000 7fb27e7f3640 -1 reset not still connected to 0x55e1884eaf70 2026-02-18T08:29:04.671 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:700 IOs, 412569600 bytes 2026-02-18T08:29:05.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:800 IOs, 438121472 bytes 2026-02-18T08:29:05.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:900 IOs, 448423936 bytes 2026-02-18T08:29:05.771 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:29:05.765+0000 7fb27e7f3640 -1 reset not still connected to 0x55e1703c8a90 2026-02-18T08:29:06.026 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:29:06.021+0000 7fb27e7f3640 -1 reset not still connected to 0x55e18895cea0 2026-02-18T08:29:06.196 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1000 IOs, 492845056 bytes 2026-02-18T08:29:06.324 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:29:06.317+0000 7fb27e7f3640 -1 reset not still connected to 0x55e18895cea0 2026-02-18T08:29:06.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1100 IOs, 515187712 bytes 2026-02-18T08:29:07.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1200 IOs, 533138432 bytes 2026-02-18T08:29:07.884 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1300 IOs, 600457216 bytes 2026-02-18T08:29:08.216 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:29:08.209+0000 7faffc95c640 -1 reset not still connected to 0x5644a41fe5b0 2026-02-18T08:29:08.216 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:29:08.209+0000 7faffc95c640 -1 reset not still connected to 0x5644a5073110 2026-02-18T08:29:08.216 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:29:08.209+0000 7faffc95c640 -1 reset not still connected to 0x5644c58264e0 2026-02-18T08:29:08.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1400 IOs, 620906496 bytes 2026-02-18T08:29:09.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1500 IOs, 691837952 bytes 2026-02-18T08:29:09.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1600 IOs, 692474880 bytes 2026-02-18T08:29:09.311 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:29:09.305+0000 7fb27e7f3640 -1 reset not still connected to 0x55e1703c8b60 2026-02-18T08:29:09.311 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:29:09.305+0000 7fb27e7f3640 -1 reset not still connected to 0x55e17071e820 2026-02-18T08:29:09.311 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:29:09.305+0000 7fb27e7f3640 -1 reset not still connected to 0x55e178bde8f0 2026-02-18T08:29:09.810 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1700 IOs, 731563008 bytes 2026-02-18T08:29:09.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1800 IOs, 747854848 bytes 2026-02-18T08:29:10.173 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1900 IOs, 753772544 bytes 2026-02-18T08:29:11.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:2000 IOs, 810911744 bytes 2026-02-18T08:29:11.327 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:2100 IOs, 825024512 bytes 2026-02-18T08:29:11.373 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ error_code=124 2026-02-18T08:29:11.373 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set -e 2026-02-18T08:29:11.373 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 124 -eq 124 ']' 2026-02-18T08:29:11.373 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:29:11.373 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T08:29:11.373 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_20 2026-02-18T08:29:11.373 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_image_and_enable_mirror cluster2 mirror image_20 journal 128M 2026-02-18T08:29:11.373 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:29:11.374 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:29:11.374 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:29:11.374 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:29:11.374 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_20 2026-02-18T08:29:11.374 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:29:11.374 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local mode=journal 2026-02-18T08:29:11.374 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n journal ']' 2026-02-18T08:29:11.374 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:29:11.374 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_image cluster2 mirror image_20 128M 2026-02-18T08:29:11.374 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:29:11.374 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:29:11.374 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:29:11.374 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:29:11.375 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_20 2026-02-18T08:29:11.375 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:29:11.375 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local size=128 2026-02-18T08:29:11.375 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n 128M ']' 2026-02-18T08:29:11.375 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ size=128M 2026-02-18T08:29:11.375 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:29:11.375 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,journaling mirror/image_20 2026-02-18T08:29:11.415 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ is_pool_mirror_mode_image mirror 2026-02-18T08:29:11.415 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:29:11.415 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' pool = image ']' 2026-02-18T08:29:11.415 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ case "${pool}" in 2026-02-18T08:29:11.415 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 1 2026-02-18T08:29:11.415 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n '' ']' 2026-02-18T08:29:11.416 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ stress_write_image cluster2 mirror image_20 2026-02-18T08:29:11.416 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:29:11.416 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:29:11.416 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_20 2026-02-18T08:29:11.416 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-18T08:29:11.417 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local duration=11 2026-02-18T08:29:11.417 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set +e 2026-02-18T08:29:11.417 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ timeout 11s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_20 --debug-rbd=20 --debug-journaler=20 2026-02-18T08:29:11.732 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:29:11.725+0000 7fb27e7f3640 -1 reset not still connected to 0x55e1703c8b60 2026-02-18T08:29:11.732 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:29:11.725+0000 7fb27e7f3640 -1 reset not still connected to 0x55e17071e820 2026-02-18T08:29:11.732 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:29:11.725+0000 7fb27e7f3640 -1 reset not still connected to 0x55e178bde8f0 2026-02-18T08:29:12.451 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:100 IOs, 31752192 bytes 2026-02-18T08:29:13.179 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:200 IOs, 68670464 bytes 2026-02-18T08:29:13.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:300 IOs, 99943424 bytes 2026-02-18T08:29:13.931 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:29:13.925+0000 7faffc95c640 -1 reset not still connected to 0x5644a84a61a0 2026-02-18T08:29:13.931 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:29:13.925+0000 7faffc95c640 -1 reset not still connected to 0x5644c58264e0 2026-02-18T08:29:15.007 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:400 IOs, 122021888 bytes 2026-02-18T08:29:15.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:500 IOs, 159306752 bytes 2026-02-18T08:29:16.623 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:600 IOs, 230303744 bytes 2026-02-18T08:29:17.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:700 IOs, 274214912 bytes 2026-02-18T08:29:17.634 INFO:tasks.ceph.cluster1.osd.1.trial034.stderr:2026-02-18T08:29:17.629+0000 7f20d7266640 -1 reset not still connected to 0x5563fcd7bee0 2026-02-18T08:29:17.635 INFO:tasks.ceph.cluster1.osd.1.trial034.stderr:2026-02-18T08:29:17.629+0000 7f20d7266640 -1 reset not still connected to 0x5563fd14cf70 2026-02-18T08:29:17.636 INFO:tasks.ceph.cluster1.osd.1.trial034.stderr:2026-02-18T08:29:17.629+0000 7f20d7266640 -1 reset not still connected to 0x5563fd730a90 2026-02-18T08:29:17.636 INFO:tasks.ceph.cluster1.osd.1.trial034.stderr:2026-02-18T08:29:17.629+0000 7f20d7266640 -1 reset not still connected to 0x5563fd7312b0 2026-02-18T08:29:17.636 INFO:tasks.ceph.cluster1.osd.1.trial034.stderr:2026-02-18T08:29:17.629+0000 7f20d7266640 -1 reset not still connected to 0x5563fe8d60d0 2026-02-18T08:29:17.637 INFO:tasks.ceph.cluster1.osd.1.trial034.stderr:2026-02-18T08:29:17.629+0000 7f20d7266640 -1 reset not still connected to 0x55641ff40dd0 2026-02-18T08:29:18.272 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:29:18.265+0000 7fb27e7f3640 -1 reset not still connected to 0x55e18817ba00 2026-02-18T08:29:18.392 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:29:18.385+0000 7fb27e7f3640 -1 reset not still connected to 0x55e170742000 2026-02-18T08:29:18.392 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:29:18.385+0000 7fb27e7f3640 -1 reset not still connected to 0x55e170c6e000 2026-02-18T08:29:18.393 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:29:18.385+0000 7fb27e7f3640 -1 reset not still connected to 0x55e171e68000 2026-02-18T08:29:18.393 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:29:18.385+0000 7fb27e7f3640 -1 reset not still connected to 0x55e171e69380 2026-02-18T08:29:18.393 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:29:18.385+0000 7fb27e7f3640 -1 reset not still connected to 0x55e183c10750 2026-02-18T08:29:18.393 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:29:18.385+0000 7fb27e7f3640 -1 reset not still connected to 0x55e18817ba00 2026-02-18T08:29:18.393 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:29:18.389+0000 7fb27e7f3640 -1 reset not still connected to 0x55e1884eaf70 2026-02-18T08:29:18.415 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:800 IOs, 341013504 bytes 2026-02-18T08:29:18.512 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:900 IOs, 341628928 bytes 2026-02-18T08:29:19.176 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1000 IOs, 375395328 bytes 2026-02-18T08:29:19.511 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1100 IOs, 389427200 bytes 2026-02-18T08:29:19.829 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1200 IOs, 428867584 bytes 2026-02-18T08:29:20.706 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1300 IOs, 470076416 bytes 2026-02-18T08:29:21.691 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1400 IOs, 521444352 bytes 2026-02-18T08:29:22.431 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ error_code=124 2026-02-18T08:29:22.431 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set -e 2026-02-18T08:29:22.432 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 124 -eq 124 ']' 2026-02-18T08:29:22.432 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:29:22.432 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T08:29:22.432 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_21 2026-02-18T08:29:22.432 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_image_and_enable_mirror cluster2 mirror image_21 journal 128M 2026-02-18T08:29:22.432 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:29:22.432 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:29:22.432 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:29:22.432 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:29:22.432 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_21 2026-02-18T08:29:22.432 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:29:22.433 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local mode=journal 2026-02-18T08:29:22.433 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n journal ']' 2026-02-18T08:29:22.433 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:29:22.433 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_image cluster2 mirror image_21 128M 2026-02-18T08:29:22.433 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:29:22.433 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:29:22.433 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:29:22.433 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:29:22.433 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_21 2026-02-18T08:29:22.433 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:29:22.433 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local size=128 2026-02-18T08:29:22.433 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n 128M ']' 2026-02-18T08:29:22.434 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ size=128M 2026-02-18T08:29:22.434 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:29:22.434 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,journaling mirror/image_21 2026-02-18T08:29:22.481 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ is_pool_mirror_mode_image mirror 2026-02-18T08:29:22.481 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:29:22.481 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' pool = image ']' 2026-02-18T08:29:22.481 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ case "${pool}" in 2026-02-18T08:29:22.481 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 1 2026-02-18T08:29:22.481 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n '' ']' 2026-02-18T08:29:22.481 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ stress_write_image cluster2 mirror image_21 2026-02-18T08:29:22.481 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:29:22.481 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:29:22.482 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_21 2026-02-18T08:29:22.482 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-18T08:29:22.483 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local duration=5 2026-02-18T08:29:22.483 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set +e 2026-02-18T08:29:22.483 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ timeout 5s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_21 --debug-rbd=20 --debug-journaler=20 2026-02-18T08:29:22.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:100 IOs, 618496 bytes 2026-02-18T08:29:22.760 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:200 IOs, 6797312 bytes 2026-02-18T08:29:23.175 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:300 IOs, 48787456 bytes 2026-02-18T08:29:24.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:400 IOs, 71037952 bytes 2026-02-18T08:29:25.194 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:500 IOs, 143630336 bytes 2026-02-18T08:29:25.462 INFO:tasks.ceph.cluster1.osd.1.trial034.stderr:2026-02-18T08:29:25.457+0000 7f20d7266640 -1 reset not still connected to 0x5563fcd931e0 2026-02-18T08:29:25.462 INFO:tasks.ceph.cluster1.osd.1.trial034.stderr:2026-02-18T08:29:25.457+0000 7f20d7266640 -1 reset not still connected to 0x5563fe231520 2026-02-18T08:29:25.462 INFO:tasks.ceph.cluster1.osd.1.trial034.stderr:2026-02-18T08:29:25.457+0000 7f20d7266640 -1 reset not still connected to 0x556404cc84e0 2026-02-18T08:29:25.462 INFO:tasks.ceph.cluster1.osd.1.trial034.stderr:2026-02-18T08:29:25.457+0000 7f20d7266640 -1 reset not still connected to 0x5564228bf520 2026-02-18T08:29:25.462 INFO:tasks.ceph.cluster1.osd.1.trial034.stderr:2026-02-18T08:29:25.457+0000 7f20d7266640 -1 reset not still connected to 0x556422af5ad0 2026-02-18T08:29:26.019 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:600 IOs, 174761984 bytes 2026-02-18T08:29:26.489 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:29:26.485+0000 7faffc95c640 -1 reset not still connected to 0x5644c1b595f0 2026-02-18T08:29:26.489 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:29:26.485+0000 7faffc95c640 -1 reset not still connected to 0x5644d9b0e410 2026-02-18T08:29:27.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:700 IOs, 262048768 bytes 2026-02-18T08:29:27.492 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ error_code=124 2026-02-18T08:29:27.493 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set -e 2026-02-18T08:29:27.493 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 124 -eq 124 ']' 2026-02-18T08:29:27.493 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:29:27.493 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T08:29:27.493 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_22 2026-02-18T08:29:27.493 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_image_and_enable_mirror cluster2 mirror image_22 journal 128M 2026-02-18T08:29:27.493 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:29:27.493 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:29:27.493 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:29:27.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:29:27.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_22 2026-02-18T08:29:27.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:29:27.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local mode=journal 2026-02-18T08:29:27.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n journal ']' 2026-02-18T08:29:27.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:29:27.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_image cluster2 mirror image_22 128M 2026-02-18T08:29:27.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:29:27.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:29:27.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:29:27.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:29:27.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_22 2026-02-18T08:29:27.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:29:27.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local size=128 2026-02-18T08:29:27.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n 128M ']' 2026-02-18T08:29:27.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ size=128M 2026-02-18T08:29:27.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:29:27.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,journaling mirror/image_22 2026-02-18T08:29:27.532 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ is_pool_mirror_mode_image mirror 2026-02-18T08:29:27.532 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:29:27.532 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' pool = image ']' 2026-02-18T08:29:27.532 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ case "${pool}" in 2026-02-18T08:29:27.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 1 2026-02-18T08:29:27.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n '' ']' 2026-02-18T08:29:27.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ stress_write_image cluster2 mirror image_22 2026-02-18T08:29:27.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:29:27.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:29:27.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_22 2026-02-18T08:29:27.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-18T08:29:27.534 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local duration=12 2026-02-18T08:29:27.534 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set +e 2026-02-18T08:29:27.534 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ timeout 12s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_22 --debug-rbd=20 --debug-journaler=20 2026-02-18T08:29:27.718 INFO:tasks.ceph.cluster1.osd.1.trial034.stderr:2026-02-18T08:29:27.713+0000 7f20d7266640 -1 reset not still connected to 0x5563fd14cf70 2026-02-18T08:29:27.719 INFO:tasks.ceph.cluster1.osd.1.trial034.stderr:2026-02-18T08:29:27.713+0000 7f20d7266640 -1 reset not still connected to 0x5563fd14d930 2026-02-18T08:29:27.719 INFO:tasks.ceph.cluster1.osd.1.trial034.stderr:2026-02-18T08:29:27.713+0000 7f20d7266640 -1 reset not still connected to 0x5563fd730a90 2026-02-18T08:29:27.719 INFO:tasks.ceph.cluster1.osd.1.trial034.stderr:2026-02-18T08:29:27.713+0000 7f20d7266640 -1 reset not still connected to 0x5563fd7312b0 2026-02-18T08:29:27.719 INFO:tasks.ceph.cluster1.osd.1.trial034.stderr:2026-02-18T08:29:27.713+0000 7f20d7266640 -1 reset not still connected to 0x5563fe8d60d0 2026-02-18T08:29:27.720 INFO:tasks.ceph.cluster1.osd.1.trial034.stderr:2026-02-18T08:29:27.713+0000 7f20d7266640 -1 reset not still connected to 0x55641ff40dd0 2026-02-18T08:29:28.896 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:100 IOs, 79859712 bytes 2026-02-18T08:29:29.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:200 IOs, 128316416 bytes 2026-02-18T08:29:30.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:300 IOs, 137455616 bytes 2026-02-18T08:29:30.884 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:400 IOs, 184964096 bytes 2026-02-18T08:29:31.351 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:500 IOs, 211289088 bytes 2026-02-18T08:29:32.353 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:600 IOs, 256126976 bytes 2026-02-18T08:29:32.604 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:700 IOs, 272052224 bytes 2026-02-18T08:29:33.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:800 IOs, 308148224 bytes 2026-02-18T08:29:33.647 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:900 IOs, 341600256 bytes 2026-02-18T08:29:34.324 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1000 IOs, 372052992 bytes 2026-02-18T08:29:34.883 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1100 IOs, 382799872 bytes 2026-02-18T08:29:35.782 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1200 IOs, 425768960 bytes 2026-02-18T08:29:36.173 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:29:36.168+0000 7faffc95c640 -1 reset not still connected to 0x5644a0dcbad0 2026-02-18T08:29:36.174 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:29:36.168+0000 7faffc95c640 -1 reset not still connected to 0x5644a41fe5b0 2026-02-18T08:29:36.174 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:29:36.168+0000 7faffc95c640 -1 reset not still connected to 0x5644a5073110 2026-02-18T08:29:36.174 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:29:36.168+0000 7faffc95c640 -1 reset not still connected to 0x5644a84a61a0 2026-02-18T08:29:36.175 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:29:36.168+0000 7faffc95c640 -1 reset not still connected to 0x5644c58264e0 2026-02-18T08:29:36.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1300 IOs, 480977920 bytes 2026-02-18T08:29:36.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1400 IOs, 493634560 bytes 2026-02-18T08:29:37.635 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1500 IOs, 571595776 bytes 2026-02-18T08:29:38.058 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:29:38.052+0000 7fb27e7f3640 -1 reset not still connected to 0x55e170c6e000 2026-02-18T08:29:38.059 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:29:38.052+0000 7fb27e7f3640 -1 reset not still connected to 0x55e171e68000 2026-02-18T08:29:38.059 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:29:38.052+0000 7fb27e7f3640 -1 reset not still connected to 0x55e171e69380 2026-02-18T08:29:38.059 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:29:38.052+0000 7fb27e7f3640 -1 reset not still connected to 0x55e183c10750 2026-02-18T08:29:38.059 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:29:38.052+0000 7fb27e7f3640 -1 reset not still connected to 0x55e18817ba00 2026-02-18T08:29:38.059 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:29:38.052+0000 7fb27e7f3640 -1 reset not still connected to 0x55e1884eaf70 2026-02-18T08:29:38.147 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1600 IOs, 629084160 bytes 2026-02-18T08:29:38.597 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:29:38.592+0000 7faffc95c640 -1 reset not still connected to 0x5644c58264e0 2026-02-18T08:29:38.808 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1700 IOs, 669822976 bytes 2026-02-18T08:29:39.468 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:29:39.460+0000 7fb27e7f3640 -1 reset not still connected to 0x55e178bde340 2026-02-18T08:29:39.468 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:29:39.460+0000 7fb27e7f3640 -1 reset not still connected to 0x55e183c11860 2026-02-18T08:29:39.468 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:29:39.460+0000 7fb27e7f3640 -1 reset not still connected to 0x55e18895cea0 2026-02-18T08:29:39.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ error_code=124 2026-02-18T08:29:39.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set -e 2026-02-18T08:29:39.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 124 -eq 124 ']' 2026-02-18T08:29:39.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:29:39.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T08:29:39.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_23 2026-02-18T08:29:39.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_image_and_enable_mirror cluster2 mirror image_23 journal 128M 2026-02-18T08:29:39.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:29:39.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:29:39.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:29:39.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:29:39.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_23 2026-02-18T08:29:39.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:29:39.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local mode=journal 2026-02-18T08:29:39.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n journal ']' 2026-02-18T08:29:39.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:29:39.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_image cluster2 mirror image_23 128M 2026-02-18T08:29:39.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:29:39.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:29:39.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:29:39.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:29:39.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_23 2026-02-18T08:29:39.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:29:39.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local size=128 2026-02-18T08:29:39.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n 128M ']' 2026-02-18T08:29:39.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ size=128M 2026-02-18T08:29:39.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:29:39.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,journaling mirror/image_23 2026-02-18T08:29:39.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ is_pool_mirror_mode_image mirror 2026-02-18T08:29:39.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:29:39.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' pool = image ']' 2026-02-18T08:29:39.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ case "${pool}" in 2026-02-18T08:29:39.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 1 2026-02-18T08:29:39.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n '' ']' 2026-02-18T08:29:39.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ stress_write_image cluster2 mirror image_23 2026-02-18T08:29:39.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:29:39.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:29:39.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_23 2026-02-18T08:29:39.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-18T08:29:39.594 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local duration=14 2026-02-18T08:29:39.594 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set +e 2026-02-18T08:29:39.594 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ timeout 14s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_23 --debug-rbd=20 --debug-journaler=20 2026-02-18T08:29:40.606 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:100 IOs, 53853184 bytes 2026-02-18T08:29:40.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:200 IOs, 60954624 bytes 2026-02-18T08:29:42.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:300 IOs, 158645248 bytes 2026-02-18T08:29:42.661 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:29:42.656+0000 7fb27e7f3640 -1 reset not still connected to 0x55e171c0b860 2026-02-18T08:29:42.661 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:29:42.656+0000 7fb27e7f3640 -1 reset not still connected to 0x55e188bae0d0 2026-02-18T08:29:43.804 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:400 IOs, 196783104 bytes 2026-02-18T08:29:44.388 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:500 IOs, 230922240 bytes 2026-02-18T08:29:45.110 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:29:45.104+0000 7fb27e7f3640 -1 reset not still connected to 0x55e170c6e000 2026-02-18T08:29:45.111 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:29:45.104+0000 7fb27e7f3640 -1 reset not still connected to 0x55e171e68000 2026-02-18T08:29:45.111 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:29:45.104+0000 7fb27e7f3640 -1 reset not still connected to 0x55e171e69380 2026-02-18T08:29:45.111 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:29:45.104+0000 7fb27e7f3640 -1 reset not still connected to 0x55e18817ba00 2026-02-18T08:29:45.529 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:600 IOs, 294646784 bytes 2026-02-18T08:29:46.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:700 IOs, 345691136 bytes 2026-02-18T08:29:47.235 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:29:47.228+0000 7fb27e7f3640 -1 reset not still connected to 0x55e183c11860 2026-02-18T08:29:47.235 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:29:47.228+0000 7fb27e7f3640 -1 reset not still connected to 0x55e18895cea0 2026-02-18T08:29:47.857 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:800 IOs, 428741632 bytes 2026-02-18T08:29:47.972 INFO:tasks.ceph.cluster1.osd.1.trial034.stderr:2026-02-18T08:29:47.964+0000 7f20d7266640 -1 reset not still connected to 0x55641ff40dd0 2026-02-18T08:29:48.302 INFO:tasks.ceph.cluster1.osd.1.trial034.stderr:2026-02-18T08:29:48.296+0000 7f20d7266640 -1 reset not still connected to 0x5563fc068340 2026-02-18T08:29:48.302 INFO:tasks.ceph.cluster1.osd.1.trial034.stderr:2026-02-18T08:29:48.296+0000 7f20d7266640 -1 reset not still connected to 0x5563fcd7b790 2026-02-18T08:29:48.302 INFO:tasks.ceph.cluster1.osd.1.trial034.stderr:2026-02-18T08:29:48.296+0000 7f20d7266640 -1 reset not still connected to 0x5563fd730a90 2026-02-18T08:29:48.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:900 IOs, 458393600 bytes 2026-02-18T08:29:49.173 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:29:49.168+0000 7fb27e7f3640 -1 reset not still connected to 0x55e170742000 2026-02-18T08:29:49.174 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:29:49.168+0000 7fb27e7f3640 -1 reset not still connected to 0x55e170c6e000 2026-02-18T08:29:49.175 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:29:49.168+0000 7fb27e7f3640 -1 reset not still connected to 0x55e171e68000 2026-02-18T08:29:49.176 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:29:49.168+0000 7fb27e7f3640 -1 reset not still connected to 0x55e171e69380 2026-02-18T08:29:49.176 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:29:49.168+0000 7fb27e7f3640 -1 reset not still connected to 0x55e183c10750 2026-02-18T08:29:49.176 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:29:49.168+0000 7fb27e7f3640 -1 reset not still connected to 0x55e18817ba00 2026-02-18T08:29:49.176 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:29:49.168+0000 7fb27e7f3640 -1 reset not still connected to 0x55e1884eaf70 2026-02-18T08:29:49.535 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1000 IOs, 508571648 bytes 2026-02-18T08:29:49.862 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1100 IOs, 522725376 bytes 2026-02-18T08:29:50.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1200 IOs, 547257344 bytes 2026-02-18T08:29:50.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1300 IOs, 579675136 bytes 2026-02-18T08:29:51.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1400 IOs, 596165632 bytes 2026-02-18T08:29:51.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1500 IOs, 612978688 bytes 2026-02-18T08:29:52.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1600 IOs, 635636736 bytes 2026-02-18T08:29:52.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1700 IOs, 636217344 bytes 2026-02-18T08:29:52.430 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1800 IOs, 654517248 bytes 2026-02-18T08:29:52.713 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1900 IOs, 680385536 bytes 2026-02-18T08:29:52.755 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.0 with skip (0.7145604450250664) > weight (0.25) 2026-02-18T08:29:52.755 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.1 with skip (0.7535550883038434) > weight (0.25) 2026-02-18T08:29:52.755 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.2 with skip (0.37333172810885884) > weight (0.25) 2026-02-18T08:29:52.755 INFO:tasks.rbd_mirror_thrash:kill cluster1.client.mirror.3 2026-02-18T08:29:52.756 INFO:tasks.rbd_mirror.cluster1.client.mirror.3:Sent signal 15 2026-02-18T08:29:52.756 INFO:tasks.rbd_mirror_thrash:waiting for 4.488201070602516 secs before reviving daemons 2026-02-18T08:29:52.756 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial163.stderr:2026-02-18T08:29:52.746+0000 7fa56ee5d640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.3 (PID: 14950) UID: 1000 2026-02-18T08:29:53.493 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:2000 IOs, 736192512 bytes 2026-02-18T08:29:53.605 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ error_code=124 2026-02-18T08:29:53.605 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set -e 2026-02-18T08:29:53.605 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 124 -eq 124 ']' 2026-02-18T08:29:53.605 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:29:53.605 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T08:29:53.605 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_24 2026-02-18T08:29:53.606 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_image_and_enable_mirror cluster2 mirror image_24 journal 128M 2026-02-18T08:29:53.606 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:29:53.606 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:29:53.606 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:29:53.606 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:29:53.606 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_24 2026-02-18T08:29:53.606 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:29:53.606 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local mode=journal 2026-02-18T08:29:53.606 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n journal ']' 2026-02-18T08:29:53.606 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:29:53.606 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_image cluster2 mirror image_24 128M 2026-02-18T08:29:53.607 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:29:53.607 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:29:53.607 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:29:53.607 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:29:53.607 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_24 2026-02-18T08:29:53.607 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:29:53.607 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local size=128 2026-02-18T08:29:53.607 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n 128M ']' 2026-02-18T08:29:53.607 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ size=128M 2026-02-18T08:29:53.607 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:29:53.607 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,journaling mirror/image_24 2026-02-18T08:29:53.648 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ is_pool_mirror_mode_image mirror 2026-02-18T08:29:53.648 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:29:53.648 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' pool = image ']' 2026-02-18T08:29:53.648 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ case "${pool}" in 2026-02-18T08:29:53.648 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 1 2026-02-18T08:29:53.648 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n '' ']' 2026-02-18T08:29:53.648 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ stress_write_image cluster2 mirror image_24 2026-02-18T08:29:53.649 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:29:53.649 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:29:53.649 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_24 2026-02-18T08:29:53.649 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-18T08:29:53.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local duration=11 2026-02-18T08:29:53.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set +e 2026-02-18T08:29:53.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ timeout 11s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_24 --debug-rbd=20 --debug-journaler=20 2026-02-18T08:29:53.672 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:29:53.664+0000 7faffc95c640 -1 reset not still connected to 0x5644a0dcb5f0 2026-02-18T08:29:53.673 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:29:53.668+0000 7faffc95c640 -1 reset not still connected to 0x5644a0dcbad0 2026-02-18T08:29:53.674 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:29:53.668+0000 7faffc95c640 -1 reset not still connected to 0x5644a41fe5b0 2026-02-18T08:29:53.675 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:29:53.668+0000 7faffc95c640 -1 reset not still connected to 0x5644a4424270 2026-02-18T08:29:53.675 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:29:53.668+0000 7faffc95c640 -1 reset not still connected to 0x5644a5073110 2026-02-18T08:29:53.675 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:29:53.668+0000 7faffc95c640 -1 reset not still connected to 0x5644a84a61a0 2026-02-18T08:29:53.675 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:29:53.668+0000 7faffc95c640 -1 reset not still connected to 0x5644c58264e0 2026-02-18T08:29:53.675 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:29:53.668+0000 7faffc95c640 -1 reset not still connected to 0x5644dfda1520 2026-02-18T08:29:53.900 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:29:53.892+0000 7fb27e7f3640 -1 reset not still connected to 0x55e170742000 2026-02-18T08:29:53.900 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:29:53.892+0000 7fb27e7f3640 -1 reset not still connected to 0x55e170c6e000 2026-02-18T08:29:53.900 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:29:53.892+0000 7fb27e7f3640 -1 reset not still connected to 0x55e171e68000 2026-02-18T08:29:53.901 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:29:53.892+0000 7fb27e7f3640 -1 reset not still connected to 0x55e171e69380 2026-02-18T08:29:53.901 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:29:53.896+0000 7fb27e7f3640 -1 reset not still connected to 0x55e183c10750 2026-02-18T08:29:53.901 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:29:53.896+0000 7fb27e7f3640 -1 reset not still connected to 0x55e18817ba00 2026-02-18T08:29:53.902 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:29:53.896+0000 7fb27e7f3640 -1 reset not still connected to 0x55e1884eaf70 2026-02-18T08:29:53.988 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:100 IOs, 14328832 bytes 2026-02-18T08:29:54.809 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:200 IOs, 64276480 bytes 2026-02-18T08:29:55.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:300 IOs, 98194432 bytes 2026-02-18T08:29:55.439 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:400 IOs, 107011072 bytes 2026-02-18T08:29:55.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:500 IOs, 127467520 bytes 2026-02-18T08:29:57.244 INFO:tasks.rbd_mirror_thrash:waiting for cluster1.client.mirror.3 2026-02-18T08:29:57.245 INFO:teuthology.orchestra.run:waiting for 600 2026-02-18T08:29:57.245 INFO:tasks.rbd_mirror_thrash:reviving cluster1.client.mirror.3 2026-02-18T08:29:57.245 INFO:tasks.rbd_mirror.cluster1.client.mirror.3:Restarting daemon 2026-02-18T08:29:57.245 DEBUG:teuthology.orchestra.run.trial163:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.3 2026-02-18T08:29:57.247 INFO:tasks.rbd_mirror.cluster1.client.mirror.3:Started 2026-02-18T08:29:57.247 INFO:tasks.rbd_mirror_thrash:waiting for 93.96133226583265 secs before thrashing 2026-02-18T08:29:57.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:600 IOs, 199997440 bytes 2026-02-18T08:29:58.574 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:700 IOs, 257183744 bytes 2026-02-18T08:29:59.112 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.0 with skip (0.38104098246641793) > weight (0.25) 2026-02-18T08:29:59.113 INFO:tasks.rbd_mirror_thrash:kill cluster2.client.mirror.1 2026-02-18T08:29:59.113 INFO:tasks.rbd_mirror.cluster2.client.mirror.1:Sent signal 15 2026-02-18T08:29:59.113 INFO:tasks.rbd_mirror_thrash:waiting for 5.287667536314698 secs before reviving daemons 2026-02-18T08:29:59.114 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.trial163.stderr:2026-02-18T08:29:59.106+0000 7f7823af0640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.1 (PID: 44681) UID: 1000 2026-02-18T08:29:59.878 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:800 IOs, 369597440 bytes 2026-02-18T08:30:01.282 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:900 IOs, 433006592 bytes 2026-02-18T08:30:01.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1000 IOs, 433647616 bytes 2026-02-18T08:30:01.955 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1100 IOs, 451813376 bytes 2026-02-18T08:30:02.514 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1200 IOs, 462947328 bytes 2026-02-18T08:30:02.713 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1300 IOs, 500412416 bytes 2026-02-18T08:30:03.596 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1400 IOs, 514215936 bytes 2026-02-18T08:30:04.401 INFO:tasks.rbd_mirror_thrash:waiting for cluster2.client.mirror.1 2026-02-18T08:30:04.402 INFO:teuthology.orchestra.run:waiting for 600 2026-02-18T08:30:04.402 INFO:tasks.rbd_mirror_thrash:reviving cluster2.client.mirror.1 2026-02-18T08:30:04.402 INFO:tasks.rbd_mirror.cluster2.client.mirror.1:Restarting daemon 2026-02-18T08:30:04.402 DEBUG:teuthology.orchestra.run.trial163:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.1 2026-02-18T08:30:04.404 INFO:tasks.rbd_mirror.cluster2.client.mirror.1:Started 2026-02-18T08:30:04.404 INFO:tasks.rbd_mirror_thrash:waiting for 83.92236501627818 secs before thrashing 2026-02-18T08:30:04.661 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ error_code=124 2026-02-18T08:30:04.661 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set -e 2026-02-18T08:30:04.661 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 124 -eq 124 ']' 2026-02-18T08:30:04.661 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:30:04.661 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T08:30:04.662 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_25 2026-02-18T08:30:04.662 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_image_and_enable_mirror cluster2 mirror image_25 journal 128M 2026-02-18T08:30:04.662 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:30:04.662 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:30:04.662 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:30:04.662 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:30:04.662 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_25 2026-02-18T08:30:04.662 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:30:04.663 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local mode=journal 2026-02-18T08:30:04.663 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n journal ']' 2026-02-18T08:30:04.663 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:30:04.663 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_image cluster2 mirror image_25 128M 2026-02-18T08:30:04.663 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:30:04.663 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:30:04.663 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:30:04.663 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:30:04.664 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_25 2026-02-18T08:30:04.664 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:30:04.664 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local size=128 2026-02-18T08:30:04.664 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n 128M ']' 2026-02-18T08:30:04.664 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ size=128M 2026-02-18T08:30:04.664 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:30:04.664 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,journaling mirror/image_25 2026-02-18T08:30:04.700 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ is_pool_mirror_mode_image mirror 2026-02-18T08:30:04.700 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:30:04.700 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' pool = image ']' 2026-02-18T08:30:04.701 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ case "${pool}" in 2026-02-18T08:30:04.701 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 1 2026-02-18T08:30:04.701 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n '' ']' 2026-02-18T08:30:04.701 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ stress_write_image cluster2 mirror image_25 2026-02-18T08:30:04.701 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:30:04.701 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:30:04.701 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_25 2026-02-18T08:30:04.701 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-18T08:30:04.702 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local duration=8 2026-02-18T08:30:04.702 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set +e 2026-02-18T08:30:04.703 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ timeout 8s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_25 --debug-rbd=20 --debug-journaler=20 2026-02-18T08:30:05.343 INFO:tasks.ceph.cluster1.osd.1.trial034.stderr:2026-02-18T08:30:05.336+0000 7f20d7266640 -1 reset not still connected to 0x556404cc84e0 2026-02-18T08:30:05.343 INFO:tasks.ceph.cluster1.osd.1.trial034.stderr:2026-02-18T08:30:05.336+0000 7f20d7266640 -1 reset not still connected to 0x5564228bf520 2026-02-18T08:30:05.343 INFO:tasks.ceph.cluster1.osd.1.trial034.stderr:2026-02-18T08:30:05.336+0000 7f20d7266640 -1 reset not still connected to 0x556422af5ad0 2026-02-18T08:30:05.667 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:100 IOs, 56442880 bytes 2026-02-18T08:30:06.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:200 IOs, 110146560 bytes 2026-02-18T08:30:06.871 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:30:06.864+0000 7fb27e7f3640 -1 reset not still connected to 0x55e170742000 2026-02-18T08:30:06.872 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:30:06.864+0000 7fb27e7f3640 -1 reset not still connected to 0x55e170c6e000 2026-02-18T08:30:06.872 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:30:06.864+0000 7fb27e7f3640 -1 reset not still connected to 0x55e171e68000 2026-02-18T08:30:06.873 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:30:06.864+0000 7fb27e7f3640 -1 reset not still connected to 0x55e171e69380 2026-02-18T08:30:06.873 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:30:06.864+0000 7fb27e7f3640 -1 reset not still connected to 0x55e183c10750 2026-02-18T08:30:06.873 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:30:06.864+0000 7fb27e7f3640 -1 reset not still connected to 0x55e18817ba00 2026-02-18T08:30:06.873 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:30:06.868+0000 7fb27e7f3640 -1 reset not still connected to 0x55e1884eaf70 2026-02-18T08:30:07.410 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:30:07.404+0000 7faffc95c640 -1 reset not still connected to 0x5644a41fe5b0 2026-02-18T08:30:07.411 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:30:07.404+0000 7faffc95c640 -1 reset not still connected to 0x5644a5073110 2026-02-18T08:30:08.316 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:300 IOs, 166544384 bytes 2026-02-18T08:30:09.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:400 IOs, 227638272 bytes 2026-02-18T08:30:10.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:500 IOs, 295298048 bytes 2026-02-18T08:30:10.999 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:600 IOs, 314597376 bytes 2026-02-18T08:30:12.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:700 IOs, 391703552 bytes 2026-02-18T08:30:12.507 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:800 IOs, 407558144 bytes 2026-02-18T08:30:12.712 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ error_code=124 2026-02-18T08:30:12.712 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set -e 2026-02-18T08:30:12.712 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 124 -eq 124 ']' 2026-02-18T08:30:12.712 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:30:12.712 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T08:30:12.712 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_26 2026-02-18T08:30:12.713 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_image_and_enable_mirror cluster2 mirror image_26 journal 128M 2026-02-18T08:30:12.713 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:30:12.713 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:30:12.713 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:30:12.713 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:30:12.713 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_26 2026-02-18T08:30:12.713 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:30:12.713 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local mode=journal 2026-02-18T08:30:12.713 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n journal ']' 2026-02-18T08:30:12.713 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:30:12.713 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_image cluster2 mirror image_26 128M 2026-02-18T08:30:12.713 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:30:12.714 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:30:12.714 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:30:12.714 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:30:12.714 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_26 2026-02-18T08:30:12.714 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:30:12.714 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local size=128 2026-02-18T08:30:12.714 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n 128M ']' 2026-02-18T08:30:12.714 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ size=128M 2026-02-18T08:30:12.714 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:30:12.714 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,journaling mirror/image_26 2026-02-18T08:30:12.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ is_pool_mirror_mode_image mirror 2026-02-18T08:30:12.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:30:12.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' pool = image ']' 2026-02-18T08:30:12.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ case "${pool}" in 2026-02-18T08:30:12.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 1 2026-02-18T08:30:12.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n '' ']' 2026-02-18T08:30:12.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ stress_write_image cluster2 mirror image_26 2026-02-18T08:30:12.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:30:12.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:30:12.758 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_26 2026-02-18T08:30:12.758 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-18T08:30:12.758 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local duration=7 2026-02-18T08:30:12.758 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set +e 2026-02-18T08:30:12.759 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ timeout 7s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_26 --debug-rbd=20 --debug-journaler=20 2026-02-18T08:30:13.373 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:100 IOs, 38577152 bytes 2026-02-18T08:30:14.101 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:200 IOs, 95488000 bytes 2026-02-18T08:30:14.774 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:300 IOs, 171394048 bytes 2026-02-18T08:30:16.290 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:400 IOs, 212720640 bytes 2026-02-18T08:30:16.463 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:500 IOs, 218037248 bytes 2026-02-18T08:30:16.955 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:30:16.951+0000 7fb27e7f3640 -1 reset not still connected to 0x55e170742000 2026-02-18T08:30:16.956 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:30:16.951+0000 7fb27e7f3640 -1 reset not still connected to 0x55e170c6e000 2026-02-18T08:30:16.956 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:30:16.951+0000 7fb27e7f3640 -1 reset not still connected to 0x55e171e68000 2026-02-18T08:30:16.957 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:30:16.951+0000 7fb27e7f3640 -1 reset not still connected to 0x55e171e69380 2026-02-18T08:30:16.957 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:30:16.951+0000 7fb27e7f3640 -1 reset not still connected to 0x55e183c10750 2026-02-18T08:30:16.957 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:30:16.951+0000 7fb27e7f3640 -1 reset not still connected to 0x55e18817ba00 2026-02-18T08:30:16.957 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:30:16.955+0000 7fb27e7f3640 -1 reset not still connected to 0x55e1884eaf70 2026-02-18T08:30:17.703 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:600 IOs, 272396288 bytes 2026-02-18T08:30:18.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:700 IOs, 297555968 bytes 2026-02-18T08:30:18.600 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:800 IOs, 325994496 bytes 2026-02-18T08:30:19.482 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:900 IOs, 377330688 bytes 2026-02-18T08:30:19.769 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ error_code=124 2026-02-18T08:30:19.769 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set -e 2026-02-18T08:30:19.770 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 124 -eq 124 ']' 2026-02-18T08:30:19.770 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:30:19.770 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T08:30:19.770 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_27 2026-02-18T08:30:19.770 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_image_and_enable_mirror cluster2 mirror image_27 journal 128M 2026-02-18T08:30:19.770 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:30:19.770 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:30:19.770 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:30:19.770 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:30:19.770 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_27 2026-02-18T08:30:19.771 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:30:19.771 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local mode=journal 2026-02-18T08:30:19.771 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n journal ']' 2026-02-18T08:30:19.771 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:30:19.771 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_image cluster2 mirror image_27 128M 2026-02-18T08:30:19.771 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:30:19.771 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:30:19.771 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:30:19.771 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:30:19.771 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_27 2026-02-18T08:30:19.771 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:30:19.772 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local size=128 2026-02-18T08:30:19.772 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n 128M ']' 2026-02-18T08:30:19.772 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ size=128M 2026-02-18T08:30:19.772 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:30:19.772 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,journaling mirror/image_27 2026-02-18T08:30:19.848 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ is_pool_mirror_mode_image mirror 2026-02-18T08:30:19.848 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:30:19.848 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' pool = image ']' 2026-02-18T08:30:19.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ case "${pool}" in 2026-02-18T08:30:19.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 1 2026-02-18T08:30:19.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n '' ']' 2026-02-18T08:30:19.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ stress_write_image cluster2 mirror image_27 2026-02-18T08:30:19.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:30:19.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:30:19.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_27 2026-02-18T08:30:19.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-18T08:30:19.850 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local duration=14 2026-02-18T08:30:19.850 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set +e 2026-02-18T08:30:19.850 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ timeout 14s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_27 --debug-rbd=20 --debug-journaler=20 2026-02-18T08:30:20.352 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:100 IOs, 17688576 bytes 2026-02-18T08:30:21.075 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:200 IOs, 51589120 bytes 2026-02-18T08:30:21.418 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:300 IOs, 87360512 bytes 2026-02-18T08:30:23.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:400 IOs, 127216640 bytes 2026-02-18T08:30:23.678 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:500 IOs, 134618112 bytes 2026-02-18T08:30:25.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:600 IOs, 219927552 bytes 2026-02-18T08:30:25.462 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:30:25.459+0000 7fb27e7f3640 -1 reset not still connected to 0x55e1703c8a90 2026-02-18T08:30:25.463 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:30:25.459+0000 7fb27e7f3640 -1 reset not still connected to 0x55e1705e44e0 2026-02-18T08:30:25.464 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:30:25.459+0000 7fb27e7f3640 -1 reset not still connected to 0x55e170742000 2026-02-18T08:30:25.464 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:30:25.459+0000 7fb27e7f3640 -1 reset not still connected to 0x55e170c6e000 2026-02-18T08:30:25.464 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:30:25.459+0000 7fb27e7f3640 -1 reset not still connected to 0x55e171e68000 2026-02-18T08:30:25.465 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:30:25.459+0000 7fb27e7f3640 -1 reset not still connected to 0x55e183c10750 2026-02-18T08:30:25.466 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:30:25.463+0000 7fb27e7f3640 -1 reset not still connected to 0x55e18817ba00 2026-02-18T08:30:25.466 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:30:25.463+0000 7fb27e7f3640 -1 reset not still connected to 0x55e1884eaf70 2026-02-18T08:30:25.609 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:30:25.607+0000 7faffc95c640 -1 reset not still connected to 0x5644c58269c0 2026-02-18T08:30:25.610 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:30:25.607+0000 7faffc95c640 -1 reset not still connected to 0x5644c8fcf790 2026-02-18T08:30:25.610 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:30:25.607+0000 7faffc95c640 -1 reset not still connected to 0x5644d9b0e9c0 2026-02-18T08:30:25.964 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:700 IOs, 276562944 bytes 2026-02-18T08:30:26.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:800 IOs, 296847360 bytes 2026-02-18T08:30:26.844 INFO:tasks.ceph.cluster1.osd.1.trial034.stderr:2026-02-18T08:30:26.839+0000 7f20d7266640 -1 reset not still connected to 0x556404cc84e0 2026-02-18T08:30:26.844 INFO:tasks.ceph.cluster1.osd.1.trial034.stderr:2026-02-18T08:30:26.839+0000 7f20d7266640 -1 reset not still connected to 0x5564228bf520 2026-02-18T08:30:26.844 INFO:tasks.ceph.cluster1.osd.1.trial034.stderr:2026-02-18T08:30:26.839+0000 7f20d7266640 -1 reset not still connected to 0x556422af5ad0 2026-02-18T08:30:27.431 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:900 IOs, 369301504 bytes 2026-02-18T08:30:27.879 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1000 IOs, 390370304 bytes 2026-02-18T08:30:28.458 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1100 IOs, 424247296 bytes 2026-02-18T08:30:28.839 INFO:tasks.ceph.cluster1.osd.1.trial034.stderr:2026-02-18T08:30:28.835+0000 7f20d7266640 -1 reset not still connected to 0x5564155c29c0 2026-02-18T08:30:29.360 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1200 IOs, 477715456 bytes 2026-02-18T08:30:29.743 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:30:29.739+0000 7fb27e7f3640 -1 reset not still connected to 0x55e18895cea0 2026-02-18T08:30:29.839 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:30:29.835+0000 7faffc95c640 -1 reset not still connected to 0x5644a5073860 2026-02-18T08:30:29.839 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:30:29.835+0000 7faffc95c640 -1 reset not still connected to 0x5644a6840410 2026-02-18T08:30:29.839 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:30:29.835+0000 7faffc95c640 -1 reset not still connected to 0x5644a716a410 2026-02-18T08:30:29.839 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:30:29.835+0000 7faffc95c640 -1 reset not still connected to 0x5644a84a6750 2026-02-18T08:30:29.840 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:30:29.835+0000 7faffc95c640 -1 reset not still connected to 0x5644a84d5930 2026-02-18T08:30:29.840 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:30:29.835+0000 7faffc95c640 -1 reset not still connected to 0x5644c1b595f0 2026-02-18T08:30:29.840 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:30:29.835+0000 7faffc95c640 -1 reset not still connected to 0x5644c8fcf380 2026-02-18T08:30:29.840 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:30:29.835+0000 7faffc95c640 -1 reset not still connected to 0x5644d9b0e410 2026-02-18T08:30:30.521 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:30:30.515+0000 7fb27e7f3640 -1 reset not still connected to 0x55e170c6e8f0 2026-02-18T08:30:30.522 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:30:30.519+0000 7fb27e7f3640 -1 reset not still connected to 0x55e178bde340 2026-02-18T08:30:30.524 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:30:30.519+0000 7fb27e7f3640 -1 reset not still connected to 0x55e183c11860 2026-02-18T08:30:30.524 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:30:30.519+0000 7fb27e7f3640 -1 reset not still connected to 0x55e18895cea0 2026-02-18T08:30:31.043 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:30:31.039+0000 7faffc95c640 -1 reset not still connected to 0x5644c0efc340 2026-02-18T08:30:31.044 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:30:31.039+0000 7faffc95c640 -1 reset not still connected to 0x5644c58269c0 2026-02-18T08:30:31.044 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:30:31.039+0000 7faffc95c640 -1 reset not still connected to 0x5644c8fcf790 2026-02-18T08:30:31.044 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:30:31.039+0000 7faffc95c640 -1 reset not still connected to 0x5644d9b0e9c0 2026-02-18T08:30:31.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1300 IOs, 558676992 bytes 2026-02-18T08:30:31.485 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1400 IOs, 578407424 bytes 2026-02-18T08:30:31.660 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1500 IOs, 586286080 bytes 2026-02-18T08:30:31.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1600 IOs, 602373120 bytes 2026-02-18T08:30:33.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1700 IOs, 656180224 bytes 2026-02-18T08:30:33.543 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:30:33.539+0000 7faffc95c640 -1 reset not still connected to 0x5644d9b0e9c0 2026-02-18T08:30:33.678 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1800 IOs, 708180992 bytes 2026-02-18T08:30:33.862 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ error_code=124 2026-02-18T08:30:33.863 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set -e 2026-02-18T08:30:33.863 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 124 -eq 124 ']' 2026-02-18T08:30:33.863 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:30:33.863 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T08:30:33.863 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_28 2026-02-18T08:30:33.863 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_image_and_enable_mirror cluster2 mirror image_28 journal 128M 2026-02-18T08:30:33.863 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:30:33.863 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:30:33.863 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:30:33.863 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:30:33.864 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_28 2026-02-18T08:30:33.864 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:30:33.864 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local mode=journal 2026-02-18T08:30:33.864 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n journal ']' 2026-02-18T08:30:33.864 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:30:33.864 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_image cluster2 mirror image_28 128M 2026-02-18T08:30:33.864 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:30:33.864 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:30:33.864 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:30:33.864 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:30:33.864 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_28 2026-02-18T08:30:33.864 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:30:33.865 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local size=128 2026-02-18T08:30:33.865 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n 128M ']' 2026-02-18T08:30:33.865 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ size=128M 2026-02-18T08:30:33.865 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:30:33.865 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,journaling mirror/image_28 2026-02-18T08:30:33.930 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ is_pool_mirror_mode_image mirror 2026-02-18T08:30:33.931 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:30:33.931 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' pool = image ']' 2026-02-18T08:30:33.931 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ case "${pool}" in 2026-02-18T08:30:33.931 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 1 2026-02-18T08:30:33.931 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n '' ']' 2026-02-18T08:30:33.931 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ stress_write_image cluster2 mirror image_28 2026-02-18T08:30:33.931 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:30:33.931 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:30:33.931 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_28 2026-02-18T08:30:33.931 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-18T08:30:33.933 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local duration=9 2026-02-18T08:30:33.933 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set +e 2026-02-18T08:30:33.933 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ timeout 9s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_28 --debug-rbd=20 --debug-journaler=20 2026-02-18T08:30:34.194 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:100 IOs, 4895744 bytes 2026-02-18T08:30:35.003 INFO:tasks.ceph.cluster1.osd.1.trial034.stderr:2026-02-18T08:30:34.999+0000 7f20d7266640 -1 reset not still connected to 0x5563fd7308f0 2026-02-18T08:30:35.003 INFO:tasks.ceph.cluster1.osd.1.trial034.stderr:2026-02-18T08:30:34.999+0000 7f20d7266640 -1 reset not still connected to 0x5563fd731450 2026-02-18T08:30:35.004 INFO:tasks.ceph.cluster1.osd.1.trial034.stderr:2026-02-18T08:30:34.999+0000 7f20d7266640 -1 reset not still connected to 0x5563fe2316c0 2026-02-18T08:30:35.004 INFO:tasks.ceph.cluster1.osd.1.trial034.stderr:2026-02-18T08:30:34.999+0000 7f20d7266640 -1 reset not still connected to 0x5563ff09dd40 2026-02-18T08:30:35.004 INFO:tasks.ceph.cluster1.osd.1.trial034.stderr:2026-02-18T08:30:34.999+0000 7f20d7266640 -1 reset not still connected to 0x55640bcd1110 2026-02-18T08:30:35.004 INFO:tasks.ceph.cluster1.osd.1.trial034.stderr:2026-02-18T08:30:34.999+0000 7f20d7266640 -1 reset not still connected to 0x5564228bfd40 2026-02-18T08:30:35.484 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:200 IOs, 56750080 bytes 2026-02-18T08:30:36.587 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:30:36.583+0000 7fb27e7f3640 -1 reset not still connected to 0x55e170742000 2026-02-18T08:30:36.589 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:30:36.583+0000 7fb27e7f3640 -1 reset not still connected to 0x55e170c6e000 2026-02-18T08:30:36.589 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:30:36.583+0000 7fb27e7f3640 -1 reset not still connected to 0x55e171e68000 2026-02-18T08:30:36.589 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:30:36.583+0000 7fb27e7f3640 -1 reset not still connected to 0x55e171e69380 2026-02-18T08:30:36.589 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:30:36.587+0000 7fb27e7f3640 -1 reset not still connected to 0x55e183c10750 2026-02-18T08:30:36.589 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:30:36.587+0000 7fb27e7f3640 -1 reset not still connected to 0x55e18817ba00 2026-02-18T08:30:36.589 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:30:36.587+0000 7fb27e7f3640 -1 reset not still connected to 0x55e1884eaf70 2026-02-18T08:30:37.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:300 IOs, 144885760 bytes 2026-02-18T08:30:37.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:400 IOs, 150272000 bytes 2026-02-18T08:30:38.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:500 IOs, 175528960 bytes 2026-02-18T08:30:38.719 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:30:38.715+0000 7fb27e7f3640 -1 reset not still connected to 0x55e18817ba00 2026-02-18T08:30:38.818 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:600 IOs, 204724224 bytes 2026-02-18T08:30:39.866 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:700 IOs, 299799552 bytes 2026-02-18T08:30:40.777 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:800 IOs, 346122240 bytes 2026-02-18T08:30:40.834 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:30:40.831+0000 7fb27e7f3640 -1 reset not still connected to 0x55e170742000 2026-02-18T08:30:40.834 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:30:40.831+0000 7fb27e7f3640 -1 reset not still connected to 0x55e170c6e000 2026-02-18T08:30:40.834 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:30:40.831+0000 7fb27e7f3640 -1 reset not still connected to 0x55e171e68000 2026-02-18T08:30:40.835 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:30:40.831+0000 7fb27e7f3640 -1 reset not still connected to 0x55e171e69380 2026-02-18T08:30:40.835 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:30:40.831+0000 7fb27e7f3640 -1 reset not still connected to 0x55e183c10750 2026-02-18T08:30:40.835 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:30:40.831+0000 7fb27e7f3640 -1 reset not still connected to 0x55e18817ba00 2026-02-18T08:30:40.835 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:30:40.831+0000 7fb27e7f3640 -1 reset not still connected to 0x55e1884eaf70 2026-02-18T08:30:41.203 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:30:41.199+0000 7fb27e7f3640 -1 reset not still connected to 0x55e18895cea0 2026-02-18T08:30:41.208 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:30:41.203+0000 7fb27e7f3640 -1 reset not still connected to 0x55e178bde340 2026-02-18T08:30:41.208 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:30:41.203+0000 7fb27e7f3640 -1 reset not still connected to 0x55e18895cea0 2026-02-18T08:30:41.800 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:900 IOs, 430042112 bytes 2026-02-18T08:30:42.841 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:30:42.835+0000 7fb27e7f3640 -1 reset not still connected to 0x55e178bde820 2026-02-18T08:30:42.841 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:30:42.839+0000 7fb27e7f3640 -1 reset not still connected to 0x55e179629ba0 2026-02-18T08:30:42.841 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:30:42.839+0000 7fb27e7f3640 -1 reset not still connected to 0x55e183c10f70 2026-02-18T08:30:42.842 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:30:42.839+0000 7fb27e7f3640 -1 reset not still connected to 0x55e188baf860 2026-02-18T08:30:42.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ error_code=124 2026-02-18T08:30:42.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set -e 2026-02-18T08:30:42.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 124 -eq 124 ']' 2026-02-18T08:30:42.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:30:42.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T08:30:42.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_29 2026-02-18T08:30:42.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_image_and_enable_mirror cluster2 mirror image_29 journal 128M 2026-02-18T08:30:42.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:30:42.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:30:42.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:30:42.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:30:42.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_29 2026-02-18T08:30:42.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:30:42.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local mode=journal 2026-02-18T08:30:42.944 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n journal ']' 2026-02-18T08:30:42.944 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:30:42.944 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_image cluster2 mirror image_29 128M 2026-02-18T08:30:42.944 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:30:42.944 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:30:42.944 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:30:42.944 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:30:42.944 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_29 2026-02-18T08:30:42.944 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:30:42.944 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local size=128 2026-02-18T08:30:42.944 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n 128M ']' 2026-02-18T08:30:42.944 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ size=128M 2026-02-18T08:30:42.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:30:42.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,journaling mirror/image_29 2026-02-18T08:30:42.986 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ is_pool_mirror_mode_image mirror 2026-02-18T08:30:42.986 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:30:42.986 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' pool = image ']' 2026-02-18T08:30:42.986 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ case "${pool}" in 2026-02-18T08:30:42.986 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 1 2026-02-18T08:30:42.986 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n '' ']' 2026-02-18T08:30:42.986 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ stress_write_image cluster2 mirror image_29 2026-02-18T08:30:42.987 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:30:42.987 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:30:42.987 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_29 2026-02-18T08:30:42.987 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-18T08:30:42.988 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local duration=8 2026-02-18T08:30:42.988 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set +e 2026-02-18T08:30:42.988 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ timeout 8s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_29 --debug-rbd=20 --debug-journaler=20 2026-02-18T08:30:43.109 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:30:43.103+0000 7fb27e7f3640 -1 reset not still connected to 0x55e178bde820 2026-02-18T08:30:43.110 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:30:43.107+0000 7fb27e7f3640 -1 reset not still connected to 0x55e179629ba0 2026-02-18T08:30:43.110 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:30:43.107+0000 7fb27e7f3640 -1 reset not still connected to 0x55e183c10f70 2026-02-18T08:30:43.110 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:30:43.107+0000 7fb27e7f3640 -1 reset not still connected to 0x55e188baf860 2026-02-18T08:30:43.290 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:30:43.287+0000 7faffc95c640 -1 reset not still connected to 0x5644a3cb4410 2026-02-18T08:30:43.290 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:30:43.287+0000 7faffc95c640 -1 reset not still connected to 0x5644a44256c0 2026-02-18T08:30:43.290 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:30:43.287+0000 7faffc95c640 -1 reset not still connected to 0x5644a716ab60 2026-02-18T08:30:43.290 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:30:43.287+0000 7faffc95c640 -1 reset not still connected to 0x5644c19b4340 2026-02-18T08:30:43.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:100 IOs, 26924032 bytes 2026-02-18T08:30:43.587 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:30:43.583+0000 7fb27e7f3640 -1 reset not still connected to 0x55e170c6e000 2026-02-18T08:30:43.589 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:30:43.587+0000 7fb27e7f3640 -1 reset not still connected to 0x55e183c10750 2026-02-18T08:30:43.589 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:30:43.587+0000 7fb27e7f3640 -1 reset not still connected to 0x55e18817ba00 2026-02-18T08:30:43.591 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:30:43.587+0000 7fb27e7f3640 -1 reset not still connected to 0x55e1884eaf70 2026-02-18T08:30:44.760 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:30:44.755+0000 7fb27e7f3640 -1 reset not still connected to 0x55e170c6e000 2026-02-18T08:30:44.760 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:30:44.755+0000 7fb27e7f3640 -1 reset not still connected to 0x55e171e68000 2026-02-18T08:30:44.761 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:30:44.759+0000 7fb27e7f3640 -1 reset not still connected to 0x55e171e69380 2026-02-18T08:30:44.761 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:30:44.759+0000 7fb27e7f3640 -1 reset not still connected to 0x55e18817ba00 2026-02-18T08:30:44.891 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:30:44.887+0000 7fb27e7f3640 -1 reset not still connected to 0x55e171c0b860 2026-02-18T08:30:44.891 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:30:44.887+0000 7fb27e7f3640 -1 reset not still connected to 0x55e188bae0d0 2026-02-18T08:30:45.069 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:200 IOs, 112207872 bytes 2026-02-18T08:30:45.266 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:30:45.263+0000 7fb27e7f3640 -1 reset not still connected to 0x55e171e69c70 2026-02-18T08:30:45.266 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:30:45.263+0000 7fb27e7f3640 -1 reset not still connected to 0x55e178bde820 2026-02-18T08:30:45.266 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:30:45.263+0000 7fb27e7f3640 -1 reset not still connected to 0x55e179629ba0 2026-02-18T08:30:45.266 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:30:45.263+0000 7fb27e7f3640 -1 reset not still connected to 0x55e183c10f70 2026-02-18T08:30:45.267 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:30:45.263+0000 7fb27e7f3640 -1 reset not still connected to 0x55e188baf860 2026-02-18T08:30:46.123 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:30:46.119+0000 7fb27e7f3640 -1 reset not still connected to 0x55e170c6e8f0 2026-02-18T08:30:46.123 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:30:46.119+0000 7fb27e7f3640 -1 reset not still connected to 0x55e171bf05b0 2026-02-18T08:30:46.124 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:30:46.119+0000 7fb27e7f3640 -1 reset not still connected to 0x55e171c0a750 2026-02-18T08:30:46.124 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:30:46.119+0000 7fb27e7f3640 -1 reset not still connected to 0x55e178bde340 2026-02-18T08:30:46.124 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:30:46.119+0000 7fb27e7f3640 -1 reset not still connected to 0x55e183c11860 2026-02-18T08:30:46.125 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:30:46.119+0000 7fb27e7f3640 -1 reset not still connected to 0x55e18895cea0 2026-02-18T08:30:46.125 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:30:46.123+0000 7fb27e7f3640 -1 reset not still connected to 0x55e1b0646000 2026-02-18T08:30:47.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:300 IOs, 232252416 bytes 2026-02-18T08:30:48.027 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:30:48.023+0000 7fb27e7f3640 -1 reset not still connected to 0x55e170ebfd40 2026-02-18T08:30:48.028 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:30:48.023+0000 7fb27e7f3640 -1 reset not still connected to 0x55e171e69c70 2026-02-18T08:30:48.028 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:30:48.023+0000 7fb27e7f3640 -1 reset not still connected to 0x55e178bde820 2026-02-18T08:30:48.029 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:30:48.023+0000 7fb27e7f3640 -1 reset not still connected to 0x55e179629ba0 2026-02-18T08:30:48.029 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:30:48.023+0000 7fb27e7f3640 -1 reset not still connected to 0x55e183c10f70 2026-02-18T08:30:48.029 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:30:48.023+0000 7fb27e7f3640 -1 reset not still connected to 0x55e188baf860 2026-02-18T08:30:48.338 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:30:48.335+0000 7fb27e7f3640 -1 reset not still connected to 0x55e170ebfd40 2026-02-18T08:30:48.338 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:30:48.335+0000 7fb27e7f3640 -1 reset not still connected to 0x55e171e69c70 2026-02-18T08:30:48.338 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:30:48.335+0000 7fb27e7f3640 -1 reset not still connected to 0x55e178bde820 2026-02-18T08:30:48.338 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:30:48.335+0000 7fb27e7f3640 -1 reset not still connected to 0x55e179629ba0 2026-02-18T08:30:48.338 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:30:48.335+0000 7fb27e7f3640 -1 reset not still connected to 0x55e183c10f70 2026-02-18T08:30:48.338 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:30:48.335+0000 7fb27e7f3640 -1 reset not still connected to 0x55e188baf860 2026-02-18T08:30:48.508 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:30:48.503+0000 7fb27e7f3640 -1 reset not still connected to 0x55e178bde340 2026-02-18T08:30:48.508 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:30:48.503+0000 7fb27e7f3640 -1 reset not still connected to 0x55e183c11860 2026-02-18T08:30:48.508 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:30:48.503+0000 7fb27e7f3640 -1 reset not still connected to 0x55e18895cea0 2026-02-18T08:30:48.622 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:400 IOs, 298074112 bytes 2026-02-18T08:30:49.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:500 IOs, 309555200 bytes 2026-02-18T08:30:50.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:600 IOs, 374714368 bytes 2026-02-18T08:30:50.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:700 IOs, 427387904 bytes 2026-02-18T08:30:50.858 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:30:50.855+0000 7faffc95c640 -1 reset not still connected to 0x5644c0efc340 2026-02-18T08:30:50.858 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:30:50.855+0000 7faffc95c640 -1 reset not still connected to 0x5644c58269c0 2026-02-18T08:30:50.858 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:30:50.855+0000 7faffc95c640 -1 reset not still connected to 0x5644c8fcf790 2026-02-18T08:30:50.858 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:30:50.855+0000 7faffc95c640 -1 reset not still connected to 0x5644d9b0e9c0 2026-02-18T08:30:50.858 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:30:50.855+0000 7faffc95c640 -1 reset not still connected to 0x5644e04e09c0 2026-02-18T08:30:51.003 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ error_code=124 2026-02-18T08:30:51.003 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set -e 2026-02-18T08:30:51.003 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 124 -eq 124 ']' 2026-02-18T08:30:51.003 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:30:51.003 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T08:30:51.003 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_30 2026-02-18T08:30:51.003 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_image_and_enable_mirror cluster2 mirror image_30 journal 128M 2026-02-18T08:30:51.003 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:30:51.003 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:30:51.004 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:30:51.004 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:30:51.004 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_30 2026-02-18T08:30:51.004 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:30:51.004 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local mode=journal 2026-02-18T08:30:51.004 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n journal ']' 2026-02-18T08:30:51.004 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:30:51.004 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_image cluster2 mirror image_30 128M 2026-02-18T08:30:51.004 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:30:51.004 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:30:51.004 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:30:51.004 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:30:51.005 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_30 2026-02-18T08:30:51.005 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:30:51.005 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local size=128 2026-02-18T08:30:51.005 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n 128M ']' 2026-02-18T08:30:51.005 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ size=128M 2026-02-18T08:30:51.005 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:30:51.005 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,journaling mirror/image_30 2026-02-18T08:30:51.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ is_pool_mirror_mode_image mirror 2026-02-18T08:30:51.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:30:51.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' pool = image ']' 2026-02-18T08:30:51.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ case "${pool}" in 2026-02-18T08:30:51.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 1 2026-02-18T08:30:51.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n '' ']' 2026-02-18T08:30:51.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ stress_write_image cluster2 mirror image_30 2026-02-18T08:30:51.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:30:51.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:30:51.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_30 2026-02-18T08:30:51.055 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-18T08:30:51.055 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local duration=11 2026-02-18T08:30:51.055 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set +e 2026-02-18T08:30:51.056 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ timeout 11s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_30 --debug-rbd=20 --debug-journaler=20 2026-02-18T08:30:51.603 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:100 IOs, 26800128 bytes 2026-02-18T08:30:51.796 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:200 IOs, 36316160 bytes 2026-02-18T08:30:51.830 INFO:tasks.ceph.cluster1.osd.1.trial034.stderr:2026-02-18T08:30:51.827+0000 7f20d7266640 -1 reset not still connected to 0x55640bcd0dd0 2026-02-18T08:30:51.830 INFO:tasks.ceph.cluster1.osd.1.trial034.stderr:2026-02-18T08:30:51.827+0000 7f20d7266640 -1 reset not still connected to 0x55640bcd1ad0 2026-02-18T08:30:51.831 INFO:tasks.ceph.cluster1.osd.1.trial034.stderr:2026-02-18T08:30:51.827+0000 7f20d7266640 -1 reset not still connected to 0x5564155c29c0 2026-02-18T08:30:52.348 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:300 IOs, 80034816 bytes 2026-02-18T08:30:53.641 INFO:tasks.ceph.cluster1.osd.1.trial034.stderr:2026-02-18T08:30:53.635+0000 7f20d7266640 -1 reset not still connected to 0x5564155c29c0 2026-02-18T08:30:53.658 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:30:53.655+0000 7fb27e7f3640 -1 reset not still connected to 0x55e1705e44e0 2026-02-18T08:30:53.659 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:30:53.655+0000 7fb27e7f3640 -1 reset not still connected to 0x55e170742000 2026-02-18T08:30:53.659 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:30:53.655+0000 7fb27e7f3640 -1 reset not still connected to 0x55e170c6e000 2026-02-18T08:30:53.659 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:30:53.655+0000 7fb27e7f3640 -1 reset not still connected to 0x55e171e68000 2026-02-18T08:30:53.659 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:30:53.655+0000 7fb27e7f3640 -1 reset not still connected to 0x55e171e69380 2026-02-18T08:30:53.659 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:30:53.655+0000 7fb27e7f3640 -1 reset not still connected to 0x55e183c10750 2026-02-18T08:30:53.659 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:30:53.655+0000 7fb27e7f3640 -1 reset not still connected to 0x55e18817ba00 2026-02-18T08:30:53.660 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:30:53.655+0000 7fb27e7f3640 -1 reset not still connected to 0x55e1884eaf70 2026-02-18T08:30:54.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:400 IOs, 127868928 bytes 2026-02-18T08:30:54.595 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:30:54.591+0000 7fb27e7f3640 -1 reset not still connected to 0x55e170c6e8f0 2026-02-18T08:30:54.596 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:30:54.591+0000 7fb27e7f3640 -1 reset not still connected to 0x55e171bf05b0 2026-02-18T08:30:54.596 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:30:54.591+0000 7fb27e7f3640 -1 reset not still connected to 0x55e178bde340 2026-02-18T08:30:54.816 INFO:tasks.ceph.cluster1.osd.1.trial034.stderr:2026-02-18T08:30:54.811+0000 7f20d7266640 -1 reset not still connected to 0x556404cc84e0 2026-02-18T08:30:54.816 INFO:tasks.ceph.cluster1.osd.1.trial034.stderr:2026-02-18T08:30:54.811+0000 7f20d7266640 -1 reset not still connected to 0x5564228bf520 2026-02-18T08:30:54.816 INFO:tasks.ceph.cluster1.osd.1.trial034.stderr:2026-02-18T08:30:54.811+0000 7f20d7266640 -1 reset not still connected to 0x556422af5ad0 2026-02-18T08:30:54.817 INFO:tasks.ceph.cluster1.osd.1.trial034.stderr:2026-02-18T08:30:54.811+0000 7f20d7266640 -1 reset not still connected to 0x55642f6a0000 2026-02-18T08:30:55.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:500 IOs, 192792576 bytes 2026-02-18T08:30:55.309 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:30:55.307+0000 7fb27e7f3640 -1 reset not still connected to 0x55e170c6e8f0 2026-02-18T08:30:55.310 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:30:55.307+0000 7fb27e7f3640 -1 reset not still connected to 0x55e171bf05b0 2026-02-18T08:30:55.310 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:30:55.307+0000 7fb27e7f3640 -1 reset not still connected to 0x55e178bde340 2026-02-18T08:30:55.311 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:30:55.307+0000 7fb27e7f3640 -1 reset not still connected to 0x55e183c11860 2026-02-18T08:30:55.311 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:30:55.307+0000 7fb27e7f3640 -1 reset not still connected to 0x55e18895cea0 2026-02-18T08:30:55.313 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:30:55.311+0000 7fb27e7f3640 -1 reset not still connected to 0x55e1b0646000 2026-02-18T08:30:55.345 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:30:55.343+0000 7fb27e7f3640 -1 reset not still connected to 0x55e170c6e8f0 2026-02-18T08:30:55.345 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:30:55.343+0000 7fb27e7f3640 -1 reset not still connected to 0x55e171bf05b0 2026-02-18T08:30:55.347 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:30:55.343+0000 7fb27e7f3640 -1 reset not still connected to 0x55e171c0a750 2026-02-18T08:30:55.347 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:30:55.343+0000 7fb27e7f3640 -1 reset not still connected to 0x55e178bde340 2026-02-18T08:30:55.347 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:30:55.343+0000 7fb27e7f3640 -1 reset not still connected to 0x55e183c11860 2026-02-18T08:30:55.347 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:30:55.343+0000 7fb27e7f3640 -1 reset not still connected to 0x55e18895cea0 2026-02-18T08:30:55.347 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:30:55.343+0000 7fb27e7f3640 -1 reset not still connected to 0x55e1b0646000 2026-02-18T08:30:56.282 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:600 IOs, 262222848 bytes 2026-02-18T08:30:56.931 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:700 IOs, 299162624 bytes 2026-02-18T08:30:57.423 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:800 IOs, 332211200 bytes 2026-02-18T08:30:57.827 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:900 IOs, 380623872 bytes 2026-02-18T08:30:59.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1000 IOs, 455570432 bytes 2026-02-18T08:30:59.416 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:30:59.411+0000 7fb27e7f3640 -1 reset not still connected to 0x55e171bf05b0 2026-02-18T08:30:59.417 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:30:59.415+0000 7fb27e7f3640 -1 reset not still connected to 0x55e171c0a750 2026-02-18T08:30:59.417 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:30:59.415+0000 7fb27e7f3640 -1 reset not still connected to 0x55e178bde340 2026-02-18T08:30:59.417 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:30:59.415+0000 7fb27e7f3640 -1 reset not still connected to 0x55e183c11860 2026-02-18T08:30:59.417 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:30:59.415+0000 7fb27e7f3640 -1 reset not still connected to 0x55e18895cea0 2026-02-18T08:30:59.418 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:30:59.415+0000 7fb27e7f3640 -1 reset not still connected to 0x55e1b0646000 2026-02-18T08:30:59.603 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:30:59.599+0000 7faffc95c640 -1 reset not still connected to 0x5644e04e0d00 2026-02-18T08:30:59.612 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1100 IOs, 484047872 bytes 2026-02-18T08:30:59.839 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:30:59.835+0000 7fb27e7f3640 -1 reset not still connected to 0x55e170742000 2026-02-18T08:30:59.839 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:30:59.835+0000 7fb27e7f3640 -1 reset not still connected to 0x55e170c6e000 2026-02-18T08:30:59.839 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:30:59.835+0000 7fb27e7f3640 -1 reset not still connected to 0x55e171e68000 2026-02-18T08:30:59.840 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:30:59.835+0000 7fb27e7f3640 -1 reset not still connected to 0x55e171e69380 2026-02-18T08:30:59.840 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:30:59.835+0000 7fb27e7f3640 -1 reset not still connected to 0x55e183c10750 2026-02-18T08:30:59.840 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:30:59.835+0000 7fb27e7f3640 -1 reset not still connected to 0x55e18817ba00 2026-02-18T08:30:59.840 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:30:59.835+0000 7fb27e7f3640 -1 reset not still connected to 0x55e1884eaf70 2026-02-18T08:31:00.364 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1200 IOs, 512367616 bytes 2026-02-18T08:31:00.953 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:31:00.951+0000 7faffc95c640 -1 reset not still connected to 0x5644c0efc340 2026-02-18T08:31:00.954 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:31:00.951+0000 7faffc95c640 -1 reset not still connected to 0x5644d9b0e9c0 2026-02-18T08:31:01.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1300 IOs, 569913344 bytes 2026-02-18T08:31:02.067 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ error_code=124 2026-02-18T08:31:02.067 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set -e 2026-02-18T08:31:02.067 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 124 -eq 124 ']' 2026-02-18T08:31:02.068 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:31:02.068 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T08:31:02.068 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_31 2026-02-18T08:31:02.068 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_image_and_enable_mirror cluster2 mirror image_31 journal 128M 2026-02-18T08:31:02.068 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:31:02.068 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:31:02.068 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:31:02.068 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:31:02.068 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_31 2026-02-18T08:31:02.068 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:31:02.068 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local mode=journal 2026-02-18T08:31:02.069 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n journal ']' 2026-02-18T08:31:02.069 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:31:02.069 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_image cluster2 mirror image_31 128M 2026-02-18T08:31:02.069 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:31:02.069 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:31:02.069 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:31:02.069 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:31:02.069 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_31 2026-02-18T08:31:02.069 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:31:02.069 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local size=128 2026-02-18T08:31:02.069 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n 128M ']' 2026-02-18T08:31:02.069 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ size=128M 2026-02-18T08:31:02.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:31:02.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,journaling mirror/image_31 2026-02-18T08:31:02.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ is_pool_mirror_mode_image mirror 2026-02-18T08:31:02.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:31:02.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' pool = image ']' 2026-02-18T08:31:02.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ case "${pool}" in 2026-02-18T08:31:02.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 1 2026-02-18T08:31:02.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n '' ']' 2026-02-18T08:31:02.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ stress_write_image cluster2 mirror image_31 2026-02-18T08:31:02.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:31:02.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:31:02.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_31 2026-02-18T08:31:02.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-18T08:31:02.110 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local duration=10 2026-02-18T08:31:02.111 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set +e 2026-02-18T08:31:02.111 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ timeout 10s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_31 --debug-rbd=20 --debug-journaler=20 2026-02-18T08:31:03.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:100 IOs, 68332544 bytes 2026-02-18T08:31:03.916 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:31:03.911+0000 7fb27e7f3640 -1 reset not still connected to 0x55e17071ec30 2026-02-18T08:31:03.916 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:31:03.911+0000 7fb27e7f3640 -1 reset not still connected to 0x55e170742ea0 2026-02-18T08:31:03.916 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:31:03.911+0000 7fb27e7f3640 -1 reset not still connected to 0x55e170c6e8f0 2026-02-18T08:31:03.916 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:31:03.911+0000 7fb27e7f3640 -1 reset not still connected to 0x55e171bf05b0 2026-02-18T08:31:03.916 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:31:03.911+0000 7fb27e7f3640 -1 reset not still connected to 0x55e171c0a750 2026-02-18T08:31:03.916 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:31:03.911+0000 7fb27e7f3640 -1 reset not still connected to 0x55e178bde340 2026-02-18T08:31:03.916 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:31:03.911+0000 7fb27e7f3640 -1 reset not still connected to 0x55e183c11860 2026-02-18T08:31:03.917 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:31:03.915+0000 7fb27e7f3640 -1 reset not still connected to 0x55e18895cea0 2026-02-18T08:31:03.917 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:31:03.915+0000 7fb27e7f3640 -1 reset not still connected to 0x55e1b0646000 2026-02-18T08:31:04.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:200 IOs, 94110720 bytes 2026-02-18T08:31:04.161 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:300 IOs, 99946496 bytes 2026-02-18T08:31:05.501 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:400 IOs, 193228800 bytes 2026-02-18T08:31:05.768 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:31:05.763+0000 7faffc95c640 -1 reset not still connected to 0x5644a3cb4750 2026-02-18T08:31:05.769 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:31:05.767+0000 7faffc95c640 -1 reset not still connected to 0x5644a44256c0 2026-02-18T08:31:05.769 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:31:05.767+0000 7faffc95c640 -1 reset not still connected to 0x5644a716ab60 2026-02-18T08:31:05.769 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:31:05.767+0000 7faffc95c640 -1 reset not still connected to 0x5644c19b4340 2026-02-18T08:31:06.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:500 IOs, 211146752 bytes 2026-02-18T08:31:06.863 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:31:06.859+0000 7faffc95c640 -1 reset not still connected to 0x5644c8fcf790 2026-02-18T08:31:06.863 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:31:06.859+0000 7faffc95c640 -1 reset not still connected to 0x5644d9b0e9c0 2026-02-18T08:31:07.318 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:31:07.315+0000 7fb27e7f3640 -1 reset not still connected to 0x55e171c0b860 2026-02-18T08:31:07.318 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:31:07.315+0000 7fb27e7f3640 -1 reset not still connected to 0x55e188bae0d0 2026-02-18T08:31:07.455 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:600 IOs, 297312256 bytes 2026-02-18T08:31:08.664 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:31:08.659+0000 7fb27e7f3640 -1 reset not still connected to 0x55e170743ee0 2026-02-18T08:31:08.664 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:31:08.659+0000 7fb27e7f3640 -1 reset not still connected to 0x55e17108f380 2026-02-18T08:31:08.665 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:31:08.663+0000 7fb27e7f3640 -1 reset not still connected to 0x55e171c0b860 2026-02-18T08:31:08.665 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:31:08.663+0000 7fb27e7f3640 -1 reset not still connected to 0x55e188bae0d0 2026-02-18T08:31:08.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:700 IOs, 365279232 bytes 2026-02-18T08:31:08.925 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:31:08.923+0000 7fb27e7f3640 -1 reset not still connected to 0x55e18817ba00 2026-02-18T08:31:08.926 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:31:08.923+0000 7fb27e7f3640 -1 reset not still connected to 0x55e1884eaf70 2026-02-18T08:31:09.697 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:800 IOs, 416214016 bytes 2026-02-18T08:31:09.729 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:31:09.727+0000 7fb27e7f3640 -1 reset not still connected to 0x55e170c6e8f0 2026-02-18T08:31:09.729 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:31:09.727+0000 7fb27e7f3640 -1 reset not still connected to 0x55e171bf05b0 2026-02-18T08:31:09.730 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:31:09.727+0000 7fb27e7f3640 -1 reset not still connected to 0x55e178bde340 2026-02-18T08:31:09.730 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:31:09.727+0000 7fb27e7f3640 -1 reset not still connected to 0x55e183c11860 2026-02-18T08:31:09.730 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:31:09.727+0000 7fb27e7f3640 -1 reset not still connected to 0x55e18895cea0 2026-02-18T08:31:10.114 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:900 IOs, 439998464 bytes 2026-02-18T08:31:10.527 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:31:10.523+0000 7fb27e7f3640 -1 reset not still connected to 0x55e171c0b860 2026-02-18T08:31:10.527 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:31:10.523+0000 7fb27e7f3640 -1 reset not still connected to 0x55e188bae0d0 2026-02-18T08:31:10.606 INFO:tasks.ceph.cluster1.osd.1.trial034.stderr:2026-02-18T08:31:10.603+0000 7f20d7266640 -1 reset not still connected to 0x556405b671e0 2026-02-18T08:31:10.606 INFO:tasks.ceph.cluster1.osd.1.trial034.stderr:2026-02-18T08:31:10.603+0000 7f20d7266640 -1 reset not still connected to 0x556408919380 2026-02-18T08:31:10.607 INFO:tasks.ceph.cluster1.osd.1.trial034.stderr:2026-02-18T08:31:10.603+0000 7f20d7266640 -1 reset not still connected to 0x5564155c2410 2026-02-18T08:31:10.836 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:31:10.831+0000 7fb27e7f3640 -1 reset not still connected to 0x55e170c6e000 2026-02-18T08:31:10.836 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:31:10.831+0000 7fb27e7f3640 -1 reset not still connected to 0x55e171e68000 2026-02-18T08:31:10.837 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:31:10.831+0000 7fb27e7f3640 -1 reset not still connected to 0x55e171e69380 2026-02-18T08:31:10.837 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:31:10.835+0000 7fb27e7f3640 -1 reset not still connected to 0x55e183c10750 2026-02-18T08:31:10.837 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:31:10.835+0000 7fb27e7f3640 -1 reset not still connected to 0x55e18817ba00 2026-02-18T08:31:10.837 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:31:10.835+0000 7fb27e7f3640 -1 reset not still connected to 0x55e1884eaf70 2026-02-18T08:31:10.968 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1000 IOs, 495908864 bytes 2026-02-18T08:31:10.996 INFO:tasks.ceph.cluster1.osd.1.trial034.stderr:2026-02-18T08:31:10.991+0000 7f20d7266640 -1 reset not still connected to 0x556422af5040 2026-02-18T08:31:10.996 INFO:tasks.ceph.cluster1.osd.1.trial034.stderr:2026-02-18T08:31:10.991+0000 7f20d7266640 -1 reset not still connected to 0x556422af5ad0 2026-02-18T08:31:10.996 INFO:tasks.ceph.cluster1.osd.1.trial034.stderr:2026-02-18T08:31:10.991+0000 7f20d7266640 -1 reset not still connected to 0x55642f6a0000 2026-02-18T08:31:11.320 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:31:11.295+0000 7fb27e7f3640 -1 reset not still connected to 0x55e170742000 2026-02-18T08:31:11.320 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:31:11.295+0000 7fb27e7f3640 -1 reset not still connected to 0x55e170c6e000 2026-02-18T08:31:11.320 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:31:11.315+0000 7fb27e7f3640 -1 reset not still connected to 0x55e183c10750 2026-02-18T08:31:11.321 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:31:11.315+0000 7fb27e7f3640 -1 reset not still connected to 0x55e18817ba00 2026-02-18T08:31:11.321 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:31:11.315+0000 7fb27e7f3640 -1 reset not still connected to 0x55e1884eaf70 2026-02-18T08:31:11.676 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1100 IOs, 530185216 bytes 2026-02-18T08:31:12.122 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ error_code=124 2026-02-18T08:31:12.122 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set -e 2026-02-18T08:31:12.122 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 124 -eq 124 ']' 2026-02-18T08:31:12.122 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:31:12.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T08:31:12.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_32 2026-02-18T08:31:12.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_image_and_enable_mirror cluster2 mirror image_32 journal 128M 2026-02-18T08:31:12.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:31:12.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:31:12.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:31:12.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:31:12.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_32 2026-02-18T08:31:12.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:31:12.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local mode=journal 2026-02-18T08:31:12.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n journal ']' 2026-02-18T08:31:12.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:31:12.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_image cluster2 mirror image_32 128M 2026-02-18T08:31:12.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:31:12.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:31:12.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:31:12.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:31:12.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_32 2026-02-18T08:31:12.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:31:12.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local size=128 2026-02-18T08:31:12.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n 128M ']' 2026-02-18T08:31:12.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ size=128M 2026-02-18T08:31:12.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:31:12.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,journaling mirror/image_32 2026-02-18T08:31:12.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ is_pool_mirror_mode_image mirror 2026-02-18T08:31:12.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:31:12.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' pool = image ']' 2026-02-18T08:31:12.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ case "${pool}" in 2026-02-18T08:31:12.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 1 2026-02-18T08:31:12.161 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n '' ']' 2026-02-18T08:31:12.161 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ stress_write_image cluster2 mirror image_32 2026-02-18T08:31:12.161 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:31:12.161 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:31:12.161 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_32 2026-02-18T08:31:12.161 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-18T08:31:12.162 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local duration=12 2026-02-18T08:31:12.162 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set +e 2026-02-18T08:31:12.162 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ timeout 12s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_32 --debug-rbd=20 --debug-journaler=20 2026-02-18T08:31:12.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:100 IOs, 21542912 bytes 2026-02-18T08:31:12.808 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:200 IOs, 27838464 bytes 2026-02-18T08:31:13.171 INFO:tasks.ceph.cluster1.osd.1.trial034.stderr:2026-02-18T08:31:13.167+0000 7f20d7266640 -1 reset not still connected to 0x5563fe2316c0 2026-02-18T08:31:13.172 INFO:tasks.ceph.cluster1.osd.1.trial034.stderr:2026-02-18T08:31:13.167+0000 7f20d7266640 -1 reset not still connected to 0x5563ff09dd40 2026-02-18T08:31:13.172 INFO:tasks.ceph.cluster1.osd.1.trial034.stderr:2026-02-18T08:31:13.167+0000 7f20d7266640 -1 reset not still connected to 0x55640bcd1110 2026-02-18T08:31:13.172 INFO:tasks.ceph.cluster1.osd.1.trial034.stderr:2026-02-18T08:31:13.167+0000 7f20d7266640 -1 reset not still connected to 0x5564155c3d40 2026-02-18T08:31:13.172 INFO:tasks.ceph.cluster1.osd.1.trial034.stderr:2026-02-18T08:31:13.167+0000 7f20d7266640 -1 reset not still connected to 0x5564228bfd40 2026-02-18T08:31:13.280 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:31:13.275+0000 7fb27e7f3640 -1 reset not still connected to 0x55e188baf860 2026-02-18T08:31:13.908 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:31:13.903+0000 7faffc95c640 -1 reset not still connected to 0x5644c8fcf790 2026-02-18T08:31:13.909 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:31:13.907+0000 7faffc95c640 -1 reset not still connected to 0x5644d9b0e9c0 2026-02-18T08:31:13.909 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:31:13.907+0000 7faffc95c640 -1 reset not still connected to 0x5644e04e09c0 2026-02-18T08:31:14.363 INFO:tasks.ceph.cluster1.osd.1.trial034.stderr:2026-02-18T08:31:14.359+0000 7f20d7266640 -1 reset not still connected to 0x5563fe2316c0 2026-02-18T08:31:14.363 INFO:tasks.ceph.cluster1.osd.1.trial034.stderr:2026-02-18T08:31:14.359+0000 7f20d7266640 -1 reset not still connected to 0x5563ff09dd40 2026-02-18T08:31:14.364 INFO:tasks.ceph.cluster1.osd.1.trial034.stderr:2026-02-18T08:31:14.359+0000 7f20d7266640 -1 reset not still connected to 0x55640bcd1110 2026-02-18T08:31:14.364 INFO:tasks.ceph.cluster1.osd.1.trial034.stderr:2026-02-18T08:31:14.359+0000 7f20d7266640 -1 reset not still connected to 0x5564155c3d40 2026-02-18T08:31:14.364 INFO:tasks.ceph.cluster1.osd.1.trial034.stderr:2026-02-18T08:31:14.359+0000 7f20d7266640 -1 reset not still connected to 0x5564228bfd40 2026-02-18T08:31:14.954 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:31:14.951+0000 7faffc95c640 -1 reset not still connected to 0x5644a716ab60 2026-02-18T08:31:14.954 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:31:14.951+0000 7faffc95c640 -1 reset not still connected to 0x5644c19b4340 2026-02-18T08:31:14.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:300 IOs, 105326592 bytes 2026-02-18T08:31:15.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:400 IOs, 114314240 bytes 2026-02-18T08:31:15.931 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:500 IOs, 140944384 bytes 2026-02-18T08:31:16.150 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:31:16.147+0000 7fb27e7f3640 -1 reset not still connected to 0x55e170c6e000 2026-02-18T08:31:16.151 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:31:16.147+0000 7fb27e7f3640 -1 reset not still connected to 0x55e171e68000 2026-02-18T08:31:16.151 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:31:16.147+0000 7fb27e7f3640 -1 reset not still connected to 0x55e171e69380 2026-02-18T08:31:16.151 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:31:16.147+0000 7fb27e7f3640 -1 reset not still connected to 0x55e183c10750 2026-02-18T08:31:16.151 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:31:16.147+0000 7fb27e7f3640 -1 reset not still connected to 0x55e18817ba00 2026-02-18T08:31:16.152 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:31:16.147+0000 7fb27e7f3640 -1 reset not still connected to 0x55e1884eaf70 2026-02-18T08:31:16.177 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:31:16.175+0000 7fb27e7f3640 -1 reset not still connected to 0x55e17071ec30 2026-02-18T08:31:16.177 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:31:16.175+0000 7fb27e7f3640 -1 reset not still connected to 0x55e170742ea0 2026-02-18T08:31:16.178 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:31:16.175+0000 7fb27e7f3640 -1 reset not still connected to 0x55e170c6e8f0 2026-02-18T08:31:16.178 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:31:16.175+0000 7fb27e7f3640 -1 reset not still connected to 0x55e171bf05b0 2026-02-18T08:31:16.178 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:31:16.175+0000 7fb27e7f3640 -1 reset not still connected to 0x55e171c0a750 2026-02-18T08:31:16.179 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:31:16.175+0000 7fb27e7f3640 -1 reset not still connected to 0x55e178bde340 2026-02-18T08:31:16.179 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:31:16.175+0000 7fb27e7f3640 -1 reset not still connected to 0x55e183c11860 2026-02-18T08:31:16.180 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:31:16.175+0000 7fb27e7f3640 -1 reset not still connected to 0x55e18895cea0 2026-02-18T08:31:16.180 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:31:16.175+0000 7fb27e7f3640 -1 reset not still connected to 0x55e1b0646000 2026-02-18T08:31:16.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:600 IOs, 172864512 bytes 2026-02-18T08:31:16.507 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:31:16.503+0000 7fb27e7f3640 -1 reset not still connected to 0x55e171c0b860 2026-02-18T08:31:16.561 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:700 IOs, 196385792 bytes 2026-02-18T08:31:17.092 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:800 IOs, 216390656 bytes 2026-02-18T08:31:17.458 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:31:17.455+0000 7faffc95c640 -1 reset not still connected to 0x5644a5073860 2026-02-18T08:31:17.458 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:31:17.455+0000 7faffc95c640 -1 reset not still connected to 0x5644a6840410 2026-02-18T08:31:17.459 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:31:17.455+0000 7faffc95c640 -1 reset not still connected to 0x5644a716a410 2026-02-18T08:31:17.459 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:31:17.455+0000 7faffc95c640 -1 reset not still connected to 0x5644a84a6750 2026-02-18T08:31:17.460 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:31:17.459+0000 7faffc95c640 -1 reset not still connected to 0x5644a84d5930 2026-02-18T08:31:17.460 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:31:17.459+0000 7faffc95c640 -1 reset not still connected to 0x5644c1b595f0 2026-02-18T08:31:17.460 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:31:17.459+0000 7faffc95c640 -1 reset not still connected to 0x5644c8fcf380 2026-02-18T08:31:17.460 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:31:17.459+0000 7faffc95c640 -1 reset not still connected to 0x5644d9b0e410 2026-02-18T08:31:18.036 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:900 IOs, 260922368 bytes 2026-02-18T08:31:18.223 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1000 IOs, 268701696 bytes 2026-02-18T08:31:19.027 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1100 IOs, 315205632 bytes 2026-02-18T08:31:19.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1200 IOs, 406274048 bytes 2026-02-18T08:31:20.048 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:31:20.043+0000 7faffc95c640 -1 reset not still connected to 0x5644d9b0e9c0 2026-02-18T08:31:20.049 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:31:20.047+0000 7faffc95c640 -1 reset not still connected to 0x5644e04e1ad0 2026-02-18T08:31:20.251 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:31:20.246+0000 7faffc95c640 -1 reset not still connected to 0x5644a716ab60 2026-02-18T08:31:20.251 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:31:20.246+0000 7faffc95c640 -1 reset not still connected to 0x5644c19b4340 2026-02-18T08:31:20.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1300 IOs, 504328192 bytes 2026-02-18T08:31:21.266 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1400 IOs, 519607296 bytes 2026-02-18T08:31:21.832 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:31:21.826+0000 7faffc95c640 -1 reset not still connected to 0x5644a5073860 2026-02-18T08:31:21.832 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:31:21.826+0000 7faffc95c640 -1 reset not still connected to 0x5644a6840410 2026-02-18T08:31:21.832 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:31:21.826+0000 7faffc95c640 -1 reset not still connected to 0x5644a716a410 2026-02-18T08:31:21.832 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:31:21.826+0000 7faffc95c640 -1 reset not still connected to 0x5644a84a6750 2026-02-18T08:31:21.833 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:31:21.830+0000 7faffc95c640 -1 reset not still connected to 0x5644a84d5930 2026-02-18T08:31:21.833 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:31:21.830+0000 7faffc95c640 -1 reset not still connected to 0x5644c1b595f0 2026-02-18T08:31:21.833 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:31:21.830+0000 7faffc95c640 -1 reset not still connected to 0x5644c8fcf380 2026-02-18T08:31:21.833 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:31:21.830+0000 7faffc95c640 -1 reset not still connected to 0x5644d9b0e410 2026-02-18T08:31:22.166 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1500 IOs, 563952640 bytes 2026-02-18T08:31:22.495 INFO:tasks.ceph.cluster1.osd.1.trial034.stderr:2026-02-18T08:31:22.490+0000 7f20d7266640 -1 reset not still connected to 0x5563fe231520 2026-02-18T08:31:22.496 INFO:tasks.ceph.cluster1.osd.1.trial034.stderr:2026-02-18T08:31:22.490+0000 7f20d7266640 -1 reset not still connected to 0x556404cc84e0 2026-02-18T08:31:22.496 INFO:tasks.ceph.cluster1.osd.1.trial034.stderr:2026-02-18T08:31:22.490+0000 7f20d7266640 -1 reset not still connected to 0x5564228bf520 2026-02-18T08:31:22.496 INFO:tasks.ceph.cluster1.osd.1.trial034.stderr:2026-02-18T08:31:22.490+0000 7f20d7266640 -1 reset not still connected to 0x556422af5040 2026-02-18T08:31:22.496 INFO:tasks.ceph.cluster1.osd.1.trial034.stderr:2026-02-18T08:31:22.490+0000 7f20d7266640 -1 reset not still connected to 0x556422af5ad0 2026-02-18T08:31:22.496 INFO:tasks.ceph.cluster1.osd.1.trial034.stderr:2026-02-18T08:31:22.490+0000 7f20d7266640 -1 reset not still connected to 0x55642f6a0000 2026-02-18T08:31:22.866 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1600 IOs, 614688768 bytes 2026-02-18T08:31:23.982 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:31:23.978+0000 7faffc95c640 -1 reset not still connected to 0x5644a716ab60 2026-02-18T08:31:23.982 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:31:23.978+0000 7faffc95c640 -1 reset not still connected to 0x5644c19b4340 2026-02-18T08:31:24.019 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1700 IOs, 682595328 bytes 2026-02-18T08:31:24.173 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ error_code=124 2026-02-18T08:31:24.173 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set -e 2026-02-18T08:31:24.173 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 124 -eq 124 ']' 2026-02-18T08:31:24.173 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:31:24.173 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T08:31:24.173 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_33 2026-02-18T08:31:24.174 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_image_and_enable_mirror cluster2 mirror image_33 journal 128M 2026-02-18T08:31:24.174 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:31:24.174 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:31:24.174 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:31:24.174 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:31:24.174 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_33 2026-02-18T08:31:24.174 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:31:24.174 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local mode=journal 2026-02-18T08:31:24.174 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n journal ']' 2026-02-18T08:31:24.174 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:31:24.174 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_image cluster2 mirror image_33 128M 2026-02-18T08:31:24.174 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:31:24.175 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:31:24.175 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:31:24.175 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:31:24.175 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_33 2026-02-18T08:31:24.175 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:31:24.175 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local size=128 2026-02-18T08:31:24.175 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n 128M ']' 2026-02-18T08:31:24.175 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ size=128M 2026-02-18T08:31:24.175 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:31:24.175 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,journaling mirror/image_33 2026-02-18T08:31:24.219 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ is_pool_mirror_mode_image mirror 2026-02-18T08:31:24.219 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:31:24.220 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' pool = image ']' 2026-02-18T08:31:24.220 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ case "${pool}" in 2026-02-18T08:31:24.220 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 1 2026-02-18T08:31:24.220 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n '' ']' 2026-02-18T08:31:24.220 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ stress_write_image cluster2 mirror image_33 2026-02-18T08:31:24.220 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:31:24.220 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:31:24.220 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_33 2026-02-18T08:31:24.220 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-18T08:31:24.221 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local duration=10 2026-02-18T08:31:24.221 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set +e 2026-02-18T08:31:24.221 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ timeout 10s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_33 --debug-rbd=20 --debug-journaler=20 2026-02-18T08:31:25.020 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:31:25.014+0000 7faffc95c640 -1 reset not still connected to 0x5644a41fe5b0 2026-02-18T08:31:25.021 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:31:25.018+0000 7faffc95c640 -1 reset not still connected to 0x5644a4424270 2026-02-18T08:31:25.021 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:31:25.018+0000 7faffc95c640 -1 reset not still connected to 0x5644a5073110 2026-02-18T08:31:25.021 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:31:25.018+0000 7faffc95c640 -1 reset not still connected to 0x5644a84a61a0 2026-02-18T08:31:25.021 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:31:25.018+0000 7faffc95c640 -1 reset not still connected to 0x5644c58264e0 2026-02-18T08:31:25.021 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:31:25.018+0000 7faffc95c640 -1 reset not still connected to 0x5644dfda1520 2026-02-18T08:31:25.021 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:31:25.018+0000 7faffc95c640 -1 reset not still connected to 0x5644e04e0d00 2026-02-18T08:31:25.366 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:100 IOs, 49553408 bytes 2026-02-18T08:31:25.969 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:200 IOs, 80023552 bytes 2026-02-18T08:31:27.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:300 IOs, 119357440 bytes 2026-02-18T08:31:28.137 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:400 IOs, 123309056 bytes 2026-02-18T08:31:28.328 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.0 with skip (0.7438377272386537) > weight (0.25) 2026-02-18T08:31:28.328 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.1 with skip (0.7054063413100659) > weight (0.25) 2026-02-18T08:31:28.328 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.2 with skip (0.34219727931293775) > weight (0.25) 2026-02-18T08:31:28.328 INFO:tasks.rbd_mirror_thrash:kill cluster2.client.mirror.3 2026-02-18T08:31:28.329 INFO:tasks.rbd_mirror.cluster2.client.mirror.3:Sent signal 15 2026-02-18T08:31:28.329 INFO:tasks.rbd_mirror_thrash:waiting for 6.675115845911392 secs before reviving daemons 2026-02-18T08:31:28.329 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.trial163.stderr:2026-02-18T08:31:28.316+0000 7febfd425640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.3 (PID: 87594) UID: 1000 2026-02-18T08:31:28.436 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:500 IOs, 135435264 bytes 2026-02-18T08:31:30.004 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:600 IOs, 251888640 bytes 2026-02-18T08:31:31.164 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:700 IOs, 329504768 bytes 2026-02-18T08:31:31.209 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.0 with skip (0.25582508559024975) > weight (0.25) 2026-02-18T08:31:31.209 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.1 with skip (0.46911035213395) > weight (0.25) 2026-02-18T08:31:31.210 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.2 with skip (0.5541411971561677) > weight (0.25) 2026-02-18T08:31:31.210 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.3 with skip (0.8455867940330279) > weight (0.25) 2026-02-18T08:31:31.210 INFO:tasks.rbd_mirror_thrash:waiting for 103.2322165085765 secs before thrashing 2026-02-18T08:31:31.922 INFO:tasks.ceph.cluster1.osd.1.trial034.stderr:2026-02-18T08:31:31.918+0000 7f20d7266640 -1 reset not still connected to 0x5564155c29c0 2026-02-18T08:31:32.036 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:800 IOs, 403010560 bytes 2026-02-18T08:31:32.392 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:31:32.386+0000 7faffc95c640 -1 reset not still connected to 0x5644c19b4340 2026-02-18T08:31:33.196 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:31:33.190+0000 7fb27e7f3640 -1 reset not still connected to 0x55e17071ec30 2026-02-18T08:31:33.196 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:31:33.190+0000 7fb27e7f3640 -1 reset not still connected to 0x55e170c6e8f0 2026-02-18T08:31:33.196 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:31:33.190+0000 7fb27e7f3640 -1 reset not still connected to 0x55e171bf05b0 2026-02-18T08:31:33.196 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:31:33.190+0000 7fb27e7f3640 -1 reset not still connected to 0x55e178bde340 2026-02-18T08:31:33.198 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:31:33.194+0000 7fb27e7f3640 -1 reset not still connected to 0x55e183c11860 2026-02-18T08:31:33.198 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:31:33.194+0000 7fb27e7f3640 -1 reset not still connected to 0x55e18895cea0 2026-02-18T08:31:33.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:900 IOs, 475217920 bytes 2026-02-18T08:31:33.848 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:31:33.842+0000 7fb27e7f3640 -1 reset not still connected to 0x55e17108f380 2026-02-18T08:31:33.850 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:31:33.846+0000 7fb27e7f3640 -1 reset not still connected to 0x55e171c0b860 2026-02-18T08:31:33.851 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:31:33.846+0000 7fb27e7f3640 -1 reset not still connected to 0x55e188bae0d0 2026-02-18T08:31:34.231 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ error_code=124 2026-02-18T08:31:34.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set -e 2026-02-18T08:31:34.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 124 -eq 124 ']' 2026-02-18T08:31:34.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:31:34.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T08:31:34.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_34 2026-02-18T08:31:34.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_image_and_enable_mirror cluster2 mirror image_34 journal 128M 2026-02-18T08:31:34.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:31:34.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:31:34.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:31:34.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:31:34.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_34 2026-02-18T08:31:34.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:31:34.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local mode=journal 2026-02-18T08:31:34.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n journal ']' 2026-02-18T08:31:34.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:31:34.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_image cluster2 mirror image_34 128M 2026-02-18T08:31:34.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:31:34.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:31:34.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:31:34.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:31:34.234 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_34 2026-02-18T08:31:34.234 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:31:34.234 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local size=128 2026-02-18T08:31:34.234 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n 128M ']' 2026-02-18T08:31:34.234 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ size=128M 2026-02-18T08:31:34.234 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:31:34.234 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,journaling mirror/image_34 2026-02-18T08:31:34.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ is_pool_mirror_mode_image mirror 2026-02-18T08:31:34.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:31:34.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' pool = image ']' 2026-02-18T08:31:34.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ case "${pool}" in 2026-02-18T08:31:34.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 1 2026-02-18T08:31:34.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n '' ']' 2026-02-18T08:31:34.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ stress_write_image cluster2 mirror image_34 2026-02-18T08:31:34.273 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:31:34.273 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:31:34.273 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_34 2026-02-18T08:31:34.273 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-18T08:31:34.273 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local duration=6 2026-02-18T08:31:34.273 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set +e 2026-02-18T08:31:34.273 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ timeout 6s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_34 --debug-rbd=20 --debug-journaler=20 2026-02-18T08:31:34.845 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:31:34.842+0000 7faffc95c640 -1 reset not still connected to 0x5644d9b0e9c0 2026-02-18T08:31:34.845 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:31:34.842+0000 7faffc95c640 -1 reset not still connected to 0x5644e04e09c0 2026-02-18T08:31:35.004 INFO:tasks.rbd_mirror_thrash:waiting for cluster2.client.mirror.3 2026-02-18T08:31:35.005 INFO:teuthology.orchestra.run:waiting for 600 2026-02-18T08:31:35.005 INFO:tasks.rbd_mirror_thrash:reviving cluster2.client.mirror.3 2026-02-18T08:31:35.005 INFO:tasks.rbd_mirror.cluster2.client.mirror.3:Restarting daemon 2026-02-18T08:31:35.005 DEBUG:teuthology.orchestra.run.trial163:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.3 2026-02-18T08:31:35.008 INFO:tasks.rbd_mirror.cluster2.client.mirror.3:Started 2026-02-18T08:31:35.008 INFO:tasks.rbd_mirror_thrash:waiting for 77.58409577575533 secs before thrashing 2026-02-18T08:31:35.095 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:100 IOs, 34526208 bytes 2026-02-18T08:31:35.166 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:31:35.162+0000 7faffc95c640 -1 reset not still connected to 0x5644d9b0e9c0 2026-02-18T08:31:35.335 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:31:35.330+0000 7faffc95c640 -1 reset not still connected to 0x5644a716ac30 2026-02-18T08:31:35.336 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:31:35.330+0000 7faffc95c640 -1 reset not still connected to 0x5644c58269c0 2026-02-18T08:31:35.336 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:31:35.330+0000 7faffc95c640 -1 reset not still connected to 0x5644d9b0e9c0 2026-02-18T08:31:35.654 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:200 IOs, 67179520 bytes 2026-02-18T08:31:35.865 INFO:tasks.ceph.cluster1.osd.1.trial034.stderr:2026-02-18T08:31:35.862+0000 7f20d7266640 -1 reset not still connected to 0x5564155c29c0 2026-02-18T08:31:36.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:300 IOs, 100219904 bytes 2026-02-18T08:31:36.698 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:31:36.694+0000 7faffc95c640 -1 reset not still connected to 0x5644c0efc340 2026-02-18T08:31:36.699 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:31:36.694+0000 7faffc95c640 -1 reset not still connected to 0x5644c58269c0 2026-02-18T08:31:36.699 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:31:36.694+0000 7faffc95c640 -1 reset not still connected to 0x5644c8fcf790 2026-02-18T08:31:36.699 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:31:36.694+0000 7faffc95c640 -1 reset not still connected to 0x5644d9b0e9c0 2026-02-18T08:31:36.699 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:31:36.694+0000 7faffc95c640 -1 reset not still connected to 0x5644e04e09c0 2026-02-18T08:31:36.700 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:31:36.694+0000 7faffc95c640 -1 reset not still connected to 0x5644e04e1ad0 2026-02-18T08:31:37.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:400 IOs, 131642368 bytes 2026-02-18T08:31:37.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:500 IOs, 141949952 bytes 2026-02-18T08:31:37.858 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:600 IOs, 184561664 bytes 2026-02-18T08:31:38.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:700 IOs, 206284800 bytes 2026-02-18T08:31:38.569 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:31:38.566+0000 7faffc95c640 -1 reset not still connected to 0x5644a41fe5b0 2026-02-18T08:31:38.570 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:31:38.566+0000 7faffc95c640 -1 reset not still connected to 0x5644a4424270 2026-02-18T08:31:38.570 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:31:38.566+0000 7faffc95c640 -1 reset not still connected to 0x5644a5073110 2026-02-18T08:31:38.570 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:31:38.566+0000 7faffc95c640 -1 reset not still connected to 0x5644a84a61a0 2026-02-18T08:31:38.570 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:31:38.566+0000 7faffc95c640 -1 reset not still connected to 0x5644c1ba7d40 2026-02-18T08:31:38.570 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:31:38.566+0000 7faffc95c640 -1 reset not still connected to 0x5644c58264e0 2026-02-18T08:31:38.570 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:31:38.566+0000 7faffc95c640 -1 reset not still connected to 0x5644dfda1520 2026-02-18T08:31:38.571 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:31:38.566+0000 7faffc95c640 -1 reset not still connected to 0x5644e04e0d00 2026-02-18T08:31:39.316 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:800 IOs, 274900992 bytes 2026-02-18T08:31:39.572 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:900 IOs, 283114496 bytes 2026-02-18T08:31:40.281 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ error_code=124 2026-02-18T08:31:40.281 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set -e 2026-02-18T08:31:40.281 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 124 -eq 124 ']' 2026-02-18T08:31:40.281 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:31:40.281 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T08:31:40.281 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_35 2026-02-18T08:31:40.281 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_image_and_enable_mirror cluster2 mirror image_35 journal 128M 2026-02-18T08:31:40.281 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:31:40.281 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:31:40.281 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:31:40.281 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:31:40.281 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_35 2026-02-18T08:31:40.281 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:31:40.281 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local mode=journal 2026-02-18T08:31:40.281 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n journal ']' 2026-02-18T08:31:40.281 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:31:40.281 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_image cluster2 mirror image_35 128M 2026-02-18T08:31:40.281 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:31:40.281 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:31:40.281 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:31:40.281 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:31:40.282 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_35 2026-02-18T08:31:40.282 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:31:40.282 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local size=128 2026-02-18T08:31:40.282 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n 128M ']' 2026-02-18T08:31:40.282 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ size=128M 2026-02-18T08:31:40.282 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:31:40.282 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,journaling mirror/image_35 2026-02-18T08:31:40.325 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ is_pool_mirror_mode_image mirror 2026-02-18T08:31:40.325 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:31:40.325 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' pool = image ']' 2026-02-18T08:31:40.325 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ case "${pool}" in 2026-02-18T08:31:40.325 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 1 2026-02-18T08:31:40.325 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n '' ']' 2026-02-18T08:31:40.325 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ stress_write_image cluster2 mirror image_35 2026-02-18T08:31:40.325 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:31:40.325 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:31:40.325 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_35 2026-02-18T08:31:40.325 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-18T08:31:40.327 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local duration=14 2026-02-18T08:31:40.327 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set +e 2026-02-18T08:31:40.327 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ timeout 14s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_35 --debug-rbd=20 --debug-journaler=20 2026-02-18T08:31:40.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:100 IOs, 14535680 bytes 2026-02-18T08:31:40.520 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:31:40.514+0000 7fb27e7f3640 -1 reset not still connected to 0x55e183c10750 2026-02-18T08:31:40.520 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:31:40.514+0000 7fb27e7f3640 -1 reset not still connected to 0x55e18817ba00 2026-02-18T08:31:40.520 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:31:40.514+0000 7fb27e7f3640 -1 reset not still connected to 0x55e1884eaf70 2026-02-18T08:31:41.094 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:31:41.090+0000 7fb27e7f3640 -1 reset not still connected to 0x55e183c11860 2026-02-18T08:31:41.095 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:31:41.090+0000 7fb27e7f3640 -1 reset not still connected to 0x55e18895cea0 2026-02-18T08:31:41.230 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:31:41.226+0000 7fb27e7f3640 -1 reset not still connected to 0x55e18895cea0 2026-02-18T08:31:41.286 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:200 IOs, 64642048 bytes 2026-02-18T08:31:41.573 INFO:tasks.ceph.cluster1.osd.1.trial034.stderr:2026-02-18T08:31:41.570+0000 7f20d7266640 -1 reset not still connected to 0x556408919380 2026-02-18T08:31:41.573 INFO:tasks.ceph.cluster1.osd.1.trial034.stderr:2026-02-18T08:31:41.570+0000 7f20d7266640 -1 reset not still connected to 0x5564155c2410 2026-02-18T08:31:41.797 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:31:41.794+0000 7fb27e7f3640 -1 reset not still connected to 0x55e18817ba00 2026-02-18T08:31:41.867 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:300 IOs, 81933312 bytes 2026-02-18T08:31:41.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:400 IOs, 83069952 bytes 2026-02-18T08:31:42.353 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:500 IOs, 105269248 bytes 2026-02-18T08:31:42.502 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:31:42.498+0000 7fb27e7f3640 -1 reset not still connected to 0x55e171e68000 2026-02-18T08:31:42.503 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:31:42.498+0000 7fb27e7f3640 -1 reset not still connected to 0x55e18817ba00 2026-02-18T08:31:43.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:600 IOs, 156749824 bytes 2026-02-18T08:31:44.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:700 IOs, 249972736 bytes 2026-02-18T08:31:45.314 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:800 IOs, 275873792 bytes 2026-02-18T08:31:46.205 INFO:tasks.ceph.cluster2.osd.2.trial163.stderr:2026-02-18T08:31:46.192+0000 7f2b60c87640 -1 reset not still connected to 0x55fbf6ab0000 2026-02-18T08:31:46.205 INFO:tasks.ceph.cluster2.osd.2.trial163.stderr:2026-02-18T08:31:46.192+0000 7f2b60c87640 -1 reset not still connected to 0x55fbf768f520 2026-02-18T08:31:46.205 INFO:tasks.ceph.cluster2.osd.2.trial163.stderr:2026-02-18T08:31:46.192+0000 7f2b60c87640 -1 reset not still connected to 0x55fbf7b96820 2026-02-18T08:31:46.205 INFO:tasks.ceph.cluster2.osd.2.trial163.stderr:2026-02-18T08:31:46.192+0000 7f2b60c87640 -1 reset not still connected to 0x55fbf7b97040 2026-02-18T08:31:46.205 INFO:tasks.ceph.cluster2.osd.2.trial163.stderr:2026-02-18T08:31:46.192+0000 7f2b60c87640 -1 reset not still connected to 0x55fbf7b975f0 2026-02-18T08:31:46.205 INFO:tasks.ceph.cluster2.osd.2.trial163.stderr:2026-02-18T08:31:46.192+0000 7f2b60c87640 -1 reset not still connected to 0x55fbf7fc2ea0 2026-02-18T08:31:46.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:900 IOs, 351578112 bytes 2026-02-18T08:31:47.670 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1000 IOs, 460444672 bytes 2026-02-18T08:31:48.259 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1100 IOs, 461050880 bytes 2026-02-18T08:31:48.263 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:31:48.258+0000 7faffc95c640 -1 reset not still connected to 0x5644d9b0e9c0 2026-02-18T08:31:48.855 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:31:48.850+0000 7faffc95c640 -1 reset not still connected to 0x5644c58264e0 2026-02-18T08:31:48.856 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:31:48.850+0000 7faffc95c640 -1 reset not still connected to 0x5644dfda1520 2026-02-18T08:31:48.856 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:31:48.850+0000 7faffc95c640 -1 reset not still connected to 0x5644e04e0d00 2026-02-18T08:31:48.987 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1200 IOs, 546935808 bytes 2026-02-18T08:31:50.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1300 IOs, 594142208 bytes 2026-02-18T08:31:50.463 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:31:50.458+0000 7fb27e7f3640 -1 reset not still connected to 0x55e183c10750 2026-02-18T08:31:50.463 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:31:50.458+0000 7fb27e7f3640 -1 reset not still connected to 0x55e18817ba00 2026-02-18T08:31:50.463 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:31:50.458+0000 7fb27e7f3640 -1 reset not still connected to 0x55e1884eaf70 2026-02-18T08:31:50.569 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:31:50.566+0000 7fb27e7f3640 -1 reset not still connected to 0x55e171e68000 2026-02-18T08:31:50.571 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:31:50.566+0000 7fb27e7f3640 -1 reset not still connected to 0x55e183c10750 2026-02-18T08:31:50.571 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:31:50.566+0000 7fb27e7f3640 -1 reset not still connected to 0x55e18817ba00 2026-02-18T08:31:50.572 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:31:50.566+0000 7fb27e7f3640 -1 reset not still connected to 0x55e1884eaf70 2026-02-18T08:31:51.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1400 IOs, 653583360 bytes 2026-02-18T08:31:51.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1500 IOs, 659750912 bytes 2026-02-18T08:31:51.587 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1600 IOs, 682633216 bytes 2026-02-18T08:31:51.802 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:31:51.798+0000 7fb27e7f3640 -1 reset not still connected to 0x55e18817ba00 2026-02-18T08:31:51.802 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:31:51.798+0000 7fb27e7f3640 -1 reset not still connected to 0x55e1884eaf70 2026-02-18T08:31:52.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1700 IOs, 712487936 bytes 2026-02-18T08:31:53.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1800 IOs, 780743680 bytes 2026-02-18T08:31:53.426 INFO:tasks.ceph.cluster1.osd.1.trial034.stderr:2026-02-18T08:31:53.422+0000 7f20d7266640 -1 reset not still connected to 0x5563fe2316c0 2026-02-18T08:31:53.426 INFO:tasks.ceph.cluster1.osd.1.trial034.stderr:2026-02-18T08:31:53.422+0000 7f20d7266640 -1 reset not still connected to 0x5563ff09dd40 2026-02-18T08:31:53.426 INFO:tasks.ceph.cluster1.osd.1.trial034.stderr:2026-02-18T08:31:53.422+0000 7f20d7266640 -1 reset not still connected to 0x55640bcd1110 2026-02-18T08:31:53.427 INFO:tasks.ceph.cluster1.osd.1.trial034.stderr:2026-02-18T08:31:53.422+0000 7f20d7266640 -1 reset not still connected to 0x5564155c3d40 2026-02-18T08:31:53.427 INFO:tasks.ceph.cluster1.osd.1.trial034.stderr:2026-02-18T08:31:53.422+0000 7f20d7266640 -1 reset not still connected to 0x5564228bfd40 2026-02-18T08:31:53.858 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:31:53.854+0000 7faffc95c640 -1 reset not still connected to 0x5644c0efc340 2026-02-18T08:31:53.859 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:31:53.854+0000 7faffc95c640 -1 reset not still connected to 0x5644c58269c0 2026-02-18T08:31:53.859 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:31:53.854+0000 7faffc95c640 -1 reset not still connected to 0x5644c8fcf790 2026-02-18T08:31:53.859 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:31:53.854+0000 7faffc95c640 -1 reset not still connected to 0x5644d9b0e9c0 2026-02-18T08:31:53.859 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:31:53.854+0000 7faffc95c640 -1 reset not still connected to 0x5644e04e09c0 2026-02-18T08:31:53.859 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:31:53.854+0000 7faffc95c640 -1 reset not still connected to 0x5644e04e1ad0 2026-02-18T08:31:54.161 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:31:54.158+0000 7fb27e7f3640 -1 reset not still connected to 0x55e18817ba00 2026-02-18T08:31:54.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ error_code=124 2026-02-18T08:31:54.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set -e 2026-02-18T08:31:54.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 124 -eq 124 ']' 2026-02-18T08:31:54.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:31:54.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T08:31:54.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_36 2026-02-18T08:31:54.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_image_and_enable_mirror cluster2 mirror image_36 journal 128M 2026-02-18T08:31:54.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:31:54.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:31:54.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:31:54.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:31:54.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_36 2026-02-18T08:31:54.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:31:54.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local mode=journal 2026-02-18T08:31:54.344 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n journal ']' 2026-02-18T08:31:54.344 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:31:54.344 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_image cluster2 mirror image_36 128M 2026-02-18T08:31:54.344 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:31:54.344 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:31:54.344 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:31:54.344 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:31:54.344 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_36 2026-02-18T08:31:54.344 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:31:54.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local size=128 2026-02-18T08:31:54.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n 128M ']' 2026-02-18T08:31:54.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ size=128M 2026-02-18T08:31:54.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:31:54.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,journaling mirror/image_36 2026-02-18T08:31:54.384 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ is_pool_mirror_mode_image mirror 2026-02-18T08:31:54.384 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:31:54.384 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' pool = image ']' 2026-02-18T08:31:54.384 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ case "${pool}" in 2026-02-18T08:31:54.384 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 1 2026-02-18T08:31:54.384 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n '' ']' 2026-02-18T08:31:54.384 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ stress_write_image cluster2 mirror image_36 2026-02-18T08:31:54.384 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:31:54.385 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:31:54.385 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_36 2026-02-18T08:31:54.385 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-18T08:31:54.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local duration=8 2026-02-18T08:31:54.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set +e 2026-02-18T08:31:54.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ timeout 8s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_36 --debug-rbd=20 --debug-journaler=20 2026-02-18T08:31:54.687 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:100 IOs, 12897280 bytes 2026-02-18T08:31:55.090 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:31:55.086+0000 7fb27e7f3640 -1 reset not still connected to 0x55e171e68000 2026-02-18T08:31:55.091 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:31:55.086+0000 7fb27e7f3640 -1 reset not still connected to 0x55e171e69380 2026-02-18T08:31:55.091 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:31:55.086+0000 7fb27e7f3640 -1 reset not still connected to 0x55e183c10750 2026-02-18T08:31:55.091 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:31:55.086+0000 7fb27e7f3640 -1 reset not still connected to 0x55e18817ba00 2026-02-18T08:31:55.092 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:31:55.086+0000 7fb27e7f3640 -1 reset not still connected to 0x55e1884eaf70 2026-02-18T08:31:55.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:200 IOs, 56641536 bytes 2026-02-18T08:31:55.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:300 IOs, 94838784 bytes 2026-02-18T08:31:56.302 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:400 IOs, 115336192 bytes 2026-02-18T08:31:57.526 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:500 IOs, 180344832 bytes 2026-02-18T08:31:58.190 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:600 IOs, 195997696 bytes 2026-02-18T08:31:58.746 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:700 IOs, 229625856 bytes 2026-02-18T08:31:58.864 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:800 IOs, 231569408 bytes 2026-02-18T08:31:59.047 INFO:tasks.ceph.cluster1.osd.1.trial034.stderr:2026-02-18T08:31:59.042+0000 7f20d7266640 -1 reset not still connected to 0x55640bcd0dd0 2026-02-18T08:31:59.047 INFO:tasks.ceph.cluster1.osd.1.trial034.stderr:2026-02-18T08:31:59.042+0000 7f20d7266640 -1 reset not still connected to 0x55640bcd1ad0 2026-02-18T08:31:59.047 INFO:tasks.ceph.cluster1.osd.1.trial034.stderr:2026-02-18T08:31:59.042+0000 7f20d7266640 -1 reset not still connected to 0x5564155c29c0 2026-02-18T08:31:59.300 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:31:59.294+0000 7faffc95c640 -1 reset not still connected to 0x5644c8fcf380 2026-02-18T08:31:59.301 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:31:59.294+0000 7faffc95c640 -1 reset not still connected to 0x5644d9b0e410 2026-02-18T08:31:59.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:900 IOs, 276043776 bytes 2026-02-18T08:31:59.804 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:31:59.802+0000 7fb27e7f3640 -1 reset not still connected to 0x55e17108f380 2026-02-18T08:31:59.805 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:31:59.802+0000 7fb27e7f3640 -1 reset not still connected to 0x55e171c0b860 2026-02-18T08:31:59.805 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:31:59.802+0000 7fb27e7f3640 -1 reset not still connected to 0x55e188bae0d0 2026-02-18T08:32:00.212 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1000 IOs, 315248640 bytes 2026-02-18T08:32:00.528 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:32:00.522+0000 7fb27e7f3640 -1 reset not still connected to 0x55e1705e44e0 2026-02-18T08:32:00.528 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:32:00.522+0000 7fb27e7f3640 -1 reset not still connected to 0x55e170742000 2026-02-18T08:32:00.528 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:32:00.522+0000 7fb27e7f3640 -1 reset not still connected to 0x55e170c6e000 2026-02-18T08:32:00.529 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:32:00.526+0000 7fb27e7f3640 -1 reset not still connected to 0x55e170c6e410 2026-02-18T08:32:00.529 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:32:00.526+0000 7fb27e7f3640 -1 reset not still connected to 0x55e171e68000 2026-02-18T08:32:00.529 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:32:00.526+0000 7fb27e7f3640 -1 reset not still connected to 0x55e171e69380 2026-02-18T08:32:00.530 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:32:00.526+0000 7fb27e7f3640 -1 reset not still connected to 0x55e183c10750 2026-02-18T08:32:00.530 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:32:00.526+0000 7fb27e7f3640 -1 reset not still connected to 0x55e18817ba00 2026-02-18T08:32:00.530 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:32:00.526+0000 7fb27e7f3640 -1 reset not still connected to 0x55e1884eaf70 2026-02-18T08:32:00.768 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1100 IOs, 336436224 bytes 2026-02-18T08:32:00.909 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:32:00.906+0000 7faffc95c640 -1 reset not still connected to 0x5644d9b0e410 2026-02-18T08:32:01.596 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:32:01.590+0000 7faffc95c640 -1 reset not still connected to 0x5644a716ac30 2026-02-18T08:32:01.596 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:32:01.594+0000 7faffc95c640 -1 reset not still connected to 0x5644a9836270 2026-02-18T08:32:01.599 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:32:01.594+0000 7faffc95c640 -1 reset not still connected to 0x5644b9a8cf70 2026-02-18T08:32:01.599 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:32:01.594+0000 7faffc95c640 -1 reset not still connected to 0x5644c0efc340 2026-02-18T08:32:01.599 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:32:01.594+0000 7faffc95c640 -1 reset not still connected to 0x5644c58269c0 2026-02-18T08:32:01.599 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:32:01.594+0000 7faffc95c640 -1 reset not still connected to 0x5644c8fcf790 2026-02-18T08:32:01.599 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:32:01.594+0000 7faffc95c640 -1 reset not still connected to 0x5644d9b0e9c0 2026-02-18T08:32:01.599 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:32:01.594+0000 7faffc95c640 -1 reset not still connected to 0x5644e04e09c0 2026-02-18T08:32:01.599 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:32:01.594+0000 7faffc95c640 -1 reset not still connected to 0x5644e04e1ad0 2026-02-18T08:32:01.827 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1200 IOs, 394572800 bytes 2026-02-18T08:32:02.395 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ error_code=124 2026-02-18T08:32:02.396 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set -e 2026-02-18T08:32:02.396 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 124 -eq 124 ']' 2026-02-18T08:32:02.396 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:32:02.396 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T08:32:02.396 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_37 2026-02-18T08:32:02.396 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_image_and_enable_mirror cluster2 mirror image_37 journal 128M 2026-02-18T08:32:02.396 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:32:02.396 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:32:02.396 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:32:02.397 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:32:02.397 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_37 2026-02-18T08:32:02.397 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:32:02.397 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local mode=journal 2026-02-18T08:32:02.397 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n journal ']' 2026-02-18T08:32:02.397 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:32:02.397 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_image cluster2 mirror image_37 128M 2026-02-18T08:32:02.397 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:32:02.397 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:32:02.397 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:32:02.397 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:32:02.397 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_37 2026-02-18T08:32:02.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:32:02.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local size=128 2026-02-18T08:32:02.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n 128M ']' 2026-02-18T08:32:02.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ size=128M 2026-02-18T08:32:02.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:32:02.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,journaling mirror/image_37 2026-02-18T08:32:02.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ is_pool_mirror_mode_image mirror 2026-02-18T08:32:02.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:32:02.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' pool = image ']' 2026-02-18T08:32:02.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ case "${pool}" in 2026-02-18T08:32:02.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 1 2026-02-18T08:32:02.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n '' ']' 2026-02-18T08:32:02.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ stress_write_image cluster2 mirror image_37 2026-02-18T08:32:02.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:32:02.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:32:02.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_37 2026-02-18T08:32:02.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-18T08:32:02.443 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local duration=10 2026-02-18T08:32:02.443 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set +e 2026-02-18T08:32:02.443 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ timeout 10s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_37 --debug-rbd=20 --debug-journaler=20 2026-02-18T08:32:03.217 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:32:03.214+0000 7fb27e7f3640 -1 reset not still connected to 0x55e170743ee0 2026-02-18T08:32:03.218 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:32:03.214+0000 7fb27e7f3640 -1 reset not still connected to 0x55e170ebe1a0 2026-02-18T08:32:03.218 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:32:03.214+0000 7fb27e7f3640 -1 reset not still connected to 0x55e17108f380 2026-02-18T08:32:03.218 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:32:03.214+0000 7fb27e7f3640 -1 reset not still connected to 0x55e171c0b860 2026-02-18T08:32:03.218 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:32:03.214+0000 7fb27e7f3640 -1 reset not still connected to 0x55e188bae0d0 2026-02-18T08:32:03.467 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:100 IOs, 48000000 bytes 2026-02-18T08:32:03.541 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:32:03.538+0000 7faffc95c640 -1 reset not still connected to 0x5644a5073110 2026-02-18T08:32:03.541 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:32:03.538+0000 7faffc95c640 -1 reset not still connected to 0x5644a84a61a0 2026-02-18T08:32:03.542 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:32:03.538+0000 7faffc95c640 -1 reset not still connected to 0x5644c1ba7d40 2026-02-18T08:32:03.542 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:32:03.538+0000 7faffc95c640 -1 reset not still connected to 0x5644c58264e0 2026-02-18T08:32:03.542 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:32:03.538+0000 7faffc95c640 -1 reset not still connected to 0x5644dfda1520 2026-02-18T08:32:03.542 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:32:03.538+0000 7faffc95c640 -1 reset not still connected to 0x5644e04e0d00 2026-02-18T08:32:03.851 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:200 IOs, 77677568 bytes 2026-02-18T08:32:04.526 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:300 IOs, 118383616 bytes 2026-02-18T08:32:05.280 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:400 IOs, 147415040 bytes 2026-02-18T08:32:06.610 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:500 IOs, 239083520 bytes 2026-02-18T08:32:07.433 INFO:tasks.ceph.cluster1.osd.1.trial034.stderr:2026-02-18T08:32:07.430+0000 7f20d7266640 -1 reset not still connected to 0x556408919380 2026-02-18T08:32:07.433 INFO:tasks.ceph.cluster1.osd.1.trial034.stderr:2026-02-18T08:32:07.430+0000 7f20d7266640 -1 reset not still connected to 0x5564155c2410 2026-02-18T08:32:07.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:600 IOs, 279358464 bytes 2026-02-18T08:32:07.780 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:700 IOs, 283732992 bytes 2026-02-18T08:32:08.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:800 IOs, 347657216 bytes 2026-02-18T08:32:09.274 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:900 IOs, 381468672 bytes 2026-02-18T08:32:09.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1000 IOs, 406584320 bytes 2026-02-18T08:32:10.350 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1100 IOs, 440637440 bytes 2026-02-18T08:32:10.827 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1200 IOs, 463099904 bytes 2026-02-18T08:32:10.854 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:32:10.850+0000 7fb27e7f3640 -1 reset not still connected to 0x55e179629ba0 2026-02-18T08:32:10.854 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:32:10.850+0000 7fb27e7f3640 -1 reset not still connected to 0x55e183c10f70 2026-02-18T08:32:10.854 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:32:10.850+0000 7fb27e7f3640 -1 reset not still connected to 0x55e188baf860 2026-02-18T08:32:10.854 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:32:10.850+0000 7fb27e7f3640 -1 reset not still connected to 0x55e1b0646a90 2026-02-18T08:32:11.131 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:32:11.126+0000 7fb27e7f3640 -1 reset not still connected to 0x55e171bf05b0 2026-02-18T08:32:11.131 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:32:11.126+0000 7fb27e7f3640 -1 reset not still connected to 0x55e171c0a750 2026-02-18T08:32:11.131 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:32:11.126+0000 7fb27e7f3640 -1 reset not still connected to 0x55e178bde340 2026-02-18T08:32:11.131 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:32:11.126+0000 7fb27e7f3640 -1 reset not still connected to 0x55e183c11860 2026-02-18T08:32:11.132 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:32:11.126+0000 7fb27e7f3640 -1 reset not still connected to 0x55e18895cea0 2026-02-18T08:32:11.132 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:32:11.126+0000 7fb27e7f3640 -1 reset not still connected to 0x55e1b0646000 2026-02-18T08:32:11.629 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:32:11.626+0000 7faffc95c640 -1 reset not still connected to 0x5644e04e1ad0 2026-02-18T08:32:11.836 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1300 IOs, 526021632 bytes 2026-02-18T08:32:12.322 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1400 IOs, 571699200 bytes 2026-02-18T08:32:12.453 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ error_code=124 2026-02-18T08:32:12.453 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set -e 2026-02-18T08:32:12.453 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 124 -eq 124 ']' 2026-02-18T08:32:12.453 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:32:12.453 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T08:32:12.454 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_38 2026-02-18T08:32:12.454 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_image_and_enable_mirror cluster2 mirror image_38 journal 128M 2026-02-18T08:32:12.454 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:32:12.454 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:32:12.454 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:32:12.454 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:32:12.454 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_38 2026-02-18T08:32:12.454 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:32:12.454 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local mode=journal 2026-02-18T08:32:12.454 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n journal ']' 2026-02-18T08:32:12.454 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:32:12.454 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_image cluster2 mirror image_38 128M 2026-02-18T08:32:12.455 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:32:12.455 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:32:12.455 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:32:12.455 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:32:12.455 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_38 2026-02-18T08:32:12.455 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:32:12.455 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local size=128 2026-02-18T08:32:12.455 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n 128M ']' 2026-02-18T08:32:12.455 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ size=128M 2026-02-18T08:32:12.455 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:32:12.455 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,journaling mirror/image_38 2026-02-18T08:32:12.500 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ is_pool_mirror_mode_image mirror 2026-02-18T08:32:12.500 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:32:12.500 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' pool = image ']' 2026-02-18T08:32:12.500 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ case "${pool}" in 2026-02-18T08:32:12.500 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 1 2026-02-18T08:32:12.500 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n '' ']' 2026-02-18T08:32:12.500 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ stress_write_image cluster2 mirror image_38 2026-02-18T08:32:12.501 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:32:12.501 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:32:12.501 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_38 2026-02-18T08:32:12.501 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-18T08:32:12.502 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local duration=6 2026-02-18T08:32:12.502 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set +e 2026-02-18T08:32:12.502 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ timeout 6s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_38 --debug-rbd=20 --debug-journaler=20 2026-02-18T08:32:12.721 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:100 IOs, 6921216 bytes 2026-02-18T08:32:12.872 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:32:12.866+0000 7faffc95c640 -1 reset not still connected to 0x5644d9b0e9c0 2026-02-18T08:32:14.170 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:200 IOs, 88969216 bytes 2026-02-18T08:32:15.068 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:300 IOs, 113180672 bytes 2026-02-18T08:32:15.771 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:400 IOs, 154747904 bytes 2026-02-18T08:32:16.989 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:500 IOs, 256006144 bytes 2026-02-18T08:32:17.597 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:32:17.590+0000 7faffc95c640 -1 reset not still connected to 0x5644a0dcbad0 2026-02-18T08:32:17.597 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:32:17.590+0000 7faffc95c640 -1 reset not still connected to 0x5644a2f8ba00 2026-02-18T08:32:17.597 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:32:17.590+0000 7faffc95c640 -1 reset not still connected to 0x5644a41fe5b0 2026-02-18T08:32:17.597 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:32:17.590+0000 7faffc95c640 -1 reset not still connected to 0x5644a4424270 2026-02-18T08:32:17.597 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:32:17.590+0000 7faffc95c640 -1 reset not still connected to 0x5644a5073110 2026-02-18T08:32:17.597 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:32:17.590+0000 7faffc95c640 -1 reset not still connected to 0x5644a84a61a0 2026-02-18T08:32:17.597 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:32:17.590+0000 7faffc95c640 -1 reset not still connected to 0x5644c1ba7d40 2026-02-18T08:32:17.598 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:32:17.590+0000 7faffc95c640 -1 reset not still connected to 0x5644c58264e0 2026-02-18T08:32:17.598 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:32:17.594+0000 7faffc95c640 -1 reset not still connected to 0x5644dfda1520 2026-02-18T08:32:17.598 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:32:17.594+0000 7faffc95c640 -1 reset not still connected to 0x5644e04e0d00 2026-02-18T08:32:18.511 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ error_code=124 2026-02-18T08:32:18.512 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set -e 2026-02-18T08:32:18.512 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 124 -eq 124 ']' 2026-02-18T08:32:18.512 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:32:18.512 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T08:32:18.512 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_39 2026-02-18T08:32:18.512 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_image_and_enable_mirror cluster2 mirror image_39 journal 128M 2026-02-18T08:32:18.512 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:32:18.512 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:32:18.512 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:32:18.512 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:32:18.513 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_39 2026-02-18T08:32:18.513 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:32:18.513 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local mode=journal 2026-02-18T08:32:18.513 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n journal ']' 2026-02-18T08:32:18.513 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:32:18.513 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_image cluster2 mirror image_39 128M 2026-02-18T08:32:18.513 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:32:18.513 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:32:18.513 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:32:18.513 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:32:18.513 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_39 2026-02-18T08:32:18.513 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:32:18.514 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local size=128 2026-02-18T08:32:18.514 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n 128M ']' 2026-02-18T08:32:18.514 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ size=128M 2026-02-18T08:32:18.514 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:32:18.514 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,journaling mirror/image_39 2026-02-18T08:32:18.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ is_pool_mirror_mode_image mirror 2026-02-18T08:32:18.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:32:18.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' pool = image ']' 2026-02-18T08:32:18.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ case "${pool}" in 2026-02-18T08:32:18.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 1 2026-02-18T08:32:18.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n '' ']' 2026-02-18T08:32:18.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ stress_write_image cluster2 mirror image_39 2026-02-18T08:32:18.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:32:18.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:32:18.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_39 2026-02-18T08:32:18.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-18T08:32:18.557 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local duration=14 2026-02-18T08:32:18.557 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set +e 2026-02-18T08:32:18.557 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ timeout 14s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_39 --debug-rbd=20 --debug-journaler=20 2026-02-18T08:32:18.910 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:100 IOs, 10746880 bytes 2026-02-18T08:32:19.385 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:200 IOs, 35492864 bytes 2026-02-18T08:32:20.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:300 IOs, 55448576 bytes 2026-02-18T08:32:20.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:400 IOs, 111227904 bytes 2026-02-18T08:32:20.955 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:32:20.949+0000 7fb27e7f3640 -1 reset not still connected to 0x55e17071ec30 2026-02-18T08:32:20.957 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:32:20.953+0000 7fb27e7f3640 -1 reset not still connected to 0x55e170742ea0 2026-02-18T08:32:20.957 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:32:20.953+0000 7fb27e7f3640 -1 reset not still connected to 0x55e170c6e8f0 2026-02-18T08:32:20.957 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:32:20.953+0000 7fb27e7f3640 -1 reset not still connected to 0x55e171bf05b0 2026-02-18T08:32:20.957 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:32:20.953+0000 7fb27e7f3640 -1 reset not still connected to 0x55e171c0a750 2026-02-18T08:32:20.957 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:32:20.953+0000 7fb27e7f3640 -1 reset not still connected to 0x55e178bde340 2026-02-18T08:32:20.957 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:32:20.953+0000 7fb27e7f3640 -1 reset not still connected to 0x55e183c11860 2026-02-18T08:32:20.957 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:32:20.953+0000 7fb27e7f3640 -1 reset not still connected to 0x55e18895cea0 2026-02-18T08:32:20.958 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:32:20.953+0000 7fb27e7f3640 -1 reset not still connected to 0x55e1b0646000 2026-02-18T08:32:21.050 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:32:21.045+0000 7faffc95c640 -1 reset not still connected to 0x5644c1b595f0 2026-02-18T08:32:21.050 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:32:21.045+0000 7faffc95c640 -1 reset not still connected to 0x5644c8fcf380 2026-02-18T08:32:21.050 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:32:21.045+0000 7faffc95c640 -1 reset not still connected to 0x5644d9b0e410 2026-02-18T08:32:21.051 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:32:21.045+0000 7faffc95c640 -1 reset not still connected to 0x5644dc3ccc30 2026-02-18T08:32:21.778 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:32:21.773+0000 7fb27e7f3640 -1 reset not still connected to 0x55e188baf860 2026-02-18T08:32:21.779 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:32:21.773+0000 7fb27e7f3640 -1 reset not still connected to 0x55e1b0646a90 2026-02-18T08:32:22.280 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:500 IOs, 147654656 bytes 2026-02-18T08:32:22.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:600 IOs, 179194880 bytes 2026-02-18T08:32:22.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:700 IOs, 179804160 bytes 2026-02-18T08:32:23.428 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:800 IOs, 222509056 bytes 2026-02-18T08:32:24.472 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:900 IOs, 274121728 bytes 2026-02-18T08:32:24.719 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:32:24.713+0000 7fb27e7f3640 -1 reset not still connected to 0x55e17108f380 2026-02-18T08:32:25.377 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1000 IOs, 352425984 bytes 2026-02-18T08:32:25.570 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:32:25.565+0000 7faffc95c640 -1 reset not still connected to 0x5644a41fe5b0 2026-02-18T08:32:25.570 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:32:25.565+0000 7faffc95c640 -1 reset not still connected to 0x5644a4424270 2026-02-18T08:32:25.570 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:32:25.565+0000 7faffc95c640 -1 reset not still connected to 0x5644a5073110 2026-02-18T08:32:25.570 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:32:25.565+0000 7faffc95c640 -1 reset not still connected to 0x5644a84a61a0 2026-02-18T08:32:25.570 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:32:25.565+0000 7faffc95c640 -1 reset not still connected to 0x5644c1ba7d40 2026-02-18T08:32:25.570 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:32:25.565+0000 7faffc95c640 -1 reset not still connected to 0x5644c58264e0 2026-02-18T08:32:25.570 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:32:25.565+0000 7faffc95c640 -1 reset not still connected to 0x5644dfda1520 2026-02-18T08:32:25.571 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:32:25.565+0000 7faffc95c640 -1 reset not still connected to 0x5644e04e0d00 2026-02-18T08:32:25.671 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1100 IOs, 383642624 bytes 2026-02-18T08:32:25.711 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:32:25.705+0000 7faffc95c640 -1 reset not still connected to 0x5644a0dcbad0 2026-02-18T08:32:25.713 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:32:25.709+0000 7faffc95c640 -1 reset not still connected to 0x5644a2f8ba00 2026-02-18T08:32:25.713 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:32:25.709+0000 7faffc95c640 -1 reset not still connected to 0x5644a41fe5b0 2026-02-18T08:32:25.713 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:32:25.709+0000 7faffc95c640 -1 reset not still connected to 0x5644a4424270 2026-02-18T08:32:25.713 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:32:25.709+0000 7faffc95c640 -1 reset not still connected to 0x5644a5073110 2026-02-18T08:32:25.714 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:32:25.709+0000 7faffc95c640 -1 reset not still connected to 0x5644a84a61a0 2026-02-18T08:32:25.714 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:32:25.709+0000 7faffc95c640 -1 reset not still connected to 0x5644c1ba7d40 2026-02-18T08:32:25.714 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:32:25.709+0000 7faffc95c640 -1 reset not still connected to 0x5644c58264e0 2026-02-18T08:32:25.714 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:32:25.709+0000 7faffc95c640 -1 reset not still connected to 0x5644dfda1520 2026-02-18T08:32:25.714 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:32:25.709+0000 7faffc95c640 -1 reset not still connected to 0x5644e04e0d00 2026-02-18T08:32:26.067 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:32:26.061+0000 7faffc95c640 -1 reset not still connected to 0x5644c8fcf790 2026-02-18T08:32:26.068 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:32:26.061+0000 7faffc95c640 -1 reset not still connected to 0x5644d9b0e9c0 2026-02-18T08:32:26.068 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:32:26.061+0000 7faffc95c640 -1 reset not still connected to 0x5644e04e09c0 2026-02-18T08:32:26.068 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:32:26.061+0000 7faffc95c640 -1 reset not still connected to 0x5644e04e1ad0 2026-02-18T08:32:26.132 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:32:26.125+0000 7faffc95c640 -1 reset not still connected to 0x5644c0efc340 2026-02-18T08:32:26.133 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:32:26.129+0000 7faffc95c640 -1 reset not still connected to 0x5644c58269c0 2026-02-18T08:32:26.133 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:32:26.129+0000 7faffc95c640 -1 reset not still connected to 0x5644c8fcf790 2026-02-18T08:32:26.133 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:32:26.129+0000 7faffc95c640 -1 reset not still connected to 0x5644d9b0e9c0 2026-02-18T08:32:26.133 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:32:26.129+0000 7faffc95c640 -1 reset not still connected to 0x5644e04e09c0 2026-02-18T08:32:26.133 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:32:26.129+0000 7faffc95c640 -1 reset not still connected to 0x5644e04e1ad0 2026-02-18T08:32:26.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1200 IOs, 446764032 bytes 2026-02-18T08:32:26.997 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:32:26.993+0000 7faffc95c640 -1 reset not still connected to 0x5644c8fcf790 2026-02-18T08:32:26.997 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:32:26.993+0000 7faffc95c640 -1 reset not still connected to 0x5644d9b0e9c0 2026-02-18T08:32:26.997 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:32:26.993+0000 7faffc95c640 -1 reset not still connected to 0x5644e04e09c0 2026-02-18T08:32:26.998 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:32:26.993+0000 7faffc95c640 -1 reset not still connected to 0x5644e04e1ad0 2026-02-18T08:32:27.399 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:32:27.393+0000 7fb27e7f3640 -1 reset not still connected to 0x55e1705e44e0 2026-02-18T08:32:27.400 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:32:27.393+0000 7fb27e7f3640 -1 reset not still connected to 0x55e170742000 2026-02-18T08:32:27.400 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:32:27.393+0000 7fb27e7f3640 -1 reset not still connected to 0x55e170c6e000 2026-02-18T08:32:27.400 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:32:27.393+0000 7fb27e7f3640 -1 reset not still connected to 0x55e170c6e410 2026-02-18T08:32:27.400 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:32:27.393+0000 7fb27e7f3640 -1 reset not still connected to 0x55e171e68000 2026-02-18T08:32:27.400 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:32:27.393+0000 7fb27e7f3640 -1 reset not still connected to 0x55e171e69380 2026-02-18T08:32:27.400 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:32:27.393+0000 7fb27e7f3640 -1 reset not still connected to 0x55e183c10750 2026-02-18T08:32:27.400 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:32:27.393+0000 7fb27e7f3640 -1 reset not still connected to 0x55e18817ba00 2026-02-18T08:32:27.401 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:32:27.393+0000 7fb27e7f3640 -1 reset not still connected to 0x55e1884eaf70 2026-02-18T08:32:28.095 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:32:28.089+0000 7faffc95c640 -1 reset not still connected to 0x5644af614c30 2026-02-18T08:32:28.095 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:32:28.089+0000 7faffc95c640 -1 reset not still connected to 0x5644c19b4340 2026-02-18T08:32:28.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1300 IOs, 549712896 bytes 2026-02-18T08:32:28.366 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:32:28.361+0000 7fb27e7f3640 -1 reset not still connected to 0x55e18817ba00 2026-02-18T08:32:28.367 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:32:28.361+0000 7fb27e7f3640 -1 reset not still connected to 0x55e1884eaf70 2026-02-18T08:32:28.688 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:32:28.681+0000 7faffc95c640 -1 reset not still connected to 0x5644a41fe5b0 2026-02-18T08:32:28.688 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:32:28.685+0000 7faffc95c640 -1 reset not still connected to 0x5644a4424270 2026-02-18T08:32:28.688 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:32:28.685+0000 7faffc95c640 -1 reset not still connected to 0x5644a5073110 2026-02-18T08:32:28.688 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:32:28.685+0000 7faffc95c640 -1 reset not still connected to 0x5644a84a61a0 2026-02-18T08:32:28.688 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:32:28.685+0000 7faffc95c640 -1 reset not still connected to 0x5644c1ba7d40 2026-02-18T08:32:28.688 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:32:28.685+0000 7faffc95c640 -1 reset not still connected to 0x5644c58264e0 2026-02-18T08:32:28.688 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:32:28.685+0000 7faffc95c640 -1 reset not still connected to 0x5644dfda1520 2026-02-18T08:32:28.688 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:32:28.685+0000 7faffc95c640 -1 reset not still connected to 0x5644e04e0d00 2026-02-18T08:32:29.383 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1400 IOs, 610763776 bytes 2026-02-18T08:32:29.527 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:32:29.521+0000 7faffc95c640 -1 reset not still connected to 0x5644a6840410 2026-02-18T08:32:29.527 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:32:29.521+0000 7faffc95c640 -1 reset not still connected to 0x5644a716a410 2026-02-18T08:32:29.528 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:32:29.521+0000 7faffc95c640 -1 reset not still connected to 0x5644a84a6750 2026-02-18T08:32:29.529 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:32:29.525+0000 7faffc95c640 -1 reset not still connected to 0x5644a84d5930 2026-02-18T08:32:29.529 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:32:29.525+0000 7faffc95c640 -1 reset not still connected to 0x5644c1b595f0 2026-02-18T08:32:29.529 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:32:29.525+0000 7faffc95c640 -1 reset not still connected to 0x5644c8fcf380 2026-02-18T08:32:29.529 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:32:29.525+0000 7faffc95c640 -1 reset not still connected to 0x5644d9b0e410 2026-02-18T08:32:29.529 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:32:29.525+0000 7faffc95c640 -1 reset not still connected to 0x5644dc3ccc30 2026-02-18T08:32:29.584 INFO:tasks.ceph.cluster1.osd.1.trial034.stderr:2026-02-18T08:32:29.577+0000 7f20d7266640 -1 reset not still connected to 0x556408919380 2026-02-18T08:32:29.584 INFO:tasks.ceph.cluster1.osd.1.trial034.stderr:2026-02-18T08:32:29.577+0000 7f20d7266640 -1 reset not still connected to 0x5564155c2410 2026-02-18T08:32:29.937 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:32:29.933+0000 7faffc95c640 -1 reset not still connected to 0x5644a3cb4750 2026-02-18T08:32:29.937 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:32:29.933+0000 7faffc95c640 -1 reset not still connected to 0x5644a4424c30 2026-02-18T08:32:29.937 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:32:29.933+0000 7faffc95c640 -1 reset not still connected to 0x5644a44256c0 2026-02-18T08:32:29.938 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:32:29.933+0000 7faffc95c640 -1 reset not still connected to 0x5644a68408f0 2026-02-18T08:32:29.938 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:32:29.933+0000 7faffc95c640 -1 reset not still connected to 0x5644a716ab60 2026-02-18T08:32:29.938 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:32:29.933+0000 7faffc95c640 -1 reset not still connected to 0x5644af614c30 2026-02-18T08:32:29.938 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:32:29.933+0000 7faffc95c640 -1 reset not still connected to 0x5644c19b4340 2026-02-18T08:32:30.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1500 IOs, 643719168 bytes 2026-02-18T08:32:30.325 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:32:30.321+0000 7fb27e7f3640 -1 reset not still connected to 0x55e179629ba0 2026-02-18T08:32:30.325 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:32:30.321+0000 7fb27e7f3640 -1 reset not still connected to 0x55e183c10f70 2026-02-18T08:32:30.325 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:32:30.321+0000 7fb27e7f3640 -1 reset not still connected to 0x55e188baf860 2026-02-18T08:32:30.326 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:32:30.321+0000 7fb27e7f3640 -1 reset not still connected to 0x55e1b0646a90 2026-02-18T08:32:30.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1600 IOs, 679895040 bytes 2026-02-18T08:32:30.662 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:32:30.657+0000 7faffc95c640 -1 reset not still connected to 0x5644a5073860 2026-02-18T08:32:30.663 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:32:30.657+0000 7faffc95c640 -1 reset not still connected to 0x5644a6840410 2026-02-18T08:32:30.663 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:32:30.657+0000 7faffc95c640 -1 reset not still connected to 0x5644a716a410 2026-02-18T08:32:30.663 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:32:30.657+0000 7faffc95c640 -1 reset not still connected to 0x5644a84a6750 2026-02-18T08:32:30.664 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:32:30.657+0000 7faffc95c640 -1 reset not still connected to 0x5644a84d5930 2026-02-18T08:32:30.664 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:32:30.657+0000 7faffc95c640 -1 reset not still connected to 0x5644c1b595f0 2026-02-18T08:32:30.664 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:32:30.657+0000 7faffc95c640 -1 reset not still connected to 0x5644c8fcf380 2026-02-18T08:32:30.665 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:32:30.657+0000 7faffc95c640 -1 reset not still connected to 0x5644d9b0e410 2026-02-18T08:32:30.665 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:32:30.657+0000 7faffc95c640 -1 reset not still connected to 0x5644dc3ccc30 2026-02-18T08:32:31.132 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1700 IOs, 709174272 bytes 2026-02-18T08:32:31.390 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:32:31.385+0000 7faffc95c640 -1 reset not still connected to 0x5644e04e1ad0 2026-02-18T08:32:31.678 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1800 IOs, 725861376 bytes 2026-02-18T08:32:31.955 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:32:31.949+0000 7fb27e7f3640 -1 reset not still connected to 0x55e171c0b860 2026-02-18T08:32:31.955 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:32:31.949+0000 7fb27e7f3640 -1 reset not still connected to 0x55e188bae0d0 2026-02-18T08:32:32.566 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ error_code=124 2026-02-18T08:32:32.567 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set -e 2026-02-18T08:32:32.567 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 124 -eq 124 ']' 2026-02-18T08:32:32.567 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:32:32.567 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T08:32:32.567 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_40 2026-02-18T08:32:32.567 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_image_and_enable_mirror cluster2 mirror image_40 journal 128M 2026-02-18T08:32:32.567 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:32:32.567 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:32:32.567 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:32:32.568 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:32:32.568 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_40 2026-02-18T08:32:32.568 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:32:32.568 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local mode=journal 2026-02-18T08:32:32.568 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n journal ']' 2026-02-18T08:32:32.568 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:32:32.568 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_image cluster2 mirror image_40 128M 2026-02-18T08:32:32.568 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:32:32.568 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:32:32.568 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:32:32.568 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:32:32.568 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_40 2026-02-18T08:32:32.569 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:32:32.569 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local size=128 2026-02-18T08:32:32.569 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n 128M ']' 2026-02-18T08:32:32.569 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ size=128M 2026-02-18T08:32:32.569 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:32:32.569 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,journaling mirror/image_40 2026-02-18T08:32:32.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ is_pool_mirror_mode_image mirror 2026-02-18T08:32:32.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:32:32.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' pool = image ']' 2026-02-18T08:32:32.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ case "${pool}" in 2026-02-18T08:32:32.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 1 2026-02-18T08:32:32.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n '' ']' 2026-02-18T08:32:32.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ stress_write_image cluster2 mirror image_40 2026-02-18T08:32:32.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:32:32.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:32:32.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_40 2026-02-18T08:32:32.610 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-18T08:32:32.610 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local duration=6 2026-02-18T08:32:32.610 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set +e 2026-02-18T08:32:32.610 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ timeout 6s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_40 --debug-rbd=20 --debug-journaler=20 2026-02-18T08:32:34.477 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:100 IOs, 95416320 bytes 2026-02-18T08:32:34.768 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:32:34.761+0000 7faffc95c640 -1 reset not still connected to 0x5644c58269c0 2026-02-18T08:32:34.769 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:32:34.765+0000 7faffc95c640 -1 reset not still connected to 0x5644c8fcf790 2026-02-18T08:32:34.769 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:32:34.765+0000 7faffc95c640 -1 reset not still connected to 0x5644d9b0e9c0 2026-02-18T08:32:35.200 INFO:tasks.ceph.cluster2.osd.0.trial163.stderr:2026-02-18T08:32:35.191+0000 7f2129686640 -1 reset not still connected to 0x5617578955f0 2026-02-18T08:32:35.200 INFO:tasks.ceph.cluster2.osd.0.trial163.stderr:2026-02-18T08:32:35.191+0000 7f2129686640 -1 reset not still connected to 0x56178357f450 2026-02-18T08:32:35.200 INFO:tasks.ceph.cluster2.osd.0.trial163.stderr:2026-02-18T08:32:35.191+0000 7f2129686640 -1 reset not still connected to 0x56178357fd40 2026-02-18T08:32:35.427 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:200 IOs, 123220992 bytes 2026-02-18T08:32:35.500 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:32:35.497+0000 7faffc95c640 -1 reset not still connected to 0x5644c0efc340 2026-02-18T08:32:35.501 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:32:35.497+0000 7faffc95c640 -1 reset not still connected to 0x5644c58269c0 2026-02-18T08:32:35.501 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:32:35.497+0000 7faffc95c640 -1 reset not still connected to 0x5644c8fcf790 2026-02-18T08:32:35.501 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:32:35.497+0000 7faffc95c640 -1 reset not still connected to 0x5644d9b0e9c0 2026-02-18T08:32:35.501 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:32:35.497+0000 7faffc95c640 -1 reset not still connected to 0x5644e04e09c0 2026-02-18T08:32:35.502 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:32:35.497+0000 7faffc95c640 -1 reset not still connected to 0x5644e04e1ad0 2026-02-18T08:32:36.244 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:32:36.241+0000 7faffc95c640 -1 reset not still connected to 0x5644a0dcbad0 2026-02-18T08:32:36.245 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:32:36.241+0000 7faffc95c640 -1 reset not still connected to 0x5644a2f8ba00 2026-02-18T08:32:36.245 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:32:36.241+0000 7faffc95c640 -1 reset not still connected to 0x5644a41fe5b0 2026-02-18T08:32:36.246 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:32:36.241+0000 7faffc95c640 -1 reset not still connected to 0x5644a4424270 2026-02-18T08:32:36.246 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:32:36.241+0000 7faffc95c640 -1 reset not still connected to 0x5644a5073110 2026-02-18T08:32:36.246 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:32:36.241+0000 7faffc95c640 -1 reset not still connected to 0x5644a84a61a0 2026-02-18T08:32:36.246 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:32:36.241+0000 7faffc95c640 -1 reset not still connected to 0x5644c1ba7d40 2026-02-18T08:32:36.246 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:32:36.241+0000 7faffc95c640 -1 reset not still connected to 0x5644c58264e0 2026-02-18T08:32:36.246 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:32:36.241+0000 7faffc95c640 -1 reset not still connected to 0x5644dfda1520 2026-02-18T08:32:36.247 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:32:36.241+0000 7faffc95c640 -1 reset not still connected to 0x5644e04e0d00 2026-02-18T08:32:36.363 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:300 IOs, 180789248 bytes 2026-02-18T08:32:36.842 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:400 IOs, 255034368 bytes 2026-02-18T08:32:38.416 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:500 IOs, 314126336 bytes 2026-02-18T08:32:38.489 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:32:38.485+0000 7fb27e7f3640 -1 reset not still connected to 0x55e170ebe1a0 2026-02-18T08:32:38.490 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:32:38.485+0000 7fb27e7f3640 -1 reset not still connected to 0x55e17108f380 2026-02-18T08:32:38.491 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:32:38.485+0000 7fb27e7f3640 -1 reset not still connected to 0x55e171c0a680 2026-02-18T08:32:38.491 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:32:38.485+0000 7fb27e7f3640 -1 reset not still connected to 0x55e171c0b860 2026-02-18T08:32:38.491 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:32:38.485+0000 7fb27e7f3640 -1 reset not still connected to 0x55e188bae0d0 2026-02-18T08:32:38.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ error_code=124 2026-02-18T08:32:38.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set -e 2026-02-18T08:32:38.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 124 -eq 124 ']' 2026-02-18T08:32:38.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:32:38.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T08:32:38.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_41 2026-02-18T08:32:38.622 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_image_and_enable_mirror cluster2 mirror image_41 journal 128M 2026-02-18T08:32:38.622 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:32:38.622 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:32:38.622 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:32:38.622 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:32:38.622 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_41 2026-02-18T08:32:38.622 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:32:38.622 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local mode=journal 2026-02-18T08:32:38.622 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n journal ']' 2026-02-18T08:32:38.622 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:32:38.622 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_image cluster2 mirror image_41 128M 2026-02-18T08:32:38.622 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:32:38.623 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:32:38.623 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:32:38.623 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:32:38.623 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_41 2026-02-18T08:32:38.623 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:32:38.623 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local size=128 2026-02-18T08:32:38.623 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n 128M ']' 2026-02-18T08:32:38.623 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ size=128M 2026-02-18T08:32:38.623 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:32:38.623 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,journaling mirror/image_41 2026-02-18T08:32:38.840 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:32:38.837+0000 7fb27e7f3640 -1 reset not still connected to 0x55e1705e44e0 2026-02-18T08:32:38.840 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:32:38.837+0000 7fb27e7f3640 -1 reset not still connected to 0x55e170742000 2026-02-18T08:32:38.841 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:32:38.837+0000 7fb27e7f3640 -1 reset not still connected to 0x55e170c6e000 2026-02-18T08:32:38.841 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:32:38.837+0000 7fb27e7f3640 -1 reset not still connected to 0x55e170c6e410 2026-02-18T08:32:38.842 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:32:38.837+0000 7fb27e7f3640 -1 reset not still connected to 0x55e170ebfe10 2026-02-18T08:32:38.842 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:32:38.837+0000 7fb27e7f3640 -1 reset not still connected to 0x55e171e68000 2026-02-18T08:32:38.842 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:32:38.837+0000 7fb27e7f3640 -1 reset not still connected to 0x55e171e69380 2026-02-18T08:32:38.842 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:32:38.837+0000 7fb27e7f3640 -1 reset not still connected to 0x55e183c10750 2026-02-18T08:32:38.842 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:32:38.837+0000 7fb27e7f3640 -1 reset not still connected to 0x55e18817ba00 2026-02-18T08:32:38.843 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:32:38.837+0000 7fb27e7f3640 -1 reset not still connected to 0x55e1884eaf70 2026-02-18T08:32:38.870 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ is_pool_mirror_mode_image mirror 2026-02-18T08:32:38.870 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:32:38.870 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' pool = image ']' 2026-02-18T08:32:38.870 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ case "${pool}" in 2026-02-18T08:32:38.870 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 1 2026-02-18T08:32:38.870 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n '' ']' 2026-02-18T08:32:38.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ stress_write_image cluster2 mirror image_41 2026-02-18T08:32:38.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:32:38.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:32:38.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_41 2026-02-18T08:32:38.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-18T08:32:38.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local duration=7 2026-02-18T08:32:38.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set +e 2026-02-18T08:32:38.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ timeout 7s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_41 --debug-rbd=20 --debug-journaler=20 2026-02-18T08:32:38.949 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:32:38.945+0000 7faffc95c640 -1 reset not still connected to 0x5644c8fcf790 2026-02-18T08:32:38.950 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:32:38.945+0000 7faffc95c640 -1 reset not still connected to 0x5644d9b0e9c0 2026-02-18T08:32:38.950 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:32:38.945+0000 7faffc95c640 -1 reset not still connected to 0x5644e04e1ad0 2026-02-18T08:32:39.459 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:32:39.453+0000 7faffc95c640 -1 reset not still connected to 0x5644a84a61a0 2026-02-18T08:32:39.459 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:32:39.453+0000 7faffc95c640 -1 reset not still connected to 0x5644c1ba7d40 2026-02-18T08:32:39.460 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:32:39.457+0000 7faffc95c640 -1 reset not still connected to 0x5644c58264e0 2026-02-18T08:32:39.461 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:32:39.457+0000 7faffc95c640 -1 reset not still connected to 0x5644dfda1520 2026-02-18T08:32:39.461 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:32:39.457+0000 7faffc95c640 -1 reset not still connected to 0x5644e04e0d00 2026-02-18T08:32:39.747 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:100 IOs, 43379712 bytes 2026-02-18T08:32:40.404 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:32:40.397+0000 7fb27e7f3640 -1 reset not still connected to 0x55e170743ee0 2026-02-18T08:32:40.404 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:32:40.401+0000 7fb27e7f3640 -1 reset not still connected to 0x55e170ebe1a0 2026-02-18T08:32:40.405 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:32:40.401+0000 7fb27e7f3640 -1 reset not still connected to 0x55e17108f380 2026-02-18T08:32:40.405 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:32:40.401+0000 7fb27e7f3640 -1 reset not still connected to 0x55e171c0a680 2026-02-18T08:32:40.405 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:32:40.401+0000 7fb27e7f3640 -1 reset not still connected to 0x55e171c0b860 2026-02-18T08:32:40.405 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:32:40.401+0000 7fb27e7f3640 -1 reset not still connected to 0x55e188bae0d0 2026-02-18T08:32:40.415 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:200 IOs, 94144512 bytes 2026-02-18T08:32:41.425 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:32:41.421+0000 7faffc95c640 -1 reset not still connected to 0x5644c58269c0 2026-02-18T08:32:41.426 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:32:41.421+0000 7faffc95c640 -1 reset not still connected to 0x5644c8fcf790 2026-02-18T08:32:41.427 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:32:41.421+0000 7faffc95c640 -1 reset not still connected to 0x5644d9b0e9c0 2026-02-18T08:32:41.427 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:32:41.421+0000 7faffc95c640 -1 reset not still connected to 0x5644e04e09c0 2026-02-18T08:32:41.427 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:32:41.421+0000 7faffc95c640 -1 reset not still connected to 0x5644e04e1ad0 2026-02-18T08:32:41.514 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:32:41.509+0000 7faffc95c640 -1 reset not still connected to 0x5644c0efc340 2026-02-18T08:32:41.515 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:32:41.509+0000 7faffc95c640 -1 reset not still connected to 0x5644c58269c0 2026-02-18T08:32:41.515 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:32:41.509+0000 7faffc95c640 -1 reset not still connected to 0x5644c8fcf790 2026-02-18T08:32:41.515 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:32:41.509+0000 7faffc95c640 -1 reset not still connected to 0x5644d9b0e9c0 2026-02-18T08:32:41.516 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:32:41.509+0000 7faffc95c640 -1 reset not still connected to 0x5644e04e09c0 2026-02-18T08:32:42.737 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:300 IOs, 209776640 bytes 2026-02-18T08:32:43.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:400 IOs, 286161920 bytes 2026-02-18T08:32:43.982 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:32:43.977+0000 7fb27e7f3640 -1 reset not still connected to 0x55e170743ee0 2026-02-18T08:32:43.983 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:32:43.977+0000 7fb27e7f3640 -1 reset not still connected to 0x55e170ebe1a0 2026-02-18T08:32:43.983 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:32:43.977+0000 7fb27e7f3640 -1 reset not still connected to 0x55e17108f380 2026-02-18T08:32:43.984 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:32:43.977+0000 7fb27e7f3640 -1 reset not still connected to 0x55e171c0a680 2026-02-18T08:32:43.984 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:32:43.977+0000 7fb27e7f3640 -1 reset not still connected to 0x55e171c0b860 2026-02-18T08:32:43.984 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:32:43.977+0000 7fb27e7f3640 -1 reset not still connected to 0x55e188bae0d0 2026-02-18T08:32:44.449 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:500 IOs, 314595328 bytes 2026-02-18T08:32:44.844 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:32:44.837+0000 7faffc95c640 -1 reset not still connected to 0x5644a84a6750 2026-02-18T08:32:44.844 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:32:44.837+0000 7faffc95c640 -1 reset not still connected to 0x5644a84d5930 2026-02-18T08:32:44.844 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:32:44.837+0000 7faffc95c640 -1 reset not still connected to 0x5644c1b595f0 2026-02-18T08:32:44.844 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:32:44.837+0000 7faffc95c640 -1 reset not still connected to 0x5644c8fcf380 2026-02-18T08:32:44.844 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:32:44.837+0000 7faffc95c640 -1 reset not still connected to 0x5644d9b0e410 2026-02-18T08:32:44.844 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:32:44.837+0000 7faffc95c640 -1 reset not still connected to 0x5644dc3ccc30 2026-02-18T08:32:45.416 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:600 IOs, 379356160 bytes 2026-02-18T08:32:45.882 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ error_code=124 2026-02-18T08:32:45.883 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set -e 2026-02-18T08:32:45.883 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 124 -eq 124 ']' 2026-02-18T08:32:45.883 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:32:45.883 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T08:32:45.883 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_42 2026-02-18T08:32:45.883 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_image_and_enable_mirror cluster2 mirror image_42 journal 128M 2026-02-18T08:32:45.883 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:32:45.883 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:32:45.883 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:32:45.883 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:32:45.883 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_42 2026-02-18T08:32:45.884 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:32:45.884 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local mode=journal 2026-02-18T08:32:45.884 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n journal ']' 2026-02-18T08:32:45.884 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:32:45.884 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_image cluster2 mirror image_42 128M 2026-02-18T08:32:45.884 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:32:45.884 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:32:45.884 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:32:45.884 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:32:45.884 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_42 2026-02-18T08:32:45.884 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:32:45.885 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local size=128 2026-02-18T08:32:45.885 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n 128M ']' 2026-02-18T08:32:45.885 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ size=128M 2026-02-18T08:32:45.885 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:32:45.885 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,journaling mirror/image_42 2026-02-18T08:32:45.923 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ is_pool_mirror_mode_image mirror 2026-02-18T08:32:45.923 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:32:45.923 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' pool = image ']' 2026-02-18T08:32:45.923 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ case "${pool}" in 2026-02-18T08:32:45.923 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 1 2026-02-18T08:32:45.923 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n '' ']' 2026-02-18T08:32:45.923 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ stress_write_image cluster2 mirror image_42 2026-02-18T08:32:45.924 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:32:45.924 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:32:45.924 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_42 2026-02-18T08:32:45.924 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-18T08:32:45.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local duration=6 2026-02-18T08:32:45.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set +e 2026-02-18T08:32:45.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ timeout 6s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_42 --debug-rbd=20 --debug-journaler=20 2026-02-18T08:32:46.285 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:32:46.281+0000 7faffc95c640 -1 reset not still connected to 0x5644c8fcf790 2026-02-18T08:32:46.285 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:32:46.281+0000 7faffc95c640 -1 reset not still connected to 0x5644d9b0e9c0 2026-02-18T08:32:46.285 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:32:46.281+0000 7faffc95c640 -1 reset not still connected to 0x5644e04e09c0 2026-02-18T08:32:46.286 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:32:46.281+0000 7faffc95c640 -1 reset not still connected to 0x5644e04e1ad0 2026-02-18T08:32:46.667 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:100 IOs, 35685376 bytes 2026-02-18T08:32:46.809 INFO:tasks.ceph.cluster1.osd.1.trial034.stderr:2026-02-18T08:32:46.805+0000 7f20d7266640 -1 reset not still connected to 0x5564155c29c0 2026-02-18T08:32:46.809 INFO:tasks.ceph.cluster1.osd.1.trial034.stderr:2026-02-18T08:32:46.805+0000 7f20d7266640 -1 reset not still connected to 0x5564228bf5f0 2026-02-18T08:32:47.395 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:200 IOs, 83937280 bytes 2026-02-18T08:32:47.619 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:32:47.613+0000 7fb27e7f3640 -1 reset not still connected to 0x55e170ebfd40 2026-02-18T08:32:47.619 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:32:47.613+0000 7fb27e7f3640 -1 reset not still connected to 0x55e171e69c70 2026-02-18T08:32:47.619 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:32:47.613+0000 7fb27e7f3640 -1 reset not still connected to 0x55e178bde820 2026-02-18T08:32:47.620 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:32:47.613+0000 7fb27e7f3640 -1 reset not still connected to 0x55e179629ba0 2026-02-18T08:32:47.620 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:32:47.613+0000 7fb27e7f3640 -1 reset not still connected to 0x55e183c10f70 2026-02-18T08:32:47.620 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:32:47.617+0000 7fb27e7f3640 -1 reset not still connected to 0x55e188baf860 2026-02-18T08:32:47.620 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:32:47.617+0000 7fb27e7f3640 -1 reset not still connected to 0x55e1b0646a90 2026-02-18T08:32:47.696 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:300 IOs, 98646016 bytes 2026-02-18T08:32:47.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:400 IOs, 99792896 bytes 2026-02-18T08:32:48.334 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:32:48.329+0000 7fb27e7f3640 -1 reset not still connected to 0x55e183c10f70 2026-02-18T08:32:48.334 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:32:48.329+0000 7fb27e7f3640 -1 reset not still connected to 0x55e188baf860 2026-02-18T08:32:49.279 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:500 IOs, 141149184 bytes 2026-02-18T08:32:50.067 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:600 IOs, 182572032 bytes 2026-02-18T08:32:50.388 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:32:50.385+0000 7fb27e7f3640 -1 reset not still connected to 0x55e170743ee0 2026-02-18T08:32:50.389 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:32:50.385+0000 7fb27e7f3640 -1 reset not still connected to 0x55e170ebe1a0 2026-02-18T08:32:50.389 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:32:50.385+0000 7fb27e7f3640 -1 reset not still connected to 0x55e17108f380 2026-02-18T08:32:50.390 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:32:50.385+0000 7fb27e7f3640 -1 reset not still connected to 0x55e171c0a680 2026-02-18T08:32:50.390 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:32:50.385+0000 7fb27e7f3640 -1 reset not still connected to 0x55e171c0b860 2026-02-18T08:32:50.390 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:32:50.385+0000 7fb27e7f3640 -1 reset not still connected to 0x55e188bae0d0 2026-02-18T08:32:50.484 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:700 IOs, 199196672 bytes 2026-02-18T08:32:50.810 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:32:50.805+0000 7fb27e7f3640 -1 reset not still connected to 0x55e170c6e8f0 2026-02-18T08:32:50.810 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:32:50.805+0000 7fb27e7f3640 -1 reset not still connected to 0x55e171bf05b0 2026-02-18T08:32:50.811 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:32:50.805+0000 7fb27e7f3640 -1 reset not still connected to 0x55e171bf1110 2026-02-18T08:32:50.811 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:32:50.805+0000 7fb27e7f3640 -1 reset not still connected to 0x55e171c0a750 2026-02-18T08:32:50.811 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:32:50.805+0000 7fb27e7f3640 -1 reset not still connected to 0x55e178bde340 2026-02-18T08:32:50.811 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:32:50.805+0000 7fb27e7f3640 -1 reset not still connected to 0x55e183c11860 2026-02-18T08:32:50.811 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:32:50.805+0000 7fb27e7f3640 -1 reset not still connected to 0x55e18895cea0 2026-02-18T08:32:50.813 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:32:50.809+0000 7fb27e7f3640 -1 reset not still connected to 0x55e1b0646000 2026-02-18T08:32:51.270 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:32:51.265+0000 7fb27e7f3640 -1 reset not still connected to 0x55e178bde340 2026-02-18T08:32:51.270 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:32:51.265+0000 7fb27e7f3640 -1 reset not still connected to 0x55e183c11860 2026-02-18T08:32:51.270 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:32:51.265+0000 7fb27e7f3640 -1 reset not still connected to 0x55e18895cea0 2026-02-18T08:32:51.297 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:800 IOs, 240753664 bytes 2026-02-18T08:32:51.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ error_code=124 2026-02-18T08:32:51.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set -e 2026-02-18T08:32:51.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 124 -eq 124 ']' 2026-02-18T08:32:51.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:32:51.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T08:32:51.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_43 2026-02-18T08:32:51.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_image_and_enable_mirror cluster2 mirror image_43 journal 128M 2026-02-18T08:32:51.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:32:51.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:32:51.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:32:51.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:32:51.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_43 2026-02-18T08:32:51.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:32:51.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local mode=journal 2026-02-18T08:32:51.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n journal ']' 2026-02-18T08:32:51.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:32:51.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_image cluster2 mirror image_43 128M 2026-02-18T08:32:51.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:32:51.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:32:51.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:32:51.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:32:51.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_43 2026-02-18T08:32:51.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:32:51.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local size=128 2026-02-18T08:32:51.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n 128M ']' 2026-02-18T08:32:51.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ size=128M 2026-02-18T08:32:51.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:32:51.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,journaling mirror/image_43 2026-02-18T08:32:52.025 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:32:52.021+0000 7fb27e7f3640 -1 reset not still connected to 0x55e16ef2b790 2026-02-18T08:32:52.025 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:32:52.021+0000 7fb27e7f3640 -1 reset not still connected to 0x55e17071ec30 2026-02-18T08:32:52.027 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:32:52.021+0000 7fb27e7f3640 -1 reset not still connected to 0x55e170742ea0 2026-02-18T08:32:52.029 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:32:52.025+0000 7fb27e7f3640 -1 reset not still connected to 0x55e170c6e8f0 2026-02-18T08:32:52.029 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:32:52.025+0000 7fb27e7f3640 -1 reset not still connected to 0x55e171bf05b0 2026-02-18T08:32:52.029 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:32:52.025+0000 7fb27e7f3640 -1 reset not still connected to 0x55e171bf1110 2026-02-18T08:32:52.029 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:32:52.025+0000 7fb27e7f3640 -1 reset not still connected to 0x55e171c0a750 2026-02-18T08:32:52.029 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:32:52.025+0000 7fb27e7f3640 -1 reset not still connected to 0x55e178bde340 2026-02-18T08:32:52.029 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:32:52.025+0000 7fb27e7f3640 -1 reset not still connected to 0x55e183c11860 2026-02-18T08:32:52.029 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:32:52.025+0000 7fb27e7f3640 -1 reset not still connected to 0x55e18895cea0 2026-02-18T08:32:52.029 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:32:52.025+0000 7fb27e7f3640 -1 reset not still connected to 0x55e1b0646000 2026-02-18T08:32:52.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ is_pool_mirror_mode_image mirror 2026-02-18T08:32:52.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:32:52.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' pool = image ']' 2026-02-18T08:32:52.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ case "${pool}" in 2026-02-18T08:32:52.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 1 2026-02-18T08:32:52.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n '' ']' 2026-02-18T08:32:52.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ stress_write_image cluster2 mirror image_43 2026-02-18T08:32:52.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:32:52.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:32:52.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_43 2026-02-18T08:32:52.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-18T08:32:52.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local duration=6 2026-02-18T08:32:52.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set +e 2026-02-18T08:32:52.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ timeout 6s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_43 --debug-rbd=20 --debug-journaler=20 2026-02-18T08:32:52.593 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.0 with skip (0.39824200448881497) > weight (0.25) 2026-02-18T08:32:52.593 INFO:tasks.rbd_mirror_thrash:kill cluster2.client.mirror.1 2026-02-18T08:32:52.593 INFO:tasks.rbd_mirror.cluster2.client.mirror.1:Sent signal 15 2026-02-18T08:32:52.594 INFO:tasks.rbd_mirror_thrash:waiting for 2.931168720646864 secs before reviving daemons 2026-02-18T08:32:52.594 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.trial163.stderr:2026-02-18T08:32:52.587+0000 7f08be64d640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.1 (PID: 88736) UID: 1000 2026-02-18T08:32:52.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:100 IOs, 44409856 bytes 2026-02-18T08:32:54.200 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:200 IOs, 110018560 bytes 2026-02-18T08:32:55.003 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:300 IOs, 152923136 bytes 2026-02-18T08:32:55.292 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:32:55.285+0000 7faffc95c640 -1 reset not still connected to 0x5644a5073110 2026-02-18T08:32:55.292 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:32:55.289+0000 7faffc95c640 -1 reset not still connected to 0x5644a84a61a0 2026-02-18T08:32:55.293 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:32:55.289+0000 7faffc95c640 -1 reset not still connected to 0x5644c1ba7d40 2026-02-18T08:32:55.293 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:32:55.289+0000 7faffc95c640 -1 reset not still connected to 0x5644c58264e0 2026-02-18T08:32:55.293 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:32:55.289+0000 7faffc95c640 -1 reset not still connected to 0x5644e04e0d00 2026-02-18T08:32:55.526 INFO:tasks.rbd_mirror_thrash:waiting for cluster2.client.mirror.1 2026-02-18T08:32:55.526 INFO:teuthology.orchestra.run:waiting for 600 2026-02-18T08:32:55.526 INFO:tasks.rbd_mirror_thrash:reviving cluster2.client.mirror.1 2026-02-18T08:32:55.526 INFO:tasks.rbd_mirror.cluster2.client.mirror.1:Restarting daemon 2026-02-18T08:32:55.526 DEBUG:teuthology.orchestra.run.trial163:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.1 2026-02-18T08:32:55.528 INFO:tasks.rbd_mirror.cluster2.client.mirror.1:Started 2026-02-18T08:32:55.529 INFO:tasks.rbd_mirror_thrash:waiting for 119.59685627917979 secs before thrashing 2026-02-18T08:32:55.813 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:400 IOs, 213748736 bytes 2026-02-18T08:32:57.585 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:500 IOs, 293635072 bytes 2026-02-18T08:32:57.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:600 IOs, 298151936 bytes 2026-02-18T08:32:58.191 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:32:58.185+0000 7faffc95c640 -1 reset not still connected to 0x5644c1b595f0 2026-02-18T08:32:58.191 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:32:58.185+0000 7faffc95c640 -1 reset not still connected to 0x5644c8fcf380 2026-02-18T08:32:58.192 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:32:58.185+0000 7faffc95c640 -1 reset not still connected to 0x5644d9b0e410 2026-02-18T08:32:58.192 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:32:58.189+0000 7faffc95c640 -1 reset not still connected to 0x5644dc3ccc30 2026-02-18T08:32:58.198 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ error_code=124 2026-02-18T08:32:58.198 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set -e 2026-02-18T08:32:58.198 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 124 -eq 124 ']' 2026-02-18T08:32:58.198 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:32:58.198 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T08:32:58.199 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_44 2026-02-18T08:32:58.199 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_image_and_enable_mirror cluster2 mirror image_44 journal 128M 2026-02-18T08:32:58.199 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:32:58.199 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:32:58.199 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:32:58.199 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:32:58.200 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_44 2026-02-18T08:32:58.200 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:32:58.200 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local mode=journal 2026-02-18T08:32:58.200 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n journal ']' 2026-02-18T08:32:58.200 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:32:58.200 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_image cluster2 mirror image_44 128M 2026-02-18T08:32:58.201 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:32:58.201 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:32:58.201 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:32:58.201 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:32:58.201 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_44 2026-02-18T08:32:58.202 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:32:58.202 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local size=128 2026-02-18T08:32:58.202 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n 128M ']' 2026-02-18T08:32:58.202 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ size=128M 2026-02-18T08:32:58.202 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:32:58.202 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,journaling mirror/image_44 2026-02-18T08:32:58.238 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ is_pool_mirror_mode_image mirror 2026-02-18T08:32:58.238 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:32:58.238 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' pool = image ']' 2026-02-18T08:32:58.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ case "${pool}" in 2026-02-18T08:32:58.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 1 2026-02-18T08:32:58.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n '' ']' 2026-02-18T08:32:58.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ stress_write_image cluster2 mirror image_44 2026-02-18T08:32:58.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:32:58.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:32:58.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_44 2026-02-18T08:32:58.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-18T08:32:58.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local duration=11 2026-02-18T08:32:58.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set +e 2026-02-18T08:32:58.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ timeout 11s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_44 --debug-rbd=20 --debug-journaler=20 2026-02-18T08:32:58.845 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:32:58.841+0000 7faffc95c640 -1 reset not still connected to 0x5644d9b0e9c0 2026-02-18T08:32:58.846 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:32:58.841+0000 7faffc95c640 -1 reset not still connected to 0x5644e04e09c0 2026-02-18T08:32:58.847 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:32:58.841+0000 7faffc95c640 -1 reset not still connected to 0x5644e04e1ad0 2026-02-18T08:32:59.122 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:32:59.117+0000 7faffc95c640 -1 reset not still connected to 0x5644c0efc340 2026-02-18T08:32:59.123 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:32:59.117+0000 7faffc95c640 -1 reset not still connected to 0x5644c58269c0 2026-02-18T08:32:59.123 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:32:59.117+0000 7faffc95c640 -1 reset not still connected to 0x5644c8fcf790 2026-02-18T08:32:59.123 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:32:59.117+0000 7faffc95c640 -1 reset not still connected to 0x5644d9b0e9c0 2026-02-18T08:32:59.123 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:32:59.117+0000 7faffc95c640 -1 reset not still connected to 0x5644e04e09c0 2026-02-18T08:32:59.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:100 IOs, 92406784 bytes 2026-02-18T08:32:59.927 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:32:59.921+0000 7faffc95c640 -1 reset not still connected to 0x5644a5073860 2026-02-18T08:32:59.928 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:32:59.921+0000 7faffc95c640 -1 reset not still connected to 0x5644a716a410 2026-02-18T08:32:59.928 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:32:59.921+0000 7faffc95c640 -1 reset not still connected to 0x5644a84a6750 2026-02-18T08:32:59.928 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:32:59.921+0000 7faffc95c640 -1 reset not still connected to 0x5644c1b595f0 2026-02-18T08:32:59.928 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:32:59.925+0000 7faffc95c640 -1 reset not still connected to 0x5644c8fcf380 2026-02-18T08:32:59.929 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:32:59.925+0000 7faffc95c640 -1 reset not still connected to 0x5644d9b0e410 2026-02-18T08:32:59.929 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:32:59.925+0000 7faffc95c640 -1 reset not still connected to 0x5644dc3ccc30 2026-02-18T08:33:00.174 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:33:00.169+0000 7fb27e7f3640 -1 reset not still connected to 0x55e171c0b860 2026-02-18T08:33:00.537 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:33:00.533+0000 7fb27e7f3640 -1 reset not still connected to 0x55e170742000 2026-02-18T08:33:00.538 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:33:00.533+0000 7fb27e7f3640 -1 reset not still connected to 0x55e170c6e000 2026-02-18T08:33:00.538 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:33:00.533+0000 7fb27e7f3640 -1 reset not still connected to 0x55e170c6e410 2026-02-18T08:33:00.539 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:33:00.533+0000 7fb27e7f3640 -1 reset not still connected to 0x55e170ebfe10 2026-02-18T08:33:00.539 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:33:00.533+0000 7fb27e7f3640 -1 reset not still connected to 0x55e171e68000 2026-02-18T08:33:00.539 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:33:00.533+0000 7fb27e7f3640 -1 reset not still connected to 0x55e171e69380 2026-02-18T08:33:00.540 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:33:00.533+0000 7fb27e7f3640 -1 reset not still connected to 0x55e171e69520 2026-02-18T08:33:00.540 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:33:00.533+0000 7fb27e7f3640 -1 reset not still connected to 0x55e183c10750 2026-02-18T08:33:00.540 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:33:00.533+0000 7fb27e7f3640 -1 reset not still connected to 0x55e18817ba00 2026-02-18T08:33:00.540 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:33:00.533+0000 7fb27e7f3640 -1 reset not still connected to 0x55e1884eaf70 2026-02-18T08:33:00.540 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:33:00.533+0000 7fb27e7f3640 -1 reset not still connected to 0x55e193b90b60 2026-02-18T08:33:00.541 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:33:00.537+0000 7fb27e7f3640 -1 reset not still connected to 0x55e16dd9fd40 2026-02-18T08:33:00.541 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:33:00.537+0000 7fb27e7f3640 -1 reset not still connected to 0x55e170693450 2026-02-18T08:33:00.541 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:33:00.537+0000 7fb27e7f3640 -1 reset not still connected to 0x55e170743ee0 2026-02-18T08:33:00.541 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:33:00.537+0000 7fb27e7f3640 -1 reset not still connected to 0x55e170ebe1a0 2026-02-18T08:33:00.541 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:33:00.537+0000 7fb27e7f3640 -1 reset not still connected to 0x55e17108f380 2026-02-18T08:33:00.541 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:33:00.537+0000 7fb27e7f3640 -1 reset not still connected to 0x55e171c0a680 2026-02-18T08:33:00.542 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:33:00.537+0000 7fb27e7f3640 -1 reset not still connected to 0x55e171c0b860 2026-02-18T08:33:00.542 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:33:00.537+0000 7fb27e7f3640 -1 reset not still connected to 0x55e188bae0d0 2026-02-18T08:33:01.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:200 IOs, 152768512 bytes 2026-02-18T08:33:01.078 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:33:01.073+0000 7faffc95c640 -1 reset not still connected to 0x5644a84a6750 2026-02-18T08:33:01.445 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:33:01.441+0000 7fb27e7f3640 -1 reset not still connected to 0x55e1705e44e0 2026-02-18T08:33:01.445 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:33:01.441+0000 7fb27e7f3640 -1 reset not still connected to 0x55e170742000 2026-02-18T08:33:01.445 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:33:01.441+0000 7fb27e7f3640 -1 reset not still connected to 0x55e170c6e000 2026-02-18T08:33:01.446 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:33:01.441+0000 7fb27e7f3640 -1 reset not still connected to 0x55e170c6e410 2026-02-18T08:33:01.447 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:33:01.441+0000 7fb27e7f3640 -1 reset not still connected to 0x55e170ebfe10 2026-02-18T08:33:01.448 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:33:01.441+0000 7fb27e7f3640 -1 reset not still connected to 0x55e171e68000 2026-02-18T08:33:01.448 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:33:01.441+0000 7fb27e7f3640 -1 reset not still connected to 0x55e171e69380 2026-02-18T08:33:01.448 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:33:01.441+0000 7fb27e7f3640 -1 reset not still connected to 0x55e171e69520 2026-02-18T08:33:01.448 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:33:01.441+0000 7fb27e7f3640 -1 reset not still connected to 0x55e183c10750 2026-02-18T08:33:01.448 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:33:01.441+0000 7fb27e7f3640 -1 reset not still connected to 0x55e18817ba00 2026-02-18T08:33:01.448 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:33:01.441+0000 7fb27e7f3640 -1 reset not still connected to 0x55e1884eaf70 2026-02-18T08:33:01.448 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:33:01.441+0000 7fb27e7f3640 -1 reset not still connected to 0x55e193b90b60 2026-02-18T08:33:01.471 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:33:01.465+0000 7fb27e7f3640 -1 reset not still connected to 0x55e18817ba00 2026-02-18T08:33:01.471 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:33:01.465+0000 7fb27e7f3640 -1 reset not still connected to 0x55e193b90b60 2026-02-18T08:33:01.480 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:300 IOs, 192843776 bytes 2026-02-18T08:33:02.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:400 IOs, 216248320 bytes 2026-02-18T08:33:02.840 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:500 IOs, 259296256 bytes 2026-02-18T08:33:03.103 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:33:03.097+0000 7fb27e7f3640 -1 reset not still connected to 0x55e183c11860 2026-02-18T08:33:03.104 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:33:03.097+0000 7fb27e7f3640 -1 reset not still connected to 0x55e18895cea0 2026-02-18T08:33:03.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:600 IOs, 281149440 bytes 2026-02-18T08:33:03.319 INFO:tasks.ceph.cluster1.osd.1.trial034.stderr:2026-02-18T08:33:03.313+0000 7f20d7266640 -1 reset not still connected to 0x5563fd7308f0 2026-02-18T08:33:03.319 INFO:tasks.ceph.cluster1.osd.1.trial034.stderr:2026-02-18T08:33:03.313+0000 7f20d7266640 -1 reset not still connected to 0x5563fd731450 2026-02-18T08:33:03.320 INFO:tasks.ceph.cluster1.osd.1.trial034.stderr:2026-02-18T08:33:03.313+0000 7f20d7266640 -1 reset not still connected to 0x5563fe1041a0 2026-02-18T08:33:03.320 INFO:tasks.ceph.cluster1.osd.1.trial034.stderr:2026-02-18T08:33:03.317+0000 7f20d7266640 -1 reset not still connected to 0x5563fe2316c0 2026-02-18T08:33:03.320 INFO:tasks.ceph.cluster1.osd.1.trial034.stderr:2026-02-18T08:33:03.317+0000 7f20d7266640 -1 reset not still connected to 0x5563ff09dd40 2026-02-18T08:33:03.320 INFO:tasks.ceph.cluster1.osd.1.trial034.stderr:2026-02-18T08:33:03.317+0000 7f20d7266640 -1 reset not still connected to 0x55640bcd1110 2026-02-18T08:33:03.321 INFO:tasks.ceph.cluster1.osd.1.trial034.stderr:2026-02-18T08:33:03.317+0000 7f20d7266640 -1 reset not still connected to 0x5564155c3d40 2026-02-18T08:33:03.321 INFO:tasks.ceph.cluster1.osd.1.trial034.stderr:2026-02-18T08:33:03.317+0000 7f20d7266640 -1 reset not still connected to 0x5564228bfd40 2026-02-18T08:33:03.472 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:700 IOs, 305292288 bytes 2026-02-18T08:33:03.518 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:33:03.513+0000 7faffc95c640 -1 reset not still connected to 0x5644a716ac30 2026-02-18T08:33:03.518 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:33:03.513+0000 7faffc95c640 -1 reset not still connected to 0x5644a9836270 2026-02-18T08:33:03.520 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:33:03.517+0000 7faffc95c640 -1 reset not still connected to 0x5644b9a8cf70 2026-02-18T08:33:03.520 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:33:03.517+0000 7faffc95c640 -1 reset not still connected to 0x5644bd70a750 2026-02-18T08:33:03.521 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:33:03.517+0000 7faffc95c640 -1 reset not still connected to 0x5644c0efc340 2026-02-18T08:33:03.521 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:33:03.517+0000 7faffc95c640 -1 reset not still connected to 0x5644c58269c0 2026-02-18T08:33:03.521 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:33:03.517+0000 7faffc95c640 -1 reset not still connected to 0x5644c8fcf790 2026-02-18T08:33:03.521 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:33:03.517+0000 7faffc95c640 -1 reset not still connected to 0x5644d9b0e9c0 2026-02-18T08:33:03.521 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:33:03.517+0000 7faffc95c640 -1 reset not still connected to 0x5644e04e09c0 2026-02-18T08:33:03.521 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:33:03.517+0000 7faffc95c640 -1 reset not still connected to 0x5644e04e1ad0 2026-02-18T08:33:03.692 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:33:03.685+0000 7faffc95c640 -1 reset not still connected to 0x5644a716ac30 2026-02-18T08:33:03.692 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:33:03.689+0000 7faffc95c640 -1 reset not still connected to 0x5644a9836270 2026-02-18T08:33:03.692 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:33:03.689+0000 7faffc95c640 -1 reset not still connected to 0x5644b9a8cf70 2026-02-18T08:33:03.693 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:33:03.689+0000 7faffc95c640 -1 reset not still connected to 0x5644bd70a750 2026-02-18T08:33:03.693 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:33:03.689+0000 7faffc95c640 -1 reset not still connected to 0x5644c0efc340 2026-02-18T08:33:03.693 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:33:03.689+0000 7faffc95c640 -1 reset not still connected to 0x5644c58269c0 2026-02-18T08:33:03.694 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:33:03.689+0000 7faffc95c640 -1 reset not still connected to 0x5644c8fcf790 2026-02-18T08:33:03.694 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:33:03.689+0000 7faffc95c640 -1 reset not still connected to 0x5644d9b0e9c0 2026-02-18T08:33:03.694 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:33:03.689+0000 7faffc95c640 -1 reset not still connected to 0x5644e04e09c0 2026-02-18T08:33:03.694 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:33:03.689+0000 7faffc95c640 -1 reset not still connected to 0x5644e04e1ad0 2026-02-18T08:33:04.122 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:800 IOs, 333952000 bytes 2026-02-18T08:33:04.232 INFO:tasks.ceph.cluster1.osd.1.trial034.stderr:2026-02-18T08:33:04.229+0000 7f20d7266640 -1 reset not still connected to 0x5563fe2316c0 2026-02-18T08:33:04.232 INFO:tasks.ceph.cluster1.osd.1.trial034.stderr:2026-02-18T08:33:04.229+0000 7f20d7266640 -1 reset not still connected to 0x5563ff09dd40 2026-02-18T08:33:04.232 INFO:tasks.ceph.cluster1.osd.1.trial034.stderr:2026-02-18T08:33:04.229+0000 7f20d7266640 -1 reset not still connected to 0x55640bcd1110 2026-02-18T08:33:04.232 INFO:tasks.ceph.cluster1.osd.1.trial034.stderr:2026-02-18T08:33:04.229+0000 7f20d7266640 -1 reset not still connected to 0x5564155c3d40 2026-02-18T08:33:04.233 INFO:tasks.ceph.cluster1.osd.1.trial034.stderr:2026-02-18T08:33:04.229+0000 7f20d7266640 -1 reset not still connected to 0x5564228bfd40 2026-02-18T08:33:04.842 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:33:04.837+0000 7fb27e7f3640 -1 reset not still connected to 0x55e171c0b860 2026-02-18T08:33:04.856 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:33:04.849+0000 7fb27e7f3640 -1 reset not still connected to 0x55e18817ba00 2026-02-18T08:33:04.856 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:33:04.853+0000 7fb27e7f3640 -1 reset not still connected to 0x55e193b90b60 2026-02-18T08:33:05.749 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:900 IOs, 399721472 bytes 2026-02-18T08:33:05.939 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:33:05.921+0000 7faffc95c640 -1 reset not still connected to 0x5644bd70a750 2026-02-18T08:33:05.939 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:33:05.921+0000 7faffc95c640 -1 reset not still connected to 0x5644c0efc340 2026-02-18T08:33:05.939 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:33:05.925+0000 7faffc95c640 -1 reset not still connected to 0x5644c58269c0 2026-02-18T08:33:05.939 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:33:05.925+0000 7faffc95c640 -1 reset not still connected to 0x5644c8fcf790 2026-02-18T08:33:05.939 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:33:05.925+0000 7faffc95c640 -1 reset not still connected to 0x5644d9b0e9c0 2026-02-18T08:33:05.939 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:33:05.925+0000 7faffc95c640 -1 reset not still connected to 0x5644e04e09c0 2026-02-18T08:33:05.940 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:33:05.925+0000 7faffc95c640 -1 reset not still connected to 0x5644e04e1ad0 2026-02-18T08:33:06.327 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:33:06.321+0000 7fb27e7f3640 -1 reset not still connected to 0x55e17108f380 2026-02-18T08:33:06.328 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:33:06.325+0000 7fb27e7f3640 -1 reset not still connected to 0x55e171c0a680 2026-02-18T08:33:06.330 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:33:06.325+0000 7fb27e7f3640 -1 reset not still connected to 0x55e171c0b860 2026-02-18T08:33:06.330 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:33:06.325+0000 7fb27e7f3640 -1 reset not still connected to 0x55e188bae0d0 2026-02-18T08:33:06.707 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1000 IOs, 439625728 bytes 2026-02-18T08:33:07.491 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1100 IOs, 520951808 bytes 2026-02-18T08:33:08.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1200 IOs, 541030400 bytes 2026-02-18T08:33:08.207 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1300 IOs, 547605504 bytes 2026-02-18T08:33:08.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1400 IOs, 573761536 bytes 2026-02-18T08:33:08.564 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:33:08.557+0000 7faffc95c640 -1 reset not still connected to 0x5644a9836270 2026-02-18T08:33:08.564 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:33:08.561+0000 7faffc95c640 -1 reset not still connected to 0x5644b9a8cf70 2026-02-18T08:33:08.565 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:33:08.561+0000 7faffc95c640 -1 reset not still connected to 0x5644bd70a750 2026-02-18T08:33:08.565 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:33:08.561+0000 7faffc95c640 -1 reset not still connected to 0x5644c0efc340 2026-02-18T08:33:08.565 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:33:08.561+0000 7faffc95c640 -1 reset not still connected to 0x5644c58269c0 2026-02-18T08:33:08.565 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:33:08.561+0000 7faffc95c640 -1 reset not still connected to 0x5644c8fcf790 2026-02-18T08:33:08.565 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:33:08.561+0000 7faffc95c640 -1 reset not still connected to 0x5644d9b0e9c0 2026-02-18T08:33:08.565 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:33:08.561+0000 7faffc95c640 -1 reset not still connected to 0x5644e04e09c0 2026-02-18T08:33:08.565 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:33:08.561+0000 7faffc95c640 -1 reset not still connected to 0x5644e04e1ad0 2026-02-18T08:33:08.753 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:33:08.749+0000 7faffc95c640 -1 reset not still connected to 0x5644a380aa90 2026-02-18T08:33:08.754 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:33:08.749+0000 7faffc95c640 -1 reset not still connected to 0x5644a5073860 2026-02-18T08:33:08.754 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:33:08.749+0000 7faffc95c640 -1 reset not still connected to 0x5644a6840410 2026-02-18T08:33:08.754 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:33:08.749+0000 7faffc95c640 -1 reset not still connected to 0x5644a716a410 2026-02-18T08:33:08.754 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:33:08.749+0000 7faffc95c640 -1 reset not still connected to 0x5644a84a6750 2026-02-18T08:33:08.755 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:33:08.749+0000 7faffc95c640 -1 reset not still connected to 0x5644a84d5930 2026-02-18T08:33:08.756 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:33:08.749+0000 7faffc95c640 -1 reset not still connected to 0x5644c1b595f0 2026-02-18T08:33:08.756 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:33:08.749+0000 7faffc95c640 -1 reset not still connected to 0x5644c8fcf380 2026-02-18T08:33:08.756 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:33:08.749+0000 7faffc95c640 -1 reset not still connected to 0x5644d9b0e410 2026-02-18T08:33:08.756 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:33:08.749+0000 7faffc95c640 -1 reset not still connected to 0x5644dc3ccc30 2026-02-18T08:33:08.861 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:33:08.857+0000 7faffc95c640 -1 reset not still connected to 0x5644bd70a750 2026-02-18T08:33:08.861 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:33:08.857+0000 7faffc95c640 -1 reset not still connected to 0x5644c0efc340 2026-02-18T08:33:08.862 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:33:08.857+0000 7faffc95c640 -1 reset not still connected to 0x5644c58269c0 2026-02-18T08:33:08.862 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:33:08.857+0000 7faffc95c640 -1 reset not still connected to 0x5644c8fcf790 2026-02-18T08:33:08.862 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:33:08.857+0000 7faffc95c640 -1 reset not still connected to 0x5644d9b0e9c0 2026-02-18T08:33:08.862 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:33:08.857+0000 7faffc95c640 -1 reset not still connected to 0x5644e04e09c0 2026-02-18T08:33:08.863 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:33:08.857+0000 7faffc95c640 -1 reset not still connected to 0x5644e04e1ad0 2026-02-18T08:33:09.248 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ error_code=124 2026-02-18T08:33:09.248 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set -e 2026-02-18T08:33:09.249 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 124 -eq 124 ']' 2026-02-18T08:33:09.249 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:33:09.249 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T08:33:09.249 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_45 2026-02-18T08:33:09.249 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_image_and_enable_mirror cluster2 mirror image_45 journal 128M 2026-02-18T08:33:09.249 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:33:09.249 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:33:09.249 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:33:09.249 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:33:09.249 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_45 2026-02-18T08:33:09.250 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:33:09.250 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local mode=journal 2026-02-18T08:33:09.250 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n journal ']' 2026-02-18T08:33:09.250 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:33:09.250 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_image cluster2 mirror image_45 128M 2026-02-18T08:33:09.250 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:33:09.250 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:33:09.250 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:33:09.250 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:33:09.250 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_45 2026-02-18T08:33:09.250 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:33:09.250 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local size=128 2026-02-18T08:33:09.251 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n 128M ']' 2026-02-18T08:33:09.251 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ size=128M 2026-02-18T08:33:09.251 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:33:09.251 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,journaling mirror/image_45 2026-02-18T08:33:09.289 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ is_pool_mirror_mode_image mirror 2026-02-18T08:33:09.289 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:33:09.289 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' pool = image ']' 2026-02-18T08:33:09.290 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ case "${pool}" in 2026-02-18T08:33:09.290 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 1 2026-02-18T08:33:09.290 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n '' ']' 2026-02-18T08:33:09.290 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ stress_write_image cluster2 mirror image_45 2026-02-18T08:33:09.290 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:33:09.290 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:33:09.290 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_45 2026-02-18T08:33:09.290 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-18T08:33:09.291 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local duration=13 2026-02-18T08:33:09.291 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set +e 2026-02-18T08:33:09.292 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ timeout 13s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_45 --debug-rbd=20 --debug-journaler=20 2026-02-18T08:33:09.662 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:33:09.657+0000 7faffc95c640 -1 reset not still connected to 0x5644a716ac30 2026-02-18T08:33:09.663 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:33:09.657+0000 7faffc95c640 -1 reset not still connected to 0x5644a9836270 2026-02-18T08:33:09.663 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:33:09.657+0000 7faffc95c640 -1 reset not still connected to 0x5644b9a8cf70 2026-02-18T08:33:09.664 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:33:09.661+0000 7faffc95c640 -1 reset not still connected to 0x5644bd70a750 2026-02-18T08:33:09.664 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:33:09.661+0000 7faffc95c640 -1 reset not still connected to 0x5644c0efc340 2026-02-18T08:33:09.664 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:33:09.661+0000 7faffc95c640 -1 reset not still connected to 0x5644c58269c0 2026-02-18T08:33:09.664 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:33:09.661+0000 7faffc95c640 -1 reset not still connected to 0x5644c8fcf790 2026-02-18T08:33:09.664 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:33:09.661+0000 7faffc95c640 -1 reset not still connected to 0x5644d9b0e9c0 2026-02-18T08:33:09.664 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:33:09.661+0000 7faffc95c640 -1 reset not still connected to 0x5644e04e09c0 2026-02-18T08:33:09.665 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:33:09.661+0000 7faffc95c640 -1 reset not still connected to 0x5644e04e1ad0 2026-02-18T08:33:09.740 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:100 IOs, 19188736 bytes 2026-02-18T08:33:10.705 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:200 IOs, 87292928 bytes 2026-02-18T08:33:11.513 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:300 IOs, 119609344 bytes 2026-02-18T08:33:11.553 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:33:11.549+0000 7faffc95c640 -1 reset not still connected to 0x5644a34c3450 2026-02-18T08:33:11.553 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:33:11.549+0000 7faffc95c640 -1 reset not still connected to 0x5644a716ac30 2026-02-18T08:33:11.553 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:33:11.549+0000 7faffc95c640 -1 reset not still connected to 0x5644a9836270 2026-02-18T08:33:11.554 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:33:11.549+0000 7faffc95c640 -1 reset not still connected to 0x5644b9a8cf70 2026-02-18T08:33:11.554 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:33:11.549+0000 7faffc95c640 -1 reset not still connected to 0x5644bd70a750 2026-02-18T08:33:11.554 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:33:11.549+0000 7faffc95c640 -1 reset not still connected to 0x5644c0efc340 2026-02-18T08:33:11.555 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:33:11.549+0000 7faffc95c640 -1 reset not still connected to 0x5644c58269c0 2026-02-18T08:33:11.555 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:33:11.549+0000 7faffc95c640 -1 reset not still connected to 0x5644c8fcf790 2026-02-18T08:33:11.555 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:33:11.549+0000 7faffc95c640 -1 reset not still connected to 0x5644d9b0e9c0 2026-02-18T08:33:11.555 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:33:11.549+0000 7faffc95c640 -1 reset not still connected to 0x5644e04e09c0 2026-02-18T08:33:11.555 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:33:11.549+0000 7faffc95c640 -1 reset not still connected to 0x5644e04e1ad0 2026-02-18T08:33:11.749 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:33:11.744+0000 7faffc95c640 -1 reset not still connected to 0x5644c1b595f0 2026-02-18T08:33:11.749 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:33:11.744+0000 7faffc95c640 -1 reset not still connected to 0x5644c8fcf380 2026-02-18T08:33:11.749 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:33:11.744+0000 7faffc95c640 -1 reset not still connected to 0x5644d9b0e410 2026-02-18T08:33:11.749 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:33:11.744+0000 7faffc95c640 -1 reset not still connected to 0x5644dc3ccc30 2026-02-18T08:33:12.699 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:33:12.692+0000 7faffc95c640 -1 reset not still connected to 0x5644a3cb4410 2026-02-18T08:33:12.699 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:33:12.692+0000 7faffc95c640 -1 reset not still connected to 0x5644a3cb4750 2026-02-18T08:33:12.700 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:33:12.692+0000 7faffc95c640 -1 reset not still connected to 0x5644a44256c0 2026-02-18T08:33:12.700 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:33:12.692+0000 7faffc95c640 -1 reset not still connected to 0x5644a716ab60 2026-02-18T08:33:12.700 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:33:12.692+0000 7faffc95c640 -1 reset not still connected to 0x5644af614c30 2026-02-18T08:33:12.700 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:33:12.692+0000 7faffc95c640 -1 reset not still connected to 0x5644b7e83930 2026-02-18T08:33:12.701 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:33:12.692+0000 7faffc95c640 -1 reset not still connected to 0x5644b9a8c1a0 2026-02-18T08:33:12.701 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:33:12.696+0000 7faffc95c640 -1 reset not still connected to 0x5644c19b4340 2026-02-18T08:33:13.070 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:33:13.064+0000 7fb27e7f3640 -1 reset not still connected to 0x55e18895cea0 2026-02-18T08:33:13.188 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:33:13.184+0000 7fb27e7f3640 -1 reset not still connected to 0x55e17071ec30 2026-02-18T08:33:13.188 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:33:13.184+0000 7fb27e7f3640 -1 reset not still connected to 0x55e170c6e8f0 2026-02-18T08:33:13.188 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:33:13.184+0000 7fb27e7f3640 -1 reset not still connected to 0x55e171bf05b0 2026-02-18T08:33:13.188 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:33:13.184+0000 7fb27e7f3640 -1 reset not still connected to 0x55e178bde340 2026-02-18T08:33:13.189 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:33:13.184+0000 7fb27e7f3640 -1 reset not still connected to 0x55e18895cea0 2026-02-18T08:33:13.406 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:400 IOs, 203463680 bytes 2026-02-18T08:33:13.619 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:33:13.612+0000 7faffc95c640 -1 reset not still connected to 0x5644d9b0e9c0 2026-02-18T08:33:13.693 INFO:tasks.ceph.cluster1.osd.1.trial034.stderr:2026-02-18T08:33:13.688+0000 7f20d7266640 -1 reset not still connected to 0x55640bcd1ad0 2026-02-18T08:33:13.695 INFO:tasks.ceph.cluster1.osd.1.trial034.stderr:2026-02-18T08:33:13.688+0000 7f20d7266640 -1 reset not still connected to 0x5564155c29c0 2026-02-18T08:33:13.696 INFO:tasks.ceph.cluster1.osd.1.trial034.stderr:2026-02-18T08:33:13.688+0000 7f20d7266640 -1 reset not still connected to 0x5564228bf5f0 2026-02-18T08:33:14.443 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.0 with skip (0.7418962157239358) > weight (0.25) 2026-02-18T08:33:14.443 INFO:tasks.rbd_mirror_thrash:kill cluster1.client.mirror.1 2026-02-18T08:33:14.443 INFO:tasks.rbd_mirror.cluster1.client.mirror.1:Sent signal 15 2026-02-18T08:33:14.443 INFO:tasks.rbd_mirror_thrash:waiting for 0.6627676778639136 secs before reviving daemons 2026-02-18T08:33:14.444 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.trial163.stderr:2026-02-18T08:33:14.434+0000 7ff3c2b99640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.1 (PID: 37642) UID: 1000 2026-02-18T08:33:14.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:500 IOs, 258748416 bytes 2026-02-18T08:33:14.822 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:600 IOs, 292966400 bytes 2026-02-18T08:33:15.107 INFO:tasks.rbd_mirror_thrash:waiting for cluster1.client.mirror.1 2026-02-18T08:33:15.107 INFO:teuthology.orchestra.run:waiting for 600 2026-02-18T08:33:15.585 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:700 IOs, 330180608 bytes 2026-02-18T08:33:15.758 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:33:15.752+0000 7fb27e7f3640 -1 reset not still connected to 0x55e16fecc1a0 2026-02-18T08:33:15.760 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:33:15.756+0000 7fb27e7f3640 -1 reset not still connected to 0x55e170ebfd40 2026-02-18T08:33:15.760 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:33:15.756+0000 7fb27e7f3640 -1 reset not still connected to 0x55e171e69c70 2026-02-18T08:33:15.760 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:33:15.756+0000 7fb27e7f3640 -1 reset not still connected to 0x55e178bde820 2026-02-18T08:33:15.761 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:33:15.756+0000 7fb27e7f3640 -1 reset not still connected to 0x55e179629ba0 2026-02-18T08:33:15.761 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:33:15.756+0000 7fb27e7f3640 -1 reset not still connected to 0x55e183c10f70 2026-02-18T08:33:15.761 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:33:15.756+0000 7fb27e7f3640 -1 reset not still connected to 0x55e188baf860 2026-02-18T08:33:15.761 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:33:15.756+0000 7fb27e7f3640 -1 reset not still connected to 0x55e1b0646a90 2026-02-18T08:33:15.982 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:33:15.976+0000 7fb27e7f3640 -1 reset not still connected to 0x55e179629ba0 2026-02-18T08:33:15.982 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:33:15.976+0000 7fb27e7f3640 -1 reset not still connected to 0x55e183c10f70 2026-02-18T08:33:15.983 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:33:15.976+0000 7fb27e7f3640 -1 reset not still connected to 0x55e188baf860 2026-02-18T08:33:15.983 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:33:15.976+0000 7fb27e7f3640 -1 reset not still connected to 0x55e1b0646a90 2026-02-18T08:33:16.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:800 IOs, 402880512 bytes 2026-02-18T08:33:16.394 INFO:tasks.ceph.cluster1.osd.1.trial034.stderr:2026-02-18T08:33:16.388+0000 7f20d7266640 -1 reset not still connected to 0x556404cc8c30 2026-02-18T08:33:16.394 INFO:tasks.ceph.cluster1.osd.1.trial034.stderr:2026-02-18T08:33:16.388+0000 7f20d7266640 -1 reset not still connected to 0x55640bcd0dd0 2026-02-18T08:33:16.395 INFO:tasks.ceph.cluster1.osd.1.trial034.stderr:2026-02-18T08:33:16.388+0000 7f20d7266640 -1 reset not still connected to 0x55640bcd1ad0 2026-02-18T08:33:16.395 INFO:tasks.ceph.cluster1.osd.1.trial034.stderr:2026-02-18T08:33:16.388+0000 7f20d7266640 -1 reset not still connected to 0x5564155c29c0 2026-02-18T08:33:16.395 INFO:tasks.ceph.cluster1.osd.1.trial034.stderr:2026-02-18T08:33:16.388+0000 7f20d7266640 -1 reset not still connected to 0x5564228bf5f0 2026-02-18T08:33:16.582 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:33:16.576+0000 7fb27e7f3640 -1 reset not still connected to 0x55e16ef2b790 2026-02-18T08:33:16.582 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:33:16.576+0000 7fb27e7f3640 -1 reset not still connected to 0x55e16fecc8f0 2026-02-18T08:33:16.583 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:33:16.576+0000 7fb27e7f3640 -1 reset not still connected to 0x55e170c6e8f0 2026-02-18T08:33:16.583 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:33:16.576+0000 7fb27e7f3640 -1 reset not still connected to 0x55e171bf05b0 2026-02-18T08:33:16.583 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:33:16.576+0000 7fb27e7f3640 -1 reset not still connected to 0x55e171bf1110 2026-02-18T08:33:16.585 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:33:16.580+0000 7fb27e7f3640 -1 reset not still connected to 0x55e171c0a750 2026-02-18T08:33:16.585 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:33:16.580+0000 7fb27e7f3640 -1 reset not still connected to 0x55e178bde340 2026-02-18T08:33:16.585 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:33:16.580+0000 7fb27e7f3640 -1 reset not still connected to 0x55e183c11860 2026-02-18T08:33:16.586 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:33:16.580+0000 7fb27e7f3640 -1 reset not still connected to 0x55e18895cea0 2026-02-18T08:33:16.587 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:33:16.580+0000 7fb27e7f3640 -1 reset not still connected to 0x55e1b0646000 2026-02-18T08:33:17.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:900 IOs, 439985152 bytes 2026-02-18T08:33:17.207 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:33:17.200+0000 7faffc95c640 -1 reset not still connected to 0x5644c19b4340 2026-02-18T08:33:17.328 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:33:17.320+0000 7fb27e7f3640 -1 reset not still connected to 0x55e178bde340 2026-02-18T08:33:17.328 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:33:17.320+0000 7fb27e7f3640 -1 reset not still connected to 0x55e183c11860 2026-02-18T08:33:17.328 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:33:17.320+0000 7fb27e7f3640 -1 reset not still connected to 0x55e18895cea0 2026-02-18T08:33:17.775 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:33:17.768+0000 7fb27e7f3640 -1 reset not still connected to 0x55e16ef2b790 2026-02-18T08:33:17.776 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:33:17.768+0000 7fb27e7f3640 -1 reset not still connected to 0x55e16fecc8f0 2026-02-18T08:33:17.776 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:33:17.772+0000 7fb27e7f3640 -1 reset not still connected to 0x55e17071ec30 2026-02-18T08:33:17.776 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:33:17.772+0000 7fb27e7f3640 -1 reset not still connected to 0x55e170742ea0 2026-02-18T08:33:17.776 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:33:17.772+0000 7fb27e7f3640 -1 reset not still connected to 0x55e170c6e8f0 2026-02-18T08:33:17.777 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:33:17.772+0000 7fb27e7f3640 -1 reset not still connected to 0x55e171bf05b0 2026-02-18T08:33:17.777 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:33:17.772+0000 7fb27e7f3640 -1 reset not still connected to 0x55e171bf1110 2026-02-18T08:33:17.777 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:33:17.772+0000 7fb27e7f3640 -1 reset not still connected to 0x55e171c0a750 2026-02-18T08:33:17.777 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:33:17.772+0000 7fb27e7f3640 -1 reset not still connected to 0x55e178bde340 2026-02-18T08:33:17.779 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:33:17.772+0000 7fb27e7f3640 -1 reset not still connected to 0x55e183c11860 2026-02-18T08:33:17.779 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:33:17.772+0000 7fb27e7f3640 -1 reset not still connected to 0x55e18895cea0 2026-02-18T08:33:17.779 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:33:17.772+0000 7fb27e7f3640 -1 reset not still connected to 0x55e1b0646000 2026-02-18T08:33:18.270 INFO:tasks.ceph.cluster1.osd.1.trial034.stderr:2026-02-18T08:33:18.264+0000 7f20d7266640 -1 reset not still connected to 0x556408919380 2026-02-18T08:33:18.270 INFO:tasks.ceph.cluster1.osd.1.trial034.stderr:2026-02-18T08:33:18.264+0000 7f20d7266640 -1 reset not still connected to 0x5564155c2410 2026-02-18T08:33:18.655 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:33:18.648+0000 7fb27e7f3640 -1 reset not still connected to 0x55e188baf860 2026-02-18T08:33:18.656 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:33:18.652+0000 7fb27e7f3640 -1 reset not still connected to 0x55e1b0646a90 2026-02-18T08:33:18.675 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1000 IOs, 503193600 bytes 2026-02-18T08:33:18.879 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1100 IOs, 506563584 bytes 2026-02-18T08:33:19.095 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1200 IOs, 535744512 bytes 2026-02-18T08:33:19.608 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:33:19.600+0000 7fb27e7f3640 -1 reset not still connected to 0x55e170693450 2026-02-18T08:33:19.608 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:33:19.604+0000 7fb27e7f3640 -1 reset not still connected to 0x55e170743ee0 2026-02-18T08:33:19.609 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:33:19.604+0000 7fb27e7f3640 -1 reset not still connected to 0x55e170ebe1a0 2026-02-18T08:33:19.609 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:33:19.604+0000 7fb27e7f3640 -1 reset not still connected to 0x55e17108f380 2026-02-18T08:33:19.609 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:33:19.604+0000 7fb27e7f3640 -1 reset not still connected to 0x55e171c0a680 2026-02-18T08:33:19.609 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:33:19.604+0000 7fb27e7f3640 -1 reset not still connected to 0x55e171c0b860 2026-02-18T08:33:19.611 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:33:19.604+0000 7fb27e7f3640 -1 reset not still connected to 0x55e17615aea0 2026-02-18T08:33:19.611 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:33:19.604+0000 7fb27e7f3640 -1 reset not still connected to 0x55e176ac6410 2026-02-18T08:33:19.611 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:33:19.604+0000 7fb27e7f3640 -1 reset not still connected to 0x55e188bae0d0 2026-02-18T08:33:19.722 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1300 IOs, 563307520 bytes 2026-02-18T08:33:20.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1400 IOs, 583846912 bytes 2026-02-18T08:33:20.466 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:33:20.460+0000 7fb27e7f3640 -1 reset not still connected to 0x55e178bde340 2026-02-18T08:33:20.466 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:33:20.460+0000 7fb27e7f3640 -1 reset not still connected to 0x55e183c11860 2026-02-18T08:33:20.466 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:33:20.460+0000 7fb27e7f3640 -1 reset not still connected to 0x55e18895cea0 2026-02-18T08:33:20.550 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:33:20.544+0000 7fb27e7f3640 -1 reset not still connected to 0x55e18895cea0 2026-02-18T08:33:20.767 INFO:tasks.ceph.cluster1.osd.1.trial034.stderr:2026-02-18T08:33:20.760+0000 7f20d7266640 -1 reset not still connected to 0x556422af5ad0 2026-02-18T08:33:20.768 INFO:tasks.ceph.cluster1.osd.1.trial034.stderr:2026-02-18T08:33:20.760+0000 7f20d7266640 -1 reset not still connected to 0x55642603b520 2026-02-18T08:33:20.768 INFO:tasks.ceph.cluster1.osd.1.trial034.stderr:2026-02-18T08:33:20.760+0000 7f20d7266640 -1 reset not still connected to 0x55642f6a1790 2026-02-18T08:33:20.768 INFO:tasks.ceph.cluster1.osd.1.trial034.stderr:2026-02-18T08:33:20.760+0000 7f20d7266640 -1 reset not still connected to 0x556431954000 2026-02-18T08:33:20.805 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial163.stderr:2026-02-18T08:33:20.794+0000 7fbd2b6e3640 -1 rbd::mirror::ImageReplayer: 0x55785d6a4a00 [3/26772eec-fd07-4fe4-bd28-93c852efe5e1] start: already running 2026-02-18T08:33:21.209 INFO:tasks.rbd_mirror_thrash:reviving cluster1.client.mirror.1 2026-02-18T08:33:21.209 INFO:tasks.rbd_mirror.cluster1.client.mirror.1:Restarting daemon 2026-02-18T08:33:21.209 DEBUG:teuthology.orchestra.run.trial163:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.1 2026-02-18T08:33:21.211 INFO:tasks.rbd_mirror.cluster1.client.mirror.1:Started 2026-02-18T08:33:21.211 INFO:tasks.rbd_mirror_thrash:waiting for 64.14247285752116 secs before thrashing 2026-02-18T08:33:21.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1500 IOs, 656760832 bytes 2026-02-18T08:33:21.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1600 IOs, 672702464 bytes 2026-02-18T08:33:22.254 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:33:22.248+0000 7fb27e7f3640 -1 reset not still connected to 0x55e171c0b860 2026-02-18T08:33:22.255 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:33:22.248+0000 7fb27e7f3640 -1 reset not still connected to 0x55e176ac6410 2026-02-18T08:33:22.257 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:33:22.252+0000 7fb27e7f3640 -1 reset not still connected to 0x55e177d12d00 2026-02-18T08:33:22.257 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:33:22.252+0000 7fb27e7f3640 -1 reset not still connected to 0x55e18895c820 2026-02-18T08:33:22.257 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:33:22.252+0000 7fb27e7f3640 -1 reset not still connected to 0x55e188baf860 2026-02-18T08:33:22.302 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ error_code=124 2026-02-18T08:33:22.302 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set -e 2026-02-18T08:33:22.302 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 124 -eq 124 ']' 2026-02-18T08:33:22.302 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:33:22.302 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T08:33:22.302 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_46 2026-02-18T08:33:22.302 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_image_and_enable_mirror cluster2 mirror image_46 journal 128M 2026-02-18T08:33:22.302 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:33:22.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:33:22.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:33:22.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:33:22.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_46 2026-02-18T08:33:22.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:33:22.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local mode=journal 2026-02-18T08:33:22.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n journal ']' 2026-02-18T08:33:22.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:33:22.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_image cluster2 mirror image_46 128M 2026-02-18T08:33:22.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:33:22.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:33:22.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:33:22.304 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:33:22.304 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_46 2026-02-18T08:33:22.304 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:33:22.304 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local size=128 2026-02-18T08:33:22.304 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n 128M ']' 2026-02-18T08:33:22.304 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ size=128M 2026-02-18T08:33:22.304 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:33:22.304 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,journaling mirror/image_46 2026-02-18T08:33:22.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ is_pool_mirror_mode_image mirror 2026-02-18T08:33:22.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:33:22.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' pool = image ']' 2026-02-18T08:33:22.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ case "${pool}" in 2026-02-18T08:33:22.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 1 2026-02-18T08:33:22.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n '' ']' 2026-02-18T08:33:22.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ stress_write_image cluster2 mirror image_46 2026-02-18T08:33:22.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:33:22.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:33:22.348 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_46 2026-02-18T08:33:22.348 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-18T08:33:22.348 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local duration=6 2026-02-18T08:33:22.348 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set +e 2026-02-18T08:33:22.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ timeout 6s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_46 --debug-rbd=20 --debug-journaler=20 2026-02-18T08:33:22.490 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:33:22.484+0000 7fb27e7f3640 -1 reset not still connected to 0x55e170693450 2026-02-18T08:33:22.491 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:33:22.484+0000 7fb27e7f3640 -1 reset not still connected to 0x55e17108f380 2026-02-18T08:33:22.491 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:33:22.484+0000 7fb27e7f3640 -1 reset not still connected to 0x55e171c0b860 2026-02-18T08:33:22.491 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:33:22.484+0000 7fb27e7f3640 -1 reset not still connected to 0x55e1737832b0 2026-02-18T08:33:22.491 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:33:22.484+0000 7fb27e7f3640 -1 reset not still connected to 0x55e17615aea0 2026-02-18T08:33:22.491 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:33:22.484+0000 7fb27e7f3640 -1 reset not still connected to 0x55e176ac6410 2026-02-18T08:33:22.491 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:33:22.484+0000 7fb27e7f3640 -1 reset not still connected to 0x55e177d12d00 2026-02-18T08:33:22.491 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:33:22.484+0000 7fb27e7f3640 -1 reset not still connected to 0x55e18895c820 2026-02-18T08:33:22.491 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:33:22.484+0000 7fb27e7f3640 -1 reset not still connected to 0x55e188baf860 2026-02-18T08:33:23.179 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:100 IOs, 57833472 bytes 2026-02-18T08:33:23.824 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:33:23.816+0000 7fb27e7f3640 -1 reset not still connected to 0x55e178bde340 2026-02-18T08:33:23.825 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:33:23.820+0000 7fb27e7f3640 -1 reset not still connected to 0x55e178bdeea0 2026-02-18T08:33:23.829 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:33:23.824+0000 7fb27e7f3640 -1 reset not still connected to 0x55e179629ba0 2026-02-18T08:33:23.829 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:33:23.824+0000 7fb27e7f3640 -1 reset not still connected to 0x55e17997b110 2026-02-18T08:33:23.831 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:33:23.824+0000 7fb27e7f3640 -1 reset not still connected to 0x55e183c11860 2026-02-18T08:33:23.833 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:33:23.828+0000 7fb27e7f3640 -1 reset not still connected to 0x55e188434c30 2026-02-18T08:33:23.833 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:33:23.828+0000 7fb27e7f3640 -1 reset not still connected to 0x55e188435380 2026-02-18T08:33:23.833 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:33:23.828+0000 7fb27e7f3640 -1 reset not still connected to 0x55e18895cea0 2026-02-18T08:33:23.833 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:33:23.828+0000 7fb27e7f3640 -1 reset not still connected to 0x55e18895d040 2026-02-18T08:33:23.833 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:33:23.828+0000 7fb27e7f3640 -1 reset not still connected to 0x55e193b8f930 2026-02-18T08:33:23.896 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:200 IOs, 104636416 bytes 2026-02-18T08:33:25.176 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:33:25.172+0000 7faffc95c640 -1 reset not still connected to 0x5644a9837790 2026-02-18T08:33:25.176 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:33:25.172+0000 7faffc95c640 -1 reset not still connected to 0x5644d360b5f0 2026-02-18T08:33:25.177 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:33:25.172+0000 7faffc95c640 -1 reset not still connected to 0x5644d9b50ea0 2026-02-18T08:33:25.178 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:33:25.172+0000 7faffc95c640 -1 reset not still connected to 0x5644d9b51110 2026-02-18T08:33:25.178 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:33:25.172+0000 7faffc95c640 -1 reset not still connected to 0x5644dc3cc4e0 2026-02-18T08:33:25.178 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:33:25.172+0000 7faffc95c640 -1 reset not still connected to 0x5644dc3cd930 2026-02-18T08:33:25.178 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:33:25.172+0000 7faffc95c640 -1 reset not still connected to 0x5644e04e0dd0 2026-02-18T08:33:25.246 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:300 IOs, 120787968 bytes 2026-02-18T08:33:25.513 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:33:25.508+0000 7faffc95c640 -1 reset not still connected to 0x5644a5073860 2026-02-18T08:33:25.514 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:33:25.508+0000 7faffc95c640 -1 reset not still connected to 0x5644a84a6750 2026-02-18T08:33:25.514 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:33:25.508+0000 7faffc95c640 -1 reset not still connected to 0x5644c0a69a00 2026-02-18T08:33:25.516 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:33:25.508+0000 7faffc95c640 -1 reset not still connected to 0x5644c0efcf70 2026-02-18T08:33:25.516 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:33:25.508+0000 7faffc95c640 -1 reset not still connected to 0x5644c19b4000 2026-02-18T08:33:25.516 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:33:25.508+0000 7faffc95c640 -1 reset not still connected to 0x5644c5827380 2026-02-18T08:33:25.516 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:33:25.508+0000 7faffc95c640 -1 reset not still connected to 0x5644c8a8e0d0 2026-02-18T08:33:25.516 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:33:25.508+0000 7faffc95c640 -1 reset not still connected to 0x5644c8a8e340 2026-02-18T08:33:25.517 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:33:25.508+0000 7faffc95c640 -1 reset not still connected to 0x5644c8a8fc70 2026-02-18T08:33:25.517 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:33:25.508+0000 7faffc95c640 -1 reset not still connected to 0x5644d16a6f70 2026-02-18T08:33:25.517 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:33:25.512+0000 7faffc95c640 -1 reset not still connected to 0x5644d236f450 2026-02-18T08:33:25.518 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:33:25.512+0000 7faffc95c640 -1 reset not still connected to 0x5644d236fa00 2026-02-18T08:33:25.518 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:33:25.512+0000 7faffc95c640 -1 reset not still connected to 0x5644d3e70340 2026-02-18T08:33:25.518 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:33:25.512+0000 7faffc95c640 -1 reset not still connected to 0x5644d7dcca90 2026-02-18T08:33:25.518 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:33:25.512+0000 7faffc95c640 -1 reset not still connected to 0x5644d9b50f70 2026-02-18T08:33:26.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:400 IOs, 161566720 bytes 2026-02-18T08:33:26.230 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:500 IOs, 181755904 bytes 2026-02-18T08:33:26.528 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:33:26.520+0000 7faffc95c640 -1 reset not still connected to 0x5644c8a8e340 2026-02-18T08:33:26.529 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:33:26.520+0000 7faffc95c640 -1 reset not still connected to 0x5644c8a8fc70 2026-02-18T08:33:26.529 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:33:26.524+0000 7faffc95c640 -1 reset not still connected to 0x5644d16a6f70 2026-02-18T08:33:26.529 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:33:26.524+0000 7faffc95c640 -1 reset not still connected to 0x5644d236f450 2026-02-18T08:33:26.529 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:33:26.524+0000 7faffc95c640 -1 reset not still connected to 0x5644d236fa00 2026-02-18T08:33:26.529 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:33:26.524+0000 7faffc95c640 -1 reset not still connected to 0x5644d3e70340 2026-02-18T08:33:26.529 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:33:26.524+0000 7faffc95c640 -1 reset not still connected to 0x5644d9b50f70 2026-02-18T08:33:27.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:600 IOs, 222286848 bytes 2026-02-18T08:33:27.920 INFO:tasks.ceph.cluster1.osd.1.trial034.stderr:2026-02-18T08:33:27.916+0000 7f20d7266640 -1 reset not still connected to 0x55640bcd1ad0 2026-02-18T08:33:27.921 INFO:tasks.ceph.cluster1.osd.1.trial034.stderr:2026-02-18T08:33:27.916+0000 7f20d7266640 -1 reset not still connected to 0x5564155c24e0 2026-02-18T08:33:27.922 INFO:tasks.ceph.cluster1.osd.1.trial034.stderr:2026-02-18T08:33:27.916+0000 7f20d7266640 -1 reset not still connected to 0x55641ff40750 2026-02-18T08:33:27.923 INFO:tasks.ceph.cluster1.osd.1.trial034.stderr:2026-02-18T08:33:27.916+0000 7f20d7266640 -1 reset not still connected to 0x556422af4410 2026-02-18T08:33:27.923 INFO:tasks.ceph.cluster1.osd.1.trial034.stderr:2026-02-18T08:33:27.916+0000 7f20d7266640 -1 reset not still connected to 0x556428803520 2026-02-18T08:33:27.923 INFO:tasks.ceph.cluster1.osd.1.trial034.stderr:2026-02-18T08:33:27.916+0000 7f20d7266640 -1 reset not still connected to 0x55642f106a90 2026-02-18T08:33:27.923 INFO:tasks.ceph.cluster1.osd.1.trial034.stderr:2026-02-18T08:33:27.916+0000 7f20d7266640 -1 reset not still connected to 0x55642f107c70 2026-02-18T08:33:28.007 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:700 IOs, 291030016 bytes 2026-02-18T08:33:28.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ error_code=124 2026-02-18T08:33:28.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set -e 2026-02-18T08:33:28.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 124 -eq 124 ']' 2026-02-18T08:33:28.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:33:28.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T08:33:28.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_47 2026-02-18T08:33:28.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_image_and_enable_mirror cluster2 mirror image_47 journal 128M 2026-02-18T08:33:28.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:33:28.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:33:28.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:33:28.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:33:28.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_47 2026-02-18T08:33:28.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:33:28.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local mode=journal 2026-02-18T08:33:28.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n journal ']' 2026-02-18T08:33:28.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:33:28.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_image cluster2 mirror image_47 128M 2026-02-18T08:33:28.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:33:28.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:33:28.360 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:33:28.360 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:33:28.360 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_47 2026-02-18T08:33:28.360 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:33:28.360 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local size=128 2026-02-18T08:33:28.360 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n 128M ']' 2026-02-18T08:33:28.360 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ size=128M 2026-02-18T08:33:28.360 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:33:28.360 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,journaling mirror/image_47 2026-02-18T08:33:28.407 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ is_pool_mirror_mode_image mirror 2026-02-18T08:33:28.407 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:33:28.408 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' pool = image ']' 2026-02-18T08:33:28.408 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ case "${pool}" in 2026-02-18T08:33:28.408 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 1 2026-02-18T08:33:28.408 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n '' ']' 2026-02-18T08:33:28.408 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ stress_write_image cluster2 mirror image_47 2026-02-18T08:33:28.408 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:33:28.408 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:33:28.408 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_47 2026-02-18T08:33:28.408 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-18T08:33:28.409 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local duration=7 2026-02-18T08:33:28.409 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set +e 2026-02-18T08:33:28.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ timeout 7s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_47 --debug-rbd=20 --debug-journaler=20 2026-02-18T08:33:28.561 INFO:tasks.ceph.cluster2.osd.0.trial163.stderr:2026-02-18T08:33:28.550+0000 7f2115a77640 -1 bdev(0x56173d927400 /var/lib/ceph/osd/cluster2-0/block) aio_submit retries 1 2026-02-18T08:33:28.562 INFO:tasks.ceph.cluster2.osd.0.trial163.stderr:2026-02-18T08:33:28.554+0000 7f2115276640 -1 bdev(0x56173d927400 /var/lib/ceph/osd/cluster2-0/block) aio_submit retries 1 2026-02-18T08:33:29.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:100 IOs, 30668800 bytes 2026-02-18T08:33:29.901 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:33:29.896+0000 7fb27e7f3640 -1 reset not still connected to 0x55e17071ec30 2026-02-18T08:33:29.901 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:33:29.896+0000 7fb27e7f3640 -1 reset not still connected to 0x55e170c6e8f0 2026-02-18T08:33:29.903 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:33:29.896+0000 7fb27e7f3640 -1 reset not still connected to 0x55e171e35d40 2026-02-18T08:33:29.903 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:33:29.896+0000 7fb27e7f3640 -1 reset not still connected to 0x55e175a309c0 2026-02-18T08:33:29.903 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:33:29.896+0000 7fb27e7f3640 -1 reset not still connected to 0x55e175f31a00 2026-02-18T08:33:29.903 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:33:29.896+0000 7fb27e7f3640 -1 reset not still connected to 0x55e178bde340 2026-02-18T08:33:29.903 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:33:29.896+0000 7fb27e7f3640 -1 reset not still connected to 0x55e178bdeea0 2026-02-18T08:33:29.905 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:33:29.900+0000 7fb27e7f3640 -1 reset not still connected to 0x55e179629ba0 2026-02-18T08:33:29.905 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:33:29.900+0000 7fb27e7f3640 -1 reset not still connected to 0x55e17997b110 2026-02-18T08:33:29.905 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:33:29.900+0000 7fb27e7f3640 -1 reset not still connected to 0x55e183c11860 2026-02-18T08:33:29.907 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:33:29.900+0000 7fb27e7f3640 -1 reset not still connected to 0x55e188435380 2026-02-18T08:33:29.907 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:33:29.900+0000 7fb27e7f3640 -1 reset not still connected to 0x55e18895cea0 2026-02-18T08:33:29.909 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:33:29.904+0000 7fb27e7f3640 -1 reset not still connected to 0x55e18895d040 2026-02-18T08:33:29.909 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:33:29.904+0000 7fb27e7f3640 -1 reset not still connected to 0x55e193b8f930 2026-02-18T08:33:30.180 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:200 IOs, 90732544 bytes 2026-02-18T08:33:30.772 INFO:tasks.ceph.cluster1.osd.1.trial034.stderr:2026-02-18T08:33:30.764+0000 7f20d7266640 -1 reset not still connected to 0x556415bf3ad0 2026-02-18T08:33:30.772 INFO:tasks.ceph.cluster1.osd.1.trial034.stderr:2026-02-18T08:33:30.764+0000 7f20d7266640 -1 reset not still connected to 0x556428803d40 2026-02-18T08:33:31.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:300 IOs, 102581248 bytes 2026-02-18T08:33:31.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:400 IOs, 138297344 bytes 2026-02-18T08:33:32.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:500 IOs, 186870784 bytes 2026-02-18T08:33:32.910 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:600 IOs, 277991424 bytes 2026-02-18T08:33:33.092 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:33:33.084+0000 7fb27e7f3640 -1 reset not still connected to 0x55e175f31a00 2026-02-18T08:33:33.092 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:33:33.084+0000 7fb27e7f3640 -1 reset not still connected to 0x55e178bde340 2026-02-18T08:33:33.092 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:33:33.084+0000 7fb27e7f3640 -1 reset not still connected to 0x55e178bdeea0 2026-02-18T08:33:33.092 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:33:33.088+0000 7fb27e7f3640 -1 reset not still connected to 0x55e179629ba0 2026-02-18T08:33:33.092 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:33:33.088+0000 7fb27e7f3640 -1 reset not still connected to 0x55e17997b110 2026-02-18T08:33:33.092 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:33:33.088+0000 7fb27e7f3640 -1 reset not still connected to 0x55e183c11860 2026-02-18T08:33:33.093 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:33:33.088+0000 7fb27e7f3640 -1 reset not still connected to 0x55e188435380 2026-02-18T08:33:33.093 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:33:33.088+0000 7fb27e7f3640 -1 reset not still connected to 0x55e18895cea0 2026-02-18T08:33:33.093 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:33:33.088+0000 7fb27e7f3640 -1 reset not still connected to 0x55e18895d040 2026-02-18T08:33:33.093 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:33:33.088+0000 7fb27e7f3640 -1 reset not still connected to 0x55e193b8f930 2026-02-18T08:33:33.425 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:33:33.420+0000 7faffc95c640 -1 reset not still connected to 0x5644d9b50ea0 2026-02-18T08:33:33.425 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:33:33.420+0000 7faffc95c640 -1 reset not still connected to 0x5644dc3cc4e0 2026-02-18T08:33:33.425 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:33:33.420+0000 7faffc95c640 -1 reset not still connected to 0x5644dc3cd930 2026-02-18T08:33:33.425 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:33:33.420+0000 7faffc95c640 -1 reset not still connected to 0x5644e04e0dd0 2026-02-18T08:33:33.549 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:33:33.544+0000 7fb27e7f3640 -1 reset not still connected to 0x55e175f31a00 2026-02-18T08:33:33.549 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:33:33.544+0000 7fb27e7f3640 -1 reset not still connected to 0x55e178bde340 2026-02-18T08:33:33.550 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:33:33.544+0000 7fb27e7f3640 -1 reset not still connected to 0x55e178bdeea0 2026-02-18T08:33:33.550 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:33:33.544+0000 7fb27e7f3640 -1 reset not still connected to 0x55e17997b110 2026-02-18T08:33:33.550 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:33:33.544+0000 7fb27e7f3640 -1 reset not still connected to 0x55e183c11860 2026-02-18T08:33:33.551 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:33:33.544+0000 7fb27e7f3640 -1 reset not still connected to 0x55e18895cea0 2026-02-18T08:33:33.551 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:33:33.544+0000 7fb27e7f3640 -1 reset not still connected to 0x55e18895d040 2026-02-18T08:33:33.551 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:33:33.544+0000 7fb27e7f3640 -1 reset not still connected to 0x55e193b8f930 2026-02-18T08:33:33.866 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:700 IOs, 324107264 bytes 2026-02-18T08:33:34.371 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:33:34.364+0000 7fb27e7f3640 -1 reset not still connected to 0x55e175a309c0 2026-02-18T08:33:34.371 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:33:34.364+0000 7fb27e7f3640 -1 reset not still connected to 0x55e175f31a00 2026-02-18T08:33:34.371 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:33:34.364+0000 7fb27e7f3640 -1 reset not still connected to 0x55e178bde340 2026-02-18T08:33:34.371 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:33:34.364+0000 7fb27e7f3640 -1 reset not still connected to 0x55e178bdeea0 2026-02-18T08:33:34.372 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:33:34.368+0000 7fb27e7f3640 -1 reset not still connected to 0x55e179629ba0 2026-02-18T08:33:34.372 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:33:34.368+0000 7fb27e7f3640 -1 reset not still connected to 0x55e17997b110 2026-02-18T08:33:34.372 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:33:34.368+0000 7fb27e7f3640 -1 reset not still connected to 0x55e183c11860 2026-02-18T08:33:34.373 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:33:34.368+0000 7fb27e7f3640 -1 reset not still connected to 0x55e188435380 2026-02-18T08:33:34.373 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:33:34.368+0000 7fb27e7f3640 -1 reset not still connected to 0x55e18895cea0 2026-02-18T08:33:34.374 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:33:34.368+0000 7fb27e7f3640 -1 reset not still connected to 0x55e18895d040 2026-02-18T08:33:34.374 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:33:34.368+0000 7fb27e7f3640 -1 reset not still connected to 0x55e193b8f930 2026-02-18T08:33:34.879 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:800 IOs, 401602560 bytes 2026-02-18T08:33:35.407 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:900 IOs, 445119488 bytes 2026-02-18T08:33:35.418 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ error_code=124 2026-02-18T08:33:35.419 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set -e 2026-02-18T08:33:35.419 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 124 -eq 124 ']' 2026-02-18T08:33:35.419 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:33:35.419 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T08:33:35.419 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_48 2026-02-18T08:33:35.419 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_image_and_enable_mirror cluster2 mirror image_48 journal 128M 2026-02-18T08:33:35.419 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:33:35.419 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:33:35.419 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:33:35.419 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:33:35.419 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_48 2026-02-18T08:33:35.420 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:33:35.420 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local mode=journal 2026-02-18T08:33:35.420 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n journal ']' 2026-02-18T08:33:35.420 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:33:35.420 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_image cluster2 mirror image_48 128M 2026-02-18T08:33:35.420 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:33:35.420 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:33:35.420 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:33:35.420 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:33:35.420 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_48 2026-02-18T08:33:35.420 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:33:35.420 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local size=128 2026-02-18T08:33:35.421 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n 128M ']' 2026-02-18T08:33:35.421 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ size=128M 2026-02-18T08:33:35.421 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:33:35.421 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,journaling mirror/image_48 2026-02-18T08:33:35.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ is_pool_mirror_mode_image mirror 2026-02-18T08:33:35.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:33:35.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' pool = image ']' 2026-02-18T08:33:35.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ case "${pool}" in 2026-02-18T08:33:35.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 1 2026-02-18T08:33:35.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n '' ']' 2026-02-18T08:33:35.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ stress_write_image cluster2 mirror image_48 2026-02-18T08:33:35.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:33:35.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:33:35.460 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_48 2026-02-18T08:33:35.460 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-18T08:33:35.460 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local duration=10 2026-02-18T08:33:35.460 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set +e 2026-02-18T08:33:35.460 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ timeout 10s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_48 --debug-rbd=20 --debug-journaler=20 2026-02-18T08:33:35.565 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:100 IOs, 624640 bytes 2026-02-18T08:33:36.553 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:200 IOs, 59430912 bytes 2026-02-18T08:33:37.086 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:300 IOs, 123486208 bytes 2026-02-18T08:33:37.183 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:33:37.176+0000 7faffc95c640 -1 reset not still connected to 0x5644b7e83c70 2026-02-18T08:33:37.185 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:33:37.180+0000 7faffc95c640 -1 reset not still connected to 0x5644bd70a270 2026-02-18T08:33:37.185 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:33:37.180+0000 7faffc95c640 -1 reset not still connected to 0x5644bd70aea0 2026-02-18T08:33:37.185 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:33:37.180+0000 7faffc95c640 -1 reset not still connected to 0x5644c5826270 2026-02-18T08:33:37.185 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:33:37.180+0000 7faffc95c640 -1 reset not still connected to 0x5644c8fcea90 2026-02-18T08:33:37.185 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:33:37.180+0000 7faffc95c640 -1 reset not still connected to 0x5644c8fcfad0 2026-02-18T08:33:37.185 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:33:37.180+0000 7faffc95c640 -1 reset not still connected to 0x5644d7dcc0d0 2026-02-18T08:33:37.185 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:33:37.180+0000 7faffc95c640 -1 reset not still connected to 0x5644dc3cd860 2026-02-18T08:33:37.185 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:33:37.180+0000 7faffc95c640 -1 reset not still connected to 0x5644dfda05b0 2026-02-18T08:33:37.191 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:33:37.184+0000 7faffc95c640 -1 reset not still connected to 0x5644d7dcc0d0 2026-02-18T08:33:37.191 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:33:37.184+0000 7faffc95c640 -1 reset not still connected to 0x5644dc3cd860 2026-02-18T08:33:37.191 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:33:37.184+0000 7faffc95c640 -1 reset not still connected to 0x5644dfda05b0 2026-02-18T08:33:38.099 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:33:38.092+0000 7fb27e7f3640 -1 reset not still connected to 0x55e18895cea0 2026-02-18T08:33:38.100 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:33:38.096+0000 7fb27e7f3640 -1 reset not still connected to 0x55e18895d040 2026-02-18T08:33:38.100 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:33:38.096+0000 7fb27e7f3640 -1 reset not still connected to 0x55e193b8f930 2026-02-18T08:33:38.287 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:400 IOs, 158901248 bytes 2026-02-18T08:33:38.651 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:33:38.644+0000 7fb27e7f3640 -1 reset not still connected to 0x55e175f31a00 2026-02-18T08:33:38.652 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:33:38.648+0000 7fb27e7f3640 -1 reset not still connected to 0x55e178bde340 2026-02-18T08:33:38.653 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:33:38.648+0000 7fb27e7f3640 -1 reset not still connected to 0x55e178bdeea0 2026-02-18T08:33:38.653 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:33:38.648+0000 7fb27e7f3640 -1 reset not still connected to 0x55e18895cea0 2026-02-18T08:33:39.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:500 IOs, 210735104 bytes 2026-02-18T08:33:39.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:600 IOs, 244407296 bytes 2026-02-18T08:33:40.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:700 IOs, 271528960 bytes 2026-02-18T08:33:40.190 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:33:40.184+0000 7fb27e7f3640 -1 reset not still connected to 0x55e176ac6340 2026-02-18T08:33:40.190 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:33:40.184+0000 7fb27e7f3640 -1 reset not still connected to 0x55e179628c30 2026-02-18T08:33:40.191 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:33:40.184+0000 7fb27e7f3640 -1 reset not still connected to 0x55e193b8f6c0 2026-02-18T08:33:40.352 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:800 IOs, 276543488 bytes 2026-02-18T08:33:40.954 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:33:40.948+0000 7fb27e7f3640 -1 reset not still connected to 0x55e188baf860 2026-02-18T08:33:41.088 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:33:41.080+0000 7faffc95c640 -1 reset not still connected to 0x5644b7e83c70 2026-02-18T08:33:41.088 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:33:41.080+0000 7faffc95c640 -1 reset not still connected to 0x5644bd70a270 2026-02-18T08:33:41.088 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:33:41.080+0000 7faffc95c640 -1 reset not still connected to 0x5644bd70aea0 2026-02-18T08:33:41.088 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:33:41.084+0000 7faffc95c640 -1 reset not still connected to 0x5644c5826270 2026-02-18T08:33:41.089 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:33:41.084+0000 7faffc95c640 -1 reset not still connected to 0x5644c8fcea90 2026-02-18T08:33:41.091 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:33:41.084+0000 7faffc95c640 -1 reset not still connected to 0x5644c8fcfad0 2026-02-18T08:33:41.092 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:33:41.084+0000 7faffc95c640 -1 reset not still connected to 0x5644d7dcc0d0 2026-02-18T08:33:41.092 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:33:41.084+0000 7faffc95c640 -1 reset not still connected to 0x5644dc3cd860 2026-02-18T08:33:41.092 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:33:41.084+0000 7faffc95c640 -1 reset not still connected to 0x5644dfda05b0 2026-02-18T08:33:41.162 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:33:41.156+0000 7faffc95c640 -1 reset not still connected to 0x5644dfda05b0 2026-02-18T08:33:41.718 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:900 IOs, 313981952 bytes 2026-02-18T08:33:42.108 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:33:42.100+0000 7faffc95c640 -1 reset not still connected to 0x5644d7dcc0d0 2026-02-18T08:33:42.108 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:33:42.100+0000 7faffc95c640 -1 reset not still connected to 0x5644dc3cd860 2026-02-18T08:33:42.108 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:33:42.100+0000 7faffc95c640 -1 reset not still connected to 0x5644dfda05b0 2026-02-18T08:33:42.113 INFO:tasks.ceph.cluster1.osd.1.trial034.stderr:2026-02-18T08:33:42.108+0000 7f20d7266640 -1 reset not still connected to 0x5564021960d0 2026-02-18T08:33:42.113 INFO:tasks.ceph.cluster1.osd.1.trial034.stderr:2026-02-18T08:33:42.108+0000 7f20d7266640 -1 reset not still connected to 0x5564088e4680 2026-02-18T08:33:42.114 INFO:tasks.ceph.cluster1.osd.1.trial034.stderr:2026-02-18T08:33:42.108+0000 7f20d7266640 -1 reset not still connected to 0x55640bcd1040 2026-02-18T08:33:42.115 INFO:tasks.ceph.cluster1.osd.1.trial034.stderr:2026-02-18T08:33:42.108+0000 7f20d7266640 -1 reset not still connected to 0x55640bcd11e0 2026-02-18T08:33:42.115 INFO:tasks.ceph.cluster1.osd.1.trial034.stderr:2026-02-18T08:33:42.108+0000 7f20d7266640 -1 reset not still connected to 0x55641ff40f70 2026-02-18T08:33:42.595 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1000 IOs, 365694976 bytes 2026-02-18T08:33:43.422 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:33:43.416+0000 7faffc95c640 -1 reset not still connected to 0x5644c1ba6a90 2026-02-18T08:33:43.422 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:33:43.416+0000 7faffc95c640 -1 reset not still connected to 0x5644c8a8f860 2026-02-18T08:33:43.423 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:33:43.416+0000 7faffc95c640 -1 reset not still connected to 0x5644d3e71d40 2026-02-18T08:33:43.423 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:33:43.416+0000 7faffc95c640 -1 reset not still connected to 0x5644d9b0e9c0 2026-02-18T08:33:44.093 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1100 IOs, 479045632 bytes 2026-02-18T08:33:44.774 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1200 IOs, 537400320 bytes 2026-02-18T08:33:45.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ error_code=124 2026-02-18T08:33:45.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set -e 2026-02-18T08:33:45.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 124 -eq 124 ']' 2026-02-18T08:33:45.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:33:45.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T08:33:45.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_49 2026-02-18T08:33:45.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_image_and_enable_mirror cluster2 mirror image_49 journal 128M 2026-02-18T08:33:45.472 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:33:45.472 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:33:45.472 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:33:45.472 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:33:45.472 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_49 2026-02-18T08:33:45.472 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:33:45.472 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local mode=journal 2026-02-18T08:33:45.472 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n journal ']' 2026-02-18T08:33:45.472 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:33:45.472 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_image cluster2 mirror image_49 128M 2026-02-18T08:33:45.472 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:33:45.472 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:33:45.473 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:33:45.473 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:33:45.473 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_49 2026-02-18T08:33:45.473 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:33:45.473 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local size=128 2026-02-18T08:33:45.473 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n 128M ']' 2026-02-18T08:33:45.473 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ size=128M 2026-02-18T08:33:45.473 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:33:45.473 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,journaling mirror/image_49 2026-02-18T08:33:45.750 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ is_pool_mirror_mode_image mirror 2026-02-18T08:33:45.751 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:33:45.751 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' pool = image ']' 2026-02-18T08:33:45.751 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ case "${pool}" in 2026-02-18T08:33:45.751 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 1 2026-02-18T08:33:45.751 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n '' ']' 2026-02-18T08:33:45.751 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ stress_write_image cluster2 mirror image_49 2026-02-18T08:33:45.751 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:33:45.751 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:33:45.751 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_49 2026-02-18T08:33:45.752 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-18T08:33:45.752 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local duration=13 2026-02-18T08:33:45.752 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set +e 2026-02-18T08:33:45.752 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ timeout 13s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_49 --debug-rbd=20 --debug-journaler=20 2026-02-18T08:33:46.245 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:100 IOs, 20409344 bytes 2026-02-18T08:33:46.314 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:33:46.308+0000 7faffc95c640 -1 reset not still connected to 0x5644a4425ad0 2026-02-18T08:33:46.314 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:33:46.308+0000 7faffc95c640 -1 reset not still connected to 0x5644a6841040 2026-02-18T08:33:46.314 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:33:46.308+0000 7faffc95c640 -1 reset not still connected to 0x5644a716ac30 2026-02-18T08:33:46.314 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:33:46.308+0000 7faffc95c640 -1 reset not still connected to 0x5644aec73040 2026-02-18T08:33:46.314 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:33:46.308+0000 7faffc95c640 -1 reset not still connected to 0x5644af70fe10 2026-02-18T08:33:46.315 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:33:46.308+0000 7faffc95c640 -1 reset not still connected to 0x5644b4738000 2026-02-18T08:33:46.315 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:33:46.308+0000 7faffc95c640 -1 reset not still connected to 0x5644bb2d1110 2026-02-18T08:33:46.315 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:33:46.308+0000 7faffc95c640 -1 reset not still connected to 0x5644c0a68b60 2026-02-18T08:33:46.315 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:33:46.308+0000 7faffc95c640 -1 reset not still connected to 0x5644c1ba6a90 2026-02-18T08:33:46.316 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:33:46.308+0000 7faffc95c640 -1 reset not still connected to 0x5644c8a8f860 2026-02-18T08:33:46.316 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:33:46.312+0000 7faffc95c640 -1 reset not still connected to 0x5644d3e71d40 2026-02-18T08:33:46.316 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:33:46.312+0000 7faffc95c640 -1 reset not still connected to 0x5644d9b0e9c0 2026-02-18T08:33:46.478 INFO:tasks.ceph.cluster1.osd.1.trial034.stderr:2026-02-18T08:33:46.472+0000 7f20d7266640 -1 reset not still connected to 0x556428803520 2026-02-18T08:33:46.478 INFO:tasks.ceph.cluster1.osd.1.trial034.stderr:2026-02-18T08:33:46.472+0000 7f20d7266640 -1 reset not still connected to 0x55642f107c70 2026-02-18T08:33:47.006 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:200 IOs, 57493504 bytes 2026-02-18T08:33:47.667 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:300 IOs, 90295296 bytes 2026-02-18T08:33:49.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:400 IOs, 108761088 bytes 2026-02-18T08:33:49.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:500 IOs, 123929600 bytes 2026-02-18T08:33:49.744 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:33:49.736+0000 7faffc95c640 -1 reset not still connected to 0x5644a4425ad0 2026-02-18T08:33:49.744 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:33:49.736+0000 7faffc95c640 -1 reset not still connected to 0x5644a6841040 2026-02-18T08:33:49.744 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:33:49.736+0000 7faffc95c640 -1 reset not still connected to 0x5644a716ac30 2026-02-18T08:33:49.744 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:33:49.736+0000 7faffc95c640 -1 reset not still connected to 0x5644aec73040 2026-02-18T08:33:49.744 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:33:49.736+0000 7faffc95c640 -1 reset not still connected to 0x5644af70fe10 2026-02-18T08:33:49.745 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:33:49.740+0000 7faffc95c640 -1 reset not still connected to 0x5644b4738000 2026-02-18T08:33:49.745 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:33:49.740+0000 7faffc95c640 -1 reset not still connected to 0x5644bb2d1110 2026-02-18T08:33:49.746 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:33:49.740+0000 7faffc95c640 -1 reset not still connected to 0x5644c0a68b60 2026-02-18T08:33:49.746 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:33:49.740+0000 7faffc95c640 -1 reset not still connected to 0x5644c1ba6a90 2026-02-18T08:33:49.746 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:33:49.740+0000 7faffc95c640 -1 reset not still connected to 0x5644c8a8f860 2026-02-18T08:33:49.746 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:33:49.740+0000 7faffc95c640 -1 reset not still connected to 0x5644d3e71d40 2026-02-18T08:33:49.746 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:33:49.740+0000 7faffc95c640 -1 reset not still connected to 0x5644d9b0e9c0 2026-02-18T08:33:49.795 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:33:49.788+0000 7faffc95c640 -1 reset not still connected to 0x5644a41ff450 2026-02-18T08:33:49.795 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:33:49.788+0000 7faffc95c640 -1 reset not still connected to 0x5644a6841040 2026-02-18T08:33:49.795 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:33:49.788+0000 7faffc95c640 -1 reset not still connected to 0x5644a716ac30 2026-02-18T08:33:49.795 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:33:49.788+0000 7faffc95c640 -1 reset not still connected to 0x5644af70fe10 2026-02-18T08:33:49.795 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:33:49.788+0000 7faffc95c640 -1 reset not still connected to 0x5644d9b0e9c0 2026-02-18T08:33:50.818 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:600 IOs, 190499840 bytes 2026-02-18T08:33:50.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:700 IOs, 213009408 bytes 2026-02-18T08:33:51.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:800 IOs, 213638144 bytes 2026-02-18T08:33:51.156 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:33:51.152+0000 7faffc95c640 -1 reset not still connected to 0x5644d9b50ea0 2026-02-18T08:33:51.156 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:33:51.152+0000 7faffc95c640 -1 reset not still connected to 0x5644dc3cd930 2026-02-18T08:33:51.156 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:33:51.152+0000 7faffc95c640 -1 reset not still connected to 0x5644e04e0dd0 2026-02-18T08:33:51.367 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:900 IOs, 239560704 bytes 2026-02-18T08:33:52.539 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1000 IOs, 275320832 bytes 2026-02-18T08:33:52.966 INFO:tasks.ceph.cluster1.osd.1.trial034.stderr:2026-02-18T08:33:52.960+0000 7f20d7266640 -1 reset not still connected to 0x5563ff09c340 2026-02-18T08:33:52.966 INFO:tasks.ceph.cluster1.osd.1.trial034.stderr:2026-02-18T08:33:52.960+0000 7f20d7266640 -1 reset not still connected to 0x5564021960d0 2026-02-18T08:33:52.967 INFO:tasks.ceph.cluster1.osd.1.trial034.stderr:2026-02-18T08:33:52.960+0000 7f20d7266640 -1 reset not still connected to 0x5564088e4680 2026-02-18T08:33:52.967 INFO:tasks.ceph.cluster1.osd.1.trial034.stderr:2026-02-18T08:33:52.960+0000 7f20d7266640 -1 reset not still connected to 0x55640bcd1040 2026-02-18T08:33:52.967 INFO:tasks.ceph.cluster1.osd.1.trial034.stderr:2026-02-18T08:33:52.960+0000 7f20d7266640 -1 reset not still connected to 0x55640bcd11e0 2026-02-18T08:33:52.967 INFO:tasks.ceph.cluster1.osd.1.trial034.stderr:2026-02-18T08:33:52.960+0000 7f20d7266640 -1 reset not still connected to 0x55641ff40f70 2026-02-18T08:33:53.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1100 IOs, 352206848 bytes 2026-02-18T08:33:53.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1200 IOs, 373655552 bytes 2026-02-18T08:33:54.915 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1300 IOs, 424863744 bytes 2026-02-18T08:33:55.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1400 IOs, 455716864 bytes 2026-02-18T08:33:55.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1500 IOs, 488955904 bytes 2026-02-18T08:33:56.114 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1600 IOs, 521316352 bytes 2026-02-18T08:33:56.146 INFO:tasks.ceph.cluster2.osd.2.trial163.stderr:2026-02-18T08:33:56.133+0000 7f2b60c87640 -1 reset not still connected to 0x55fc1b45cb60 2026-02-18T08:33:56.146 INFO:tasks.ceph.cluster2.osd.2.trial163.stderr:2026-02-18T08:33:56.133+0000 7f2b60c87640 -1 reset not still connected to 0x55fc2c9a01a0 2026-02-18T08:33:56.276 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:33:56.268+0000 7fb27e7f3640 -1 reset not still connected to 0x55e17108eea0 2026-02-18T08:33:56.276 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:33:56.268+0000 7fb27e7f3640 -1 reset not still connected to 0x55e178bde820 2026-02-18T08:33:56.276 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:33:56.272+0000 7fb27e7f3640 -1 reset not still connected to 0x55e183c10f70 2026-02-18T08:33:56.276 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:33:56.272+0000 7fb27e7f3640 -1 reset not still connected to 0x55e18817b2b0 2026-02-18T08:33:56.277 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:33:56.272+0000 7fb27e7f3640 -1 reset not still connected to 0x55e18895d380 2026-02-18T08:33:56.277 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:33:56.272+0000 7fb27e7f3640 -1 reset not still connected to 0x55e188baf5f0 2026-02-18T08:33:56.877 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1700 IOs, 576067584 bytes 2026-02-18T08:33:57.236 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:33:57.232+0000 7fb27e7f3640 -1 reset not still connected to 0x55e179628c30 2026-02-18T08:33:57.236 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:33:57.232+0000 7fb27e7f3640 -1 reset not still connected to 0x55e193b8f6c0 2026-02-18T08:33:57.733 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:33:57.728+0000 7fb27e7f3640 -1 reset not still connected to 0x55e17615aea0 2026-02-18T08:33:57.733 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:33:57.728+0000 7fb27e7f3640 -1 reset not still connected to 0x55e176ac6410 2026-02-18T08:33:57.734 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:33:57.728+0000 7fb27e7f3640 -1 reset not still connected to 0x55e177d12d00 2026-02-18T08:33:57.734 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:33:57.728+0000 7fb27e7f3640 -1 reset not still connected to 0x55e179983ee0 2026-02-18T08:33:57.735 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:33:57.728+0000 7fb27e7f3640 -1 reset not still connected to 0x55e18895c820 2026-02-18T08:33:57.735 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:33:57.728+0000 7fb27e7f3640 -1 reset not still connected to 0x55e188baf860 2026-02-18T08:33:57.736 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:33:57.732+0000 7fb27e7f3640 -1 reset not still connected to 0x55e1b06469c0 2026-02-18T08:33:57.905 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:33:57.900+0000 7fb27e7f3640 -1 reset not still connected to 0x55e171c0be10 2026-02-18T08:33:57.906 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:33:57.900+0000 7fb27e7f3640 -1 reset not still connected to 0x55e17615a8f0 2026-02-18T08:33:57.906 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:33:57.900+0000 7fb27e7f3640 -1 reset not still connected to 0x55e176ac6340 2026-02-18T08:33:57.906 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:33:57.900+0000 7fb27e7f3640 -1 reset not still connected to 0x55e179628c30 2026-02-18T08:33:57.906 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:33:57.900+0000 7fb27e7f3640 -1 reset not still connected to 0x55e193b8f6c0 2026-02-18T08:33:58.016 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:33:58.012+0000 7faffc95c640 -1 reset not still connected to 0x5644dc3cd930 2026-02-18T08:33:58.298 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1800 IOs, 649643008 bytes 2026-02-18T08:33:58.763 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ error_code=124 2026-02-18T08:33:58.763 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set -e 2026-02-18T08:33:58.763 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 124 -eq 124 ']' 2026-02-18T08:33:58.763 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:33:58.763 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T08:33:58.763 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_50 2026-02-18T08:33:58.763 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_image_and_enable_mirror cluster2 mirror image_50 journal 128M 2026-02-18T08:33:58.763 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:33:58.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:33:58.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:33:58.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:33:58.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_50 2026-02-18T08:33:58.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:33:58.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local mode=journal 2026-02-18T08:33:58.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n journal ']' 2026-02-18T08:33:58.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:33:58.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_image cluster2 mirror image_50 128M 2026-02-18T08:33:58.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:33:58.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:33:58.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:33:58.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:33:58.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_50 2026-02-18T08:33:58.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:33:58.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local size=128 2026-02-18T08:33:58.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n 128M ']' 2026-02-18T08:33:58.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ size=128M 2026-02-18T08:33:58.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:33:58.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,journaling mirror/image_50 2026-02-18T08:33:58.789 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:33:58.784+0000 7fb27e7f3640 -1 reset not still connected to 0x55e178bde820 2026-02-18T08:33:58.790 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:33:58.784+0000 7fb27e7f3640 -1 reset not still connected to 0x55e183c10f70 2026-02-18T08:33:58.790 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:33:58.784+0000 7fb27e7f3640 -1 reset not still connected to 0x55e18817b2b0 2026-02-18T08:33:58.790 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:33:58.784+0000 7fb27e7f3640 -1 reset not still connected to 0x55e18895d380 2026-02-18T08:33:58.790 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:33:58.784+0000 7fb27e7f3640 -1 reset not still connected to 0x55e188baf5f0 2026-02-18T08:33:58.804 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ is_pool_mirror_mode_image mirror 2026-02-18T08:33:58.804 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:33:58.804 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' pool = image ']' 2026-02-18T08:33:58.804 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ case "${pool}" in 2026-02-18T08:33:58.804 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 1 2026-02-18T08:33:58.804 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n '' ']' 2026-02-18T08:33:58.804 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ stress_write_image cluster2 mirror image_50 2026-02-18T08:33:58.804 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:33:58.805 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:33:58.805 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_50 2026-02-18T08:33:58.805 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-18T08:33:58.805 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local duration=6 2026-02-18T08:33:58.805 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set +e 2026-02-18T08:33:58.806 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ timeout 6s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_50 --debug-rbd=20 --debug-journaler=20 2026-02-18T08:33:58.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:100 IOs, 3747840 bytes 2026-02-18T08:33:59.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:200 IOs, 21214208 bytes 2026-02-18T08:33:59.646 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:300 IOs, 42180608 bytes 2026-02-18T08:33:59.895 INFO:tasks.ceph.cluster1.osd.1.trial034.stderr:2026-02-18T08:33:59.888+0000 7f20d7266640 -1 reset not still connected to 0x55640ccb9d40 2026-02-18T08:33:59.896 INFO:tasks.ceph.cluster1.osd.1.trial034.stderr:2026-02-18T08:33:59.888+0000 7f20d7266640 -1 reset not still connected to 0x556415bf3380 2026-02-18T08:33:59.896 INFO:tasks.ceph.cluster1.osd.1.trial034.stderr:2026-02-18T08:33:59.888+0000 7f20d7266640 -1 reset not still connected to 0x55642256d6c0 2026-02-18T08:33:59.896 INFO:tasks.ceph.cluster1.osd.1.trial034.stderr:2026-02-18T08:33:59.888+0000 7f20d7266640 -1 reset not still connected to 0x556422af5ad0 2026-02-18T08:33:59.896 INFO:tasks.ceph.cluster1.osd.1.trial034.stderr:2026-02-18T08:33:59.892+0000 7f20d7266640 -1 reset not still connected to 0x55642603b520 2026-02-18T08:33:59.896 INFO:tasks.ceph.cluster1.osd.1.trial034.stderr:2026-02-18T08:33:59.892+0000 7f20d7266640 -1 reset not still connected to 0x556431954000 2026-02-18T08:34:00.102 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:400 IOs, 81013760 bytes 2026-02-18T08:34:00.697 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:34:00.696+0000 7fb27e7f3640 -1 reset not still connected to 0x55e193b8f6c0 2026-02-18T08:34:00.698 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:34:00.696+0000 7fb27e7f3640 -1 reset not still connected to 0x55e1703c9520 2026-02-18T08:34:00.698 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:34:00.696+0000 7fb27e7f3640 -1 reset not still connected to 0x55e170c6ec30 2026-02-18T08:34:00.698 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:34:00.696+0000 7fb27e7f3640 -1 reset not still connected to 0x55e17615a8f0 2026-02-18T08:34:00.698 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:34:00.696+0000 7fb27e7f3640 -1 reset not still connected to 0x55e176ac6340 2026-02-18T08:34:00.699 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:34:00.696+0000 7fb27e7f3640 -1 reset not still connected to 0x55e193b8f6c0 2026-02-18T08:34:00.766 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:34:00.764+0000 7fb27e7f3640 -1 reset not still connected to 0x55e188baf5f0 2026-02-18T08:34:01.022 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:34:01.020+0000 7fb27e7f3640 -1 reset not still connected to 0x55e179628c30 2026-02-18T08:34:01.282 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:500 IOs, 147345408 bytes 2026-02-18T08:34:01.914 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:34:01.912+0000 7fb27e7f3640 -1 reset not still connected to 0x55e170ebfd40 2026-02-18T08:34:01.914 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:34:01.912+0000 7fb27e7f3640 -1 reset not still connected to 0x55e17108eea0 2026-02-18T08:34:01.914 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:34:01.912+0000 7fb27e7f3640 -1 reset not still connected to 0x55e178bde820 2026-02-18T08:34:01.914 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:34:01.912+0000 7fb27e7f3640 -1 reset not still connected to 0x55e183c10f70 2026-02-18T08:34:01.915 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:34:01.912+0000 7fb27e7f3640 -1 reset not still connected to 0x55e18817b2b0 2026-02-18T08:34:01.915 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:34:01.912+0000 7fb27e7f3640 -1 reset not still connected to 0x55e18895d380 2026-02-18T08:34:01.915 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:34:01.912+0000 7fb27e7f3640 -1 reset not still connected to 0x55e188baf5f0 2026-02-18T08:34:02.294 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:600 IOs, 199973888 bytes 2026-02-18T08:34:02.663 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:700 IOs, 253361152 bytes 2026-02-18T08:34:02.733 INFO:tasks.ceph.cluster1.osd.1.trial034.stderr:2026-02-18T08:34:02.731+0000 7f20d7266640 -1 reset not still connected to 0x556422af5ad0 2026-02-18T08:34:02.734 INFO:tasks.ceph.cluster1.osd.1.trial034.stderr:2026-02-18T08:34:02.731+0000 7f20d7266640 -1 reset not still connected to 0x55642603b520 2026-02-18T08:34:02.734 INFO:tasks.ceph.cluster1.osd.1.trial034.stderr:2026-02-18T08:34:02.731+0000 7f20d7266640 -1 reset not still connected to 0x556431954000 2026-02-18T08:34:03.818 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:800 IOs, 298338304 bytes 2026-02-18T08:34:04.594 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:900 IOs, 339834880 bytes 2026-02-18T08:34:04.815 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ error_code=124 2026-02-18T08:34:04.815 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set -e 2026-02-18T08:34:04.815 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 124 -eq 124 ']' 2026-02-18T08:34:04.815 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:34:04.815 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_pool_images cluster2 mirror 50 2026-02-18T08:34:04.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:34:04.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:34:04.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image_count=50 2026-02-18T08:34:04.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-18T08:34:04.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local count 2026-02-18T08:34:04.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local last_count=0 2026-02-18T08:34:04.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:34:04.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 1 40 2026-02-18T08:34:04.817 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in `seq 1 40` 2026-02-18T08:34:04.817 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 1 -ne 1 2026-02-18T08:34:04.817 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 -p mirror mirror pool status 2026-02-18T08:34:04.817 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep 'images: ' 2026-02-18T08:34:04.817 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut '-d ' -f 2 2026-02-18T08:34:04.838 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd: failed to query services: (13) Permission denied 2026-02-18T08:34:04.842 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ count=50 2026-02-18T08:34:04.842 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 50 = 50 2026-02-18T08:34:04.842 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:34:04.842 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_pool_healthy cluster2 mirror 2026-02-18T08:34:04.842 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:34:04.842 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:34:04.842 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-18T08:34:04.842 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local state 2026-02-18T08:34:04.842 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 1 40 2026-02-18T08:34:04.843 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in `seq 1 40` 2026-02-18T08:34:04.843 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 1 -ne 1 2026-02-18T08:34:04.843 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 -p mirror mirror pool status 2026-02-18T08:34:04.844 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep 'image health:' 2026-02-18T08:34:04.844 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut '-d ' -f 3 2026-02-18T08:34:04.865 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd: failed to query services: (13) Permission denied 2026-02-18T08:34:04.868 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ state=OK 2026-02-18T08:34:04.868 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test OK = ERROR 2026-02-18T08:34:04.868 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test OK = OK 2026-02-18T08:34:04.868 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:34:04.868 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_pool_images cluster1 mirror 50 2026-02-18T08:34:04.869 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:34:04.869 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:34:04.869 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image_count=50 2026-02-18T08:34:04.869 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-18T08:34:04.869 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local count 2026-02-18T08:34:04.869 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local last_count=0 2026-02-18T08:34:04.869 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:34:04.869 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 1 40 2026-02-18T08:34:04.870 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in `seq 1 40` 2026-02-18T08:34:04.870 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 1 -ne 1 2026-02-18T08:34:04.870 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster1 -p mirror mirror pool status 2026-02-18T08:34:04.870 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep 'images: ' 2026-02-18T08:34:04.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut '-d ' -f 2 2026-02-18T08:34:04.896 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd: failed to query services: (13) Permission denied 2026-02-18T08:34:04.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ count=50 2026-02-18T08:34:04.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 50 = 50 2026-02-18T08:34:04.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:34:04.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_pool_healthy cluster1 mirror 2026-02-18T08:34:04.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:34:04.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:34:04.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-18T08:34:04.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local state 2026-02-18T08:34:04.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 1 40 2026-02-18T08:34:04.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in `seq 1 40` 2026-02-18T08:34:04.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 1 -ne 1 2026-02-18T08:34:04.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster1 -p mirror mirror pool status 2026-02-18T08:34:04.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep 'image health:' 2026-02-18T08:34:04.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut '-d ' -f 3 2026-02-18T08:34:04.924 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd: failed to query services: (13) Permission denied 2026-02-18T08:34:04.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ state=WARNING 2026-02-18T08:34:04.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test WARNING = ERROR 2026-02-18T08:34:04.928 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test WARNING = OK 2026-02-18T08:34:04.928 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in `seq 1 40` 2026-02-18T08:34:04.928 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 2 -ne 1 2026-02-18T08:34:04.928 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 30 2026-02-18T08:34:05.844 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:34:05.843+0000 7fb27e7f3640 -1 reset not still connected to 0x55e171c0b860 2026-02-18T08:34:05.844 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:34:05.843+0000 7fb27e7f3640 -1 reset not still connected to 0x55e17615aea0 2026-02-18T08:34:05.844 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:34:05.843+0000 7fb27e7f3640 -1 reset not still connected to 0x55e176ac6410 2026-02-18T08:34:05.846 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:34:05.843+0000 7fb27e7f3640 -1 reset not still connected to 0x55e177d12d00 2026-02-18T08:34:05.846 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:34:05.843+0000 7fb27e7f3640 -1 reset not still connected to 0x55e179983ee0 2026-02-18T08:34:05.846 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:34:05.843+0000 7fb27e7f3640 -1 reset not still connected to 0x55e18895c820 2026-02-18T08:34:05.846 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:34:05.843+0000 7fb27e7f3640 -1 reset not still connected to 0x55e188baf860 2026-02-18T08:34:05.847 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:34:05.843+0000 7fb27e7f3640 -1 reset not still connected to 0x55e1b06469c0 2026-02-18T08:34:05.924 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:34:05.919+0000 7fb27e7f3640 -1 reset not still connected to 0x55e171c0b860 2026-02-18T08:34:05.924 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:34:05.923+0000 7fb27e7f3640 -1 reset not still connected to 0x55e17615aea0 2026-02-18T08:34:05.925 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:34:05.923+0000 7fb27e7f3640 -1 reset not still connected to 0x55e176ac6410 2026-02-18T08:34:05.925 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:34:05.923+0000 7fb27e7f3640 -1 reset not still connected to 0x55e177d12d00 2026-02-18T08:34:05.925 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:34:05.923+0000 7fb27e7f3640 -1 reset not still connected to 0x55e179983ee0 2026-02-18T08:34:05.926 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:34:05.923+0000 7fb27e7f3640 -1 reset not still connected to 0x55e18895c820 2026-02-18T08:34:05.926 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:34:05.923+0000 7fb27e7f3640 -1 reset not still connected to 0x55e188baf860 2026-02-18T08:34:05.926 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:34:05.923+0000 7fb27e7f3640 -1 reset not still connected to 0x55e1b06469c0 2026-02-18T08:34:05.955 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:34:05.951+0000 7fb27e7f3640 -1 reset not still connected to 0x55e17615a8f0 2026-02-18T08:34:05.955 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:34:05.951+0000 7fb27e7f3640 -1 reset not still connected to 0x55e176ac6340 2026-02-18T08:34:05.955 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:34:05.951+0000 7fb27e7f3640 -1 reset not still connected to 0x55e179628c30 2026-02-18T08:34:05.956 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:34:05.951+0000 7fb27e7f3640 -1 reset not still connected to 0x55e193b8f6c0 2026-02-18T08:34:06.007 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:34:06.003+0000 7fb27e7f3640 -1 reset not still connected to 0x55e193b8f930 2026-02-18T08:34:06.463 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:34:06.459+0000 7fb27e7f3640 -1 reset not still connected to 0x55e170c6ec30 2026-02-18T08:34:06.464 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:34:06.463+0000 7fb27e7f3640 -1 reset not still connected to 0x55e176ac6340 2026-02-18T08:34:06.464 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:34:06.463+0000 7fb27e7f3640 -1 reset not still connected to 0x55e179628c30 2026-02-18T08:34:06.464 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:34:06.463+0000 7fb27e7f3640 -1 reset not still connected to 0x55e193b8f6c0 2026-02-18T08:34:07.419 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:34:07.415+0000 7fb27e7f3640 -1 reset not still connected to 0x55e183c11860 2026-02-18T08:34:07.421 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:34:07.419+0000 7fb27e7f3640 -1 reset not still connected to 0x55e18895cea0 2026-02-18T08:34:07.421 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:34:07.419+0000 7fb27e7f3640 -1 reset not still connected to 0x55e18895d040 2026-02-18T08:34:07.421 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:34:07.419+0000 7fb27e7f3640 -1 reset not still connected to 0x55e193b8f930 2026-02-18T08:34:07.825 INFO:tasks.ceph.cluster1.osd.1.trial034.stderr:2026-02-18T08:34:07.823+0000 7f20d7266640 -1 reset not still connected to 0x556405b671e0 2026-02-18T08:34:07.826 INFO:tasks.ceph.cluster1.osd.1.trial034.stderr:2026-02-18T08:34:07.823+0000 7f20d7266640 -1 reset not still connected to 0x556415bf3ad0 2026-02-18T08:34:07.826 INFO:tasks.ceph.cluster1.osd.1.trial034.stderr:2026-02-18T08:34:07.823+0000 7f20d7266640 -1 reset not still connected to 0x556428803d40 2026-02-18T08:34:08.389 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:34:08.387+0000 7fb27e7f3640 -1 reset not still connected to 0x55e179628c30 2026-02-18T08:34:08.389 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:34:08.387+0000 7fb27e7f3640 -1 reset not still connected to 0x55e193b8f6c0 2026-02-18T08:34:08.425 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:34:08.423+0000 7faffc95c640 -1 reset not still connected to 0x5644bd70aea0 2026-02-18T08:34:08.425 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:34:08.423+0000 7faffc95c640 -1 reset not still connected to 0x5644c8fcea90 2026-02-18T08:34:08.426 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:34:08.423+0000 7faffc95c640 -1 reset not still connected to 0x5644c8fcfad0 2026-02-18T08:34:08.426 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:34:08.423+0000 7faffc95c640 -1 reset not still connected to 0x5644d7dcc0d0 2026-02-18T08:34:08.427 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:34:08.423+0000 7faffc95c640 -1 reset not still connected to 0x5644dc3cd040 2026-02-18T08:34:08.427 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:34:08.423+0000 7faffc95c640 -1 reset not still connected to 0x5644dc3cd860 2026-02-18T08:34:08.427 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:34:08.423+0000 7faffc95c640 -1 reset not still connected to 0x5644dfda05b0 2026-02-18T08:34:08.525 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:34:08.523+0000 7faffc95c640 -1 reset not still connected to 0x5644a4425ad0 2026-02-18T08:34:08.525 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:34:08.523+0000 7faffc95c640 -1 reset not still connected to 0x5644a6841040 2026-02-18T08:34:08.525 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:34:08.523+0000 7faffc95c640 -1 reset not still connected to 0x5644a716ac30 2026-02-18T08:34:08.526 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:34:08.523+0000 7faffc95c640 -1 reset not still connected to 0x5644aec73040 2026-02-18T08:34:08.526 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:34:08.523+0000 7faffc95c640 -1 reset not still connected to 0x5644af70fe10 2026-02-18T08:34:08.526 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:34:08.523+0000 7faffc95c640 -1 reset not still connected to 0x5644b4738000 2026-02-18T08:34:08.527 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:34:08.523+0000 7faffc95c640 -1 reset not still connected to 0x5644bb2d1110 2026-02-18T08:34:08.527 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:34:08.523+0000 7faffc95c640 -1 reset not still connected to 0x5644c0a68b60 2026-02-18T08:34:08.527 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:34:08.523+0000 7faffc95c640 -1 reset not still connected to 0x5644c1ba6a90 2026-02-18T08:34:08.527 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:34:08.523+0000 7faffc95c640 -1 reset not still connected to 0x5644c8a8f860 2026-02-18T08:34:08.527 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:34:08.523+0000 7faffc95c640 -1 reset not still connected to 0x5644d3e71d40 2026-02-18T08:34:08.527 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:34:08.523+0000 7faffc95c640 -1 reset not still connected to 0x5644d9b0e9c0 2026-02-18T08:34:08.657 INFO:tasks.ceph.cluster1.osd.1.trial034.stderr:2026-02-18T08:34:08.655+0000 7f20d7266640 -1 reset not still connected to 0x55640ccb9d40 2026-02-18T08:34:08.658 INFO:tasks.ceph.cluster1.osd.1.trial034.stderr:2026-02-18T08:34:08.655+0000 7f20d7266640 -1 reset not still connected to 0x556422af5ad0 2026-02-18T08:34:10.558 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:34:10.555+0000 7faffc95c640 -1 reset not still connected to 0x5644a84a6750 2026-02-18T08:34:10.558 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:34:10.555+0000 7faffc95c640 -1 reset not still connected to 0x5644c5827380 2026-02-18T08:34:14.299 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:34:14.295+0000 7fb27e7f3640 -1 reset not still connected to 0x55e176ac6410 2026-02-18T08:34:14.299 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:34:14.295+0000 7fb27e7f3640 -1 reset not still connected to 0x55e177d12d00 2026-02-18T08:34:14.300 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:34:14.295+0000 7fb27e7f3640 -1 reset not still connected to 0x55e179983ee0 2026-02-18T08:34:14.300 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:34:14.299+0000 7fb27e7f3640 -1 reset not still connected to 0x55e18895c820 2026-02-18T08:34:14.300 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:34:14.299+0000 7fb27e7f3640 -1 reset not still connected to 0x55e188baf860 2026-02-18T08:34:14.301 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:34:14.299+0000 7fb27e7f3640 -1 reset not still connected to 0x55e1b06469c0 2026-02-18T08:34:14.462 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:34:14.459+0000 7fb27e7f3640 -1 reset not still connected to 0x55e175f31a00 2026-02-18T08:34:14.462 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:34:14.459+0000 7fb27e7f3640 -1 reset not still connected to 0x55e178bde340 2026-02-18T08:34:14.464 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:34:14.463+0000 7fb27e7f3640 -1 reset not still connected to 0x55e178bdeea0 2026-02-18T08:34:14.464 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:34:14.463+0000 7fb27e7f3640 -1 reset not still connected to 0x55e179629ba0 2026-02-18T08:34:14.464 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:34:14.463+0000 7fb27e7f3640 -1 reset not still connected to 0x55e17997b110 2026-02-18T08:34:14.464 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:34:14.463+0000 7fb27e7f3640 -1 reset not still connected to 0x55e183c11860 2026-02-18T08:34:14.466 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:34:14.463+0000 7fb27e7f3640 -1 reset not still connected to 0x55e188435380 2026-02-18T08:34:14.466 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:34:14.463+0000 7fb27e7f3640 -1 reset not still connected to 0x55e18895cea0 2026-02-18T08:34:14.466 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:34:14.463+0000 7fb27e7f3640 -1 reset not still connected to 0x55e18895d040 2026-02-18T08:34:14.467 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:34:14.463+0000 7fb27e7f3640 -1 reset not still connected to 0x55e193b8f930 2026-02-18T08:34:16.565 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:34:16.563+0000 7faffc95c640 -1 reset not still connected to 0x5644b7e83c70 2026-02-18T08:34:16.565 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:34:16.563+0000 7faffc95c640 -1 reset not still connected to 0x5644bd70a270 2026-02-18T08:34:16.565 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:34:16.563+0000 7faffc95c640 -1 reset not still connected to 0x5644bd70aea0 2026-02-18T08:34:16.566 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:34:16.563+0000 7faffc95c640 -1 reset not still connected to 0x5644c8fcea90 2026-02-18T08:34:16.567 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:34:16.563+0000 7faffc95c640 -1 reset not still connected to 0x5644c8fcfad0 2026-02-18T08:34:16.567 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:34:16.563+0000 7faffc95c640 -1 reset not still connected to 0x5644d7dcc0d0 2026-02-18T08:34:16.567 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:34:16.563+0000 7faffc95c640 -1 reset not still connected to 0x5644dc3cd860 2026-02-18T08:34:16.567 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:34:16.563+0000 7faffc95c640 -1 reset not still connected to 0x5644dfda05b0 2026-02-18T08:34:17.460 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:34:17.459+0000 7faffc95c640 -1 reset not still connected to 0x5644bd70aea0 2026-02-18T08:34:17.461 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:34:17.459+0000 7faffc95c640 -1 reset not still connected to 0x5644c8fcea90 2026-02-18T08:34:17.461 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:34:17.459+0000 7faffc95c640 -1 reset not still connected to 0x5644c8fcfad0 2026-02-18T08:34:17.461 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:34:17.459+0000 7faffc95c640 -1 reset not still connected to 0x5644d7dcc0d0 2026-02-18T08:34:17.462 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:34:17.459+0000 7faffc95c640 -1 reset not still connected to 0x5644dc3cd860 2026-02-18T08:34:17.462 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:34:17.459+0000 7faffc95c640 -1 reset not still connected to 0x5644dfda05b0 2026-02-18T08:34:17.893 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:34:17.891+0000 7fb27e7f3640 -1 reset not still connected to 0x55e179628c30 2026-02-18T08:34:17.893 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:34:17.891+0000 7fb27e7f3640 -1 reset not still connected to 0x55e193b8f6c0 2026-02-18T08:34:18.404 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:34:18.403+0000 7faffc95c640 -1 reset not still connected to 0x5644a6841040 2026-02-18T08:34:18.406 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:34:18.403+0000 7faffc95c640 -1 reset not still connected to 0x5644a716ac30 2026-02-18T08:34:18.406 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:34:18.403+0000 7faffc95c640 -1 reset not still connected to 0x5644aec73040 2026-02-18T08:34:18.407 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:34:18.403+0000 7faffc95c640 -1 reset not still connected to 0x5644af70fe10 2026-02-18T08:34:18.407 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:34:18.403+0000 7faffc95c640 -1 reset not still connected to 0x5644b4738000 2026-02-18T08:34:18.407 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:34:18.403+0000 7faffc95c640 -1 reset not still connected to 0x5644bb2d1110 2026-02-18T08:34:18.407 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:34:18.403+0000 7faffc95c640 -1 reset not still connected to 0x5644c0a68b60 2026-02-18T08:34:18.407 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:34:18.403+0000 7faffc95c640 -1 reset not still connected to 0x5644c1ba6a90 2026-02-18T08:34:18.407 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:34:18.403+0000 7faffc95c640 -1 reset not still connected to 0x5644c8a8f860 2026-02-18T08:34:18.407 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:34:18.403+0000 7faffc95c640 -1 reset not still connected to 0x5644d3e71d40 2026-02-18T08:34:18.407 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:34:18.403+0000 7faffc95c640 -1 reset not still connected to 0x5644d9b0e9c0 2026-02-18T08:34:18.828 INFO:tasks.ceph.cluster1.osd.1.trial034.stderr:2026-02-18T08:34:18.827+0000 7f20d7266640 -1 reset not still connected to 0x55640bcd1040 2026-02-18T08:34:18.829 INFO:tasks.ceph.cluster1.osd.1.trial034.stderr:2026-02-18T08:34:18.827+0000 7f20d7266640 -1 reset not still connected to 0x55640bcd11e0 2026-02-18T08:34:18.829 INFO:tasks.ceph.cluster1.osd.1.trial034.stderr:2026-02-18T08:34:18.827+0000 7f20d7266640 -1 reset not still connected to 0x55641ff40f70 2026-02-18T08:34:19.058 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:34:19.055+0000 7faffc95c640 -1 reset not still connected to 0x5644c8fcea90 2026-02-18T08:34:19.058 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:34:19.055+0000 7faffc95c640 -1 reset not still connected to 0x5644c8fcfad0 2026-02-18T08:34:19.058 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:34:19.055+0000 7faffc95c640 -1 reset not still connected to 0x5644d7dcc0d0 2026-02-18T08:34:19.058 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:34:19.055+0000 7faffc95c640 -1 reset not still connected to 0x5644dc3cd040 2026-02-18T08:34:19.059 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:34:19.055+0000 7faffc95c640 -1 reset not still connected to 0x5644dc3cd860 2026-02-18T08:34:19.059 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:34:19.055+0000 7faffc95c640 -1 reset not still connected to 0x5644dfda05b0 2026-02-18T08:34:19.135 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:34:19.131+0000 7faffc95c640 -1 reset not still connected to 0x5644d9b0e9c0 2026-02-18T08:34:19.275 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:34:19.271+0000 7faffc95c640 -1 reset not still connected to 0x5644c5826270 2026-02-18T08:34:19.275 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:34:19.271+0000 7faffc95c640 -1 reset not still connected to 0x5644c8fcea90 2026-02-18T08:34:19.275 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:34:19.271+0000 7faffc95c640 -1 reset not still connected to 0x5644c8fcfad0 2026-02-18T08:34:19.275 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:34:19.271+0000 7faffc95c640 -1 reset not still connected to 0x5644d7dcc0d0 2026-02-18T08:34:19.276 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:34:19.275+0000 7faffc95c640 -1 reset not still connected to 0x5644dc3cd040 2026-02-18T08:34:19.276 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:34:19.275+0000 7faffc95c640 -1 reset not still connected to 0x5644dc3cd860 2026-02-18T08:34:19.276 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:34:19.275+0000 7faffc95c640 -1 reset not still connected to 0x5644dfda05b0 2026-02-18T08:34:20.659 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:34:20.655+0000 7faffc95c640 -1 reset not still connected to 0x5644d7dcc0d0 2026-02-18T08:34:20.659 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:34:20.655+0000 7faffc95c640 -1 reset not still connected to 0x5644dc3cd860 2026-02-18T08:34:20.660 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:34:20.655+0000 7faffc95c640 -1 reset not still connected to 0x5644dfda05b0 2026-02-18T08:34:23.845 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:34:23.843+0000 7faffc95c640 -1 reset not still connected to 0x5644dfda05b0 2026-02-18T08:34:25.355 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.0 with skip (0.7598448124507887) > weight (0.25) 2026-02-18T08:34:25.355 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.1 with skip (0.5553094412000036) > weight (0.25) 2026-02-18T08:34:25.355 INFO:tasks.rbd_mirror_thrash:kill cluster1.client.mirror.2 2026-02-18T08:34:25.355 INFO:tasks.rbd_mirror.cluster1.client.mirror.2:Sent signal 15 2026-02-18T08:34:25.355 INFO:tasks.rbd_mirror_thrash:waiting for 9.993563635842579 secs before reviving daemons 2026-02-18T08:34:25.356 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.trial163.stderr:2026-02-18T08:34:25.345+0000 7f1c57b7e640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.2 (PID: 26151) UID: 1000 2026-02-18T08:34:25.929 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:34:25.927+0000 7fb27e7f3640 -1 reset not still connected to 0x55e188baf5f0 2026-02-18T08:34:26.552 INFO:tasks.ceph.cluster1.osd.1.trial034.stderr:2026-02-18T08:34:26.551+0000 7f20d7266640 -1 reset not still connected to 0x5563ff09c340 2026-02-18T08:34:26.552 INFO:tasks.ceph.cluster1.osd.1.trial034.stderr:2026-02-18T08:34:26.551+0000 7f20d7266640 -1 reset not still connected to 0x5564021960d0 2026-02-18T08:34:26.552 INFO:tasks.ceph.cluster1.osd.1.trial034.stderr:2026-02-18T08:34:26.551+0000 7f20d7266640 -1 reset not still connected to 0x5564088e4680 2026-02-18T08:34:26.553 INFO:tasks.ceph.cluster1.osd.1.trial034.stderr:2026-02-18T08:34:26.551+0000 7f20d7266640 -1 reset not still connected to 0x55640bcd1040 2026-02-18T08:34:26.553 INFO:tasks.ceph.cluster1.osd.1.trial034.stderr:2026-02-18T08:34:26.551+0000 7f20d7266640 -1 reset not still connected to 0x55640bcd11e0 2026-02-18T08:34:26.553 INFO:tasks.ceph.cluster1.osd.1.trial034.stderr:2026-02-18T08:34:26.551+0000 7f20d7266640 -1 reset not still connected to 0x55641ff40f70 2026-02-18T08:34:27.732 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:34:27.731+0000 7faffc95c640 -1 reset not still connected to 0x5644dc3cd860 2026-02-18T08:34:27.733 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:34:27.731+0000 7faffc95c640 -1 reset not still connected to 0x5644dfda05b0 2026-02-18T08:34:27.859 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:34:27.855+0000 7fb27e7f3640 -1 reset not still connected to 0x55e17615a8f0 2026-02-18T08:34:27.859 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:34:27.855+0000 7fb27e7f3640 -1 reset not still connected to 0x55e176ac6340 2026-02-18T08:34:27.859 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:34:27.855+0000 7fb27e7f3640 -1 reset not still connected to 0x55e179628c30 2026-02-18T08:34:27.860 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:34:27.855+0000 7fb27e7f3640 -1 reset not still connected to 0x55e193b8f6c0 2026-02-18T08:34:30.445 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:34:30.443+0000 7faffc95c640 -1 reset not still connected to 0x5644bd70aea0 2026-02-18T08:34:30.445 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:34:30.443+0000 7faffc95c640 -1 reset not still connected to 0x5644c8fcea90 2026-02-18T08:34:30.447 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:34:30.443+0000 7faffc95c640 -1 reset not still connected to 0x5644c8fcfad0 2026-02-18T08:34:30.447 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:34:30.443+0000 7faffc95c640 -1 reset not still connected to 0x5644d7dcc0d0 2026-02-18T08:34:30.447 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:34:30.443+0000 7faffc95c640 -1 reset not still connected to 0x5644dc3cd040 2026-02-18T08:34:30.447 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:34:30.443+0000 7faffc95c640 -1 reset not still connected to 0x5644dc3cd860 2026-02-18T08:34:30.447 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:34:30.443+0000 7faffc95c640 -1 reset not still connected to 0x5644dfda05b0 2026-02-18T08:34:31.163 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:34:31.159+0000 7faffc95c640 -1 reset not still connected to 0x5644d9b0e9c0 2026-02-18T08:34:34.930 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster1 -p mirror mirror pool status 2026-02-18T08:34:34.930 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep 'image health:' 2026-02-18T08:34:34.930 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut '-d ' -f 3 2026-02-18T08:34:34.955 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd: failed to query services: (13) Permission denied 2026-02-18T08:34:34.958 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ state=OK 2026-02-18T08:34:34.958 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test OK = ERROR 2026-02-18T08:34:34.958 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test OK = OK 2026-02-18T08:34:34.958 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:34:34.958 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ testlog 'TEST: compare many images' 2026-02-18T08:34:34.959 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ tee -a /tmp/tmp.vybh1m9yJF/rbd-mirror.test.log 2026-02-18T08:34:34.959 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ date '+%F %T' 2026-02-18T08:34:34.959 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ echo 2026-02-18 08:34:34 TEST: compare many images 2026-02-18T08:34:34.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:2026-02-18 08:34:34 TEST: compare many images 2026-02-18T08:34:34.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 1 50 2026-02-18T08:34:34.962 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T08:34:34.962 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_1 2026-02-18T08:34:34.962 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_snap cluster2 mirror image_1 snap 2026-02-18T08:34:34.962 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:34:34.962 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:34:34.962 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_1 2026-02-18T08:34:34.962 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap 2026-02-18T08:34:34.962 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 -p mirror snap create image_1@snap --debug-rbd=20 --debug-journaler=20 2026-02-18T08:34:35.350 INFO:tasks.rbd_mirror_thrash:waiting for cluster1.client.mirror.2 2026-02-18T08:34:35.350 INFO:teuthology.orchestra.run:waiting for 600 2026-02-18T08:34:35.351 INFO:tasks.rbd_mirror_thrash:reviving cluster1.client.mirror.2 2026-02-18T08:34:35.351 INFO:tasks.rbd_mirror.cluster1.client.mirror.2:Restarting daemon 2026-02-18T08:34:35.351 DEBUG:teuthology.orchestra.run.trial163:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.2 2026-02-18T08:34:35.353 INFO:tasks.rbd_mirror.cluster1.client.mirror.2:Started 2026-02-18T08:34:35.353 INFO:tasks.rbd_mirror_thrash:waiting for 68.33121512108701 secs before thrashing 2026-02-18T08:34:39.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_replay_started cluster1 mirror image_1 2026-02-18T08:34:39.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:34:39.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:34:39.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_1 2026-02-18T08:34:39.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_replay_state cluster1 mirror image_1 started 2026-02-18T08:34:39.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:34:39.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:34:39.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_1 2026-02-18T08:34:39.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local state=started 2026-02-18T08:34:39.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-18T08:34:39.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:34:39.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1 2026-02-18T08:34:40.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test_image_replay_state cluster1 mirror image_1 started 2026-02-18T08:34:40.027 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:34:40.027 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:34:40.027 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_1 2026-02-18T08:34:40.027 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local test_state=started 2026-02-18T08:34:40.027 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local status_result 2026-02-18T08:34:40.027 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_state=stopped 2026-02-18T08:34:40.027 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_1 2026-02-18T08:34:40.027 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -i state 2026-02-18T08:34:40.027 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_instance=cluster1 2026-02-18T08:34:40.028 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-18T08:34:40.028 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:34:40.028 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance=cluster1 2026-02-18T08:34:40.028 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local loop_instance 2026-02-18T08:34:40.028 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:34:40.028 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sleep 0 2026-02-18T08:34:40.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:34:40.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ seq 0 3 2026-02-18T08:34:40.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:34:40.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_1 2026-02-18T08:34:40.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:0 2026-02-18T08:34:40.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-18T08:34:40.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:34:40.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:34:40.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:34:40.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-18T08:34:40.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:34:40.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:34:40.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:34:40.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:34:40.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-18T08:34:40.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:34:40.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:34:40.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:34:40.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:34:40.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:0 2026-02-18T08:34:40.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-18T08:34:40.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-18T08:34:40.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:34:40.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:0 2026-02-18T08:34:40.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T08:34:40.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-18T08:34:40.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:34:40.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:34:40.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-18T08:34:40.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-18T08:34:40.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-18T08:34:40.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=0 2026-02-18T08:34:40.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=0 2026-02-18T08:34:40.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:34:40.040 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:34:40.041 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:34:40.041 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:34:40.041 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_1 2026-02-18T08:34:40.094 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_1'] 2026-02-18T08:34:40.094 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status : get status for rbd mirror 2026-02-18T08:34:40.094 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:34:40.097 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:34:40.097 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_1 2026-02-18T08:34:40.097 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:1 2026-02-18T08:34:40.097 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-18T08:34:40.097 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:34:40.097 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:34:40.097 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:34:40.097 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-18T08:34:40.097 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:34:40.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:34:40.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:34:40.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:34:40.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-18T08:34:40.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:34:40.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:34:40.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:34:40.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:34:40.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:1 2026-02-18T08:34:40.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-18T08:34:40.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-18T08:34:40.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:34:40.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:1 2026-02-18T08:34:40.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T08:34:40.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-18T08:34:40.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:34:40.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:34:40.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-18T08:34:40.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-18T08:34:40.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-18T08:34:40.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=1 2026-02-18T08:34:40.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=1 2026-02-18T08:34:40.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:34:40.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:34:40.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:34:40.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:34:40.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_1 2026-02-18T08:34:40.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ return 0 2026-02-18T08:34:40.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ status_result=' "state": "Replaying"' 2026-02-18T08:34:40.169 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ echo ' "state": "Replaying"' 2026-02-18T08:34:40.169 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ grep -i Replaying 2026-02-18T08:34:40.170 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: "state": "Replaying" 2026-02-18T08:34:40.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ current_state=started 2026-02-18T08:34:40.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test started = started 2026-02-18T08:34:40.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:34:40.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror image_1 2026-02-18T08:34:40.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_cluster=cluster1 2026-02-18T08:34:40.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:34:40.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_pool=mirror 2026-02-18T08:34:40.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local remote_pool=mirror 2026-02-18T08:34:40.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_1 2026-02-18T08:34:40.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' journal = journal ']' 2026-02-18T08:34:40.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror mirror image_1 2026-02-18T08:34:40.173 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_cluster=cluster1 2026-02-18T08:34:40.173 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:34:40.173 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_pool=mirror 2026-02-18T08:34:40.173 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local remote_pool=mirror 2026-02-18T08:34:40.173 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_1 2026-02-18T08:34:40.173 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-02-18T08:34:40.173 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-02-18T08:34:40.173 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:34:40.173 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:34:40.173 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:34:40.373 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_1 2026-02-18T08:34:40.373 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:34:40.373 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:34:40.373 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_1 2026-02-18T08:34:40.373 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:34:40.373 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_1 ']' 2026-02-18T08:34:40.373 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_1' 2026-02-18T08:34:40.374 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_1 2026-02-18T08:34:40.374 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:34:40.374 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:34:40.374 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:34:40.374 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:34:40.374 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:34:40.374 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:34:40.374 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:34:40.374 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:34:40.377 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:34:40.377 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:34:40.377 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_1 2026-02-18T08:34:40.377 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:34:40.377 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:34:40.377 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:34:40.377 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:34:40.377 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:34:40.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:34:40.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:34:40.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:34:40.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:34:40.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:34:40.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:34:40.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:34:40.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:34:40.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:34:40.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:34:40.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:34:40.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:34:40.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:34:40.379 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:34:40.379 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:34:40.379 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:34:40.379 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:34:40.379 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:34:40.379 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:34:40.379 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:34:40.379 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:34:40.379 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:34:40.379 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:34:40.379 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:34:40.379 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:34:40.387 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:34:40.388 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:34:40.388 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:34:40.388 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_1 2026-02-18T08:34:40.440 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_1'] 2026-02-18T08:34:40.440 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:34:40.440 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:34:40.443 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:34:40.443 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_1 2026-02-18T08:34:40.443 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:34:40.443 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:34:40.443 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:34:40.443 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:34:40.443 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:34:40.443 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:34:40.443 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:34:40.443 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:34:40.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:34:40.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:34:40.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:34:40.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:34:40.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:34:40.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:34:40.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:34:40.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:34:40.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:34:40.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:34:40.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:34:40.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:34:40.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:34:40.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:34:40.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:34:40.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:34:40.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:34:40.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:34:40.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:34:40.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:34:40.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:34:40.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:34:40.453 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:34:40.454 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:34:40.454 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:34:40.454 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_1 2026-02-18T08:34:40.510 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:34:40.512 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_1 2026-02-18T08:34:40.512 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:34:40.512 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:34:40.512 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_1 2026-02-18T08:34:40.512 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_1 '' 2026-02-18T08:34:40.512 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:34:40.512 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:34:40.512 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_1 2026-02-18T08:34:40.512 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:34:40.512 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_1.status 2026-02-18T08:34:40.513 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_1.status 2026-02-18T08:34:40.513 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:34:40.513 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_1.status 2026-02-18T08:34:40.514 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_1 2026-02-18T08:34:40.514 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_1.status 2026-02-18T08:34:40.540 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 11 2026-02-18T08:34:40.540 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 11 2026-02-18T08:34:40.540 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:34:40.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=2, entry_tid=1], [object_number=44, tag_tid=2, entry_tid=0], [object_number=47, tag_tid=1, entry_tid=49083], [object_number=46, tag_tid=1, entry_tid=49082]]], state=connected] 2026-02-18T08:34:40.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=44, tag_tid=1, entry_tid=49084], [object_number=47, tag_tid=1, entry_tid=49083], [object_number=46, tag_tid=1, entry_tid=49082], [object_number=45, tag_tid=1, entry_tid=49081]]], state=connected] 2026-02-18T08:34:40.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_1.status 2026-02-18T08:34:40.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=45, tag_tid=2, entry_tid=1' 2026-02-18T08:34:40.545 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_1 2026-02-18T08:34:40.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:34:40.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:34:40.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_1 2026-02-18T08:34:40.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_1 '..*' 2026-02-18T08:34:40.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:34:40.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:34:40.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_1 2026-02-18T08:34:40.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:34:40.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_1.status 2026-02-18T08:34:40.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_1.status 2026-02-18T08:34:40.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:34:40.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_1.status 2026-02-18T08:34:40.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_1 2026-02-18T08:34:40.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_1.status 2026-02-18T08:34:40.574 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 11 2026-02-18T08:34:40.574 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 11 2026-02-18T08:34:40.574 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:34:40.574 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=2, entry_tid=1], [object_number=44, tag_tid=2, entry_tid=0], [object_number=47, tag_tid=1, entry_tid=49083], [object_number=46, tag_tid=1, entry_tid=49082]]], state=connected] 2026-02-18T08:34:40.574 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=44, tag_tid=1, entry_tid=49084], [object_number=47, tag_tid=1, entry_tid=49083], [object_number=46, tag_tid=1, entry_tid=49082], [object_number=45, tag_tid=1, entry_tid=49081]]], state=connected] 2026-02-18T08:34:40.576 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_1.status 2026-02-18T08:34:40.578 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=44, tag_tid=1, entry_tid=49084' 2026-02-18T08:34:40.578 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=45, tag_tid=2, entry_tid=1' -a 'object_number=45, tag_tid=2, entry_tid=1' = 'object_number=44, tag_tid=1, entry_tid=49084' 2026-02-18T08:34:40.578 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=44, tag_tid=1, entry_tid=49084' '!=' '' 2026-02-18T08:34:40.578 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:34:40.578 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=44, tag_tid=1, entry_tid=49084' = '' 2026-02-18T08:34:40.578 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=44, tag_tid=1, entry_tid=49084' 2026-02-18T08:34:40.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=45, tag_tid=2, entry_tid=1' 2026-02-18T08:34:40.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:34:40.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:34:40.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=2 2026-02-18T08:34:40.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=44, tag_tid=1, entry_tid=49084' 2026-02-18T08:34:40.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:34:40.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:34:40.582 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=1 2026-02-18T08:34:40.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=45, tag_tid=2, entry_tid=1' 2026-02-18T08:34:40.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:34:40.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:34:40.585 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:34:40.585 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=44, tag_tid=1, entry_tid=49084' 2026-02-18T08:34:40.586 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:34:40.586 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:34:40.587 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=49084 2026-02-18T08:34:40.587 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 2 = 1 -a 1 -le 49084 2026-02-18T08:34:40.587 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:34:40.587 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:34:40.587 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:34:40.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_1 2026-02-18T08:34:40.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:34:40.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:34:40.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_1 2026-02-18T08:34:40.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:34:40.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_1 ']' 2026-02-18T08:34:40.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_1' 2026-02-18T08:34:40.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_1 2026-02-18T08:34:40.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:34:40.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:34:40.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:34:40.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:34:40.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:34:40.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:34:40.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:34:40.793 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:34:40.793 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:34:40.793 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:34:40.793 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_1 2026-02-18T08:34:40.793 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:34:40.793 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:34:40.793 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:34:40.794 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:34:40.794 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:34:40.794 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:34:40.794 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:34:40.794 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:34:40.794 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:34:40.794 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:34:40.794 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:34:40.794 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:34:40.794 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:34:40.794 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:34:40.794 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:34:40.795 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:34:40.795 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:34:40.795 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:34:40.795 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:34:40.795 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:34:40.795 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:34:40.795 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:34:40.795 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:34:40.795 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:34:40.795 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:34:40.795 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:34:40.796 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:34:40.796 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:34:40.796 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:34:40.796 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:34:40.805 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:34:40.805 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:34:40.805 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:34:40.805 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_1 2026-02-18T08:34:40.857 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_1'] 2026-02-18T08:34:40.858 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:34:40.858 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:34:40.861 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:34:40.861 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_1 2026-02-18T08:34:40.861 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:34:40.861 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:34:40.861 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:34:40.861 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:34:40.861 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:34:40.861 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:34:40.861 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:34:40.861 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:34:40.861 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:34:40.862 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:34:40.862 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:34:40.862 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:34:40.862 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:34:40.862 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:34:40.862 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:34:40.862 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:34:40.862 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:34:40.862 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:34:40.862 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:34:40.862 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:34:40.862 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:34:40.863 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:34:40.863 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:34:40.863 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:34:40.863 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:34:40.863 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:34:40.863 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:34:40.863 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:34:40.863 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:34:40.863 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:34:40.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:34:40.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:34:40.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:34:40.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_1 2026-02-18T08:34:40.930 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:34:40.931 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_1 2026-02-18T08:34:40.931 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:34:40.931 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:34:40.931 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_1 2026-02-18T08:34:40.931 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_1 '' 2026-02-18T08:34:40.931 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:34:40.931 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:34:40.932 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_1 2026-02-18T08:34:40.932 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:34:40.932 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_1.status 2026-02-18T08:34:40.932 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_1.status 2026-02-18T08:34:40.932 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:34:40.932 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_1.status 2026-02-18T08:34:40.933 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_1 2026-02-18T08:34:40.933 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_1.status 2026-02-18T08:34:40.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 11 2026-02-18T08:34:40.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 11 2026-02-18T08:34:40.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:34:40.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=2, entry_tid=1], [object_number=44, tag_tid=2, entry_tid=0], [object_number=47, tag_tid=1, entry_tid=49083], [object_number=46, tag_tid=1, entry_tid=49082]]], state=connected] 2026-02-18T08:34:40.961 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=44, tag_tid=1, entry_tid=49084], [object_number=47, tag_tid=1, entry_tid=49083], [object_number=46, tag_tid=1, entry_tid=49082], [object_number=45, tag_tid=1, entry_tid=49081]]], state=connected] 2026-02-18T08:34:40.963 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_1.status 2026-02-18T08:34:40.964 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=45, tag_tid=2, entry_tid=1' 2026-02-18T08:34:40.966 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_1 2026-02-18T08:34:40.966 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:34:40.966 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:34:40.966 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_1 2026-02-18T08:34:40.966 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_1 '..*' 2026-02-18T08:34:40.966 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:34:40.966 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:34:40.966 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_1 2026-02-18T08:34:40.966 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:34:40.966 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_1.status 2026-02-18T08:34:40.967 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_1.status 2026-02-18T08:34:40.967 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:34:40.968 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_1.status 2026-02-18T08:34:40.968 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_1 2026-02-18T08:34:40.968 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_1.status 2026-02-18T08:34:40.994 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 11 2026-02-18T08:34:40.994 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 11 2026-02-18T08:34:40.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:34:40.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=2, entry_tid=1], [object_number=44, tag_tid=2, entry_tid=0], [object_number=47, tag_tid=1, entry_tid=49083], [object_number=46, tag_tid=1, entry_tid=49082]]], state=connected] 2026-02-18T08:34:40.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=44, tag_tid=1, entry_tid=49084], [object_number=47, tag_tid=1, entry_tid=49083], [object_number=46, tag_tid=1, entry_tid=49082], [object_number=45, tag_tid=1, entry_tid=49081]]], state=connected] 2026-02-18T08:34:40.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_1.status 2026-02-18T08:34:40.999 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=44, tag_tid=1, entry_tid=49084' 2026-02-18T08:34:40.999 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=45, tag_tid=2, entry_tid=1' -a 'object_number=45, tag_tid=2, entry_tid=1' = 'object_number=44, tag_tid=1, entry_tid=49084' 2026-02-18T08:34:40.999 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=44, tag_tid=1, entry_tid=49084' '!=' 'object_number=44, tag_tid=1, entry_tid=49084' 2026-02-18T08:34:40.999 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:34:40.999 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.4 2026-02-18T08:34:41.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_1 2026-02-18T08:34:41.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:34:41.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:34:41.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_1 2026-02-18T08:34:41.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:34:41.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_1 ']' 2026-02-18T08:34:41.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_1' 2026-02-18T08:34:41.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_1 2026-02-18T08:34:41.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:34:41.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:34:41.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:34:41.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:34:41.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:34:41.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:34:41.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:34:41.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:34:41.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:34:41.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:34:41.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_1 2026-02-18T08:34:41.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:34:41.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:34:41.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:34:41.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:34:41.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:34:41.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:34:41.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:34:41.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:34:41.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:34:41.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:34:41.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:34:41.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:34:41.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:34:41.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:34:41.403 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:34:41.403 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:34:41.403 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:34:41.403 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:34:41.403 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:34:41.403 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:34:41.403 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:34:41.403 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:34:41.403 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:34:41.403 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:34:41.403 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:34:41.403 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:34:41.403 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:34:41.403 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:34:41.403 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:34:41.403 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:34:41.416 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:34:41.416 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:34:41.416 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:34:41.416 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_1 2026-02-18T08:34:41.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_1'] 2026-02-18T08:34:41.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:34:41.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:34:41.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:34:41.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_1 2026-02-18T08:34:41.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:34:41.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:34:41.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:34:41.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:34:41.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:34:41.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:34:41.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:34:41.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:34:41.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:34:41.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:34:41.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:34:41.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:34:41.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:34:41.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:34:41.472 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:34:41.472 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:34:41.472 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:34:41.472 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:34:41.472 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:34:41.472 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:34:41.472 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:34:41.472 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:34:41.472 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:34:41.472 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:34:41.472 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:34:41.472 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:34:41.472 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:34:41.472 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:34:41.472 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:34:41.472 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:34:41.483 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:34:41.484 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:34:41.484 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:34:41.484 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_1 2026-02-18T08:34:41.551 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:34:41.551 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_1 2026-02-18T08:34:41.551 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:34:41.551 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:34:41.551 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_1 2026-02-18T08:34:41.551 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_1 '' 2026-02-18T08:34:41.551 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:34:41.551 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:34:41.552 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_1 2026-02-18T08:34:41.552 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:34:41.552 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_1.status 2026-02-18T08:34:41.552 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_1.status 2026-02-18T08:34:41.552 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:34:41.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_1.status 2026-02-18T08:34:41.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_1 2026-02-18T08:34:41.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_1.status 2026-02-18T08:34:41.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 11 2026-02-18T08:34:41.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 11 2026-02-18T08:34:41.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:34:41.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=2, entry_tid=1], [object_number=44, tag_tid=2, entry_tid=0], [object_number=47, tag_tid=1, entry_tid=49083], [object_number=46, tag_tid=1, entry_tid=49082]]], state=connected] 2026-02-18T08:34:41.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=44, tag_tid=1, entry_tid=49084], [object_number=47, tag_tid=1, entry_tid=49083], [object_number=46, tag_tid=1, entry_tid=49082], [object_number=45, tag_tid=1, entry_tid=49081]]], state=connected] 2026-02-18T08:34:41.594 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_1.status 2026-02-18T08:34:41.596 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=45, tag_tid=2, entry_tid=1' 2026-02-18T08:34:41.597 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_1 2026-02-18T08:34:41.597 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:34:41.597 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:34:41.597 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_1 2026-02-18T08:34:41.597 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_1 '..*' 2026-02-18T08:34:41.597 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:34:41.597 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:34:41.597 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_1 2026-02-18T08:34:41.597 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:34:41.597 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_1.status 2026-02-18T08:34:41.598 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_1.status 2026-02-18T08:34:41.598 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:34:41.599 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_1.status 2026-02-18T08:34:41.599 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_1.status 2026-02-18T08:34:41.599 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_1 2026-02-18T08:34:41.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 11 2026-02-18T08:34:41.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 11 2026-02-18T08:34:41.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:34:41.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=2, entry_tid=1], [object_number=44, tag_tid=2, entry_tid=0], [object_number=47, tag_tid=1, entry_tid=49083], [object_number=46, tag_tid=1, entry_tid=49082]]], state=connected] 2026-02-18T08:34:41.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=44, tag_tid=1, entry_tid=49084], [object_number=47, tag_tid=1, entry_tid=49083], [object_number=46, tag_tid=1, entry_tid=49082], [object_number=45, tag_tid=1, entry_tid=49081]]], state=connected] 2026-02-18T08:34:41.629 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_1.status 2026-02-18T08:34:41.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=44, tag_tid=1, entry_tid=49084' 2026-02-18T08:34:41.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=45, tag_tid=2, entry_tid=1' -a 'object_number=45, tag_tid=2, entry_tid=1' = 'object_number=44, tag_tid=1, entry_tid=49084' 2026-02-18T08:34:41.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=44, tag_tid=1, entry_tid=49084' '!=' 'object_number=44, tag_tid=1, entry_tid=49084' 2026-02-18T08:34:41.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:34:41.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.8 2026-02-18T08:34:42.434 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_1 2026-02-18T08:34:42.434 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:34:42.434 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:34:42.434 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_1 2026-02-18T08:34:42.434 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:34:42.434 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_1 ']' 2026-02-18T08:34:42.434 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_1' 2026-02-18T08:34:42.434 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_1 2026-02-18T08:34:42.435 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:34:42.435 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:34:42.435 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:34:42.435 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:34:42.435 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:34:42.435 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:34:42.435 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:34:42.435 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:34:42.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:34:42.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:34:42.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_1 2026-02-18T08:34:42.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:34:42.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:34:42.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:34:42.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:34:42.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:34:42.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:34:42.439 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:34:42.439 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:34:42.439 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:34:42.439 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:34:42.439 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:34:42.439 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:34:42.439 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:34:42.439 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:34:42.439 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:34:42.439 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:34:42.439 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:34:42.439 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:34:42.439 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:34:42.440 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:34:42.440 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:34:42.440 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:34:42.440 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:34:42.440 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:34:42.440 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:34:42.440 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:34:42.440 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:34:42.440 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:34:42.440 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:34:42.440 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:34:42.449 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:34:42.449 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:34:42.449 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:34:42.449 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_1 2026-02-18T08:34:42.510 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_1'] 2026-02-18T08:34:42.510 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:34:42.510 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:34:42.512 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:34:42.513 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_1 2026-02-18T08:34:42.513 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:34:42.513 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:34:42.513 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:34:42.513 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:34:42.513 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:34:42.513 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:34:42.513 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:34:42.513 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:34:42.513 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:34:42.513 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:34:42.514 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:34:42.514 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:34:42.514 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:34:42.514 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:34:42.514 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:34:42.514 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:34:42.514 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:34:42.514 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:34:42.514 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:34:42.514 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:34:42.514 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:34:42.514 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:34:42.514 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:34:42.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:34:42.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:34:42.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:34:42.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:34:42.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:34:42.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:34:42.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:34:42.524 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:34:42.524 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:34:42.524 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:34:42.524 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_1 2026-02-18T08:34:42.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:34:42.584 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_1 2026-02-18T08:34:42.585 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:34:42.585 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:34:42.585 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_1 2026-02-18T08:34:42.585 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_1 '' 2026-02-18T08:34:42.585 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:34:42.585 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:34:42.585 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_1 2026-02-18T08:34:42.585 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:34:42.585 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_1.status 2026-02-18T08:34:42.585 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_1.status 2026-02-18T08:34:42.585 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:34:42.586 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_1.status 2026-02-18T08:34:42.586 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_1 2026-02-18T08:34:42.587 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_1.status 2026-02-18T08:34:42.623 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 11 2026-02-18T08:34:42.623 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 11 2026-02-18T08:34:42.623 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:34:42.623 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=2, entry_tid=1], [object_number=44, tag_tid=2, entry_tid=0], [object_number=47, tag_tid=1, entry_tid=49083], [object_number=46, tag_tid=1, entry_tid=49082]]], state=connected] 2026-02-18T08:34:42.623 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=44, tag_tid=1, entry_tid=49084], [object_number=47, tag_tid=1, entry_tid=49083], [object_number=46, tag_tid=1, entry_tid=49082], [object_number=45, tag_tid=1, entry_tid=49081]]], state=connected] 2026-02-18T08:34:42.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_1.status 2026-02-18T08:34:42.628 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=45, tag_tid=2, entry_tid=1' 2026-02-18T08:34:42.629 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_1 2026-02-18T08:34:42.629 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:34:42.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:34:42.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_1 2026-02-18T08:34:42.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_1 '..*' 2026-02-18T08:34:42.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:34:42.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:34:42.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_1 2026-02-18T08:34:42.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:34:42.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_1.status 2026-02-18T08:34:42.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_1.status 2026-02-18T08:34:42.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:34:42.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_1.status 2026-02-18T08:34:42.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_1 2026-02-18T08:34:42.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_1.status 2026-02-18T08:34:42.660 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 11 2026-02-18T08:34:42.660 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 11 2026-02-18T08:34:42.660 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:34:42.660 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=2, entry_tid=1], [object_number=44, tag_tid=2, entry_tid=0], [object_number=47, tag_tid=1, entry_tid=49083], [object_number=46, tag_tid=1, entry_tid=49082]]], state=connected] 2026-02-18T08:34:42.660 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=44, tag_tid=1, entry_tid=49084], [object_number=47, tag_tid=1, entry_tid=49083], [object_number=46, tag_tid=1, entry_tid=49082], [object_number=45, tag_tid=1, entry_tid=49081]]], state=connected] 2026-02-18T08:34:42.663 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_1.status 2026-02-18T08:34:42.664 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=44, tag_tid=1, entry_tid=49084' 2026-02-18T08:34:42.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=45, tag_tid=2, entry_tid=1' -a 'object_number=45, tag_tid=2, entry_tid=1' = 'object_number=44, tag_tid=1, entry_tid=49084' 2026-02-18T08:34:42.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=44, tag_tid=1, entry_tid=49084' '!=' 'object_number=44, tag_tid=1, entry_tid=49084' 2026-02-18T08:34:42.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:34:42.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1.6 2026-02-18T08:34:43.023 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:34:43.019+0000 7faffc95c640 -1 reset not still connected to 0x5644dc3cd860 2026-02-18T08:34:43.023 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:34:43.019+0000 7faffc95c640 -1 reset not still connected to 0x5644dfda05b0 2026-02-18T08:34:44.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_1 2026-02-18T08:34:44.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:34:44.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:34:44.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_1 2026-02-18T08:34:44.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:34:44.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_1 ']' 2026-02-18T08:34:44.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_1' 2026-02-18T08:34:44.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_1 2026-02-18T08:34:44.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:34:44.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:34:44.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:34:44.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:34:44.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:34:44.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:34:44.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:34:44.270 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:34:44.270 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:34:44.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:34:44.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_1 2026-02-18T08:34:44.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:34:44.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:34:44.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:34:44.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:34:44.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:34:44.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:34:44.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:34:44.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:34:44.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:34:44.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:34:44.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:34:44.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:34:44.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:34:44.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:34:44.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:34:44.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:34:44.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:34:44.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:34:44.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:34:44.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:34:44.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:34:44.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:34:44.273 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:34:44.273 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:34:44.273 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:34:44.273 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:34:44.273 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:34:44.273 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:34:44.273 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:34:44.273 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:34:44.281 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:34:44.282 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:34:44.282 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:34:44.282 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_1 2026-02-18T08:34:44.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_1'] 2026-02-18T08:34:44.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:34:44.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:34:44.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:34:44.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_1 2026-02-18T08:34:44.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:34:44.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:34:44.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:34:44.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:34:44.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:34:44.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:34:44.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:34:44.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:34:44.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:34:44.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:34:44.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:34:44.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:34:44.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:34:44.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:34:44.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:34:44.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:34:44.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:34:44.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:34:44.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:34:44.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:34:44.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:34:44.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:34:44.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:34:44.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:34:44.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:34:44.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:34:44.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:34:44.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:34:44.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:34:44.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:34:44.348 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:34:44.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:34:44.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:34:44.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_1 2026-02-18T08:34:44.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:34:44.412 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_1 2026-02-18T08:34:44.412 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:34:44.412 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:34:44.413 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_1 2026-02-18T08:34:44.413 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_1 '' 2026-02-18T08:34:44.413 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:34:44.413 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:34:44.413 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_1 2026-02-18T08:34:44.413 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:34:44.413 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_1.status 2026-02-18T08:34:44.413 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_1.status 2026-02-18T08:34:44.414 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:34:44.414 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_1.status 2026-02-18T08:34:44.415 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_1.status 2026-02-18T08:34:44.415 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_1 2026-02-18T08:34:44.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 11 2026-02-18T08:34:44.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 11 2026-02-18T08:34:44.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:34:44.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=2, entry_tid=1], [object_number=44, tag_tid=2, entry_tid=0], [object_number=47, tag_tid=1, entry_tid=49083], [object_number=46, tag_tid=1, entry_tid=49082]]], state=connected] 2026-02-18T08:34:44.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=45, tag_tid=2, entry_tid=1], [object_number=44, tag_tid=2, entry_tid=0], [object_number=47, tag_tid=1, entry_tid=49083], [object_number=46, tag_tid=1, entry_tid=49082]]], state=connected] 2026-02-18T08:34:44.446 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_1.status 2026-02-18T08:34:44.448 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=45, tag_tid=2, entry_tid=1' 2026-02-18T08:34:44.449 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_1 2026-02-18T08:34:44.449 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:34:44.449 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:34:44.449 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_1 2026-02-18T08:34:44.449 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_1 '..*' 2026-02-18T08:34:44.449 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:34:44.449 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:34:44.449 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_1 2026-02-18T08:34:44.449 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:34:44.450 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_1.status 2026-02-18T08:34:44.450 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_1.status 2026-02-18T08:34:44.450 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:34:44.451 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_1.status 2026-02-18T08:34:44.451 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_1 2026-02-18T08:34:44.451 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_1.status 2026-02-18T08:34:44.480 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 11 2026-02-18T08:34:44.480 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 11 2026-02-18T08:34:44.480 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:34:44.480 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=2, entry_tid=1], [object_number=44, tag_tid=2, entry_tid=0], [object_number=47, tag_tid=1, entry_tid=49083], [object_number=46, tag_tid=1, entry_tid=49082]]], state=connected] 2026-02-18T08:34:44.480 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=45, tag_tid=2, entry_tid=1], [object_number=44, tag_tid=2, entry_tid=0], [object_number=47, tag_tid=1, entry_tid=49083], [object_number=46, tag_tid=1, entry_tid=49082]]], state=connected] 2026-02-18T08:34:44.483 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_1.status 2026-02-18T08:34:44.485 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=45, tag_tid=2, entry_tid=1' 2026-02-18T08:34:44.485 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=45, tag_tid=2, entry_tid=1' -a 'object_number=45, tag_tid=2, entry_tid=1' = 'object_number=45, tag_tid=2, entry_tid=1' 2026-02-18T08:34:44.485 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:34:44.485 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_snap_present cluster1 mirror image_1 snap 2026-02-18T08:34:44.485 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:34:44.486 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:34:44.486 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_1 2026-02-18T08:34:44.486 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap 2026-02-18T08:34:44.486 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-18T08:34:44.486 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-18T08:34:44.486 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1 2026-02-18T08:34:45.390 INFO:tasks.ceph.cluster1.osd.1.trial034.stderr:2026-02-18T08:34:45.387+0000 7f20d7266640 -1 reset not still connected to 0x556422af5ad0 2026-02-18T08:34:45.391 INFO:tasks.ceph.cluster1.osd.1.trial034.stderr:2026-02-18T08:34:45.387+0000 7f20d7266640 -1 reset not still connected to 0x556431954000 2026-02-18T08:34:45.486 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster1 info mirror/image_1@snap 2026-02-18T08:34:45.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:rbd image 'image_1': 2026-02-18T08:34:45.516 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: size 128 MiB in 32 objects 2026-02-18T08:34:45.516 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: order 22 (4 MiB objects) 2026-02-18T08:34:45.516 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: snapshot_count: 1 2026-02-18T08:34:45.516 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: id: 10ed6b0d9a98 2026-02-18T08:34:45.516 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: block_name_prefix: rbd_data.10ed6b0d9a98 2026-02-18T08:34:45.516 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: format: 2 2026-02-18T08:34:45.516 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: features: layering, exclusive-lock, journaling 2026-02-18T08:34:45.516 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: op_features: 2026-02-18T08:34:45.516 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: flags: 2026-02-18T08:34:45.516 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: create_timestamp: Wed Feb 18 08:25:40 2026 2026-02-18T08:34:45.516 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: access_timestamp: Wed Feb 18 08:25:40 2026 2026-02-18T08:34:45.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: modify_timestamp: Wed Feb 18 08:26:41 2026 2026-02-18T08:34:45.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: protected: False 2026-02-18T08:34:45.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: journal: 10ed6b0d9a98 2026-02-18T08:34:45.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring state: enabled 2026-02-18T08:34:45.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring mode: journal 2026-02-18T08:34:45.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring global id: 6cf12cb6-02f6-4151-9b83-28f3d01070c1 2026-02-18T08:34:45.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring primary: false 2026-02-18T08:34:45.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:34:45.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ compare_image_snaps mirror image_1 snap 2026-02-18T08:34:45.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:34:45.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_1 2026-02-18T08:34:45.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap 2026-02-18T08:34:45.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local ret=0 2026-02-18T08:34:45.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local rmt_export=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_1.export 2026-02-18T08:34:45.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loc_export=/tmp/tmp.vybh1m9yJF/cluster1-mirror-image_1.export 2026-02-18T08:34:45.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_1.export /tmp/tmp.vybh1m9yJF/cluster1-mirror-image_1.export 2026-02-18T08:34:45.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 -p mirror export image_1@snap /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_1.export 2026-02-18T08:34:45.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-18T08:34:45.700 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster1 -p mirror export image_1@snap /tmp/tmp.vybh1m9yJF/cluster1-mirror-image_1.export 2026-02-18T08:34:45.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-18T08:34:45.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmp /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_1.export /tmp/tmp.vybh1m9yJF/cluster1-mirror-image_1.export 2026-02-18T08:34:45.950 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_1.export /tmp/tmp.vybh1m9yJF/cluster1-mirror-image_1.export 2026-02-18T08:34:45.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:34:45.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T08:34:45.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_2 2026-02-18T08:34:45.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_snap cluster2 mirror image_2 snap 2026-02-18T08:34:45.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:34:45.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:34:45.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_2 2026-02-18T08:34:45.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap 2026-02-18T08:34:45.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 -p mirror snap create image_2@snap --debug-rbd=20 --debug-journaler=20 2026-02-18T08:34:47.144 INFO:tasks.ceph.cluster1.osd.1.trial034.stderr:2026-02-18T08:34:47.143+0000 7f20d7266640 -1 reset not still connected to 0x55641ff40f70 2026-02-18T08:34:48.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_replay_started cluster1 mirror image_2 2026-02-18T08:34:48.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:34:48.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:34:48.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_2 2026-02-18T08:34:48.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_replay_state cluster1 mirror image_2 started 2026-02-18T08:34:48.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:34:48.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:34:48.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_2 2026-02-18T08:34:48.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local state=started 2026-02-18T08:34:48.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-18T08:34:48.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:34:48.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1 2026-02-18T08:34:49.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test_image_replay_state cluster1 mirror image_2 started 2026-02-18T08:34:49.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:34:49.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:34:49.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_2 2026-02-18T08:34:49.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local test_state=started 2026-02-18T08:34:49.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local status_result 2026-02-18T08:34:49.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_state=stopped 2026-02-18T08:34:49.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_2 2026-02-18T08:34:49.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_instance=cluster1 2026-02-18T08:34:49.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -i state 2026-02-18T08:34:49.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-18T08:34:49.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:34:49.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance=cluster1 2026-02-18T08:34:49.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local loop_instance 2026-02-18T08:34:49.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:34:49.027 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sleep 0 2026-02-18T08:34:49.027 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:34:49.027 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ seq 0 3 2026-02-18T08:34:49.029 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:34:49.029 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_2 2026-02-18T08:34:49.029 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:0 2026-02-18T08:34:49.029 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-18T08:34:49.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:34:49.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:34:49.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:34:49.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-18T08:34:49.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:34:49.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:34:49.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:34:49.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:34:49.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-18T08:34:49.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:34:49.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:34:49.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:34:49.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:34:49.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:0 2026-02-18T08:34:49.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-18T08:34:49.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-18T08:34:49.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:34:49.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:0 2026-02-18T08:34:49.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T08:34:49.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-18T08:34:49.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:34:49.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:34:49.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-18T08:34:49.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-18T08:34:49.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-18T08:34:49.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=0 2026-02-18T08:34:49.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=0 2026-02-18T08:34:49.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:34:49.039 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:34:49.039 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:34:49.040 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:34:49.040 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_2 2026-02-18T08:34:49.093 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:no valid command found; 1 closest matches: 2026-02-18T08:34:49.093 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status mirror/image_24 2026-02-18T08:34:49.093 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:34:49.096 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:34:49.096 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_2 2026-02-18T08:34:49.096 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:1 2026-02-18T08:34:49.096 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-18T08:34:49.096 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:34:49.096 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:34:49.096 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:34:49.096 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-18T08:34:49.096 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:34:49.096 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:34:49.097 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:34:49.097 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:34:49.097 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-18T08:34:49.097 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:34:49.097 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:34:49.097 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:34:49.097 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:34:49.097 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:1 2026-02-18T08:34:49.097 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-18T08:34:49.097 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-18T08:34:49.097 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:34:49.097 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:1 2026-02-18T08:34:49.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T08:34:49.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-18T08:34:49.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:34:49.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:34:49.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-18T08:34:49.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-18T08:34:49.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-18T08:34:49.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=1 2026-02-18T08:34:49.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=1 2026-02-18T08:34:49.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:34:49.105 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:34:49.106 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:34:49.106 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:34:49.106 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_2 2026-02-18T08:34:49.162 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_2'] 2026-02-18T08:34:49.162 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status : get status for rbd mirror 2026-02-18T08:34:49.162 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:34:49.165 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:34:49.165 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_2 2026-02-18T08:34:49.165 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:2 2026-02-18T08:34:49.166 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-18T08:34:49.166 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:34:49.166 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:34:49.166 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:34:49.166 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-18T08:34:49.166 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:34:49.166 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:34:49.166 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:34:49.166 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:34:49.166 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-18T08:34:49.166 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:34:49.167 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:34:49.167 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:34:49.167 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:34:49.167 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:2 2026-02-18T08:34:49.167 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-18T08:34:49.167 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-18T08:34:49.167 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:34:49.167 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:2 2026-02-18T08:34:49.167 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T08:34:49.167 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-18T08:34:49.167 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:34:49.167 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:34:49.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-18T08:34:49.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-18T08:34:49.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-18T08:34:49.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=2 2026-02-18T08:34:49.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=2 2026-02-18T08:34:49.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:34:49.177 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:34:49.177 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:34:49.178 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:34:49.178 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_2 2026-02-18T08:34:49.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:no valid command found; 1 closest matches: 2026-02-18T08:34:49.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status mirror/image_20 2026-02-18T08:34:49.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:34:49.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:34:49.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/image_2 2026-02-18T08:34:49.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:3 2026-02-18T08:34:49.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-18T08:34:49.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:34:49.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:34:49.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:34:49.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-18T08:34:49.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:34:49.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:34:49.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:34:49.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:34:49.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-18T08:34:49.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:34:49.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:34:49.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:34:49.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:34:49.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:3 2026-02-18T08:34:49.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-18T08:34:49.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-18T08:34:49.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:34:49.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:3 2026-02-18T08:34:49.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T08:34:49.244 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-18T08:34:49.244 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:34:49.244 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:34:49.244 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-18T08:34:49.244 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-18T08:34:49.244 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-18T08:34:49.244 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=3 2026-02-18T08:34:49.244 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=3 2026-02-18T08:34:49.244 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:34:49.252 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:34:49.253 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:34:49.253 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:34:49.254 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/image_2 2026-02-18T08:34:49.315 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ return 0 2026-02-18T08:34:49.316 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ status_result=' "state": "Replaying"' 2026-02-18T08:34:49.316 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ echo ' "state": "Replaying"' 2026-02-18T08:34:49.316 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ grep -i Replaying 2026-02-18T08:34:49.318 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: "state": "Replaying" 2026-02-18T08:34:49.319 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ current_state=started 2026-02-18T08:34:49.319 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test started = started 2026-02-18T08:34:49.319 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:34:49.319 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror image_2 2026-02-18T08:34:49.320 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_cluster=cluster1 2026-02-18T08:34:49.320 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:34:49.320 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_pool=mirror 2026-02-18T08:34:49.320 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local remote_pool=mirror 2026-02-18T08:34:49.320 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_2 2026-02-18T08:34:49.320 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' journal = journal ']' 2026-02-18T08:34:49.320 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror mirror image_2 2026-02-18T08:34:49.320 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_cluster=cluster1 2026-02-18T08:34:49.320 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:34:49.320 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_pool=mirror 2026-02-18T08:34:49.320 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local remote_pool=mirror 2026-02-18T08:34:49.320 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_2 2026-02-18T08:34:49.321 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-02-18T08:34:49.321 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-02-18T08:34:49.321 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:34:49.321 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:34:49.321 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:34:49.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_2 2026-02-18T08:34:49.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:34:49.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:34:49.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_2 2026-02-18T08:34:49.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:34:49.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_2 ']' 2026-02-18T08:34:49.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_2' 2026-02-18T08:34:49.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_2 2026-02-18T08:34:49.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:34:49.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:34:49.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:34:49.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:34:49.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:34:49.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:34:49.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:34:49.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:34:49.524 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:34:49.524 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:34:49.525 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_2 2026-02-18T08:34:49.525 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:34:49.525 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:34:49.525 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:34:49.525 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:34:49.525 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:34:49.525 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:34:49.525 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:34:49.525 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:34:49.525 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:34:49.525 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:34:49.526 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:34:49.526 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:34:49.526 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:34:49.526 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:34:49.526 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:34:49.526 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:34:49.526 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:34:49.526 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:34:49.526 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:34:49.526 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:34:49.526 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:34:49.526 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:34:49.527 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:34:49.527 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:34:49.527 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:34:49.527 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:34:49.527 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:34:49.527 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:34:49.527 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:34:49.527 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:34:49.536 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:34:49.536 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:34:49.536 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:34:49.536 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_2 2026-02-18T08:34:49.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:no valid command found; 1 closest matches: 2026-02-18T08:34:49.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush mirror/image_24 2026-02-18T08:34:49.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:34:49.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:34:49.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_2 2026-02-18T08:34:49.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:34:49.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:34:49.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:34:49.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:34:49.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:34:49.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:34:49.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:34:49.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:34:49.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:34:49.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:34:49.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:34:49.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:34:49.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:34:49.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:34:49.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:34:49.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:34:49.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:34:49.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:34:49.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:34:49.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:34:49.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:34:49.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:34:49.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:34:49.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:34:49.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:34:49.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:34:49.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:34:49.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:34:49.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:34:49.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:34:49.601 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:34:49.601 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:34:49.602 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:34:49.602 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_2 2026-02-18T08:34:49.654 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_2'] 2026-02-18T08:34:49.654 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:34:49.654 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:34:49.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:34:49.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_2 2026-02-18T08:34:49.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:34:49.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:34:49.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:34:49.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:34:49.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:34:49.658 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:34:49.658 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:34:49.658 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:34:49.658 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:34:49.658 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:34:49.658 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:34:49.658 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:34:49.658 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:34:49.658 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:34:49.658 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:34:49.658 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:34:49.659 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:34:49.659 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:34:49.659 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:34:49.659 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:34:49.659 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:34:49.659 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:34:49.659 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:34:49.659 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:34:49.659 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:34:49.659 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:34:49.659 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:34:49.659 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:34:49.660 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:34:49.660 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:34:49.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:34:49.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:34:49.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:34:49.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_2 2026-02-18T08:34:49.718 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:no valid command found; 1 closest matches: 2026-02-18T08:34:49.718 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush mirror/image_20 2026-02-18T08:34:49.718 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:34:49.721 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:34:49.721 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/image_2 2026-02-18T08:34:49.721 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:34:49.721 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:34:49.721 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:34:49.721 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:34:49.721 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:34:49.721 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:34:49.721 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:34:49.721 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:34:49.722 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:34:49.722 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:34:49.722 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:34:49.722 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:34:49.722 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:34:49.722 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:34:49.722 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:34:49.722 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:34:49.722 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:34:49.722 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:34:49.722 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:34:49.722 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:34:49.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:34:49.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:34:49.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:34:49.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:34:49.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:34:49.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:34:49.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:34:49.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:34:49.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:34:49.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:34:49.732 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:34:49.733 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:34:49.733 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:34:49.733 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/image_2 2026-02-18T08:34:49.799 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:34:49.799 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_2 2026-02-18T08:34:49.800 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:34:49.800 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:34:49.800 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_2 2026-02-18T08:34:49.800 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_2 '' 2026-02-18T08:34:49.800 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:34:49.800 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:34:49.800 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_2 2026-02-18T08:34:49.800 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:34:49.800 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_2.status 2026-02-18T08:34:49.801 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_2.status 2026-02-18T08:34:49.801 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:34:49.801 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_2.status 2026-02-18T08:34:49.802 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_2 2026-02-18T08:34:49.802 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_2.status 2026-02-18T08:34:49.828 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 7 2026-02-18T08:34:49.828 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 7 2026-02-18T08:34:49.828 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:34:49.828 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=29, tag_tid=2, entry_tid=1], [object_number=28, tag_tid=2, entry_tid=0], [object_number=31, tag_tid=1, entry_tid=32431], [object_number=30, tag_tid=1, entry_tid=32430]]], state=connected] 2026-02-18T08:34:49.829 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=29, tag_tid=1, entry_tid=32433], [object_number=28, tag_tid=1, entry_tid=32432], [object_number=31, tag_tid=1, entry_tid=32431], [object_number=30, tag_tid=1, entry_tid=32430]]], state=connected] 2026-02-18T08:34:49.831 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_2.status 2026-02-18T08:34:49.832 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=29, tag_tid=2, entry_tid=1' 2026-02-18T08:34:49.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_2 2026-02-18T08:34:49.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:34:49.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:34:49.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_2 2026-02-18T08:34:49.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_2 '..*' 2026-02-18T08:34:49.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:34:49.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:34:49.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_2 2026-02-18T08:34:49.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:34:49.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_2.status 2026-02-18T08:34:49.835 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_2.status 2026-02-18T08:34:49.835 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:34:49.836 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_2.status 2026-02-18T08:34:49.836 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_2 2026-02-18T08:34:49.836 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_2.status 2026-02-18T08:34:49.861 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 7 2026-02-18T08:34:49.861 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 7 2026-02-18T08:34:49.862 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:34:49.862 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=29, tag_tid=2, entry_tid=1], [object_number=28, tag_tid=2, entry_tid=0], [object_number=31, tag_tid=1, entry_tid=32431], [object_number=30, tag_tid=1, entry_tid=32430]]], state=connected] 2026-02-18T08:34:49.862 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=29, tag_tid=1, entry_tid=32433], [object_number=28, tag_tid=1, entry_tid=32432], [object_number=31, tag_tid=1, entry_tid=32431], [object_number=30, tag_tid=1, entry_tid=32430]]], state=connected] 2026-02-18T08:34:49.864 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_2.status 2026-02-18T08:34:49.866 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=29, tag_tid=1, entry_tid=32433' 2026-02-18T08:34:49.866 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=29, tag_tid=2, entry_tid=1' -a 'object_number=29, tag_tid=2, entry_tid=1' = 'object_number=29, tag_tid=1, entry_tid=32433' 2026-02-18T08:34:49.866 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=29, tag_tid=1, entry_tid=32433' '!=' '' 2026-02-18T08:34:49.866 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:34:49.866 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=29, tag_tid=1, entry_tid=32433' = '' 2026-02-18T08:34:49.866 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=29, tag_tid=1, entry_tid=32433' 2026-02-18T08:34:49.867 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=29, tag_tid=2, entry_tid=1' 2026-02-18T08:34:49.867 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:34:49.867 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:34:49.868 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=2 2026-02-18T08:34:49.868 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=29, tag_tid=1, entry_tid=32433' 2026-02-18T08:34:49.868 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:34:49.869 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:34:49.870 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=1 2026-02-18T08:34:49.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=29, tag_tid=2, entry_tid=1' 2026-02-18T08:34:49.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:34:49.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:34:49.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:34:49.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=29, tag_tid=1, entry_tid=32433' 2026-02-18T08:34:49.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:34:49.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:34:49.875 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=32433 2026-02-18T08:34:49.875 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 2 = 1 -a 1 -le 32433 2026-02-18T08:34:49.875 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:34:49.875 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:34:49.875 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:34:50.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_2 2026-02-18T08:34:50.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:34:50.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:34:50.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_2 2026-02-18T08:34:50.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:34:50.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_2 ']' 2026-02-18T08:34:50.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_2' 2026-02-18T08:34:50.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_2 2026-02-18T08:34:50.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:34:50.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:34:50.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:34:50.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:34:50.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:34:50.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:34:50.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:34:50.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:34:50.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:34:50.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:34:50.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_2 2026-02-18T08:34:50.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:34:50.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:34:50.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:34:50.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:34:50.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:34:50.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:34:50.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:34:50.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:34:50.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:34:50.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:34:50.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:34:50.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:34:50.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:34:50.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:34:50.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:34:50.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:34:50.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:34:50.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:34:50.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:34:50.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:34:50.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:34:50.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:34:50.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:34:50.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:34:50.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:34:50.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:34:50.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:34:50.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:34:50.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:34:50.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:34:50.092 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:34:50.092 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:34:50.093 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:34:50.093 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_2 2026-02-18T08:34:50.145 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:no valid command found; 1 closest matches: 2026-02-18T08:34:50.145 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush mirror/image_24 2026-02-18T08:34:50.145 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:34:50.147 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:34:50.148 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_2 2026-02-18T08:34:50.148 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:34:50.148 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:34:50.148 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:34:50.148 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:34:50.148 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:34:50.148 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:34:50.148 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:34:50.148 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:34:50.148 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:34:50.148 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:34:50.149 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:34:50.149 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:34:50.149 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:34:50.149 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:34:50.149 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:34:50.149 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:34:50.149 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:34:50.149 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:34:50.149 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:34:50.149 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:34:50.149 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:34:50.149 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:34:50.150 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:34:50.150 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:34:50.150 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:34:50.150 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:34:50.150 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:34:50.150 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:34:50.150 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:34:50.150 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:34:50.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:34:50.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:34:50.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:34:50.161 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_2 2026-02-18T08:34:50.210 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_2'] 2026-02-18T08:34:50.210 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:34:50.210 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:34:50.213 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:34:50.213 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_2 2026-02-18T08:34:50.213 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:34:50.213 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:34:50.213 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:34:50.214 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:34:50.214 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:34:50.214 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:34:50.214 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:34:50.214 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:34:50.214 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:34:50.214 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:34:50.214 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:34:50.214 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:34:50.214 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:34:50.214 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:34:50.214 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:34:50.215 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:34:50.215 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:34:50.215 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:34:50.215 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:34:50.215 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:34:50.215 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:34:50.215 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:34:50.215 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:34:50.215 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:34:50.215 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:34:50.215 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:34:50.215 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:34:50.215 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:34:50.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:34:50.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:34:50.225 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:34:50.225 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:34:50.225 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:34:50.225 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_2 2026-02-18T08:34:50.273 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:no valid command found; 1 closest matches: 2026-02-18T08:34:50.273 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush mirror/image_20 2026-02-18T08:34:50.273 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:34:50.276 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:34:50.276 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/image_2 2026-02-18T08:34:50.276 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:34:50.276 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:34:50.276 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:34:50.277 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:34:50.277 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:34:50.277 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:34:50.277 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:34:50.277 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:34:50.277 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:34:50.277 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:34:50.277 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:34:50.277 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:34:50.277 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:34:50.277 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:34:50.277 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:34:50.278 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:34:50.278 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:34:50.278 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:34:50.278 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:34:50.278 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:34:50.278 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:34:50.278 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:34:50.278 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:34:50.278 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:34:50.278 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:34:50.278 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:34:50.278 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:34:50.279 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:34:50.279 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:34:50.279 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:34:50.288 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:34:50.288 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:34:50.288 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:34:50.288 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/image_2 2026-02-18T08:34:50.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:34:50.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_2 2026-02-18T08:34:50.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:34:50.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:34:50.348 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_2 2026-02-18T08:34:50.348 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_2 '' 2026-02-18T08:34:50.348 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:34:50.348 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:34:50.348 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_2 2026-02-18T08:34:50.348 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:34:50.348 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_2.status 2026-02-18T08:34:50.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_2.status 2026-02-18T08:34:50.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:34:50.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_2.status 2026-02-18T08:34:50.350 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_2 2026-02-18T08:34:50.350 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_2.status 2026-02-18T08:34:50.377 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 7 2026-02-18T08:34:50.377 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 7 2026-02-18T08:34:50.377 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:34:50.377 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=29, tag_tid=2, entry_tid=1], [object_number=28, tag_tid=2, entry_tid=0], [object_number=31, tag_tid=1, entry_tid=32431], [object_number=30, tag_tid=1, entry_tid=32430]]], state=connected] 2026-02-18T08:34:50.377 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=29, tag_tid=1, entry_tid=32433], [object_number=28, tag_tid=1, entry_tid=32432], [object_number=31, tag_tid=1, entry_tid=32431], [object_number=30, tag_tid=1, entry_tid=32430]]], state=connected] 2026-02-18T08:34:50.379 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_2.status 2026-02-18T08:34:50.380 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=29, tag_tid=2, entry_tid=1' 2026-02-18T08:34:50.382 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_2 2026-02-18T08:34:50.382 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:34:50.382 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:34:50.382 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_2 2026-02-18T08:34:50.382 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_2 '..*' 2026-02-18T08:34:50.382 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:34:50.382 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:34:50.382 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_2 2026-02-18T08:34:50.382 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:34:50.382 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_2.status 2026-02-18T08:34:50.383 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_2.status 2026-02-18T08:34:50.383 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:34:50.383 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_2.status 2026-02-18T08:34:50.384 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_2 2026-02-18T08:34:50.384 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_2.status 2026-02-18T08:34:50.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 7 2026-02-18T08:34:50.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 7 2026-02-18T08:34:50.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:34:50.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=29, tag_tid=2, entry_tid=1], [object_number=28, tag_tid=2, entry_tid=0], [object_number=31, tag_tid=1, entry_tid=32431], [object_number=30, tag_tid=1, entry_tid=32430]]], state=connected] 2026-02-18T08:34:50.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=29, tag_tid=1, entry_tid=32433], [object_number=28, tag_tid=1, entry_tid=32432], [object_number=31, tag_tid=1, entry_tid=32431], [object_number=30, tag_tid=1, entry_tid=32430]]], state=connected] 2026-02-18T08:34:50.412 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_2.status 2026-02-18T08:34:50.414 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=29, tag_tid=1, entry_tid=32433' 2026-02-18T08:34:50.414 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=29, tag_tid=2, entry_tid=1' -a 'object_number=29, tag_tid=2, entry_tid=1' = 'object_number=29, tag_tid=1, entry_tid=32433' 2026-02-18T08:34:50.414 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=29, tag_tid=1, entry_tid=32433' '!=' 'object_number=29, tag_tid=1, entry_tid=32433' 2026-02-18T08:34:50.415 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:34:50.415 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.4 2026-02-18T08:34:50.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_2 2026-02-18T08:34:50.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:34:50.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:34:50.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_2 2026-02-18T08:34:50.817 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:34:50.817 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_2 ']' 2026-02-18T08:34:50.817 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_2' 2026-02-18T08:34:50.817 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_2 2026-02-18T08:34:50.817 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:34:50.817 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:34:50.817 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:34:50.817 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:34:50.817 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:34:50.817 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:34:50.817 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:34:50.820 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:34:50.820 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:34:50.820 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:34:50.820 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_2 2026-02-18T08:34:50.821 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:34:50.821 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:34:50.821 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:34:50.821 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:34:50.821 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:34:50.821 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:34:50.821 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:34:50.821 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:34:50.821 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:34:50.821 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:34:50.821 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:34:50.821 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:34:50.822 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:34:50.822 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:34:50.822 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:34:50.822 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:34:50.822 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:34:50.822 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:34:50.822 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:34:50.822 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:34:50.822 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:34:50.822 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:34:50.822 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:34:50.822 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:34:50.823 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:34:50.823 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:34:50.823 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:34:50.823 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:34:50.823 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:34:50.823 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:34:50.832 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:34:50.832 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:34:50.832 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:34:50.832 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_2 2026-02-18T08:34:50.885 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:no valid command found; 1 closest matches: 2026-02-18T08:34:50.886 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush mirror/image_24 2026-02-18T08:34:50.886 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:34:50.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:34:50.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_2 2026-02-18T08:34:50.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:34:50.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:34:50.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:34:50.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:34:50.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:34:50.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:34:50.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:34:50.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:34:50.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:34:50.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:34:50.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:34:50.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:34:50.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:34:50.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:34:50.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:34:50.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:34:50.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:34:50.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:34:50.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:34:50.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:34:50.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:34:50.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:34:50.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:34:50.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:34:50.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:34:50.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:34:50.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:34:50.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:34:50.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:34:50.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:34:50.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:34:50.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:34:50.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:34:50.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_2 2026-02-18T08:34:50.954 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_2'] 2026-02-18T08:34:50.954 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:34:50.954 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:34:50.956 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:34:50.956 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_2 2026-02-18T08:34:50.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:34:50.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:34:50.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:34:50.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:34:50.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:34:50.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:34:50.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:34:50.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:34:50.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:34:50.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:34:50.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:34:50.958 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:34:50.958 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:34:50.958 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:34:50.958 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:34:50.958 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:34:50.958 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:34:50.958 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:34:50.958 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:34:50.958 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:34:50.958 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:34:50.958 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:34:50.958 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:34:50.958 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:34:50.959 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:34:50.959 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:34:50.959 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:34:50.959 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:34:50.959 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:34:50.959 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:34:50.968 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:34:50.969 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:34:50.969 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:34:50.969 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_2 2026-02-18T08:34:51.019 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:no valid command found; 1 closest matches: 2026-02-18T08:34:51.019 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush mirror/image_20 2026-02-18T08:34:51.019 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:34:51.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:34:51.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/image_2 2026-02-18T08:34:51.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:34:51.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:34:51.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:34:51.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:34:51.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:34:51.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:34:51.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:34:51.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:34:51.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:34:51.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:34:51.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:34:51.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:34:51.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:34:51.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:34:51.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:34:51.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:34:51.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:34:51.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:34:51.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:34:51.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:34:51.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:34:51.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:34:51.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:34:51.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:34:51.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:34:51.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:34:51.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:34:51.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:34:51.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:34:51.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:34:51.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:34:51.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:34:51.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:34:51.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/image_2 2026-02-18T08:34:51.092 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:34:51.094 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_2 2026-02-18T08:34:51.094 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:34:51.094 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:34:51.094 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_2 2026-02-18T08:34:51.094 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_2 '' 2026-02-18T08:34:51.094 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:34:51.094 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:34:51.094 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_2 2026-02-18T08:34:51.094 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:34:51.095 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_2.status 2026-02-18T08:34:51.095 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_2.status 2026-02-18T08:34:51.095 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:34:51.095 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_2.status 2026-02-18T08:34:51.096 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_2 2026-02-18T08:34:51.096 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_2.status 2026-02-18T08:34:51.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 7 2026-02-18T08:34:51.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 7 2026-02-18T08:34:51.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:34:51.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=29, tag_tid=2, entry_tid=1], [object_number=28, tag_tid=2, entry_tid=0], [object_number=31, tag_tid=1, entry_tid=32431], [object_number=30, tag_tid=1, entry_tid=32430]]], state=connected] 2026-02-18T08:34:51.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=29, tag_tid=1, entry_tid=32433], [object_number=28, tag_tid=1, entry_tid=32432], [object_number=31, tag_tid=1, entry_tid=32431], [object_number=30, tag_tid=1, entry_tid=32430]]], state=connected] 2026-02-18T08:34:51.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_2.status 2026-02-18T08:34:51.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=29, tag_tid=2, entry_tid=1' 2026-02-18T08:34:51.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_2 2026-02-18T08:34:51.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:34:51.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:34:51.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_2 2026-02-18T08:34:51.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_2 '..*' 2026-02-18T08:34:51.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:34:51.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:34:51.129 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_2 2026-02-18T08:34:51.129 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:34:51.129 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_2.status 2026-02-18T08:34:51.129 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_2.status 2026-02-18T08:34:51.129 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:34:51.130 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_2.status 2026-02-18T08:34:51.130 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_2 2026-02-18T08:34:51.131 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_2.status 2026-02-18T08:34:51.167 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 7 2026-02-18T08:34:51.167 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 7 2026-02-18T08:34:51.167 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:34:51.167 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=29, tag_tid=2, entry_tid=1], [object_number=28, tag_tid=2, entry_tid=0], [object_number=31, tag_tid=1, entry_tid=32431], [object_number=30, tag_tid=1, entry_tid=32430]]], state=connected] 2026-02-18T08:34:51.167 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=29, tag_tid=1, entry_tid=32433], [object_number=28, tag_tid=1, entry_tid=32432], [object_number=31, tag_tid=1, entry_tid=32431], [object_number=30, tag_tid=1, entry_tid=32430]]], state=connected] 2026-02-18T08:34:51.171 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_2.status 2026-02-18T08:34:51.173 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=29, tag_tid=1, entry_tid=32433' 2026-02-18T08:34:51.173 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=29, tag_tid=2, entry_tid=1' -a 'object_number=29, tag_tid=2, entry_tid=1' = 'object_number=29, tag_tid=1, entry_tid=32433' 2026-02-18T08:34:51.173 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=29, tag_tid=1, entry_tid=32433' '!=' 'object_number=29, tag_tid=1, entry_tid=32433' 2026-02-18T08:34:51.173 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:34:51.173 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.8 2026-02-18T08:34:51.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_2 2026-02-18T08:34:51.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:34:51.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:34:51.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_2 2026-02-18T08:34:51.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:34:51.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_2 ']' 2026-02-18T08:34:51.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_2' 2026-02-18T08:34:51.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_2 2026-02-18T08:34:51.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:34:51.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:34:51.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:34:51.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:34:51.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:34:51.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:34:51.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:34:51.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:34:51.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:34:51.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:34:51.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_2 2026-02-18T08:34:51.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:34:51.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:34:51.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:34:51.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:34:51.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:34:51.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:34:51.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:34:51.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:34:51.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:34:51.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:34:51.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:34:51.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:34:51.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:34:51.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:34:51.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:34:51.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:34:51.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:34:51.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:34:51.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:34:51.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:34:51.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:34:51.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:34:51.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:34:51.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:34:51.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:34:51.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:34:51.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:34:51.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:34:51.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:34:51.982 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:34:51.990 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:34:51.990 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:34:51.990 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:34:51.990 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_2 2026-02-18T08:34:52.041 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:no valid command found; 1 closest matches: 2026-02-18T08:34:52.041 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush mirror/image_24 2026-02-18T08:34:52.041 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:34:52.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:34:52.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_2 2026-02-18T08:34:52.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:34:52.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:34:52.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:34:52.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:34:52.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:34:52.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:34:52.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:34:52.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:34:52.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:34:52.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:34:52.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:34:52.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:34:52.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:34:52.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:34:52.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:34:52.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:34:52.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:34:52.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:34:52.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:34:52.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:34:52.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:34:52.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:34:52.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:34:52.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:34:52.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:34:52.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:34:52.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:34:52.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:34:52.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:34:52.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:34:52.056 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:34:52.056 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:34:52.056 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:34:52.056 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_2 2026-02-18T08:34:52.119 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_2'] 2026-02-18T08:34:52.119 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:34:52.120 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:34:52.122 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:34:52.122 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_2 2026-02-18T08:34:52.122 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:34:52.122 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:34:52.122 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:34:52.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:34:52.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:34:52.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:34:52.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:34:52.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:34:52.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:34:52.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:34:52.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:34:52.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:34:52.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:34:52.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:34:52.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:34:52.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:34:52.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:34:52.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:34:52.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:34:52.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:34:52.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:34:52.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:34:52.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:34:52.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:34:52.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:34:52.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:34:52.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:34:52.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:34:52.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:34:52.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:34:52.132 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:34:52.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:34:52.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:34:52.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_2 2026-02-18T08:34:52.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:no valid command found; 1 closest matches: 2026-02-18T08:34:52.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush mirror/image_20 2026-02-18T08:34:52.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:34:52.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:34:52.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/image_2 2026-02-18T08:34:52.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:34:52.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:34:52.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:34:52.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:34:52.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:34:52.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:34:52.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:34:52.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:34:52.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:34:52.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:34:52.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:34:52.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:34:52.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:34:52.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:34:52.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:34:52.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:34:52.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:34:52.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:34:52.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:34:52.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:34:52.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:34:52.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:34:52.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:34:52.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:34:52.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:34:52.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:34:52.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:34:52.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:34:52.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:34:52.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:34:52.195 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:34:52.195 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:34:52.196 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:34:52.196 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/image_2 2026-02-18T08:34:52.252 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:34:52.254 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_2 2026-02-18T08:34:52.254 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:34:52.254 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:34:52.254 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_2 2026-02-18T08:34:52.254 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_2 '' 2026-02-18T08:34:52.254 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:34:52.254 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:34:52.254 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_2 2026-02-18T08:34:52.254 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:34:52.254 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_2.status 2026-02-18T08:34:52.255 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_2.status 2026-02-18T08:34:52.255 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:34:52.256 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_2.status 2026-02-18T08:34:52.256 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_2 2026-02-18T08:34:52.256 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_2.status 2026-02-18T08:34:52.293 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 7 2026-02-18T08:34:52.293 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 7 2026-02-18T08:34:52.293 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:34:52.293 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=29, tag_tid=2, entry_tid=1], [object_number=28, tag_tid=2, entry_tid=0], [object_number=31, tag_tid=1, entry_tid=32431], [object_number=30, tag_tid=1, entry_tid=32430]]], state=connected] 2026-02-18T08:34:52.293 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=29, tag_tid=1, entry_tid=32433], [object_number=28, tag_tid=1, entry_tid=32432], [object_number=31, tag_tid=1, entry_tid=32431], [object_number=30, tag_tid=1, entry_tid=32430]]], state=connected] 2026-02-18T08:34:52.297 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_2.status 2026-02-18T08:34:52.298 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=29, tag_tid=2, entry_tid=1' 2026-02-18T08:34:52.299 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_2 2026-02-18T08:34:52.299 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:34:52.299 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:34:52.299 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_2 2026-02-18T08:34:52.299 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_2 '..*' 2026-02-18T08:34:52.299 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:34:52.300 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:34:52.300 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_2 2026-02-18T08:34:52.300 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:34:52.300 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_2.status 2026-02-18T08:34:52.300 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_2.status 2026-02-18T08:34:52.300 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:34:52.301 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_2.status 2026-02-18T08:34:52.301 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_2 2026-02-18T08:34:52.302 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_2.status 2026-02-18T08:34:52.535 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 7 2026-02-18T08:34:52.536 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 7 2026-02-18T08:34:52.536 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:34:52.536 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=29, tag_tid=2, entry_tid=1], [object_number=28, tag_tid=2, entry_tid=0], [object_number=31, tag_tid=1, entry_tid=32431], [object_number=30, tag_tid=1, entry_tid=32430]]], state=connected] 2026-02-18T08:34:52.536 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=29, tag_tid=1, entry_tid=32433], [object_number=28, tag_tid=1, entry_tid=32432], [object_number=31, tag_tid=1, entry_tid=32431], [object_number=30, tag_tid=1, entry_tid=32430]]], state=connected] 2026-02-18T08:34:52.539 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_2.status 2026-02-18T08:34:52.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=29, tag_tid=1, entry_tid=32433' 2026-02-18T08:34:52.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=29, tag_tid=2, entry_tid=1' -a 'object_number=29, tag_tid=2, entry_tid=1' = 'object_number=29, tag_tid=1, entry_tid=32433' 2026-02-18T08:34:52.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=29, tag_tid=1, entry_tid=32433' '!=' 'object_number=29, tag_tid=1, entry_tid=32433' 2026-02-18T08:34:52.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:34:52.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1.6 2026-02-18T08:34:54.144 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_2 2026-02-18T08:34:54.144 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:34:54.144 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:34:54.144 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_2 2026-02-18T08:34:54.144 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:34:54.144 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_2 ']' 2026-02-18T08:34:54.144 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_2' 2026-02-18T08:34:54.145 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_2 2026-02-18T08:34:54.145 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:34:54.145 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:34:54.145 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:34:54.145 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:34:54.145 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:34:54.145 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:34:54.145 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:34:54.148 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:34:54.148 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:34:54.148 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:34:54.148 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_2 2026-02-18T08:34:54.148 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:34:54.148 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:34:54.149 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:34:54.149 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:34:54.149 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:34:54.149 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:34:54.149 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:34:54.149 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:34:54.149 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:34:54.149 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:34:54.149 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:34:54.149 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:34:54.149 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:34:54.149 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:34:54.150 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:34:54.150 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:34:54.150 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:34:54.150 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:34:54.150 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:34:54.150 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:34:54.150 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:34:54.150 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:34:54.150 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:34:54.150 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:34:54.150 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:34:54.150 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:34:54.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:34:54.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:34:54.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:34:54.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:34:54.158 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:34:54.158 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:34:54.158 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:34:54.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_2 2026-02-18T08:34:54.210 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:no valid command found; 1 closest matches: 2026-02-18T08:34:54.210 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush mirror/image_24 2026-02-18T08:34:54.211 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:34:54.214 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:34:54.214 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_2 2026-02-18T08:34:54.214 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:34:54.214 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:34:54.214 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:34:54.214 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:34:54.214 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:34:54.214 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:34:54.214 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:34:54.215 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:34:54.215 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:34:54.215 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:34:54.215 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:34:54.215 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:34:54.215 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:34:54.215 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:34:54.215 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:34:54.215 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:34:54.215 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:34:54.215 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:34:54.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:34:54.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:34:54.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:34:54.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:34:54.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:34:54.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:34:54.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:34:54.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:34:54.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:34:54.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:34:54.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:34:54.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:34:54.225 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:34:54.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:34:54.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:34:54.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_2 2026-02-18T08:34:54.280 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_2'] 2026-02-18T08:34:54.280 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:34:54.280 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:34:54.283 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:34:54.283 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_2 2026-02-18T08:34:54.283 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:34:54.283 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:34:54.283 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:34:54.283 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:34:54.283 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:34:54.283 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:34:54.283 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:34:54.284 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:34:54.284 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:34:54.284 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:34:54.284 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:34:54.284 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:34:54.284 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:34:54.284 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:34:54.284 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:34:54.284 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:34:54.284 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:34:54.284 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:34:54.284 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:34:54.285 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:34:54.285 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:34:54.285 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:34:54.285 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:34:54.285 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:34:54.285 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:34:54.285 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:34:54.285 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:34:54.285 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:34:54.285 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:34:54.285 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:34:54.294 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:34:54.294 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:34:54.294 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:34:54.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_2 2026-02-18T08:34:54.351 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:no valid command found; 1 closest matches: 2026-02-18T08:34:54.351 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush mirror/image_20 2026-02-18T08:34:54.351 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:34:54.354 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:34:54.354 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/image_2 2026-02-18T08:34:54.354 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:34:54.354 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:34:54.354 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:34:54.355 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:34:54.355 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:34:54.355 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:34:54.355 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:34:54.355 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:34:54.355 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:34:54.355 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:34:54.355 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:34:54.355 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:34:54.355 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:34:54.355 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:34:54.355 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:34:54.356 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:34:54.356 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:34:54.356 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:34:54.356 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:34:54.356 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:34:54.356 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:34:54.356 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:34:54.356 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:34:54.356 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:34:54.356 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:34:54.356 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:34:54.356 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:34:54.356 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:34:54.357 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:34:54.357 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:34:54.365 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:34:54.366 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:34:54.366 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:34:54.366 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/image_2 2026-02-18T08:34:54.422 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:34:54.423 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_2 2026-02-18T08:34:54.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:34:54.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:34:54.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_2 2026-02-18T08:34:54.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_2 '' 2026-02-18T08:34:54.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:34:54.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:34:54.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_2 2026-02-18T08:34:54.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:34:54.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_2.status 2026-02-18T08:34:54.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_2.status 2026-02-18T08:34:54.425 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:34:54.425 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_2.status 2026-02-18T08:34:54.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_2 2026-02-18T08:34:54.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_2.status 2026-02-18T08:34:54.452 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 7 2026-02-18T08:34:54.452 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 7 2026-02-18T08:34:54.452 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:34:54.452 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=29, tag_tid=2, entry_tid=1], [object_number=28, tag_tid=2, entry_tid=0], [object_number=31, tag_tid=1, entry_tid=32431], [object_number=30, tag_tid=1, entry_tid=32430]]], state=connected] 2026-02-18T08:34:54.452 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=29, tag_tid=1, entry_tid=32433], [object_number=28, tag_tid=1, entry_tid=32432], [object_number=31, tag_tid=1, entry_tid=32431], [object_number=30, tag_tid=1, entry_tid=32430]]], state=connected] 2026-02-18T08:34:54.454 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_2.status 2026-02-18T08:34:54.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=29, tag_tid=2, entry_tid=1' 2026-02-18T08:34:54.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_2 2026-02-18T08:34:54.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:34:54.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:34:54.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_2 2026-02-18T08:34:54.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_2 '..*' 2026-02-18T08:34:54.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:34:54.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:34:54.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_2 2026-02-18T08:34:54.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:34:54.458 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_2.status 2026-02-18T08:34:54.458 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_2.status 2026-02-18T08:34:54.458 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:34:54.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_2.status 2026-02-18T08:34:54.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_2 2026-02-18T08:34:54.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_2.status 2026-02-18T08:34:54.485 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 7 2026-02-18T08:34:54.486 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 7 2026-02-18T08:34:54.486 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:34:54.486 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=29, tag_tid=2, entry_tid=1], [object_number=28, tag_tid=2, entry_tid=0], [object_number=31, tag_tid=1, entry_tid=32431], [object_number=30, tag_tid=1, entry_tid=32430]]], state=connected] 2026-02-18T08:34:54.486 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=29, tag_tid=1, entry_tid=32433], [object_number=28, tag_tid=1, entry_tid=32432], [object_number=31, tag_tid=1, entry_tid=32431], [object_number=30, tag_tid=1, entry_tid=32430]]], state=connected] 2026-02-18T08:34:54.488 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_2.status 2026-02-18T08:34:54.490 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=29, tag_tid=1, entry_tid=32433' 2026-02-18T08:34:54.490 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=29, tag_tid=2, entry_tid=1' -a 'object_number=29, tag_tid=2, entry_tid=1' = 'object_number=29, tag_tid=1, entry_tid=32433' 2026-02-18T08:34:54.490 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=29, tag_tid=1, entry_tid=32433' '!=' 'object_number=29, tag_tid=1, entry_tid=32433' 2026-02-18T08:34:54.490 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:34:54.490 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 2 2026-02-18T08:34:54.556 INFO:tasks.ceph.cluster1.osd.1.trial034.stderr:2026-02-18T08:34:54.554+0000 7f20d7266640 -1 reset not still connected to 0x55641ff40f70 2026-02-18T08:34:55.126 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.0 with skip (0.4107963042411328) > weight (0.25) 2026-02-18T08:34:55.126 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.1 with skip (0.9376373257131583) > weight (0.25) 2026-02-18T08:34:55.127 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.2 with skip (0.4886431886603716) > weight (0.25) 2026-02-18T08:34:55.127 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.3 with skip (0.658487634310733) > weight (0.25) 2026-02-18T08:34:55.127 INFO:tasks.rbd_mirror_thrash:waiting for 95.30638380110491 secs before thrashing 2026-02-18T08:34:56.492 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_2 2026-02-18T08:34:56.492 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:34:56.492 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:34:56.492 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_2 2026-02-18T08:34:56.493 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:34:56.493 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_2 ']' 2026-02-18T08:34:56.493 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_2' 2026-02-18T08:34:56.493 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_2 2026-02-18T08:34:56.493 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:34:56.493 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:34:56.493 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:34:56.493 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:34:56.493 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:34:56.493 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:34:56.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:34:56.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:34:56.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:34:56.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:34:56.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_2 2026-02-18T08:34:56.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:34:56.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:34:56.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:34:56.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:34:56.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:34:56.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:34:56.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:34:56.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:34:56.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:34:56.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:34:56.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:34:56.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:34:56.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:34:56.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:34:56.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:34:56.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:34:56.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:34:56.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:34:56.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:34:56.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:34:56.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:34:56.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:34:56.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:34:56.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:34:56.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:34:56.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:34:56.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:34:56.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:34:56.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:34:56.500 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:34:56.508 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:34:56.508 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:34:56.509 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:34:56.509 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_2 2026-02-18T08:34:56.561 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:no valid command found; 1 closest matches: 2026-02-18T08:34:56.561 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush mirror/image_24 2026-02-18T08:34:56.561 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:34:56.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:34:56.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_2 2026-02-18T08:34:56.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:34:56.565 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:34:56.565 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:34:56.565 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:34:56.565 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:34:56.565 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:34:56.565 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:34:56.565 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:34:56.565 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:34:56.565 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:34:56.565 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:34:56.565 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:34:56.565 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:34:56.566 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:34:56.566 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:34:56.566 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:34:56.566 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:34:56.566 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:34:56.566 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:34:56.566 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:34:56.566 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:34:56.566 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:34:56.566 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:34:56.566 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:34:56.567 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:34:56.567 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:34:56.567 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:34:56.567 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:34:56.567 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:34:56.567 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:34:56.575 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:34:56.576 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:34:56.576 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:34:56.576 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_2 2026-02-18T08:34:56.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_2'] 2026-02-18T08:34:56.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:34:56.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:34:56.636 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:34:56.637 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_2 2026-02-18T08:34:56.637 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:34:56.637 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:34:56.637 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:34:56.637 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:34:56.637 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:34:56.637 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:34:56.637 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:34:56.637 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:34:56.637 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:34:56.637 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:34:56.638 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:34:56.638 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:34:56.638 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:34:56.638 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:34:56.638 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:34:56.638 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:34:56.638 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:34:56.638 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:34:56.638 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:34:56.638 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:34:56.638 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:34:56.638 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:34:56.639 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:34:56.639 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:34:56.639 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:34:56.639 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:34:56.639 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:34:56.639 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:34:56.639 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:34:56.639 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:34:56.649 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:34:56.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:34:56.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:34:56.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_2 2026-02-18T08:34:56.703 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:no valid command found; 2 closest matches: 2026-02-18T08:34:56.703 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush mirror/image_20 2026-02-18T08:34:56.703 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush mirror/image_29 2026-02-18T08:34:56.703 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:34:56.706 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:34:56.706 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/image_2 2026-02-18T08:34:56.706 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:34:56.706 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:34:56.706 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:34:56.706 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:34:56.706 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:34:56.706 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:34:56.707 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:34:56.707 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:34:56.707 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:34:56.707 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:34:56.707 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:34:56.707 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:34:56.707 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:34:56.707 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:34:56.707 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:34:56.707 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:34:56.707 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:34:56.707 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:34:56.708 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:34:56.708 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:34:56.708 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:34:56.708 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:34:56.708 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:34:56.708 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:34:56.708 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:34:56.708 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:34:56.708 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:34:56.708 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:34:56.708 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:34:56.708 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:34:56.718 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:34:56.718 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:34:56.718 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:34:56.718 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/image_2 2026-02-18T08:34:56.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:34:56.776 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_2 2026-02-18T08:34:56.776 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:34:56.776 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:34:56.777 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_2 2026-02-18T08:34:56.777 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_2 '' 2026-02-18T08:34:56.777 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:34:56.777 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:34:56.777 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_2 2026-02-18T08:34:56.777 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:34:56.777 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_2.status 2026-02-18T08:34:56.777 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_2.status 2026-02-18T08:34:56.777 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:34:56.778 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_2.status 2026-02-18T08:34:56.778 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_2.status 2026-02-18T08:34:56.779 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_2 2026-02-18T08:34:56.807 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 7 2026-02-18T08:34:56.807 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 7 2026-02-18T08:34:56.807 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:34:56.807 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=29, tag_tid=2, entry_tid=1], [object_number=28, tag_tid=2, entry_tid=0], [object_number=31, tag_tid=1, entry_tid=32431], [object_number=30, tag_tid=1, entry_tid=32430]]], state=connected] 2026-02-18T08:34:56.807 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=29, tag_tid=1, entry_tid=32433], [object_number=28, tag_tid=1, entry_tid=32432], [object_number=31, tag_tid=1, entry_tid=32431], [object_number=30, tag_tid=1, entry_tid=32430]]], state=connected] 2026-02-18T08:34:56.810 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_2.status 2026-02-18T08:34:56.811 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=29, tag_tid=2, entry_tid=1' 2026-02-18T08:34:56.812 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_2 2026-02-18T08:34:56.812 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:34:56.812 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:34:56.813 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_2 2026-02-18T08:34:56.813 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_2 '..*' 2026-02-18T08:34:56.813 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:34:56.813 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:34:56.813 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_2 2026-02-18T08:34:56.813 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:34:56.813 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_2.status 2026-02-18T08:34:56.813 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_2.status 2026-02-18T08:34:56.813 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:34:56.814 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_2.status 2026-02-18T08:34:56.814 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_2 2026-02-18T08:34:56.814 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_2.status 2026-02-18T08:34:56.841 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 7 2026-02-18T08:34:56.842 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 7 2026-02-18T08:34:56.842 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:34:56.842 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=29, tag_tid=2, entry_tid=1], [object_number=28, tag_tid=2, entry_tid=0], [object_number=31, tag_tid=1, entry_tid=32431], [object_number=30, tag_tid=1, entry_tid=32430]]], state=connected] 2026-02-18T08:34:56.842 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=29, tag_tid=1, entry_tid=32433], [object_number=28, tag_tid=1, entry_tid=32432], [object_number=31, tag_tid=1, entry_tid=32431], [object_number=30, tag_tid=1, entry_tid=32430]]], state=connected] 2026-02-18T08:34:56.844 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_2.status 2026-02-18T08:34:56.845 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=29, tag_tid=1, entry_tid=32433' 2026-02-18T08:34:56.845 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=29, tag_tid=2, entry_tid=1' -a 'object_number=29, tag_tid=2, entry_tid=1' = 'object_number=29, tag_tid=1, entry_tid=32433' 2026-02-18T08:34:56.845 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=29, tag_tid=1, entry_tid=32433' '!=' 'object_number=29, tag_tid=1, entry_tid=32433' 2026-02-18T08:34:56.845 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:34:56.845 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 2 2026-02-18T08:34:58.848 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_2 2026-02-18T08:34:58.848 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:34:58.848 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:34:58.848 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_2 2026-02-18T08:34:58.848 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:34:58.848 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_2 ']' 2026-02-18T08:34:58.848 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_2' 2026-02-18T08:34:58.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_2 2026-02-18T08:34:58.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:34:58.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:34:58.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:34:58.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:34:58.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:34:58.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:34:58.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:34:58.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:34:58.852 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:34:58.852 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:34:58.852 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_2 2026-02-18T08:34:58.852 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:34:58.852 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:34:58.852 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:34:58.852 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:34:58.853 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:34:58.853 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:34:58.853 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:34:58.853 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:34:58.853 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:34:58.853 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:34:58.853 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:34:58.853 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:34:58.853 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:34:58.853 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:34:58.853 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:34:58.853 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:34:58.854 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:34:58.854 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:34:58.854 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:34:58.854 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:34:58.854 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:34:58.854 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:34:58.854 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:34:58.854 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:34:58.854 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:34:58.854 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:34:58.854 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:34:58.854 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:34:58.855 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:34:58.855 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:34:58.862 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:34:58.863 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:34:58.863 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:34:58.863 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_2 2026-02-18T08:34:58.915 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:no valid command found; 1 closest matches: 2026-02-18T08:34:58.915 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush mirror/image_24 2026-02-18T08:34:58.915 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:34:58.918 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:34:58.918 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_2 2026-02-18T08:34:58.918 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:34:58.918 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:34:58.918 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:34:58.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:34:58.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:34:58.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:34:58.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:34:58.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:34:58.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:34:58.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:34:58.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:34:58.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:34:58.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:34:58.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:34:58.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:34:58.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:34:58.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:34:58.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:34:58.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:34:58.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:34:58.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:34:58.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:34:58.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:34:58.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:34:58.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:34:58.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:34:58.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:34:58.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:34:58.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:34:58.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:34:58.928 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:34:58.929 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:34:58.929 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:34:58.929 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_2 2026-02-18T08:34:58.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_2'] 2026-02-18T08:34:58.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:34:58.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:34:58.986 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:34:58.986 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_2 2026-02-18T08:34:58.986 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:34:58.986 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:34:58.987 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:34:58.987 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:34:58.987 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:34:58.987 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:34:58.987 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:34:58.987 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:34:58.987 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:34:58.987 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:34:58.987 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:34:58.987 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:34:58.987 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:34:58.987 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:34:58.988 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:34:58.988 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:34:58.988 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:34:58.988 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:34:58.988 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:34:58.988 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:34:58.988 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:34:58.988 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:34:58.988 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:34:58.988 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:34:58.988 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:34:58.988 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:34:58.989 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:34:58.989 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:34:58.989 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:34:58.989 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:34:58.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:34:58.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:34:58.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:34:58.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_2 2026-02-18T08:34:59.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:no valid command found; 2 closest matches: 2026-02-18T08:34:59.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush mirror/image_20 2026-02-18T08:34:59.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush mirror/image_29 2026-02-18T08:34:59.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:34:59.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:34:59.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/image_2 2026-02-18T08:34:59.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:34:59.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:34:59.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:34:59.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:34:59.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:34:59.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:34:59.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:34:59.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:34:59.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:34:59.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:34:59.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:34:59.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:34:59.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:34:59.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:34:59.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:34:59.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:34:59.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:34:59.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:34:59.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:34:59.055 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:34:59.055 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:34:59.055 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:34:59.055 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:34:59.055 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:34:59.055 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:34:59.055 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:34:59.055 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:34:59.055 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:34:59.055 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:34:59.055 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:34:59.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:34:59.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:34:59.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:34:59.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/image_2 2026-02-18T08:34:59.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:34:59.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_2 2026-02-18T08:34:59.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:34:59.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:34:59.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_2 2026-02-18T08:34:59.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_2 '' 2026-02-18T08:34:59.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:34:59.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:34:59.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_2 2026-02-18T08:34:59.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:34:59.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_2.status 2026-02-18T08:34:59.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_2.status 2026-02-18T08:34:59.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:34:59.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_2.status 2026-02-18T08:34:59.129 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_2 2026-02-18T08:34:59.129 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_2.status 2026-02-18T08:34:59.156 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 7 2026-02-18T08:34:59.156 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 7 2026-02-18T08:34:59.156 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:34:59.156 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=29, tag_tid=2, entry_tid=1], [object_number=28, tag_tid=2, entry_tid=0], [object_number=31, tag_tid=1, entry_tid=32431], [object_number=30, tag_tid=1, entry_tid=32430]]], state=connected] 2026-02-18T08:34:59.157 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=29, tag_tid=2, entry_tid=1], [object_number=28, tag_tid=2, entry_tid=0], [object_number=31, tag_tid=1, entry_tid=32431], [object_number=30, tag_tid=1, entry_tid=32430]]], state=connected] 2026-02-18T08:34:59.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_2.status 2026-02-18T08:34:59.161 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=29, tag_tid=2, entry_tid=1' 2026-02-18T08:34:59.162 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_2 2026-02-18T08:34:59.162 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:34:59.162 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:34:59.162 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_2 2026-02-18T08:34:59.162 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_2 '..*' 2026-02-18T08:34:59.162 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:34:59.162 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:34:59.162 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_2 2026-02-18T08:34:59.162 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:34:59.163 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_2.status 2026-02-18T08:34:59.163 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_2.status 2026-02-18T08:34:59.163 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:34:59.164 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_2.status 2026-02-18T08:34:59.164 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_2 2026-02-18T08:34:59.164 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_2.status 2026-02-18T08:34:59.198 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 7 2026-02-18T08:34:59.198 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 7 2026-02-18T08:34:59.198 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:34:59.198 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=29, tag_tid=2, entry_tid=1], [object_number=28, tag_tid=2, entry_tid=0], [object_number=31, tag_tid=1, entry_tid=32431], [object_number=30, tag_tid=1, entry_tid=32430]]], state=connected] 2026-02-18T08:34:59.199 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=29, tag_tid=2, entry_tid=1], [object_number=28, tag_tid=2, entry_tid=0], [object_number=31, tag_tid=1, entry_tid=32431], [object_number=30, tag_tid=1, entry_tid=32430]]], state=connected] 2026-02-18T08:34:59.202 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_2.status 2026-02-18T08:34:59.203 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=29, tag_tid=2, entry_tid=1' 2026-02-18T08:34:59.204 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=29, tag_tid=2, entry_tid=1' -a 'object_number=29, tag_tid=2, entry_tid=1' = 'object_number=29, tag_tid=2, entry_tid=1' 2026-02-18T08:34:59.204 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:34:59.204 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_snap_present cluster1 mirror image_2 snap 2026-02-18T08:34:59.204 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:34:59.204 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:34:59.204 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_2 2026-02-18T08:34:59.204 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap 2026-02-18T08:34:59.204 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-18T08:34:59.204 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-18T08:34:59.204 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1 2026-02-18T08:35:00.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster1 info mirror/image_2@snap 2026-02-18T08:35:00.234 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:rbd image 'image_2': 2026-02-18T08:35:00.234 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: size 128 MiB in 32 objects 2026-02-18T08:35:00.234 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: order 22 (4 MiB objects) 2026-02-18T08:35:00.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: snapshot_count: 1 2026-02-18T08:35:00.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: id: 10ac25661eb9 2026-02-18T08:35:00.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: block_name_prefix: rbd_data.10ac25661eb9 2026-02-18T08:35:00.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: format: 2 2026-02-18T08:35:00.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: features: layering, exclusive-lock, journaling 2026-02-18T08:35:00.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: op_features: 2026-02-18T08:35:00.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: flags: 2026-02-18T08:35:00.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: create_timestamp: Wed Feb 18 08:25:53 2026 2026-02-18T08:35:00.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: access_timestamp: Wed Feb 18 08:25:53 2026 2026-02-18T08:35:00.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: modify_timestamp: Wed Feb 18 08:26:54 2026 2026-02-18T08:35:00.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: protected: False 2026-02-18T08:35:00.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: journal: 10ac25661eb9 2026-02-18T08:35:00.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring state: enabled 2026-02-18T08:35:00.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring mode: journal 2026-02-18T08:35:00.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring global id: 53065475-7c34-42c7-97cc-de762b8052bd 2026-02-18T08:35:00.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring primary: false 2026-02-18T08:35:00.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:35:00.238 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ compare_image_snaps mirror image_2 snap 2026-02-18T08:35:00.238 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:35:00.238 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_2 2026-02-18T08:35:00.238 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap 2026-02-18T08:35:00.238 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local ret=0 2026-02-18T08:35:00.238 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local rmt_export=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_2.export 2026-02-18T08:35:00.238 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loc_export=/tmp/tmp.vybh1m9yJF/cluster1-mirror-image_2.export 2026-02-18T08:35:00.238 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_2.export /tmp/tmp.vybh1m9yJF/cluster1-mirror-image_2.export 2026-02-18T08:35:00.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 -p mirror export image_2@snap /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_2.export 2026-02-18T08:35:00.379 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-18T08:35:00.389 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster1 -p mirror export image_2@snap /tmp/tmp.vybh1m9yJF/cluster1-mirror-image_2.export 2026-02-18T08:35:00.582 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-18T08:35:00.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmp /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_2.export /tmp/tmp.vybh1m9yJF/cluster1-mirror-image_2.export 2026-02-18T08:35:00.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_2.export /tmp/tmp.vybh1m9yJF/cluster1-mirror-image_2.export 2026-02-18T08:35:00.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:35:00.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T08:35:00.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_3 2026-02-18T08:35:00.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_snap cluster2 mirror image_3 snap 2026-02-18T08:35:00.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:35:00.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:35:00.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_3 2026-02-18T08:35:00.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap 2026-02-18T08:35:00.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 -p mirror snap create image_3@snap --debug-rbd=20 --debug-journaler=20 2026-02-18T08:35:00.902 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:35:00.898+0000 7faffc95c640 -1 reset not still connected to 0x5644aec73110 2026-02-18T08:35:00.902 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:35:00.898+0000 7faffc95c640 -1 reset not still connected to 0x5644b7e83c70 2026-02-18T08:35:00.902 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:35:00.898+0000 7faffc95c640 -1 reset not still connected to 0x5644bd70a270 2026-02-18T08:35:00.902 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:35:00.898+0000 7faffc95c640 -1 reset not still connected to 0x5644bd70aea0 2026-02-18T08:35:00.903 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:35:00.898+0000 7faffc95c640 -1 reset not still connected to 0x5644c5826270 2026-02-18T08:35:00.903 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:35:00.898+0000 7faffc95c640 -1 reset not still connected to 0x5644c58275f0 2026-02-18T08:35:00.904 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:35:00.898+0000 7faffc95c640 -1 reset not still connected to 0x5644c8fcea90 2026-02-18T08:35:00.904 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:35:00.898+0000 7faffc95c640 -1 reset not still connected to 0x5644c8fcfad0 2026-02-18T08:35:00.904 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:35:00.898+0000 7faffc95c640 -1 reset not still connected to 0x5644d7dcc0d0 2026-02-18T08:35:00.904 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:35:00.902+0000 7faffc95c640 -1 reset not still connected to 0x5644dc3cd040 2026-02-18T08:35:00.904 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:35:00.902+0000 7faffc95c640 -1 reset not still connected to 0x5644dc3cd860 2026-02-18T08:35:00.904 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:35:00.902+0000 7faffc95c640 -1 reset not still connected to 0x5644dfda05b0 2026-02-18T08:35:03.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_replay_started cluster1 mirror image_3 2026-02-18T08:35:03.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:35:03.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:35:03.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_3 2026-02-18T08:35:03.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_replay_state cluster1 mirror image_3 started 2026-02-18T08:35:03.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:35:03.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:35:03.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_3 2026-02-18T08:35:03.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local state=started 2026-02-18T08:35:03.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-18T08:35:03.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:35:03.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1 2026-02-18T08:35:04.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test_image_replay_state cluster1 mirror image_3 started 2026-02-18T08:35:04.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:35:04.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:35:04.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_3 2026-02-18T08:35:04.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local test_state=started 2026-02-18T08:35:04.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local status_result 2026-02-18T08:35:04.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_state=stopped 2026-02-18T08:35:04.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_3 2026-02-18T08:35:04.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_instance=cluster1 2026-02-18T08:35:04.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-18T08:35:04.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:35:04.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance=cluster1 2026-02-18T08:35:04.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local loop_instance 2026-02-18T08:35:04.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:35:04.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sleep 0 2026-02-18T08:35:04.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -i state 2026-02-18T08:35:04.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:35:04.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ seq 0 3 2026-02-18T08:35:04.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:35:04.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_3 2026-02-18T08:35:04.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:0 2026-02-18T08:35:04.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-18T08:35:04.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:35:04.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:35:04.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:35:04.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-18T08:35:04.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:35:04.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:35:04.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:35:04.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:35:04.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-18T08:35:04.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:35:04.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:35:04.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:35:04.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:35:04.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:0 2026-02-18T08:35:04.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-18T08:35:04.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-18T08:35:04.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:35:04.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:0 2026-02-18T08:35:04.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T08:35:04.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-18T08:35:04.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:35:04.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:35:04.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-18T08:35:04.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-18T08:35:04.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-18T08:35:04.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=0 2026-02-18T08:35:04.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=0 2026-02-18T08:35:04.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:35:04.060 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:35:04.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:35:04.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:35:04.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_3 2026-02-18T08:35:04.122 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ return 0 2026-02-18T08:35:04.122 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ status_result=' "state": "Replaying"' 2026-02-18T08:35:04.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ echo ' "state": "Replaying"' 2026-02-18T08:35:04.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ grep -i Replaying 2026-02-18T08:35:04.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: "state": "Replaying" 2026-02-18T08:35:04.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ current_state=started 2026-02-18T08:35:04.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test started = started 2026-02-18T08:35:04.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:35:04.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror image_3 2026-02-18T08:35:04.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_cluster=cluster1 2026-02-18T08:35:04.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:35:04.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_pool=mirror 2026-02-18T08:35:04.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local remote_pool=mirror 2026-02-18T08:35:04.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_3 2026-02-18T08:35:04.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' journal = journal ']' 2026-02-18T08:35:04.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror mirror image_3 2026-02-18T08:35:04.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_cluster=cluster1 2026-02-18T08:35:04.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:35:04.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_pool=mirror 2026-02-18T08:35:04.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local remote_pool=mirror 2026-02-18T08:35:04.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_3 2026-02-18T08:35:04.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-02-18T08:35:04.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-02-18T08:35:04.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:35:04.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:35:04.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:35:04.327 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_3 2026-02-18T08:35:04.327 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:35:04.327 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:35:04.327 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_3 2026-02-18T08:35:04.327 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:35:04.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_3 ']' 2026-02-18T08:35:04.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_3' 2026-02-18T08:35:04.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_3 2026-02-18T08:35:04.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:35:04.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:35:04.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:35:04.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:35:04.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:35:04.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:35:04.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:35:04.329 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:35:04.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:35:04.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:35:04.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_3 2026-02-18T08:35:04.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:35:04.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:35:04.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:35:04.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:35:04.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:35:04.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:35:04.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:35:04.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:35:04.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:35:04.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:35:04.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:35:04.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:35:04.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:35:04.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:35:04.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:35:04.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:35:04.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:35:04.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:35:04.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:35:04.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:35:04.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:35:04.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:35:04.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:35:04.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:35:04.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:35:04.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:35:04.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:35:04.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:35:04.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:35:04.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:35:04.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:35:04.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:35:04.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:35:04.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_3 2026-02-18T08:35:04.399 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:35:04.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_3 2026-02-18T08:35:04.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:35:04.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:35:04.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_3 2026-02-18T08:35:04.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_3 '' 2026-02-18T08:35:04.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:35:04.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:35:04.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_3 2026-02-18T08:35:04.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:35:04.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_3.status 2026-02-18T08:35:04.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_3.status 2026-02-18T08:35:04.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:35:04.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_3.status 2026-02-18T08:35:04.403 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_3 2026-02-18T08:35:04.403 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_3.status 2026-02-18T08:35:04.430 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 10 2026-02-18T08:35:04.430 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 10 2026-02-18T08:35:04.430 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:35:04.430 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=41, tag_tid=2, entry_tid=1], [object_number=40, tag_tid=2, entry_tid=0], [object_number=43, tag_tid=1, entry_tid=45599], [object_number=42, tag_tid=1, entry_tid=45598]]], state=connected] 2026-02-18T08:35:04.430 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=41, tag_tid=1, entry_tid=45601], [object_number=40, tag_tid=1, entry_tid=45600], [object_number=43, tag_tid=1, entry_tid=45599], [object_number=42, tag_tid=1, entry_tid=45598]]], state=connected] 2026-02-18T08:35:04.433 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_3.status 2026-02-18T08:35:04.434 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=41, tag_tid=2, entry_tid=1' 2026-02-18T08:35:04.436 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_3 2026-02-18T08:35:04.436 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:35:04.436 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:35:04.436 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_3 2026-02-18T08:35:04.436 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_3 '..*' 2026-02-18T08:35:04.436 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:35:04.436 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:35:04.436 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_3 2026-02-18T08:35:04.436 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:35:04.437 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_3.status 2026-02-18T08:35:04.437 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_3.status 2026-02-18T08:35:04.437 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:35:04.437 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_3.status 2026-02-18T08:35:04.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_3 2026-02-18T08:35:04.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_3.status 2026-02-18T08:35:04.465 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 10 2026-02-18T08:35:04.465 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 10 2026-02-18T08:35:04.465 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:35:04.465 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=41, tag_tid=2, entry_tid=1], [object_number=40, tag_tid=2, entry_tid=0], [object_number=43, tag_tid=1, entry_tid=45599], [object_number=42, tag_tid=1, entry_tid=45598]]], state=connected] 2026-02-18T08:35:04.465 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=41, tag_tid=1, entry_tid=45601], [object_number=40, tag_tid=1, entry_tid=45600], [object_number=43, tag_tid=1, entry_tid=45599], [object_number=42, tag_tid=1, entry_tid=45598]]], state=connected] 2026-02-18T08:35:04.467 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_3.status 2026-02-18T08:35:04.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=41, tag_tid=1, entry_tid=45601' 2026-02-18T08:35:04.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=41, tag_tid=2, entry_tid=1' -a 'object_number=41, tag_tid=2, entry_tid=1' = 'object_number=41, tag_tid=1, entry_tid=45601' 2026-02-18T08:35:04.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=41, tag_tid=1, entry_tid=45601' '!=' '' 2026-02-18T08:35:04.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:35:04.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=41, tag_tid=1, entry_tid=45601' = '' 2026-02-18T08:35:04.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=41, tag_tid=1, entry_tid=45601' 2026-02-18T08:35:04.470 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=41, tag_tid=2, entry_tid=1' 2026-02-18T08:35:04.470 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:35:04.470 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:35:04.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=2 2026-02-18T08:35:04.472 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=41, tag_tid=1, entry_tid=45601' 2026-02-18T08:35:04.472 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:35:04.472 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:35:04.473 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=1 2026-02-18T08:35:04.474 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=41, tag_tid=2, entry_tid=1' 2026-02-18T08:35:04.474 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:35:04.474 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:35:04.475 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:35:04.476 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=41, tag_tid=1, entry_tid=45601' 2026-02-18T08:35:04.476 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:35:04.477 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:35:04.478 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=45601 2026-02-18T08:35:04.478 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 2 = 1 -a 1 -le 45601 2026-02-18T08:35:04.478 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:35:04.478 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:35:04.478 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:35:04.679 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_3 2026-02-18T08:35:04.680 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:35:04.680 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:35:04.680 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_3 2026-02-18T08:35:04.680 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:35:04.680 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_3 ']' 2026-02-18T08:35:04.680 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_3' 2026-02-18T08:35:04.680 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_3 2026-02-18T08:35:04.680 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:35:04.680 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:35:04.681 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:35:04.681 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:35:04.681 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:35:04.681 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:35:04.681 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:35:04.684 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:35:04.684 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:35:04.684 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:35:04.684 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_3 2026-02-18T08:35:04.684 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:35:04.684 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:35:04.684 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:35:04.684 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:35:04.684 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:35:04.684 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:35:04.685 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:35:04.685 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:35:04.685 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:35:04.685 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:35:04.685 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:35:04.685 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:35:04.685 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:35:04.685 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:35:04.685 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:35:04.685 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:35:04.685 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:35:04.685 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:35:04.685 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:35:04.686 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:35:04.686 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:35:04.686 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:35:04.686 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:35:04.686 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:35:04.686 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:35:04.686 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:35:04.686 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:35:04.686 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:35:04.686 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:35:04.686 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:35:04.695 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:35:04.696 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:35:04.696 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:35:04.696 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_3 2026-02-18T08:35:04.752 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:35:04.753 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_3 2026-02-18T08:35:04.753 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:35:04.753 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:35:04.753 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_3 2026-02-18T08:35:04.753 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_3 '' 2026-02-18T08:35:04.754 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:35:04.754 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:35:04.754 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_3 2026-02-18T08:35:04.754 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:35:04.754 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_3.status 2026-02-18T08:35:04.754 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_3.status 2026-02-18T08:35:04.754 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:35:04.755 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_3.status 2026-02-18T08:35:04.755 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_3 2026-02-18T08:35:04.756 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_3.status 2026-02-18T08:35:04.785 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 10 2026-02-18T08:35:04.785 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 10 2026-02-18T08:35:04.785 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:35:04.785 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=41, tag_tid=2, entry_tid=1], [object_number=40, tag_tid=2, entry_tid=0], [object_number=43, tag_tid=1, entry_tid=45599], [object_number=42, tag_tid=1, entry_tid=45598]]], state=connected] 2026-02-18T08:35:04.785 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=41, tag_tid=1, entry_tid=45601], [object_number=40, tag_tid=1, entry_tid=45600], [object_number=43, tag_tid=1, entry_tid=45599], [object_number=42, tag_tid=1, entry_tid=45598]]], state=connected] 2026-02-18T08:35:04.787 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_3.status 2026-02-18T08:35:04.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=41, tag_tid=2, entry_tid=1' 2026-02-18T08:35:04.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_3 2026-02-18T08:35:04.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:35:04.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:35:04.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_3 2026-02-18T08:35:04.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_3 '..*' 2026-02-18T08:35:04.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:35:04.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:35:04.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_3 2026-02-18T08:35:04.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:35:04.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_3.status 2026-02-18T08:35:04.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_3.status 2026-02-18T08:35:04.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:35:04.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_3.status 2026-02-18T08:35:04.793 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_3 2026-02-18T08:35:04.793 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_3.status 2026-02-18T08:35:04.827 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 10 2026-02-18T08:35:04.827 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 10 2026-02-18T08:35:04.827 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:35:04.827 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=41, tag_tid=2, entry_tid=1], [object_number=40, tag_tid=2, entry_tid=0], [object_number=43, tag_tid=1, entry_tid=45599], [object_number=42, tag_tid=1, entry_tid=45598]]], state=connected] 2026-02-18T08:35:04.827 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=41, tag_tid=1, entry_tid=45601], [object_number=40, tag_tid=1, entry_tid=45600], [object_number=43, tag_tid=1, entry_tid=45599], [object_number=42, tag_tid=1, entry_tid=45598]]], state=connected] 2026-02-18T08:35:04.830 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_3.status 2026-02-18T08:35:04.832 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=41, tag_tid=1, entry_tid=45601' 2026-02-18T08:35:04.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=41, tag_tid=2, entry_tid=1' -a 'object_number=41, tag_tid=2, entry_tid=1' = 'object_number=41, tag_tid=1, entry_tid=45601' 2026-02-18T08:35:04.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=41, tag_tid=1, entry_tid=45601' '!=' 'object_number=41, tag_tid=1, entry_tid=45601' 2026-02-18T08:35:04.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:35:04.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.4 2026-02-18T08:35:05.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_3 2026-02-18T08:35:05.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:35:05.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:35:05.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_3 2026-02-18T08:35:05.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:35:05.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_3 ']' 2026-02-18T08:35:05.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_3' 2026-02-18T08:35:05.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_3 2026-02-18T08:35:05.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:35:05.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:35:05.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:35:05.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:35:05.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:35:05.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:35:05.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:35:05.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:35:05.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:35:05.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:35:05.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_3 2026-02-18T08:35:05.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:35:05.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:35:05.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:35:05.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:35:05.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:35:05.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:35:05.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:35:05.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:35:05.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:35:05.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:35:05.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:35:05.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:35:05.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:35:05.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:35:05.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:35:05.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:35:05.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:35:05.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:35:05.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:35:05.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:35:05.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:35:05.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:35:05.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:35:05.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:35:05.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:35:05.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:35:05.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:35:05.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:35:05.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:35:05.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:35:05.249 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:35:05.250 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:35:05.250 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:35:05.250 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_3 2026-02-18T08:35:05.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:35:05.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_3 2026-02-18T08:35:05.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:35:05.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:35:05.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_3 2026-02-18T08:35:05.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_3 '' 2026-02-18T08:35:05.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:35:05.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:35:05.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_3 2026-02-18T08:35:05.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:35:05.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_3.status 2026-02-18T08:35:05.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_3.status 2026-02-18T08:35:05.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:35:05.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_3.status 2026-02-18T08:35:05.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_3 2026-02-18T08:35:05.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_3.status 2026-02-18T08:35:05.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 10 2026-02-18T08:35:05.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 10 2026-02-18T08:35:05.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:35:05.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=41, tag_tid=2, entry_tid=1], [object_number=40, tag_tid=2, entry_tid=0], [object_number=43, tag_tid=1, entry_tid=45599], [object_number=42, tag_tid=1, entry_tid=45598]]], state=connected] 2026-02-18T08:35:05.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=41, tag_tid=1, entry_tid=45601], [object_number=40, tag_tid=1, entry_tid=45600], [object_number=43, tag_tid=1, entry_tid=45599], [object_number=42, tag_tid=1, entry_tid=45598]]], state=connected] 2026-02-18T08:35:05.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_3.status 2026-02-18T08:35:05.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=41, tag_tid=2, entry_tid=1' 2026-02-18T08:35:05.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_3 2026-02-18T08:35:05.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:35:05.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:35:05.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_3 2026-02-18T08:35:05.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_3 '..*' 2026-02-18T08:35:05.344 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:35:05.344 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:35:05.344 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_3 2026-02-18T08:35:05.344 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:35:05.344 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_3.status 2026-02-18T08:35:05.344 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_3.status 2026-02-18T08:35:05.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:35:05.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_3.status 2026-02-18T08:35:05.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_3 2026-02-18T08:35:05.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_3.status 2026-02-18T08:35:05.372 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 10 2026-02-18T08:35:05.372 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 10 2026-02-18T08:35:05.372 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:35:05.372 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=41, tag_tid=2, entry_tid=1], [object_number=40, tag_tid=2, entry_tid=0], [object_number=43, tag_tid=1, entry_tid=45599], [object_number=42, tag_tid=1, entry_tid=45598]]], state=connected] 2026-02-18T08:35:05.373 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=41, tag_tid=1, entry_tid=45601], [object_number=40, tag_tid=1, entry_tid=45600], [object_number=43, tag_tid=1, entry_tid=45599], [object_number=42, tag_tid=1, entry_tid=45598]]], state=connected] 2026-02-18T08:35:05.375 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_3.status 2026-02-18T08:35:05.377 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=41, tag_tid=1, entry_tid=45601' 2026-02-18T08:35:05.377 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=41, tag_tid=2, entry_tid=1' -a 'object_number=41, tag_tid=2, entry_tid=1' = 'object_number=41, tag_tid=1, entry_tid=45601' 2026-02-18T08:35:05.377 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=41, tag_tid=1, entry_tid=45601' '!=' 'object_number=41, tag_tid=1, entry_tid=45601' 2026-02-18T08:35:05.377 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:35:05.377 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.8 2026-02-18T08:35:06.179 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_3 2026-02-18T08:35:06.180 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:35:06.180 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:35:06.180 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_3 2026-02-18T08:35:06.180 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:35:06.180 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_3 ']' 2026-02-18T08:35:06.180 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_3' 2026-02-18T08:35:06.180 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_3 2026-02-18T08:35:06.180 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:35:06.180 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:35:06.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:35:06.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:35:06.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:35:06.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:35:06.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:35:06.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:35:06.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:35:06.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:35:06.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_3 2026-02-18T08:35:06.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:35:06.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:35:06.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:35:06.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:35:06.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:35:06.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:35:06.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:35:06.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:35:06.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:35:06.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:35:06.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:35:06.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:35:06.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:35:06.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:35:06.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:35:06.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:35:06.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:35:06.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:35:06.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:35:06.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:35:06.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:35:06.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:35:06.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:35:06.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:35:06.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:35:06.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:35:06.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:35:06.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:35:06.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:35:06.187 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:35:06.194 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:35:06.194 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:35:06.194 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:35:06.195 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_3 2026-02-18T08:35:06.253 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:35:06.253 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_3 2026-02-18T08:35:06.254 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:35:06.254 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:35:06.254 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_3 2026-02-18T08:35:06.254 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_3 '' 2026-02-18T08:35:06.254 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:35:06.254 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:35:06.254 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_3 2026-02-18T08:35:06.254 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:35:06.254 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_3.status 2026-02-18T08:35:06.255 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_3.status 2026-02-18T08:35:06.255 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:35:06.256 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_3.status 2026-02-18T08:35:06.256 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_3 2026-02-18T08:35:06.256 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_3.status 2026-02-18T08:35:06.284 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 10 2026-02-18T08:35:06.284 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 10 2026-02-18T08:35:06.284 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:35:06.284 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=41, tag_tid=2, entry_tid=1], [object_number=40, tag_tid=2, entry_tid=0], [object_number=43, tag_tid=1, entry_tid=45599], [object_number=42, tag_tid=1, entry_tid=45598]]], state=connected] 2026-02-18T08:35:06.285 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=41, tag_tid=1, entry_tid=45601], [object_number=40, tag_tid=1, entry_tid=45600], [object_number=43, tag_tid=1, entry_tid=45599], [object_number=42, tag_tid=1, entry_tid=45598]]], state=connected] 2026-02-18T08:35:06.287 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_3.status 2026-02-18T08:35:06.288 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=41, tag_tid=2, entry_tid=1' 2026-02-18T08:35:06.289 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_3 2026-02-18T08:35:06.290 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:35:06.290 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:35:06.290 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_3 2026-02-18T08:35:06.290 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_3 '..*' 2026-02-18T08:35:06.290 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:35:06.290 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:35:06.290 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_3 2026-02-18T08:35:06.290 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:35:06.290 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_3.status 2026-02-18T08:35:06.291 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_3.status 2026-02-18T08:35:06.291 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:35:06.292 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_3.status 2026-02-18T08:35:06.292 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_3 2026-02-18T08:35:06.292 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_3.status 2026-02-18T08:35:06.329 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 10 2026-02-18T08:35:06.329 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 10 2026-02-18T08:35:06.329 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:35:06.329 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=41, tag_tid=2, entry_tid=1], [object_number=40, tag_tid=2, entry_tid=0], [object_number=43, tag_tid=1, entry_tid=45599], [object_number=42, tag_tid=1, entry_tid=45598]]], state=connected] 2026-02-18T08:35:06.329 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=41, tag_tid=1, entry_tid=45601], [object_number=40, tag_tid=1, entry_tid=45600], [object_number=43, tag_tid=1, entry_tid=45599], [object_number=42, tag_tid=1, entry_tid=45598]]], state=connected] 2026-02-18T08:35:06.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_3.status 2026-02-18T08:35:06.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=41, tag_tid=1, entry_tid=45601' 2026-02-18T08:35:06.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=41, tag_tid=2, entry_tid=1' -a 'object_number=41, tag_tid=2, entry_tid=1' = 'object_number=41, tag_tid=1, entry_tid=45601' 2026-02-18T08:35:06.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=41, tag_tid=1, entry_tid=45601' '!=' 'object_number=41, tag_tid=1, entry_tid=45601' 2026-02-18T08:35:06.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:35:06.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1.6 2026-02-18T08:35:07.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_3 2026-02-18T08:35:07.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:35:07.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:35:07.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_3 2026-02-18T08:35:07.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:35:07.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_3 ']' 2026-02-18T08:35:07.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_3' 2026-02-18T08:35:07.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_3 2026-02-18T08:35:07.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:35:07.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:35:07.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:35:07.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:35:07.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:35:07.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:35:07.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:35:07.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:35:07.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:35:07.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:35:07.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_3 2026-02-18T08:35:07.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:35:07.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:35:07.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:35:07.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:35:07.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:35:07.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:35:07.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:35:07.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:35:07.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:35:07.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:35:07.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:35:07.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:35:07.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:35:07.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:35:07.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:35:07.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:35:07.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:35:07.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:35:07.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:35:07.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:35:07.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:35:07.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:35:07.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:35:07.944 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:35:07.944 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:35:07.944 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:35:07.944 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:35:07.944 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:35:07.944 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:35:07.944 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:35:07.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:35:07.952 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:35:07.952 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:35:07.952 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_3 2026-02-18T08:35:08.010 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:35:08.012 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_3 2026-02-18T08:35:08.012 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:35:08.012 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:35:08.012 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_3 2026-02-18T08:35:08.012 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_3 '' 2026-02-18T08:35:08.012 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:35:08.012 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:35:08.012 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_3 2026-02-18T08:35:08.012 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:35:08.012 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_3.status 2026-02-18T08:35:08.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_3.status 2026-02-18T08:35:08.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:35:08.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_3.status 2026-02-18T08:35:08.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_3 2026-02-18T08:35:08.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_3.status 2026-02-18T08:35:08.040 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 10 2026-02-18T08:35:08.040 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 10 2026-02-18T08:35:08.040 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:35:08.040 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=41, tag_tid=2, entry_tid=1], [object_number=40, tag_tid=2, entry_tid=0], [object_number=43, tag_tid=1, entry_tid=45599], [object_number=42, tag_tid=1, entry_tid=45598]]], state=connected] 2026-02-18T08:35:08.041 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=41, tag_tid=1, entry_tid=45601], [object_number=40, tag_tid=1, entry_tid=45600], [object_number=43, tag_tid=1, entry_tid=45599], [object_number=42, tag_tid=1, entry_tid=45598]]], state=connected] 2026-02-18T08:35:08.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_3.status 2026-02-18T08:35:08.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=41, tag_tid=2, entry_tid=1' 2026-02-18T08:35:08.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_3 2026-02-18T08:35:08.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:35:08.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:35:08.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_3 2026-02-18T08:35:08.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_3 '..*' 2026-02-18T08:35:08.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:35:08.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:35:08.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_3 2026-02-18T08:35:08.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:35:08.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_3.status 2026-02-18T08:35:08.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_3.status 2026-02-18T08:35:08.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:35:08.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_3.status 2026-02-18T08:35:08.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_3 2026-02-18T08:35:08.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_3.status 2026-02-18T08:35:08.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 10 2026-02-18T08:35:08.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 10 2026-02-18T08:35:08.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:35:08.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=41, tag_tid=2, entry_tid=1], [object_number=40, tag_tid=2, entry_tid=0], [object_number=43, tag_tid=1, entry_tid=45599], [object_number=42, tag_tid=1, entry_tid=45598]]], state=connected] 2026-02-18T08:35:08.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=41, tag_tid=1, entry_tid=45601], [object_number=40, tag_tid=1, entry_tid=45600], [object_number=43, tag_tid=1, entry_tid=45599], [object_number=42, tag_tid=1, entry_tid=45598]]], state=connected] 2026-02-18T08:35:08.086 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_3.status 2026-02-18T08:35:08.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=41, tag_tid=1, entry_tid=45601' 2026-02-18T08:35:08.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=41, tag_tid=2, entry_tid=1' -a 'object_number=41, tag_tid=2, entry_tid=1' = 'object_number=41, tag_tid=1, entry_tid=45601' 2026-02-18T08:35:08.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=41, tag_tid=1, entry_tid=45601' '!=' 'object_number=41, tag_tid=1, entry_tid=45601' 2026-02-18T08:35:08.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:35:08.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 2 2026-02-18T08:35:10.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_3 2026-02-18T08:35:10.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:35:10.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:35:10.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_3 2026-02-18T08:35:10.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:35:10.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_3 ']' 2026-02-18T08:35:10.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_3' 2026-02-18T08:35:10.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_3 2026-02-18T08:35:10.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:35:10.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:35:10.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:35:10.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:35:10.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:35:10.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:35:10.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:35:10.093 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:35:10.093 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:35:10.093 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:35:10.094 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_3 2026-02-18T08:35:10.094 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:35:10.094 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:35:10.094 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:35:10.094 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:35:10.094 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:35:10.094 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:35:10.094 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:35:10.094 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:35:10.094 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:35:10.094 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:35:10.095 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:35:10.095 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:35:10.095 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:35:10.095 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:35:10.095 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:35:10.095 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:35:10.095 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:35:10.095 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:35:10.095 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:35:10.095 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:35:10.095 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:35:10.095 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:35:10.096 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:35:10.096 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:35:10.096 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:35:10.096 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:35:10.096 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:35:10.096 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:35:10.096 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:35:10.096 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:35:10.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:35:10.105 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:35:10.105 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:35:10.105 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_3 2026-02-18T08:35:10.163 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:35:10.164 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_3 2026-02-18T08:35:10.165 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:35:10.165 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:35:10.165 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_3 2026-02-18T08:35:10.165 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_3 '' 2026-02-18T08:35:10.165 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:35:10.165 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:35:10.165 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_3 2026-02-18T08:35:10.165 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:35:10.165 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_3.status 2026-02-18T08:35:10.165 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_3.status 2026-02-18T08:35:10.166 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:35:10.166 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_3.status 2026-02-18T08:35:10.167 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_3 2026-02-18T08:35:10.167 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_3.status 2026-02-18T08:35:10.194 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 10 2026-02-18T08:35:10.194 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 10 2026-02-18T08:35:10.194 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:35:10.195 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=41, tag_tid=2, entry_tid=1], [object_number=40, tag_tid=2, entry_tid=0], [object_number=43, tag_tid=1, entry_tid=45599], [object_number=42, tag_tid=1, entry_tid=45598]]], state=connected] 2026-02-18T08:35:10.195 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=41, tag_tid=1, entry_tid=45601], [object_number=40, tag_tid=1, entry_tid=45600], [object_number=43, tag_tid=1, entry_tid=45599], [object_number=42, tag_tid=1, entry_tid=45598]]], state=connected] 2026-02-18T08:35:10.197 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_3.status 2026-02-18T08:35:10.198 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=41, tag_tid=2, entry_tid=1' 2026-02-18T08:35:10.199 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_3 2026-02-18T08:35:10.199 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:35:10.199 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:35:10.199 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_3 2026-02-18T08:35:10.199 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_3 '..*' 2026-02-18T08:35:10.200 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:35:10.200 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:35:10.200 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_3 2026-02-18T08:35:10.200 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:35:10.200 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_3.status 2026-02-18T08:35:10.200 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_3.status 2026-02-18T08:35:10.201 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:35:10.201 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_3.status 2026-02-18T08:35:10.201 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_3 2026-02-18T08:35:10.201 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_3.status 2026-02-18T08:35:10.227 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 10 2026-02-18T08:35:10.227 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 10 2026-02-18T08:35:10.227 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:35:10.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=41, tag_tid=2, entry_tid=1], [object_number=40, tag_tid=2, entry_tid=0], [object_number=43, tag_tid=1, entry_tid=45599], [object_number=42, tag_tid=1, entry_tid=45598]]], state=connected] 2026-02-18T08:35:10.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=41, tag_tid=1, entry_tid=45601], [object_number=40, tag_tid=1, entry_tid=45600], [object_number=43, tag_tid=1, entry_tid=45599], [object_number=42, tag_tid=1, entry_tid=45598]]], state=connected] 2026-02-18T08:35:10.230 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_3.status 2026-02-18T08:35:10.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=41, tag_tid=1, entry_tid=45601' 2026-02-18T08:35:10.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=41, tag_tid=2, entry_tid=1' -a 'object_number=41, tag_tid=2, entry_tid=1' = 'object_number=41, tag_tid=1, entry_tid=45601' 2026-02-18T08:35:10.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=41, tag_tid=1, entry_tid=45601' '!=' 'object_number=41, tag_tid=1, entry_tid=45601' 2026-02-18T08:35:10.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:35:10.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 2 2026-02-18T08:35:12.234 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_3 2026-02-18T08:35:12.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:35:12.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:35:12.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_3 2026-02-18T08:35:12.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:35:12.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_3 ']' 2026-02-18T08:35:12.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_3' 2026-02-18T08:35:12.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_3 2026-02-18T08:35:12.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:35:12.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:35:12.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:35:12.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:35:12.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:35:12.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:35:12.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:35:12.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:35:12.238 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:35:12.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:35:12.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_3 2026-02-18T08:35:12.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:35:12.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:35:12.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:35:12.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:35:12.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:35:12.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:35:12.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:35:12.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:35:12.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:35:12.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:35:12.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:35:12.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:35:12.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:35:12.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:35:12.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:35:12.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:35:12.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:35:12.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:35:12.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:35:12.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:35:12.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:35:12.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:35:12.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:35:12.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:35:12.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:35:12.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:35:12.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:35:12.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:35:12.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:35:12.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:35:12.249 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:35:12.250 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:35:12.250 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:35:12.250 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_3 2026-02-18T08:35:12.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:35:12.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_3 2026-02-18T08:35:12.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:35:12.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:35:12.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_3 2026-02-18T08:35:12.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_3 '' 2026-02-18T08:35:12.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:35:12.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:35:12.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_3 2026-02-18T08:35:12.314 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:35:12.314 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_3.status 2026-02-18T08:35:12.314 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_3.status 2026-02-18T08:35:12.314 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:35:12.314 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_3.status 2026-02-18T08:35:12.315 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_3 2026-02-18T08:35:12.315 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_3.status 2026-02-18T08:35:12.344 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 10 2026-02-18T08:35:12.344 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 10 2026-02-18T08:35:12.344 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:35:12.344 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=41, tag_tid=2, entry_tid=1], [object_number=40, tag_tid=2, entry_tid=0], [object_number=43, tag_tid=1, entry_tid=45599], [object_number=42, tag_tid=1, entry_tid=45598]]], state=connected] 2026-02-18T08:35:12.344 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=41, tag_tid=2, entry_tid=1], [object_number=40, tag_tid=2, entry_tid=0], [object_number=43, tag_tid=1, entry_tid=45599], [object_number=42, tag_tid=1, entry_tid=45598]]], state=connected] 2026-02-18T08:35:12.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_3.status 2026-02-18T08:35:12.348 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=41, tag_tid=2, entry_tid=1' 2026-02-18T08:35:12.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_3 2026-02-18T08:35:12.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:35:12.350 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:35:12.350 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_3 2026-02-18T08:35:12.350 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_3 '..*' 2026-02-18T08:35:12.350 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:35:12.350 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:35:12.350 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_3 2026-02-18T08:35:12.350 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:35:12.350 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_3.status 2026-02-18T08:35:12.351 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_3.status 2026-02-18T08:35:12.351 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:35:12.351 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_3.status 2026-02-18T08:35:12.352 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_3.status 2026-02-18T08:35:12.352 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_3 2026-02-18T08:35:12.380 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 10 2026-02-18T08:35:12.380 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 10 2026-02-18T08:35:12.380 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:35:12.380 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=41, tag_tid=2, entry_tid=1], [object_number=40, tag_tid=2, entry_tid=0], [object_number=43, tag_tid=1, entry_tid=45599], [object_number=42, tag_tid=1, entry_tid=45598]]], state=connected] 2026-02-18T08:35:12.380 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=41, tag_tid=2, entry_tid=1], [object_number=40, tag_tid=2, entry_tid=0], [object_number=43, tag_tid=1, entry_tid=45599], [object_number=42, tag_tid=1, entry_tid=45598]]], state=connected] 2026-02-18T08:35:12.383 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_3.status 2026-02-18T08:35:12.384 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=41, tag_tid=2, entry_tid=1' 2026-02-18T08:35:12.385 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=41, tag_tid=2, entry_tid=1' -a 'object_number=41, tag_tid=2, entry_tid=1' = 'object_number=41, tag_tid=2, entry_tid=1' 2026-02-18T08:35:12.385 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:35:12.385 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_snap_present cluster1 mirror image_3 snap 2026-02-18T08:35:12.385 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:35:12.385 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:35:12.385 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_3 2026-02-18T08:35:12.385 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap 2026-02-18T08:35:12.385 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-18T08:35:12.385 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-18T08:35:12.385 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1 2026-02-18T08:35:13.385 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster1 info mirror/image_3@snap 2026-02-18T08:35:13.416 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:rbd image 'image_3': 2026-02-18T08:35:13.421 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: size 128 MiB in 32 objects 2026-02-18T08:35:13.421 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: order 22 (4 MiB objects) 2026-02-18T08:35:13.421 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: snapshot_count: 1 2026-02-18T08:35:13.422 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: id: 1104b23151bc 2026-02-18T08:35:13.422 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: block_name_prefix: rbd_data.1104b23151bc 2026-02-18T08:35:13.422 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: format: 2 2026-02-18T08:35:13.422 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: features: layering, exclusive-lock, journaling 2026-02-18T08:35:13.422 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: op_features: 2026-02-18T08:35:13.422 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: flags: 2026-02-18T08:35:13.422 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: create_timestamp: Wed Feb 18 08:26:03 2026 2026-02-18T08:35:13.422 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: access_timestamp: Wed Feb 18 08:26:03 2026 2026-02-18T08:35:13.423 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: modify_timestamp: Wed Feb 18 08:27:04 2026 2026-02-18T08:35:13.423 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: protected: False 2026-02-18T08:35:13.423 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: journal: 1104b23151bc 2026-02-18T08:35:13.423 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring state: enabled 2026-02-18T08:35:13.423 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring mode: journal 2026-02-18T08:35:13.423 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring global id: 21775cf4-5b98-4f72-bda0-761662e45fd2 2026-02-18T08:35:13.423 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring primary: false 2026-02-18T08:35:13.425 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:35:13.425 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ compare_image_snaps mirror image_3 snap 2026-02-18T08:35:13.425 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:35:13.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_3 2026-02-18T08:35:13.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap 2026-02-18T08:35:13.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local ret=0 2026-02-18T08:35:13.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local rmt_export=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_3.export 2026-02-18T08:35:13.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loc_export=/tmp/tmp.vybh1m9yJF/cluster1-mirror-image_3.export 2026-02-18T08:35:13.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_3.export /tmp/tmp.vybh1m9yJF/cluster1-mirror-image_3.export 2026-02-18T08:35:13.427 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 -p mirror export image_3@snap /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_3.export 2026-02-18T08:35:13.577 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-18T08:35:13.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster1 -p mirror export image_3@snap /tmp/tmp.vybh1m9yJF/cluster1-mirror-image_3.export 2026-02-18T08:35:13.806 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-18T08:35:13.815 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmp /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_3.export /tmp/tmp.vybh1m9yJF/cluster1-mirror-image_3.export 2026-02-18T08:35:13.863 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_3.export /tmp/tmp.vybh1m9yJF/cluster1-mirror-image_3.export 2026-02-18T08:35:13.909 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:35:13.909 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T08:35:13.910 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_4 2026-02-18T08:35:13.910 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_snap cluster2 mirror image_4 snap 2026-02-18T08:35:13.910 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:35:13.910 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:35:13.910 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_4 2026-02-18T08:35:13.910 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap 2026-02-18T08:35:13.910 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 -p mirror snap create image_4@snap --debug-rbd=20 --debug-journaler=20 2026-02-18T08:35:16.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_replay_started cluster1 mirror image_4 2026-02-18T08:35:16.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:35:16.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:35:16.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_4 2026-02-18T08:35:16.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_replay_state cluster1 mirror image_4 started 2026-02-18T08:35:16.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:35:16.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:35:16.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_4 2026-02-18T08:35:16.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local state=started 2026-02-18T08:35:16.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-18T08:35:16.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:35:16.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1 2026-02-18T08:35:17.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test_image_replay_state cluster1 mirror image_4 started 2026-02-18T08:35:17.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:35:17.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:35:17.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_4 2026-02-18T08:35:17.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local test_state=started 2026-02-18T08:35:17.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local status_result 2026-02-18T08:35:17.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_state=stopped 2026-02-18T08:35:17.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_4 2026-02-18T08:35:17.055 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -i state 2026-02-18T08:35:17.055 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_instance=cluster1 2026-02-18T08:35:17.055 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-18T08:35:17.055 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:35:17.055 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance=cluster1 2026-02-18T08:35:17.055 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local loop_instance 2026-02-18T08:35:17.055 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:35:17.055 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sleep 0 2026-02-18T08:35:17.059 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:35:17.059 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ seq 0 3 2026-02-18T08:35:17.059 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:35:17.059 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_4 2026-02-18T08:35:17.059 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:0 2026-02-18T08:35:17.059 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-18T08:35:17.059 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:35:17.059 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:35:17.059 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:35:17.060 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-18T08:35:17.060 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:35:17.060 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:35:17.060 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:35:17.060 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:35:17.060 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-18T08:35:17.060 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:35:17.060 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:35:17.060 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:35:17.060 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:35:17.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:0 2026-02-18T08:35:17.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-18T08:35:17.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-18T08:35:17.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:35:17.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:0 2026-02-18T08:35:17.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T08:35:17.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-18T08:35:17.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:35:17.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:35:17.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-18T08:35:17.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-18T08:35:17.062 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-18T08:35:17.062 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=0 2026-02-18T08:35:17.062 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=0 2026-02-18T08:35:17.062 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:35:17.069 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:35:17.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:35:17.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:35:17.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_4 2026-02-18T08:35:17.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:no valid command found; 2 closest matches: 2026-02-18T08:35:17.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status mirror/image_43 2026-02-18T08:35:17.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status mirror/image_46 2026-02-18T08:35:17.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:35:17.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:35:17.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_4 2026-02-18T08:35:17.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:1 2026-02-18T08:35:17.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-18T08:35:17.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:35:17.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:35:17.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:35:17.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-18T08:35:17.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:35:17.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:35:17.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:35:17.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:35:17.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-18T08:35:17.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:35:17.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:35:17.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:35:17.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:35:17.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:1 2026-02-18T08:35:17.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-18T08:35:17.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-18T08:35:17.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:35:17.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:1 2026-02-18T08:35:17.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T08:35:17.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-18T08:35:17.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:35:17.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:35:17.129 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-18T08:35:17.129 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-18T08:35:17.129 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-18T08:35:17.129 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=1 2026-02-18T08:35:17.129 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=1 2026-02-18T08:35:17.129 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:35:17.136 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:35:17.136 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:35:17.137 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:35:17.137 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_4 2026-02-18T08:35:17.195 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ return 0 2026-02-18T08:35:17.196 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ status_result=' "state": "Replaying"' 2026-02-18T08:35:17.197 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ echo ' "state": "Replaying"' 2026-02-18T08:35:17.197 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ grep -i Replaying 2026-02-18T08:35:17.198 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: "state": "Replaying" 2026-02-18T08:35:17.199 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ current_state=started 2026-02-18T08:35:17.200 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test started = started 2026-02-18T08:35:17.200 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:35:17.200 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror image_4 2026-02-18T08:35:17.200 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_cluster=cluster1 2026-02-18T08:35:17.200 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:35:17.200 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_pool=mirror 2026-02-18T08:35:17.200 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local remote_pool=mirror 2026-02-18T08:35:17.200 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_4 2026-02-18T08:35:17.200 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' journal = journal ']' 2026-02-18T08:35:17.200 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror mirror image_4 2026-02-18T08:35:17.201 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_cluster=cluster1 2026-02-18T08:35:17.201 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:35:17.201 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_pool=mirror 2026-02-18T08:35:17.201 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local remote_pool=mirror 2026-02-18T08:35:17.201 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_4 2026-02-18T08:35:17.201 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-02-18T08:35:17.201 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-02-18T08:35:17.201 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:35:17.201 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:35:17.201 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:35:17.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_4 2026-02-18T08:35:17.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:35:17.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:35:17.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_4 2026-02-18T08:35:17.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:35:17.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_4 ']' 2026-02-18T08:35:17.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_4' 2026-02-18T08:35:17.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_4 2026-02-18T08:35:17.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:35:17.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:35:17.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:35:17.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:35:17.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:35:17.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:35:17.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:35:17.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:35:17.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:35:17.405 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:35:17.405 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_4 2026-02-18T08:35:17.406 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:35:17.406 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:35:17.406 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:35:17.406 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:35:17.406 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:35:17.406 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:35:17.406 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:35:17.406 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:35:17.406 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:35:17.407 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:35:17.407 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:35:17.407 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:35:17.407 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:35:17.407 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:35:17.407 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:35:17.407 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:35:17.407 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:35:17.407 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:35:17.407 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:35:17.408 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:35:17.408 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:35:17.408 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:35:17.408 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:35:17.408 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:35:17.408 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:35:17.408 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:35:17.408 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:35:17.408 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:35:17.408 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:35:17.408 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:35:17.416 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:35:17.416 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:35:17.416 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:35:17.416 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_4 2026-02-18T08:35:17.470 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:no valid command found; 2 closest matches: 2026-02-18T08:35:17.470 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush mirror/image_43 2026-02-18T08:35:17.470 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush mirror/image_46 2026-02-18T08:35:17.470 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:35:17.473 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:35:17.473 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_4 2026-02-18T08:35:17.473 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:35:17.474 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:35:17.474 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:35:17.474 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:35:17.474 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:35:17.474 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:35:17.474 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:35:17.474 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:35:17.474 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:35:17.474 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:35:17.475 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:35:17.475 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:35:17.475 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:35:17.475 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:35:17.475 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:35:17.475 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:35:17.475 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:35:17.475 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:35:17.475 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:35:17.475 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:35:17.475 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:35:17.476 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:35:17.476 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:35:17.476 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:35:17.476 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:35:17.476 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:35:17.476 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:35:17.476 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:35:17.476 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:35:17.476 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:35:17.485 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:35:17.485 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:35:17.486 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:35:17.486 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_4 2026-02-18T08:35:17.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:35:17.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_4 2026-02-18T08:35:17.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:35:17.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:35:17.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_4 2026-02-18T08:35:17.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_4 '' 2026-02-18T08:35:17.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:35:17.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:35:17.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_4 2026-02-18T08:35:17.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:35:17.545 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_4.status 2026-02-18T08:35:17.545 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_4.status 2026-02-18T08:35:17.545 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:35:17.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_4.status 2026-02-18T08:35:17.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_4 2026-02-18T08:35:17.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_4.status 2026-02-18T08:35:17.574 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 7 2026-02-18T08:35:17.574 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 7 2026-02-18T08:35:17.574 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:35:17.574 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=29, tag_tid=2, entry_tid=1], [object_number=28, tag_tid=2, entry_tid=0], [object_number=31, tag_tid=1, entry_tid=31819], [object_number=30, tag_tid=1, entry_tid=31818]]], state=connected] 2026-02-18T08:35:17.574 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=31, tag_tid=1, entry_tid=31819], [object_number=30, tag_tid=1, entry_tid=31818], [object_number=29, tag_tid=1, entry_tid=31817], [object_number=28, tag_tid=1, entry_tid=31816]]], state=connected] 2026-02-18T08:35:17.576 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_4.status 2026-02-18T08:35:17.577 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=29, tag_tid=2, entry_tid=1' 2026-02-18T08:35:17.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_4 2026-02-18T08:35:17.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:35:17.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:35:17.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_4 2026-02-18T08:35:17.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_4 '..*' 2026-02-18T08:35:17.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:35:17.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:35:17.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_4 2026-02-18T08:35:17.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:35:17.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_4.status 2026-02-18T08:35:17.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_4.status 2026-02-18T08:35:17.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:35:17.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_4.status 2026-02-18T08:35:17.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_4 2026-02-18T08:35:17.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_4.status 2026-02-18T08:35:17.607 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 7 2026-02-18T08:35:17.607 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 7 2026-02-18T08:35:17.607 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:35:17.607 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=29, tag_tid=2, entry_tid=1], [object_number=28, tag_tid=2, entry_tid=0], [object_number=31, tag_tid=1, entry_tid=31819], [object_number=30, tag_tid=1, entry_tid=31818]]], state=connected] 2026-02-18T08:35:17.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=31, tag_tid=1, entry_tid=31819], [object_number=30, tag_tid=1, entry_tid=31818], [object_number=29, tag_tid=1, entry_tid=31817], [object_number=28, tag_tid=1, entry_tid=31816]]], state=connected] 2026-02-18T08:35:17.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_4.status 2026-02-18T08:35:17.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=31, tag_tid=1, entry_tid=31819' 2026-02-18T08:35:17.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=29, tag_tid=2, entry_tid=1' -a 'object_number=29, tag_tid=2, entry_tid=1' = 'object_number=31, tag_tid=1, entry_tid=31819' 2026-02-18T08:35:17.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=31, tag_tid=1, entry_tid=31819' '!=' '' 2026-02-18T08:35:17.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:35:17.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=31, tag_tid=1, entry_tid=31819' = '' 2026-02-18T08:35:17.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=31, tag_tid=1, entry_tid=31819' 2026-02-18T08:35:17.618 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=29, tag_tid=2, entry_tid=1' 2026-02-18T08:35:17.618 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:35:17.618 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:35:17.618 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=2 2026-02-18T08:35:17.620 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=31, tag_tid=1, entry_tid=31819' 2026-02-18T08:35:17.620 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:35:17.620 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:35:17.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=1 2026-02-18T08:35:17.622 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=29, tag_tid=2, entry_tid=1' 2026-02-18T08:35:17.622 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:35:17.622 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:35:17.623 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:35:17.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=31, tag_tid=1, entry_tid=31819' 2026-02-18T08:35:17.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:35:17.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:35:17.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=31819 2026-02-18T08:35:17.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 2 = 1 -a 1 -le 31819 2026-02-18T08:35:17.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:35:17.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:35:17.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:35:17.828 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_4 2026-02-18T08:35:17.828 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:35:17.828 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:35:17.828 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_4 2026-02-18T08:35:17.828 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:35:17.828 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_4 ']' 2026-02-18T08:35:17.828 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_4' 2026-02-18T08:35:17.829 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_4 2026-02-18T08:35:17.829 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:35:17.829 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:35:17.829 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:35:17.829 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:35:17.829 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:35:17.829 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:35:17.829 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:35:17.829 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:35:17.829 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:35:17.832 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:35:17.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_4 2026-02-18T08:35:17.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:35:17.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:35:17.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:35:17.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:35:17.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:35:17.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:35:17.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:35:17.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:35:17.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:35:17.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:35:17.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:35:17.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:35:17.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:35:17.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:35:17.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:35:17.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:35:17.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:35:17.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:35:17.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:35:17.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:35:17.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:35:17.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:35:17.835 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:35:17.835 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:35:17.835 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:35:17.835 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:35:17.835 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:35:17.835 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:35:17.835 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:35:17.835 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:35:17.842 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:35:17.842 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:35:17.842 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:35:17.842 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_4 2026-02-18T08:35:17.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:no valid command found; 2 closest matches: 2026-02-18T08:35:17.894 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush mirror/image_43 2026-02-18T08:35:17.894 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush mirror/image_46 2026-02-18T08:35:17.894 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:35:17.896 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:35:17.897 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_4 2026-02-18T08:35:17.897 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:35:17.897 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:35:17.897 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:35:17.897 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:35:17.897 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:35:17.897 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:35:17.897 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:35:17.897 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:35:17.897 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:35:17.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:35:17.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:35:17.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:35:17.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:35:17.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:35:17.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:35:17.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:35:17.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:35:17.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:35:17.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:35:17.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:35:17.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:35:17.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:35:17.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:35:17.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:35:17.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:35:17.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:35:17.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:35:17.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:35:17.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:35:17.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:35:17.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:35:17.909 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:35:17.909 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:35:17.909 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_4 2026-02-18T08:35:17.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:35:17.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_4 2026-02-18T08:35:17.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:35:17.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:35:17.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_4 2026-02-18T08:35:17.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_4 '' 2026-02-18T08:35:17.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:35:17.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:35:17.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_4 2026-02-18T08:35:17.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:35:17.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_4.status 2026-02-18T08:35:17.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_4.status 2026-02-18T08:35:17.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:35:17.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_4.status 2026-02-18T08:35:17.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_4 2026-02-18T08:35:17.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_4.status 2026-02-18T08:35:18.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 7 2026-02-18T08:35:18.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 7 2026-02-18T08:35:18.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:35:18.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=29, tag_tid=2, entry_tid=1], [object_number=28, tag_tid=2, entry_tid=0], [object_number=31, tag_tid=1, entry_tid=31819], [object_number=30, tag_tid=1, entry_tid=31818]]], state=connected] 2026-02-18T08:35:18.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=31, tag_tid=1, entry_tid=31819], [object_number=30, tag_tid=1, entry_tid=31818], [object_number=29, tag_tid=1, entry_tid=31817], [object_number=28, tag_tid=1, entry_tid=31816]]], state=connected] 2026-02-18T08:35:18.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_4.status 2026-02-18T08:35:18.019 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=29, tag_tid=2, entry_tid=1' 2026-02-18T08:35:18.020 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_4 2026-02-18T08:35:18.020 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:35:18.020 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:35:18.020 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_4 2026-02-18T08:35:18.020 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_4 '..*' 2026-02-18T08:35:18.021 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:35:18.021 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:35:18.021 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_4 2026-02-18T08:35:18.021 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:35:18.021 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_4.status 2026-02-18T08:35:18.021 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_4.status 2026-02-18T08:35:18.021 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:35:18.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_4.status 2026-02-18T08:35:18.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_4 2026-02-18T08:35:18.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_4.status 2026-02-18T08:35:18.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 7 2026-02-18T08:35:18.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 7 2026-02-18T08:35:18.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:35:18.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=29, tag_tid=2, entry_tid=1], [object_number=28, tag_tid=2, entry_tid=0], [object_number=31, tag_tid=1, entry_tid=31819], [object_number=30, tag_tid=1, entry_tid=31818]]], state=connected] 2026-02-18T08:35:18.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=31, tag_tid=1, entry_tid=31819], [object_number=30, tag_tid=1, entry_tid=31818], [object_number=29, tag_tid=1, entry_tid=31817], [object_number=28, tag_tid=1, entry_tid=31816]]], state=connected] 2026-02-18T08:35:18.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_4.status 2026-02-18T08:35:18.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=31, tag_tid=1, entry_tid=31819' 2026-02-18T08:35:18.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=29, tag_tid=2, entry_tid=1' -a 'object_number=29, tag_tid=2, entry_tid=1' = 'object_number=31, tag_tid=1, entry_tid=31819' 2026-02-18T08:35:18.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=31, tag_tid=1, entry_tid=31819' '!=' 'object_number=31, tag_tid=1, entry_tid=31819' 2026-02-18T08:35:18.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:35:18.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.4 2026-02-18T08:35:18.455 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_4 2026-02-18T08:35:18.455 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:35:18.455 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:35:18.455 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_4 2026-02-18T08:35:18.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:35:18.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_4 ']' 2026-02-18T08:35:18.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_4' 2026-02-18T08:35:18.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_4 2026-02-18T08:35:18.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:35:18.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:35:18.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:35:18.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:35:18.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:35:18.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:35:18.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:35:18.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:35:18.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:35:18.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:35:18.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_4 2026-02-18T08:35:18.460 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:35:18.460 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:35:18.460 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:35:18.460 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:35:18.460 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:35:18.460 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:35:18.460 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:35:18.460 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:35:18.460 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:35:18.460 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:35:18.460 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:35:18.460 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:35:18.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:35:18.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:35:18.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:35:18.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:35:18.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:35:18.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:35:18.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:35:18.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:35:18.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:35:18.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:35:18.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:35:18.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:35:18.462 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:35:18.462 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:35:18.462 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:35:18.462 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:35:18.462 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:35:18.462 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:35:18.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:35:18.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:35:18.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:35:18.470 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_4 2026-02-18T08:35:18.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:no valid command found; 2 closest matches: 2026-02-18T08:35:18.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush mirror/image_43 2026-02-18T08:35:18.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush mirror/image_46 2026-02-18T08:35:18.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:35:18.525 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:35:18.525 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_4 2026-02-18T08:35:18.525 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:35:18.525 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:35:18.526 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:35:18.526 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:35:18.526 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:35:18.526 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:35:18.526 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:35:18.526 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:35:18.526 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:35:18.526 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:35:18.526 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:35:18.526 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:35:18.526 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:35:18.526 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:35:18.526 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:35:18.526 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:35:18.527 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:35:18.527 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:35:18.527 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:35:18.527 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:35:18.527 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:35:18.527 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:35:18.527 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:35:18.527 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:35:18.527 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:35:18.527 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:35:18.527 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:35:18.527 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:35:18.527 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:35:18.527 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:35:18.536 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:35:18.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:35:18.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:35:18.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_4 2026-02-18T08:35:18.596 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:35:18.596 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_4 2026-02-18T08:35:18.596 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:35:18.597 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:35:18.597 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_4 2026-02-18T08:35:18.597 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_4 '' 2026-02-18T08:35:18.597 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:35:18.597 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:35:18.597 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_4 2026-02-18T08:35:18.597 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:35:18.597 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_4.status 2026-02-18T08:35:18.597 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_4.status 2026-02-18T08:35:18.597 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:35:18.600 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_4.status 2026-02-18T08:35:18.600 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_4 2026-02-18T08:35:18.600 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_4.status 2026-02-18T08:35:18.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 7 2026-02-18T08:35:18.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 7 2026-02-18T08:35:18.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:35:18.628 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=29, tag_tid=2, entry_tid=1], [object_number=28, tag_tid=2, entry_tid=0], [object_number=31, tag_tid=1, entry_tid=31819], [object_number=30, tag_tid=1, entry_tid=31818]]], state=connected] 2026-02-18T08:35:18.628 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=31, tag_tid=1, entry_tid=31819], [object_number=30, tag_tid=1, entry_tid=31818], [object_number=29, tag_tid=1, entry_tid=31817], [object_number=28, tag_tid=1, entry_tid=31816]]], state=connected] 2026-02-18T08:35:18.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_4.status 2026-02-18T08:35:18.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=29, tag_tid=2, entry_tid=1' 2026-02-18T08:35:18.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_4 2026-02-18T08:35:18.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:35:18.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:35:18.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_4 2026-02-18T08:35:18.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_4 '..*' 2026-02-18T08:35:18.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:35:18.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:35:18.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_4 2026-02-18T08:35:18.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:35:18.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_4.status 2026-02-18T08:35:18.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_4.status 2026-02-18T08:35:18.635 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:35:18.636 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_4.status 2026-02-18T08:35:18.636 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_4 2026-02-18T08:35:18.636 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_4.status 2026-02-18T08:35:18.671 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 7 2026-02-18T08:35:18.671 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 7 2026-02-18T08:35:18.671 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:35:18.671 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=29, tag_tid=2, entry_tid=1], [object_number=28, tag_tid=2, entry_tid=0], [object_number=31, tag_tid=1, entry_tid=31819], [object_number=30, tag_tid=1, entry_tid=31818]]], state=connected] 2026-02-18T08:35:18.672 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=31, tag_tid=1, entry_tid=31819], [object_number=30, tag_tid=1, entry_tid=31818], [object_number=29, tag_tid=1, entry_tid=31817], [object_number=28, tag_tid=1, entry_tid=31816]]], state=connected] 2026-02-18T08:35:18.674 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_4.status 2026-02-18T08:35:18.676 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=31, tag_tid=1, entry_tid=31819' 2026-02-18T08:35:18.676 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=29, tag_tid=2, entry_tid=1' -a 'object_number=29, tag_tid=2, entry_tid=1' = 'object_number=31, tag_tid=1, entry_tid=31819' 2026-02-18T08:35:18.676 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=31, tag_tid=1, entry_tid=31819' '!=' 'object_number=31, tag_tid=1, entry_tid=31819' 2026-02-18T08:35:18.676 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:35:18.676 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.8 2026-02-18T08:35:19.478 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_4 2026-02-18T08:35:19.478 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:35:19.478 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:35:19.478 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_4 2026-02-18T08:35:19.478 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:35:19.478 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_4 ']' 2026-02-18T08:35:19.479 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_4' 2026-02-18T08:35:19.479 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_4 2026-02-18T08:35:19.479 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:35:19.479 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:35:19.479 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:35:19.479 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:35:19.479 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:35:19.479 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:35:19.479 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:35:19.479 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:35:19.482 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:35:19.482 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:35:19.482 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_4 2026-02-18T08:35:19.482 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:35:19.482 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:35:19.482 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:35:19.482 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:35:19.483 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:35:19.483 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:35:19.483 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:35:19.483 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:35:19.483 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:35:19.483 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:35:19.483 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:35:19.483 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:35:19.483 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:35:19.483 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:35:19.483 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:35:19.483 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:35:19.484 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:35:19.484 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:35:19.484 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:35:19.484 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:35:19.484 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:35:19.484 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:35:19.484 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:35:19.484 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:35:19.484 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:35:19.484 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:35:19.484 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:35:19.484 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:35:19.485 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:35:19.485 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:35:19.492 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:35:19.492 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:35:19.493 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:35:19.493 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_4 2026-02-18T08:35:19.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:no valid command found; 2 closest matches: 2026-02-18T08:35:19.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush mirror/image_43 2026-02-18T08:35:19.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush mirror/image_46 2026-02-18T08:35:19.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:35:19.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:35:19.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_4 2026-02-18T08:35:19.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:35:19.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:35:19.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:35:19.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:35:19.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:35:19.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:35:19.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:35:19.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:35:19.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:35:19.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:35:19.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:35:19.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:35:19.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:35:19.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:35:19.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:35:19.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:35:19.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:35:19.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:35:19.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:35:19.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:35:19.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:35:19.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:35:19.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:35:19.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:35:19.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:35:19.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:35:19.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:35:19.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:35:19.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:35:19.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:35:19.560 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:35:19.560 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:35:19.560 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:35:19.561 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_4 2026-02-18T08:35:19.622 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:35:19.623 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_4 2026-02-18T08:35:19.623 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:35:19.623 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:35:19.623 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_4 2026-02-18T08:35:19.623 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_4 '' 2026-02-18T08:35:19.623 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:35:19.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:35:19.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_4 2026-02-18T08:35:19.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:35:19.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_4.status 2026-02-18T08:35:19.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_4.status 2026-02-18T08:35:19.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:35:19.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_4.status 2026-02-18T08:35:19.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_4 2026-02-18T08:35:19.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_4.status 2026-02-18T08:35:19.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 7 2026-02-18T08:35:19.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 7 2026-02-18T08:35:19.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:35:19.653 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=29, tag_tid=2, entry_tid=1], [object_number=28, tag_tid=2, entry_tid=0], [object_number=31, tag_tid=1, entry_tid=31819], [object_number=30, tag_tid=1, entry_tid=31818]]], state=connected] 2026-02-18T08:35:19.653 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=29, tag_tid=2, entry_tid=1], [object_number=28, tag_tid=2, entry_tid=0], [object_number=31, tag_tid=1, entry_tid=31819], [object_number=30, tag_tid=1, entry_tid=31818]]], state=connected] 2026-02-18T08:35:19.655 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_4.status 2026-02-18T08:35:19.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=29, tag_tid=2, entry_tid=1' 2026-02-18T08:35:19.658 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_4 2026-02-18T08:35:19.658 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:35:19.658 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:35:19.658 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_4 2026-02-18T08:35:19.658 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_4 '..*' 2026-02-18T08:35:19.659 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:35:19.659 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:35:19.659 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_4 2026-02-18T08:35:19.659 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:35:19.659 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_4.status 2026-02-18T08:35:19.659 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_4.status 2026-02-18T08:35:19.660 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:35:19.660 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_4.status 2026-02-18T08:35:19.660 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_4 2026-02-18T08:35:19.661 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_4.status 2026-02-18T08:35:19.687 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 7 2026-02-18T08:35:19.687 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 7 2026-02-18T08:35:19.687 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:35:19.687 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=29, tag_tid=2, entry_tid=1], [object_number=28, tag_tid=2, entry_tid=0], [object_number=31, tag_tid=1, entry_tid=31819], [object_number=30, tag_tid=1, entry_tid=31818]]], state=connected] 2026-02-18T08:35:19.687 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=29, tag_tid=2, entry_tid=1], [object_number=28, tag_tid=2, entry_tid=0], [object_number=31, tag_tid=1, entry_tid=31819], [object_number=30, tag_tid=1, entry_tid=31818]]], state=connected] 2026-02-18T08:35:19.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_4.status 2026-02-18T08:35:19.691 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=29, tag_tid=2, entry_tid=1' 2026-02-18T08:35:19.692 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=29, tag_tid=2, entry_tid=1' -a 'object_number=29, tag_tid=2, entry_tid=1' = 'object_number=29, tag_tid=2, entry_tid=1' 2026-02-18T08:35:19.692 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:35:19.692 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_snap_present cluster1 mirror image_4 snap 2026-02-18T08:35:19.692 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:35:19.692 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:35:19.692 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_4 2026-02-18T08:35:19.692 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap 2026-02-18T08:35:19.692 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-18T08:35:19.692 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-18T08:35:19.692 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1 2026-02-18T08:35:20.693 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster1 info mirror/image_4@snap 2026-02-18T08:35:20.722 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:rbd image 'image_4': 2026-02-18T08:35:20.722 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: size 128 MiB in 32 objects 2026-02-18T08:35:20.722 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: order 22 (4 MiB objects) 2026-02-18T08:35:20.722 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: snapshot_count: 1 2026-02-18T08:35:20.722 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: id: 11c0fbc1a5e6 2026-02-18T08:35:20.722 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: block_name_prefix: rbd_data.11c0fbc1a5e6 2026-02-18T08:35:20.722 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: format: 2 2026-02-18T08:35:20.722 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: features: layering, exclusive-lock, journaling 2026-02-18T08:35:20.722 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: op_features: 2026-02-18T08:35:20.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: flags: 2026-02-18T08:35:20.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: create_timestamp: Wed Feb 18 08:26:14 2026 2026-02-18T08:35:20.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: access_timestamp: Wed Feb 18 08:26:14 2026 2026-02-18T08:35:20.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: modify_timestamp: Wed Feb 18 08:27:23 2026 2026-02-18T08:35:20.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: protected: False 2026-02-18T08:35:20.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: journal: 11c0fbc1a5e6 2026-02-18T08:35:20.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring state: enabled 2026-02-18T08:35:20.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring mode: journal 2026-02-18T08:35:20.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring global id: e63f9967-b4a3-4e94-9111-2e2fc5eb02bc 2026-02-18T08:35:20.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring primary: false 2026-02-18T08:35:20.725 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:35:20.725 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ compare_image_snaps mirror image_4 snap 2026-02-18T08:35:20.725 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:35:20.725 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_4 2026-02-18T08:35:20.725 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap 2026-02-18T08:35:20.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local ret=0 2026-02-18T08:35:20.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local rmt_export=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_4.export 2026-02-18T08:35:20.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loc_export=/tmp/tmp.vybh1m9yJF/cluster1-mirror-image_4.export 2026-02-18T08:35:20.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_4.export /tmp/tmp.vybh1m9yJF/cluster1-mirror-image_4.export 2026-02-18T08:35:20.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 -p mirror export image_4@snap /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_4.export 2026-02-18T08:35:20.864 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-18T08:35:20.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster1 -p mirror export image_4@snap /tmp/tmp.vybh1m9yJF/cluster1-mirror-image_4.export 2026-02-18T08:35:21.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-18T08:35:21.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmp /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_4.export /tmp/tmp.vybh1m9yJF/cluster1-mirror-image_4.export 2026-02-18T08:35:21.118 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_4.export /tmp/tmp.vybh1m9yJF/cluster1-mirror-image_4.export 2026-02-18T08:35:21.163 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:35:21.163 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T08:35:21.163 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_5 2026-02-18T08:35:21.163 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_snap cluster2 mirror image_5 snap 2026-02-18T08:35:21.164 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:35:21.164 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:35:21.164 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_5 2026-02-18T08:35:21.164 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap 2026-02-18T08:35:21.164 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 -p mirror snap create image_5@snap --debug-rbd=20 --debug-journaler=20 2026-02-18T08:35:22.055 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:35:22.050+0000 7faffc95c640 -1 reset not still connected to 0x5644b7e821a0 2026-02-18T08:35:22.056 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:35:22.054+0000 7faffc95c640 -1 reset not still connected to 0x5644b7e83860 2026-02-18T08:35:22.056 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:35:22.054+0000 7faffc95c640 -1 reset not still connected to 0x5644c8fcfe10 2026-02-18T08:35:22.056 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:35:22.054+0000 7faffc95c640 -1 reset not still connected to 0x5644d460c1a0 2026-02-18T08:35:22.056 INFO:tasks.ceph.cluster1.osd.0.trial034.stderr:2026-02-18T08:35:22.054+0000 7faffc95c640 -1 reset not still connected to 0x5644d9b51ba0 2026-02-18T08:35:25.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_replay_started cluster1 mirror image_5 2026-02-18T08:35:25.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:35:25.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:35:25.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_5 2026-02-18T08:35:25.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_replay_state cluster1 mirror image_5 started 2026-02-18T08:35:25.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:35:25.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:35:25.074 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_5 2026-02-18T08:35:25.074 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local state=started 2026-02-18T08:35:25.074 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-18T08:35:25.074 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:35:25.074 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1 2026-02-18T08:35:26.074 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test_image_replay_state cluster1 mirror image_5 started 2026-02-18T08:35:26.074 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:35:26.074 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:35:26.074 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_5 2026-02-18T08:35:26.075 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local test_state=started 2026-02-18T08:35:26.075 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local status_result 2026-02-18T08:35:26.075 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_state=stopped 2026-02-18T08:35:26.076 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_5 2026-02-18T08:35:26.076 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -i state 2026-02-18T08:35:26.076 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_instance=cluster1 2026-02-18T08:35:26.076 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-18T08:35:26.076 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:35:26.076 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance=cluster1 2026-02-18T08:35:26.076 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local loop_instance 2026-02-18T08:35:26.076 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:35:26.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sleep 0 2026-02-18T08:35:26.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:35:26.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ seq 0 3 2026-02-18T08:35:26.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:35:26.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_5 2026-02-18T08:35:26.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:0 2026-02-18T08:35:26.080 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-18T08:35:26.080 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:35:26.080 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:35:26.080 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:35:26.080 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-18T08:35:26.080 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:35:26.080 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:35:26.080 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:35:26.080 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:35:26.080 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-18T08:35:26.080 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:35:26.080 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:35:26.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:35:26.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:35:26.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:0 2026-02-18T08:35:26.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-18T08:35:26.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-18T08:35:26.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:35:26.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:0 2026-02-18T08:35:26.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T08:35:26.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-18T08:35:26.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:35:26.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:35:26.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-18T08:35:26.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-18T08:35:26.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-18T08:35:26.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=0 2026-02-18T08:35:26.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=0 2026-02-18T08:35:26.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:35:26.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:35:26.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:35:26.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:35:26.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_5 2026-02-18T08:35:26.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:no valid command found; 1 closest matches: 2026-02-18T08:35:26.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status mirror/image_50 2026-02-18T08:35:26.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:35:26.157 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:35:26.158 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_5 2026-02-18T08:35:26.158 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:1 2026-02-18T08:35:26.158 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-18T08:35:26.158 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:35:26.158 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:35:26.158 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:35:26.158 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-18T08:35:26.158 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:35:26.158 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:35:26.158 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:35:26.158 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:35:26.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-18T08:35:26.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:35:26.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:35:26.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:35:26.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:35:26.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:1 2026-02-18T08:35:26.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-18T08:35:26.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-18T08:35:26.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:35:26.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:1 2026-02-18T08:35:26.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T08:35:26.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-18T08:35:26.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:35:26.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:35:26.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-18T08:35:26.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-18T08:35:26.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-18T08:35:26.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=1 2026-02-18T08:35:26.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=1 2026-02-18T08:35:26.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:35:26.167 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:35:26.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:35:26.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:35:26.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_5 2026-02-18T08:35:26.222 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_5'] 2026-02-18T08:35:26.223 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status : get status for rbd mirror 2026-02-18T08:35:26.223 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:35:26.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:35:26.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_5 2026-02-18T08:35:26.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:2 2026-02-18T08:35:26.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-18T08:35:26.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:35:26.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:35:26.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:35:26.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-18T08:35:26.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:35:26.227 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:35:26.227 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:35:26.227 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:35:26.227 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-18T08:35:26.227 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:35:26.227 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:35:26.227 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:35:26.227 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:35:26.227 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:2 2026-02-18T08:35:26.227 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-18T08:35:26.227 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-18T08:35:26.227 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:35:26.227 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:2 2026-02-18T08:35:26.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T08:35:26.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-18T08:35:26.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:35:26.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:35:26.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-18T08:35:26.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-18T08:35:26.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-18T08:35:26.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=2 2026-02-18T08:35:26.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=2 2026-02-18T08:35:26.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:35:26.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:35:26.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:35:26.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:35:26.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_5 2026-02-18T08:35:26.298 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_5'] 2026-02-18T08:35:26.298 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status : get status for rbd mirror 2026-02-18T08:35:26.298 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:35:26.301 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:35:26.301 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/image_5 2026-02-18T08:35:26.301 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:3 2026-02-18T08:35:26.301 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-18T08:35:26.301 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:35:26.301 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:35:26.301 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:35:26.301 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-18T08:35:26.301 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:35:26.301 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:35:26.302 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:35:26.302 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:35:26.302 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-18T08:35:26.302 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:35:26.302 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:35:26.302 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:35:26.302 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:35:26.302 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:3 2026-02-18T08:35:26.302 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-18T08:35:26.302 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-18T08:35:26.302 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:35:26.302 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:3 2026-02-18T08:35:26.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T08:35:26.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-18T08:35:26.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:35:26.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:35:26.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-18T08:35:26.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-18T08:35:26.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-18T08:35:26.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=3 2026-02-18T08:35:26.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=3 2026-02-18T08:35:26.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:35:26.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:35:26.312 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:35:26.312 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:35:26.312 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/image_5 2026-02-18T08:35:26.374 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ return 0 2026-02-18T08:35:26.375 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ status_result=' "state": "Replaying"' 2026-02-18T08:35:26.376 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ echo ' "state": "Replaying"' 2026-02-18T08:35:26.376 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ grep -i Replaying 2026-02-18T08:35:26.377 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: "state": "Replaying" 2026-02-18T08:35:26.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ current_state=started 2026-02-18T08:35:26.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test started = started 2026-02-18T08:35:26.379 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:35:26.379 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror image_5 2026-02-18T08:35:26.379 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_cluster=cluster1 2026-02-18T08:35:26.379 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:35:26.379 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_pool=mirror 2026-02-18T08:35:26.379 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local remote_pool=mirror 2026-02-18T08:35:26.379 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_5 2026-02-18T08:35:26.379 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' journal = journal ']' 2026-02-18T08:35:26.379 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror mirror image_5 2026-02-18T08:35:26.379 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_cluster=cluster1 2026-02-18T08:35:26.379 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:35:26.380 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_pool=mirror 2026-02-18T08:35:26.380 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local remote_pool=mirror 2026-02-18T08:35:26.380 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_5 2026-02-18T08:35:26.380 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-02-18T08:35:26.380 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-02-18T08:35:26.380 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:35:26.380 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:35:26.380 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:35:26.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_5 2026-02-18T08:35:26.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:35:26.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:35:26.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_5 2026-02-18T08:35:26.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:35:26.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_5 ']' 2026-02-18T08:35:26.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_5' 2026-02-18T08:35:26.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_5 2026-02-18T08:35:26.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:35:26.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:35:26.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:35:26.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:35:26.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:35:26.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:35:26.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:35:26.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:35:26.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:35:26.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:35:26.584 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_5 2026-02-18T08:35:26.584 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:35:26.584 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:35:26.584 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:35:26.584 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:35:26.584 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:35:26.584 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:35:26.584 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:35:26.584 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:35:26.584 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:35:26.584 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:35:26.584 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:35:26.585 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:35:26.585 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:35:26.585 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:35:26.585 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:35:26.585 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:35:26.585 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:35:26.585 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:35:26.585 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:35:26.585 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:35:26.585 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:35:26.585 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:35:26.585 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:35:26.586 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:35:26.586 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:35:26.586 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:35:26.586 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:35:26.586 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:35:26.586 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:35:26.586 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:35:26.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:35:26.594 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:35:26.594 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:35:26.594 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_5 2026-02-18T08:35:26.661 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:no valid command found; 1 closest matches: 2026-02-18T08:35:26.661 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush mirror/image_50 2026-02-18T08:35:26.661 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:35:26.664 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:35:26.664 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_5 2026-02-18T08:35:26.664 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:35:26.664 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:35:26.664 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:35:26.664 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:35:26.664 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:35:26.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:35:26.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:35:26.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:35:26.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:35:26.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:35:26.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:35:26.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:35:26.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:35:26.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:35:26.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:35:26.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:35:26.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:35:26.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:35:26.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:35:26.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:35:26.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:35:26.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:35:26.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:35:26.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:35:26.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:35:26.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:35:26.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:35:26.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:35:26.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:35:26.667 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:35:26.675 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:35:26.675 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:35:26.675 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:35:26.675 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_5 2026-02-18T08:35:26.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_5'] 2026-02-18T08:35:26.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:35:26.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:35:26.729 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:35:26.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_5 2026-02-18T08:35:26.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:35:26.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:35:26.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:35:26.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:35:26.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:35:26.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:35:26.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:35:26.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:35:26.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:35:26.731 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:35:26.731 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:35:26.731 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:35:26.731 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:35:26.731 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:35:26.731 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:35:26.731 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:35:26.731 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:35:26.731 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:35:26.731 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:35:26.731 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:35:26.731 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:35:26.732 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:35:26.732 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:35:26.732 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:35:26.732 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:35:26.732 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:35:26.732 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:35:26.732 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:35:26.732 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:35:26.732 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:35:26.740 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:35:26.740 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:35:26.740 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:35:26.740 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_5 2026-02-18T08:35:26.800 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_5'] 2026-02-18T08:35:26.800 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:35:26.800 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:35:26.801 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:35:26.801 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/image_5 2026-02-18T08:35:26.801 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:35:26.801 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:35:26.801 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:35:26.801 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:35:26.801 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:35:26.801 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:35:26.801 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:35:26.802 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:35:26.802 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:35:26.802 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:35:26.802 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:35:26.802 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:35:26.802 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:35:26.802 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:35:26.802 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:35:26.802 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:35:26.802 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:35:26.802 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:35:26.802 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:35:26.802 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:35:26.802 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:35:26.802 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:35:26.802 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:35:26.802 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:35:26.802 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:35:26.802 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:35:26.802 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:35:26.803 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:35:26.803 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:35:26.803 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:35:26.812 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:35:26.813 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:35:26.813 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:35:26.813 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/image_5 2026-02-18T08:35:26.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:35:26.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_5 2026-02-18T08:35:26.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:35:26.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:35:26.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_5 2026-02-18T08:35:26.874 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_5 '' 2026-02-18T08:35:26.874 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:35:26.874 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:35:26.874 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_5 2026-02-18T08:35:26.874 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:35:26.874 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_5.status 2026-02-18T08:35:26.874 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_5.status 2026-02-18T08:35:26.874 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:35:26.875 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_5.status 2026-02-18T08:35:26.875 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_5 2026-02-18T08:35:26.875 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_5.status 2026-02-18T08:35:26.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 10 2026-02-18T08:35:26.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 10 2026-02-18T08:35:26.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:35:26.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=41, tag_tid=2, entry_tid=1], [object_number=40, tag_tid=2, entry_tid=0], [object_number=42, tag_tid=1, entry_tid=45754], [object_number=43, tag_tid=1, entry_tid=45751]]], state=connected] 2026-02-18T08:35:26.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=42, tag_tid=1, entry_tid=45754], [object_number=41, tag_tid=1, entry_tid=45753], [object_number=40, tag_tid=1, entry_tid=45752], [object_number=43, tag_tid=1, entry_tid=45751]]], state=connected] 2026-02-18T08:35:26.905 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_5.status 2026-02-18T08:35:26.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=41, tag_tid=2, entry_tid=1' 2026-02-18T08:35:26.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_5 2026-02-18T08:35:26.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:35:26.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:35:26.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_5 2026-02-18T08:35:26.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_5 '..*' 2026-02-18T08:35:26.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:35:26.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:35:26.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_5 2026-02-18T08:35:26.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:35:26.909 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_5.status 2026-02-18T08:35:26.909 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_5.status 2026-02-18T08:35:26.909 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:35:26.910 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_5.status 2026-02-18T08:35:26.910 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_5 2026-02-18T08:35:26.910 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_5.status 2026-02-18T08:35:26.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 10 2026-02-18T08:35:26.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 10 2026-02-18T08:35:26.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:35:26.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=41, tag_tid=2, entry_tid=1], [object_number=40, tag_tid=2, entry_tid=0], [object_number=42, tag_tid=1, entry_tid=45754], [object_number=43, tag_tid=1, entry_tid=45751]]], state=connected] 2026-02-18T08:35:26.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=42, tag_tid=1, entry_tid=45754], [object_number=41, tag_tid=1, entry_tid=45753], [object_number=40, tag_tid=1, entry_tid=45752], [object_number=43, tag_tid=1, entry_tid=45751]]], state=connected] 2026-02-18T08:35:26.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_5.status 2026-02-18T08:35:26.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=42, tag_tid=1, entry_tid=45754' 2026-02-18T08:35:26.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=41, tag_tid=2, entry_tid=1' -a 'object_number=41, tag_tid=2, entry_tid=1' = 'object_number=42, tag_tid=1, entry_tid=45754' 2026-02-18T08:35:26.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=42, tag_tid=1, entry_tid=45754' '!=' '' 2026-02-18T08:35:26.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:35:26.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=42, tag_tid=1, entry_tid=45754' = '' 2026-02-18T08:35:26.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=42, tag_tid=1, entry_tid=45754' 2026-02-18T08:35:26.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=41, tag_tid=2, entry_tid=1' 2026-02-18T08:35:26.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:35:26.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:35:26.944 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=2 2026-02-18T08:35:26.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=42, tag_tid=1, entry_tid=45754' 2026-02-18T08:35:26.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:35:26.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:35:26.947 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=1 2026-02-18T08:35:26.947 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=41, tag_tid=2, entry_tid=1' 2026-02-18T08:35:26.948 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:35:26.948 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:35:26.949 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:35:26.949 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=42, tag_tid=1, entry_tid=45754' 2026-02-18T08:35:26.950 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:35:26.950 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:35:26.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=45754 2026-02-18T08:35:26.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 2 = 1 -a 1 -le 45754 2026-02-18T08:35:26.952 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:35:26.952 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:35:26.952 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:35:27.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_5 2026-02-18T08:35:27.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:35:27.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:35:27.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_5 2026-02-18T08:35:27.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:35:27.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_5 ']' 2026-02-18T08:35:27.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_5' 2026-02-18T08:35:27.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_5 2026-02-18T08:35:27.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:35:27.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:35:27.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:35:27.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:35:27.155 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:35:27.155 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:35:27.155 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:35:27.155 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:35:27.158 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:35:27.158 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:35:27.158 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_5 2026-02-18T08:35:27.158 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:35:27.158 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:35:27.158 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:35:27.158 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:35:27.158 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:35:27.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:35:27.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:35:27.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:35:27.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:35:27.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:35:27.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:35:27.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:35:27.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:35:27.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:35:27.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:35:27.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:35:27.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:35:27.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:35:27.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:35:27.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:35:27.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:35:27.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:35:27.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:35:27.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:35:27.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:35:27.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:35:27.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:35:27.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:35:27.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:35:27.161 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:35:27.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:35:27.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:35:27.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:35:27.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_5 2026-02-18T08:35:27.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:no valid command found; 1 closest matches: 2026-02-18T08:35:27.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush mirror/image_50 2026-02-18T08:35:27.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:35:27.231 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:35:27.231 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_5 2026-02-18T08:35:27.231 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:35:27.231 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:35:27.231 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:35:27.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:35:27.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:35:27.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:35:27.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:35:27.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:35:27.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:35:27.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:35:27.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:35:27.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:35:27.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:35:27.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:35:27.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:35:27.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:35:27.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:35:27.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:35:27.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:35:27.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:35:27.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:35:27.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:35:27.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:35:27.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:35:27.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:35:27.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:35:27.234 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:35:27.234 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:35:27.234 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:35:27.234 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:35:27.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:35:27.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:35:27.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:35:27.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_5 2026-02-18T08:35:27.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_5'] 2026-02-18T08:35:27.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:35:27.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:35:27.299 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:35:27.299 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_5 2026-02-18T08:35:27.299 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:35:27.299 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:35:27.299 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:35:27.299 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:35:27.299 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:35:27.299 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:35:27.300 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:35:27.300 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:35:27.300 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:35:27.300 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:35:27.300 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:35:27.300 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:35:27.300 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:35:27.300 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:35:27.300 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:35:27.300 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:35:27.300 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:35:27.300 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:35:27.301 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:35:27.301 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:35:27.301 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:35:27.301 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:35:27.301 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:35:27.301 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:35:27.301 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:35:27.301 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:35:27.301 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:35:27.301 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:35:27.301 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:35:27.301 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:35:27.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:35:27.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:35:27.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:35:27.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_5 2026-02-18T08:35:27.368 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_5'] 2026-02-18T08:35:27.368 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:35:27.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:35:27.371 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:35:27.371 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/image_5 2026-02-18T08:35:27.372 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:35:27.372 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:35:27.372 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:35:27.372 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:35:27.372 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:35:27.372 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:35:27.372 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:35:27.372 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:35:27.372 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:35:27.372 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:35:27.372 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:35:27.373 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:35:27.373 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:35:27.373 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:35:27.373 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:35:27.373 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:35:27.373 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:35:27.373 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:35:27.373 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:35:27.373 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:35:27.373 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:35:27.373 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:35:27.373 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:35:27.374 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:35:27.374 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:35:27.374 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:35:27.374 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:35:27.374 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:35:27.374 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:35:27.374 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:35:27.383 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:35:27.383 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:35:27.383 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:35:27.383 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/image_5 2026-02-18T08:35:27.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:35:27.443 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_5 2026-02-18T08:35:27.443 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:35:27.443 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:35:27.443 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_5 2026-02-18T08:35:27.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_5 '' 2026-02-18T08:35:27.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:35:27.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:35:27.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_5 2026-02-18T08:35:27.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:35:27.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_5.status 2026-02-18T08:35:27.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_5.status 2026-02-18T08:35:27.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:35:27.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_5.status 2026-02-18T08:35:27.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_5 2026-02-18T08:35:27.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_5.status 2026-02-18T08:35:27.472 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 10 2026-02-18T08:35:27.472 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 10 2026-02-18T08:35:27.472 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:35:27.472 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=41, tag_tid=2, entry_tid=1], [object_number=40, tag_tid=2, entry_tid=0], [object_number=42, tag_tid=1, entry_tid=45754], [object_number=43, tag_tid=1, entry_tid=45751]]], state=connected] 2026-02-18T08:35:27.472 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=42, tag_tid=1, entry_tid=45754], [object_number=41, tag_tid=1, entry_tid=45753], [object_number=40, tag_tid=1, entry_tid=45752], [object_number=43, tag_tid=1, entry_tid=45751]]], state=connected] 2026-02-18T08:35:27.474 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_5.status 2026-02-18T08:35:27.476 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=41, tag_tid=2, entry_tid=1' 2026-02-18T08:35:27.477 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_5 2026-02-18T08:35:27.477 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:35:27.477 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:35:27.477 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_5 2026-02-18T08:35:27.477 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_5 '..*' 2026-02-18T08:35:27.478 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:35:27.478 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:35:27.478 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_5 2026-02-18T08:35:27.478 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:35:27.478 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_5.status 2026-02-18T08:35:27.478 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_5.status 2026-02-18T08:35:27.479 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:35:27.479 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_5.status 2026-02-18T08:35:27.479 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_5 2026-02-18T08:35:27.479 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_5.status 2026-02-18T08:35:27.513 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 10 2026-02-18T08:35:27.513 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 10 2026-02-18T08:35:27.513 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:35:27.513 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=41, tag_tid=2, entry_tid=1], [object_number=40, tag_tid=2, entry_tid=0], [object_number=42, tag_tid=1, entry_tid=45754], [object_number=43, tag_tid=1, entry_tid=45751]]], state=connected] 2026-02-18T08:35:27.513 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=42, tag_tid=1, entry_tid=45754], [object_number=41, tag_tid=1, entry_tid=45753], [object_number=40, tag_tid=1, entry_tid=45752], [object_number=43, tag_tid=1, entry_tid=45751]]], state=connected] 2026-02-18T08:35:27.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_5.status 2026-02-18T08:35:27.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=42, tag_tid=1, entry_tid=45754' 2026-02-18T08:35:27.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=41, tag_tid=2, entry_tid=1' -a 'object_number=41, tag_tid=2, entry_tid=1' = 'object_number=42, tag_tid=1, entry_tid=45754' 2026-02-18T08:35:27.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=42, tag_tid=1, entry_tid=45754' '!=' 'object_number=42, tag_tid=1, entry_tid=45754' 2026-02-18T08:35:27.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:35:27.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.4 2026-02-18T08:35:27.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_5 2026-02-18T08:35:27.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:35:27.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:35:27.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_5 2026-02-18T08:35:27.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:35:27.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_5 ']' 2026-02-18T08:35:27.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_5' 2026-02-18T08:35:27.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_5 2026-02-18T08:35:27.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:35:27.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:35:27.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:35:27.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:35:27.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:35:27.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:35:27.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:35:27.923 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:35:27.923 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:35:27.923 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:35:27.923 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_5 2026-02-18T08:35:27.923 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:35:27.923 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:35:27.923 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:35:27.923 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:35:27.923 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:35:27.923 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:35:27.923 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:35:27.923 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:35:27.923 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:35:27.924 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:35:27.924 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:35:27.924 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:35:27.924 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:35:27.924 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:35:27.924 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:35:27.924 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:35:27.924 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:35:27.924 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:35:27.924 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:35:27.924 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:35:27.924 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:35:27.924 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:35:27.924 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:35:27.924 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:35:27.924 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:35:27.924 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:35:27.924 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:35:27.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:35:27.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:35:27.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:35:27.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:35:27.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:35:27.936 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:35:27.936 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_5 2026-02-18T08:35:27.989 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:no valid command found; 1 closest matches: 2026-02-18T08:35:27.989 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush mirror/image_50 2026-02-18T08:35:27.989 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:35:27.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:35:27.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_5 2026-02-18T08:35:27.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:35:27.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:35:27.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:35:27.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:35:27.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:35:27.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:35:27.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:35:27.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:35:27.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:35:27.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:35:27.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:35:27.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:35:27.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:35:27.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:35:27.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:35:27.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:35:27.992 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:35:27.992 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:35:27.992 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:35:27.992 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:35:27.992 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:35:27.992 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:35:27.992 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:35:27.992 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:35:27.992 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:35:27.992 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:35:27.992 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:35:27.992 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:35:27.992 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:35:27.992 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:35:28.003 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:35:28.003 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:35:28.003 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:35:28.003 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_5 2026-02-18T08:35:28.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_5'] 2026-02-18T08:35:28.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:35:28.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:35:28.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:35:28.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_5 2026-02-18T08:35:28.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:35:28.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:35:28.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:35:28.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:35:28.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:35:28.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:35:28.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:35:28.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:35:28.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:35:28.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:35:28.067 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:35:28.067 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:35:28.067 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:35:28.067 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:35:28.067 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:35:28.067 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:35:28.067 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:35:28.067 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:35:28.067 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:35:28.067 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:35:28.067 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:35:28.067 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:35:28.068 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:35:28.068 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:35:28.068 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:35:28.068 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:35:28.068 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:35:28.068 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:35:28.068 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:35:28.068 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:35:28.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:35:28.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:35:28.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:35:28.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_5 2026-02-18T08:35:28.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_5'] 2026-02-18T08:35:28.136 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:35:28.136 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:35:28.138 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:35:28.138 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/image_5 2026-02-18T08:35:28.138 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:35:28.139 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:35:28.139 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:35:28.139 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:35:28.139 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:35:28.139 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:35:28.139 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:35:28.139 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:35:28.139 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:35:28.139 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:35:28.139 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:35:28.139 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:35:28.139 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:35:28.140 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:35:28.140 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:35:28.140 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:35:28.140 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:35:28.140 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:35:28.140 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:35:28.140 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:35:28.140 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:35:28.140 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:35:28.140 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:35:28.140 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:35:28.140 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:35:28.141 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:35:28.141 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:35:28.141 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:35:28.141 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:35:28.141 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:35:28.149 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:35:28.149 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:35:28.149 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:35:28.149 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/image_5 2026-02-18T08:35:28.207 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:35:28.208 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_5 2026-02-18T08:35:28.208 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:35:28.208 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:35:28.208 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_5 2026-02-18T08:35:28.208 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_5 '' 2026-02-18T08:35:28.208 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:35:28.209 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:35:28.209 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_5 2026-02-18T08:35:28.209 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:35:28.209 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_5.status 2026-02-18T08:35:28.209 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_5.status 2026-02-18T08:35:28.209 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:35:28.210 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_5.status 2026-02-18T08:35:28.210 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_5 2026-02-18T08:35:28.210 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_5.status 2026-02-18T08:35:28.246 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 10 2026-02-18T08:35:28.247 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 10 2026-02-18T08:35:28.247 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:35:28.247 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=41, tag_tid=2, entry_tid=1], [object_number=40, tag_tid=2, entry_tid=0], [object_number=42, tag_tid=1, entry_tid=45754], [object_number=43, tag_tid=1, entry_tid=45751]]], state=connected] 2026-02-18T08:35:28.247 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=42, tag_tid=1, entry_tid=45754], [object_number=41, tag_tid=1, entry_tid=45753], [object_number=40, tag_tid=1, entry_tid=45752], [object_number=43, tag_tid=1, entry_tid=45751]]], state=connected] 2026-02-18T08:35:28.250 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_5.status 2026-02-18T08:35:28.252 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=41, tag_tid=2, entry_tid=1' 2026-02-18T08:35:28.253 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_5 2026-02-18T08:35:28.253 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:35:28.253 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:35:28.253 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_5 2026-02-18T08:35:28.253 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_5 '..*' 2026-02-18T08:35:28.253 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:35:28.254 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:35:28.254 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_5 2026-02-18T08:35:28.254 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:35:28.254 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_5.status 2026-02-18T08:35:28.254 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_5.status 2026-02-18T08:35:28.254 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:35:28.255 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_5.status 2026-02-18T08:35:28.255 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_5 2026-02-18T08:35:28.255 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_5.status 2026-02-18T08:35:28.290 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 10 2026-02-18T08:35:28.290 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 10 2026-02-18T08:35:28.290 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:35:28.290 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=41, tag_tid=2, entry_tid=1], [object_number=40, tag_tid=2, entry_tid=0], [object_number=42, tag_tid=1, entry_tid=45754], [object_number=43, tag_tid=1, entry_tid=45751]]], state=connected] 2026-02-18T08:35:28.290 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=42, tag_tid=1, entry_tid=45754], [object_number=41, tag_tid=1, entry_tid=45753], [object_number=40, tag_tid=1, entry_tid=45752], [object_number=43, tag_tid=1, entry_tid=45751]]], state=connected] 2026-02-18T08:35:28.293 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_5.status 2026-02-18T08:35:28.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=42, tag_tid=1, entry_tid=45754' 2026-02-18T08:35:28.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=41, tag_tid=2, entry_tid=1' -a 'object_number=41, tag_tid=2, entry_tid=1' = 'object_number=42, tag_tid=1, entry_tid=45754' 2026-02-18T08:35:28.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=42, tag_tid=1, entry_tid=45754' '!=' 'object_number=42, tag_tid=1, entry_tid=45754' 2026-02-18T08:35:28.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:35:28.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.8 2026-02-18T08:35:29.097 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_5 2026-02-18T08:35:29.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:35:29.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:35:29.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_5 2026-02-18T08:35:29.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:35:29.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_5 ']' 2026-02-18T08:35:29.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_5' 2026-02-18T08:35:29.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_5 2026-02-18T08:35:29.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:35:29.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:35:29.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:35:29.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:35:29.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:35:29.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:35:29.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:35:29.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:35:29.102 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:35:29.102 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:35:29.102 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_5 2026-02-18T08:35:29.102 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:35:29.102 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:35:29.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:35:29.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:35:29.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:35:29.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:35:29.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:35:29.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:35:29.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:35:29.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:35:29.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:35:29.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:35:29.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:35:29.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:35:29.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:35:29.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:35:29.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:35:29.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:35:29.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:35:29.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:35:29.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:35:29.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:35:29.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:35:29.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:35:29.105 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:35:29.105 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:35:29.105 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:35:29.105 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:35:29.105 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:35:29.105 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:35:29.112 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:35:29.112 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:35:29.113 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:35:29.113 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_5 2026-02-18T08:35:29.173 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:no valid command found; 1 closest matches: 2026-02-18T08:35:29.173 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush mirror/image_50 2026-02-18T08:35:29.173 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:35:29.176 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:35:29.176 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_5 2026-02-18T08:35:29.176 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:35:29.176 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:35:29.176 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:35:29.177 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:35:29.177 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:35:29.177 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:35:29.177 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:35:29.177 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:35:29.177 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:35:29.177 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:35:29.177 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:35:29.177 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:35:29.177 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:35:29.177 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:35:29.177 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:35:29.178 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:35:29.178 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:35:29.178 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:35:29.178 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:35:29.178 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:35:29.178 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:35:29.178 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:35:29.178 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:35:29.178 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:35:29.178 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:35:29.178 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:35:29.178 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:35:29.178 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:35:29.179 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:35:29.179 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:35:29.187 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:35:29.188 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:35:29.188 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:35:29.188 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_5 2026-02-18T08:35:29.248 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_5'] 2026-02-18T08:35:29.248 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:35:29.249 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:35:29.252 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:35:29.252 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_5 2026-02-18T08:35:29.252 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:35:29.252 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:35:29.252 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:35:29.252 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:35:29.252 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:35:29.252 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:35:29.253 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:35:29.253 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:35:29.253 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:35:29.253 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:35:29.253 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:35:29.253 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:35:29.253 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:35:29.253 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:35:29.253 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:35:29.253 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:35:29.253 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:35:29.254 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:35:29.254 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:35:29.254 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:35:29.254 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:35:29.254 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:35:29.254 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:35:29.254 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:35:29.254 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:35:29.254 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:35:29.254 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:35:29.254 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:35:29.255 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:35:29.255 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:35:29.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:35:29.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:35:29.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:35:29.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_5 2026-02-18T08:35:29.315 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_5'] 2026-02-18T08:35:29.316 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:35:29.316 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:35:29.318 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:35:29.319 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/image_5 2026-02-18T08:35:29.319 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:35:29.319 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:35:29.319 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:35:29.319 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:35:29.319 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:35:29.319 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:35:29.319 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:35:29.319 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:35:29.319 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:35:29.319 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:35:29.320 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:35:29.320 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:35:29.320 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:35:29.320 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:35:29.320 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:35:29.320 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:35:29.320 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:35:29.320 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:35:29.320 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:35:29.320 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:35:29.320 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:35:29.321 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:35:29.321 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:35:29.321 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:35:29.321 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:35:29.321 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:35:29.321 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:35:29.321 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:35:29.321 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:35:29.321 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:35:29.329 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:35:29.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:35:29.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:35:29.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/image_5 2026-02-18T08:35:29.399 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:35:29.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_5 2026-02-18T08:35:29.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:35:29.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:35:29.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_5 2026-02-18T08:35:29.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_5 '' 2026-02-18T08:35:29.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:35:29.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:35:29.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_5 2026-02-18T08:35:29.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:35:29.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_5.status 2026-02-18T08:35:29.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_5.status 2026-02-18T08:35:29.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:35:29.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_5.status 2026-02-18T08:35:29.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_5 2026-02-18T08:35:29.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_5.status 2026-02-18T08:35:29.429 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 10 2026-02-18T08:35:29.429 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 10 2026-02-18T08:35:29.429 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:35:29.429 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=41, tag_tid=2, entry_tid=1], [object_number=40, tag_tid=2, entry_tid=0], [object_number=42, tag_tid=1, entry_tid=45754], [object_number=43, tag_tid=1, entry_tid=45751]]], state=connected] 2026-02-18T08:35:29.429 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=42, tag_tid=1, entry_tid=45754], [object_number=41, tag_tid=1, entry_tid=45753], [object_number=40, tag_tid=1, entry_tid=45752], [object_number=43, tag_tid=1, entry_tid=45751]]], state=connected] 2026-02-18T08:35:29.432 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_5.status 2026-02-18T08:35:29.433 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=41, tag_tid=2, entry_tid=1' 2026-02-18T08:35:29.434 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_5 2026-02-18T08:35:29.434 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:35:29.435 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:35:29.435 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_5 2026-02-18T08:35:29.435 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_5 '..*' 2026-02-18T08:35:29.435 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:35:29.435 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:35:29.435 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_5 2026-02-18T08:35:29.435 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:35:29.435 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_5.status 2026-02-18T08:35:29.436 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_5.status 2026-02-18T08:35:29.436 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:35:29.436 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_5.status 2026-02-18T08:35:29.437 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_5 2026-02-18T08:35:29.437 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_5.status 2026-02-18T08:35:29.472 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 10 2026-02-18T08:35:29.472 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 10 2026-02-18T08:35:29.472 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:35:29.472 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=41, tag_tid=2, entry_tid=1], [object_number=40, tag_tid=2, entry_tid=0], [object_number=42, tag_tid=1, entry_tid=45754], [object_number=43, tag_tid=1, entry_tid=45751]]], state=connected] 2026-02-18T08:35:29.472 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=42, tag_tid=1, entry_tid=45754], [object_number=41, tag_tid=1, entry_tid=45753], [object_number=40, tag_tid=1, entry_tid=45752], [object_number=43, tag_tid=1, entry_tid=45751]]], state=connected] 2026-02-18T08:35:29.475 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_5.status 2026-02-18T08:35:29.477 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=42, tag_tid=1, entry_tid=45754' 2026-02-18T08:35:29.478 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=41, tag_tid=2, entry_tid=1' -a 'object_number=41, tag_tid=2, entry_tid=1' = 'object_number=42, tag_tid=1, entry_tid=45754' 2026-02-18T08:35:29.478 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=42, tag_tid=1, entry_tid=45754' '!=' 'object_number=42, tag_tid=1, entry_tid=45754' 2026-02-18T08:35:29.478 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:35:29.478 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1.6 2026-02-18T08:35:31.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_5 2026-02-18T08:35:31.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:35:31.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:35:31.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_5 2026-02-18T08:35:31.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:35:31.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_5 ']' 2026-02-18T08:35:31.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_5' 2026-02-18T08:35:31.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_5 2026-02-18T08:35:31.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:35:31.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:35:31.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:35:31.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:35:31.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:35:31.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:35:31.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:35:31.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:35:31.080 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:35:31.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:35:31.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_5 2026-02-18T08:35:31.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:35:31.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:35:31.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:35:31.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:35:31.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:35:31.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:35:31.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:35:31.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:35:31.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:35:31.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:35:31.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:35:31.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:35:31.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:35:31.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:35:31.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:35:31.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:35:31.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:35:31.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:35:31.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:35:31.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:35:31.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:35:31.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:35:31.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:35:31.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:35:31.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:35:31.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:35:31.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:35:31.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:35:31.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:35:31.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:35:31.093 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:35:31.094 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:35:31.094 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:35:31.094 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_5 2026-02-18T08:35:31.157 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:no valid command found; 1 closest matches: 2026-02-18T08:35:31.157 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush mirror/image_50 2026-02-18T08:35:31.157 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:35:31.158 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:35:31.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_5 2026-02-18T08:35:31.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:35:31.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:35:31.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:35:31.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:35:31.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:35:31.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:35:31.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:35:31.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:35:31.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:35:31.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:35:31.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:35:31.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:35:31.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:35:31.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:35:31.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:35:31.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:35:31.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:35:31.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:35:31.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:35:31.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:35:31.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:35:31.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:35:31.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:35:31.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:35:31.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:35:31.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:35:31.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:35:31.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:35:31.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:35:31.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:35:31.171 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:35:31.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:35:31.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:35:31.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_5 2026-02-18T08:35:31.224 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_5'] 2026-02-18T08:35:31.224 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:35:31.224 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:35:31.225 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:35:31.225 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_5 2026-02-18T08:35:31.225 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:35:31.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:35:31.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:35:31.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:35:31.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:35:31.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:35:31.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:35:31.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:35:31.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:35:31.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:35:31.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:35:31.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:35:31.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:35:31.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:35:31.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:35:31.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:35:31.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:35:31.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:35:31.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:35:31.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:35:31.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:35:31.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:35:31.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:35:31.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:35:31.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:35:31.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:35:31.227 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:35:31.227 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:35:31.227 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:35:31.227 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:35:31.238 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:35:31.238 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:35:31.238 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:35:31.238 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_5 2026-02-18T08:35:31.291 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_5'] 2026-02-18T08:35:31.291 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:35:31.291 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:35:31.293 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:35:31.294 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/image_5 2026-02-18T08:35:31.294 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:35:31.294 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:35:31.294 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:35:31.294 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:35:31.294 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:35:31.294 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:35:31.294 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:35:31.294 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:35:31.294 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:35:31.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:35:31.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:35:31.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:35:31.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:35:31.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:35:31.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:35:31.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:35:31.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:35:31.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:35:31.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:35:31.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:35:31.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:35:31.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:35:31.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:35:31.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:35:31.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:35:31.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:35:31.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:35:31.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:35:31.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:35:31.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:35:31.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:35:31.304 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:35:31.304 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:35:31.304 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/image_5 2026-02-18T08:35:31.373 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:35:31.373 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_5 2026-02-18T08:35:31.373 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:35:31.373 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:35:31.373 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_5 2026-02-18T08:35:31.374 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_5 '' 2026-02-18T08:35:31.374 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:35:31.374 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:35:31.374 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_5 2026-02-18T08:35:31.374 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:35:31.374 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_5.status 2026-02-18T08:35:31.375 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_5.status 2026-02-18T08:35:31.375 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:35:31.375 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_5.status 2026-02-18T08:35:31.375 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_5 2026-02-18T08:35:31.376 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_5.status 2026-02-18T08:35:31.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 10 2026-02-18T08:35:31.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 10 2026-02-18T08:35:31.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:35:31.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=41, tag_tid=2, entry_tid=1], [object_number=40, tag_tid=2, entry_tid=0], [object_number=42, tag_tid=1, entry_tid=45754], [object_number=43, tag_tid=1, entry_tid=45751]]], state=connected] 2026-02-18T08:35:31.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=42, tag_tid=1, entry_tid=45754], [object_number=41, tag_tid=1, entry_tid=45753], [object_number=40, tag_tid=1, entry_tid=45752], [object_number=43, tag_tid=1, entry_tid=45751]]], state=connected] 2026-02-18T08:35:31.406 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_5.status 2026-02-18T08:35:31.408 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=41, tag_tid=2, entry_tid=1' 2026-02-18T08:35:31.409 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_5 2026-02-18T08:35:31.409 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:35:31.409 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:35:31.409 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_5 2026-02-18T08:35:31.409 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_5 '..*' 2026-02-18T08:35:31.409 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:35:31.409 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:35:31.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_5 2026-02-18T08:35:31.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:35:31.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_5.status 2026-02-18T08:35:31.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_5.status 2026-02-18T08:35:31.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:35:31.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_5.status 2026-02-18T08:35:31.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_5 2026-02-18T08:35:31.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_5.status 2026-02-18T08:35:31.437 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 10 2026-02-18T08:35:31.437 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 10 2026-02-18T08:35:31.437 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:35:31.437 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=41, tag_tid=2, entry_tid=1], [object_number=40, tag_tid=2, entry_tid=0], [object_number=42, tag_tid=1, entry_tid=45754], [object_number=43, tag_tid=1, entry_tid=45751]]], state=connected] 2026-02-18T08:35:31.437 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=42, tag_tid=1, entry_tid=45754], [object_number=41, tag_tid=1, entry_tid=45753], [object_number=40, tag_tid=1, entry_tid=45752], [object_number=43, tag_tid=1, entry_tid=45751]]], state=connected] 2026-02-18T08:35:31.439 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_5.status 2026-02-18T08:35:31.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=42, tag_tid=1, entry_tid=45754' 2026-02-18T08:35:31.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=41, tag_tid=2, entry_tid=1' -a 'object_number=41, tag_tid=2, entry_tid=1' = 'object_number=42, tag_tid=1, entry_tid=45754' 2026-02-18T08:35:31.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=42, tag_tid=1, entry_tid=45754' '!=' 'object_number=42, tag_tid=1, entry_tid=45754' 2026-02-18T08:35:31.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:35:31.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 2 2026-02-18T08:35:33.443 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_5 2026-02-18T08:35:33.443 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:35:33.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:35:33.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_5 2026-02-18T08:35:33.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:35:33.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_5 ']' 2026-02-18T08:35:33.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_5' 2026-02-18T08:35:33.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_5 2026-02-18T08:35:33.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:35:33.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:35:33.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:35:33.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:35:33.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:35:33.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:35:33.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:35:33.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:35:33.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:35:33.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:35:33.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_5 2026-02-18T08:35:33.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:35:33.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:35:33.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:35:33.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:35:33.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:35:33.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:35:33.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:35:33.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:35:33.448 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:35:33.448 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:35:33.448 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:35:33.448 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:35:33.448 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:35:33.448 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:35:33.448 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:35:33.448 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:35:33.448 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:35:33.448 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:35:33.448 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:35:33.448 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:35:33.448 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:35:33.448 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:35:33.448 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:35:33.449 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:35:33.449 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:35:33.449 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:35:33.449 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:35:33.449 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:35:33.449 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:35:33.449 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:35:33.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:35:33.458 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:35:33.458 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:35:33.458 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_5 2026-02-18T08:35:33.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:no valid command found; 1 closest matches: 2026-02-18T08:35:33.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush mirror/image_50 2026-02-18T08:35:33.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:35:33.523 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:35:33.523 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_5 2026-02-18T08:35:33.523 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:35:33.523 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:35:33.523 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:35:33.523 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:35:33.523 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:35:33.523 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:35:33.523 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:35:33.524 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:35:33.524 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:35:33.524 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:35:33.524 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:35:33.524 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:35:33.524 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:35:33.524 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:35:33.524 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:35:33.524 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:35:33.524 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:35:33.524 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:35:33.524 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:35:33.524 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:35:33.524 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:35:33.524 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:35:33.525 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:35:33.525 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:35:33.525 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:35:33.525 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:35:33.525 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:35:33.525 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:35:33.525 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:35:33.525 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:35:33.534 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:35:33.535 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:35:33.535 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:35:33.535 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_5 2026-02-18T08:35:33.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_5'] 2026-02-18T08:35:33.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:35:33.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:35:33.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:35:33.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_5 2026-02-18T08:35:33.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:35:33.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:35:33.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:35:33.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:35:33.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:35:33.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:35:33.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:35:33.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:35:33.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:35:33.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:35:33.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:35:33.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:35:33.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:35:33.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:35:33.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:35:33.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:35:33.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:35:33.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:35:33.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:35:33.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:35:33.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:35:33.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:35:33.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:35:33.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:35:33.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:35:33.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:35:33.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:35:33.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:35:33.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:35:33.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:35:33.600 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:35:33.601 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:35:33.601 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:35:33.601 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_5 2026-02-18T08:35:33.660 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_5'] 2026-02-18T08:35:33.660 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:35:33.660 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:35:33.663 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:35:33.663 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/image_5 2026-02-18T08:35:33.663 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:35:33.663 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:35:33.663 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:35:33.663 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:35:33.663 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:35:33.663 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:35:33.663 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:35:33.663 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:35:33.663 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:35:33.663 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:35:33.663 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:35:33.664 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:35:33.664 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:35:33.664 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:35:33.664 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:35:33.664 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:35:33.664 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:35:33.664 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:35:33.664 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:35:33.664 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:35:33.664 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:35:33.664 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:35:33.664 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:35:33.664 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:35:33.664 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:35:33.664 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:35:33.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:35:33.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:35:33.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:35:33.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:35:33.674 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:35:33.675 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:35:33.675 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:35:33.675 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/image_5 2026-02-18T08:35:33.736 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:35:33.737 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_5 2026-02-18T08:35:33.737 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:35:33.737 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:35:33.737 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_5 2026-02-18T08:35:33.737 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_5 '' 2026-02-18T08:35:33.737 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:35:33.737 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:35:33.738 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_5 2026-02-18T08:35:33.738 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:35:33.738 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_5.status 2026-02-18T08:35:33.738 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_5.status 2026-02-18T08:35:33.738 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:35:33.739 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_5.status 2026-02-18T08:35:33.739 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_5 2026-02-18T08:35:33.739 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_5.status 2026-02-18T08:35:33.766 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 10 2026-02-18T08:35:33.766 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 10 2026-02-18T08:35:33.766 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:35:33.766 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=41, tag_tid=2, entry_tid=1], [object_number=40, tag_tid=2, entry_tid=0], [object_number=42, tag_tid=1, entry_tid=45754], [object_number=43, tag_tid=1, entry_tid=45751]]], state=connected] 2026-02-18T08:35:33.766 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=41, tag_tid=2, entry_tid=1], [object_number=40, tag_tid=2, entry_tid=0], [object_number=42, tag_tid=1, entry_tid=45754], [object_number=43, tag_tid=1, entry_tid=45751]]], state=connected] 2026-02-18T08:35:33.768 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_5.status 2026-02-18T08:35:33.770 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=41, tag_tid=2, entry_tid=1' 2026-02-18T08:35:33.771 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_5 2026-02-18T08:35:33.771 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:35:33.771 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:35:33.771 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_5 2026-02-18T08:35:33.771 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_5 '..*' 2026-02-18T08:35:33.771 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:35:33.772 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:35:33.772 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_5 2026-02-18T08:35:33.772 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:35:33.772 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_5.status 2026-02-18T08:35:33.772 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_5.status 2026-02-18T08:35:33.772 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:35:33.773 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_5.status 2026-02-18T08:35:33.773 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_5 2026-02-18T08:35:33.774 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_5.status 2026-02-18T08:35:33.808 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 10 2026-02-18T08:35:33.808 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 10 2026-02-18T08:35:33.808 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:35:33.808 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=41, tag_tid=2, entry_tid=1], [object_number=40, tag_tid=2, entry_tid=0], [object_number=42, tag_tid=1, entry_tid=45754], [object_number=43, tag_tid=1, entry_tid=45751]]], state=connected] 2026-02-18T08:35:33.809 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=41, tag_tid=2, entry_tid=1], [object_number=40, tag_tid=2, entry_tid=0], [object_number=42, tag_tid=1, entry_tid=45754], [object_number=43, tag_tid=1, entry_tid=45751]]], state=connected] 2026-02-18T08:35:33.812 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_5.status 2026-02-18T08:35:33.814 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=41, tag_tid=2, entry_tid=1' 2026-02-18T08:35:33.814 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=41, tag_tid=2, entry_tid=1' -a 'object_number=41, tag_tid=2, entry_tid=1' = 'object_number=41, tag_tid=2, entry_tid=1' 2026-02-18T08:35:33.814 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:35:33.814 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_snap_present cluster1 mirror image_5 snap 2026-02-18T08:35:33.814 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:35:33.814 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:35:33.814 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_5 2026-02-18T08:35:33.814 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap 2026-02-18T08:35:33.814 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-18T08:35:33.814 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-18T08:35:33.815 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1 2026-02-18T08:35:34.815 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster1 info mirror/image_5@snap 2026-02-18T08:35:34.845 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:rbd image 'image_5': 2026-02-18T08:35:34.845 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: size 128 MiB in 32 objects 2026-02-18T08:35:34.845 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: order 22 (4 MiB objects) 2026-02-18T08:35:34.845 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: snapshot_count: 1 2026-02-18T08:35:34.846 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: id: 10ac136e7675 2026-02-18T08:35:34.846 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: block_name_prefix: rbd_data.10ac136e7675 2026-02-18T08:35:34.846 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: format: 2 2026-02-18T08:35:34.846 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: features: layering, exclusive-lock, journaling 2026-02-18T08:35:34.846 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: op_features: 2026-02-18T08:35:34.846 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: flags: 2026-02-18T08:35:34.846 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: create_timestamp: Wed Feb 18 08:26:22 2026 2026-02-18T08:35:34.846 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: access_timestamp: Wed Feb 18 08:26:22 2026 2026-02-18T08:35:34.846 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: modify_timestamp: Wed Feb 18 08:27:23 2026 2026-02-18T08:35:34.846 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: protected: False 2026-02-18T08:35:34.846 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: journal: 10ac136e7675 2026-02-18T08:35:34.847 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring state: enabled 2026-02-18T08:35:34.847 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring mode: journal 2026-02-18T08:35:34.847 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring global id: 46909d04-c7f4-4bce-8641-e7f5b0b7aae3 2026-02-18T08:35:34.847 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring primary: false 2026-02-18T08:35:34.848 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:35:34.848 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ compare_image_snaps mirror image_5 snap 2026-02-18T08:35:34.848 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:35:34.848 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_5 2026-02-18T08:35:34.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap 2026-02-18T08:35:34.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local ret=0 2026-02-18T08:35:34.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local rmt_export=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_5.export 2026-02-18T08:35:34.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loc_export=/tmp/tmp.vybh1m9yJF/cluster1-mirror-image_5.export 2026-02-18T08:35:34.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_5.export /tmp/tmp.vybh1m9yJF/cluster1-mirror-image_5.export 2026-02-18T08:35:34.850 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 -p mirror export image_5@snap /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_5.export 2026-02-18T08:35:34.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-18T08:35:35.007 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster1 -p mirror export image_5@snap /tmp/tmp.vybh1m9yJF/cluster1-mirror-image_5.export 2026-02-18T08:35:35.201 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-18T08:35:35.214 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmp /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_5.export /tmp/tmp.vybh1m9yJF/cluster1-mirror-image_5.export 2026-02-18T08:35:35.270 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_5.export /tmp/tmp.vybh1m9yJF/cluster1-mirror-image_5.export 2026-02-18T08:35:35.320 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:35:35.321 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T08:35:35.321 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_6 2026-02-18T08:35:35.321 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_snap cluster2 mirror image_6 snap 2026-02-18T08:35:35.321 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:35:35.321 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:35:35.321 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_6 2026-02-18T08:35:35.322 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap 2026-02-18T08:35:35.322 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 -p mirror snap create image_6@snap --debug-rbd=20 --debug-journaler=20 2026-02-18T08:35:37.076 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_replay_started cluster1 mirror image_6 2026-02-18T08:35:37.076 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:35:37.076 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:35:37.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_6 2026-02-18T08:35:37.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_replay_state cluster1 mirror image_6 started 2026-02-18T08:35:37.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:35:37.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:35:37.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_6 2026-02-18T08:35:37.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local state=started 2026-02-18T08:35:37.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-18T08:35:37.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:35:37.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1 2026-02-18T08:35:38.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test_image_replay_state cluster1 mirror image_6 started 2026-02-18T08:35:38.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:35:38.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:35:38.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_6 2026-02-18T08:35:38.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local test_state=started 2026-02-18T08:35:38.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local status_result 2026-02-18T08:35:38.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_state=stopped 2026-02-18T08:35:38.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_6 2026-02-18T08:35:38.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -i state 2026-02-18T08:35:38.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_instance=cluster1 2026-02-18T08:35:38.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-18T08:35:38.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:35:38.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance=cluster1 2026-02-18T08:35:38.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local loop_instance 2026-02-18T08:35:38.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:35:38.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sleep 0 2026-02-18T08:35:38.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:35:38.080 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ seq 0 3 2026-02-18T08:35:38.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:35:38.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_6 2026-02-18T08:35:38.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:0 2026-02-18T08:35:38.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-18T08:35:38.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:35:38.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:35:38.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:35:38.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-18T08:35:38.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:35:38.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:35:38.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:35:38.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:35:38.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-18T08:35:38.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:35:38.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:35:38.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:35:38.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:35:38.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:0 2026-02-18T08:35:38.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-18T08:35:38.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-18T08:35:38.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:35:38.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:0 2026-02-18T08:35:38.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T08:35:38.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-18T08:35:38.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:35:38.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:35:38.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-18T08:35:38.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-18T08:35:38.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-18T08:35:38.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=0 2026-02-18T08:35:38.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=0 2026-02-18T08:35:38.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:35:38.093 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:35:38.094 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:35:38.094 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:35:38.094 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_6 2026-02-18T08:35:38.149 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_6'] 2026-02-18T08:35:38.150 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status : get status for rbd mirror 2026-02-18T08:35:38.150 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:35:38.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:35:38.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_6 2026-02-18T08:35:38.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:1 2026-02-18T08:35:38.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-18T08:35:38.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:35:38.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:35:38.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:35:38.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-18T08:35:38.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:35:38.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:35:38.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:35:38.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:35:38.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-18T08:35:38.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:35:38.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:35:38.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:35:38.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:35:38.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:1 2026-02-18T08:35:38.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-18T08:35:38.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-18T08:35:38.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:35:38.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:1 2026-02-18T08:35:38.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T08:35:38.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-18T08:35:38.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:35:38.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:35:38.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-18T08:35:38.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-18T08:35:38.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-18T08:35:38.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=1 2026-02-18T08:35:38.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=1 2026-02-18T08:35:38.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:35:38.163 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:35:38.164 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:35:38.164 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:35:38.164 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_6 2026-02-18T08:35:38.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ return 0 2026-02-18T08:35:38.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ status_result=' "state": "Replaying"' 2026-02-18T08:35:38.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ echo ' "state": "Replaying"' 2026-02-18T08:35:38.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ grep -i Replaying 2026-02-18T08:35:38.238 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: "state": "Replaying" 2026-02-18T08:35:38.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ current_state=started 2026-02-18T08:35:38.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test started = started 2026-02-18T08:35:38.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:35:38.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror image_6 2026-02-18T08:35:38.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_cluster=cluster1 2026-02-18T08:35:38.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:35:38.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_pool=mirror 2026-02-18T08:35:38.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local remote_pool=mirror 2026-02-18T08:35:38.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_6 2026-02-18T08:35:38.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' journal = journal ']' 2026-02-18T08:35:38.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror mirror image_6 2026-02-18T08:35:38.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_cluster=cluster1 2026-02-18T08:35:38.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:35:38.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_pool=mirror 2026-02-18T08:35:38.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local remote_pool=mirror 2026-02-18T08:35:38.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_6 2026-02-18T08:35:38.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-02-18T08:35:38.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-02-18T08:35:38.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:35:38.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:35:38.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:35:38.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_6 2026-02-18T08:35:38.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:35:38.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:35:38.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_6 2026-02-18T08:35:38.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:35:38.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_6 ']' 2026-02-18T08:35:38.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_6' 2026-02-18T08:35:38.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_6 2026-02-18T08:35:38.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:35:38.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:35:38.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:35:38.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:35:38.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:35:38.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:35:38.443 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:35:38.443 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:35:38.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:35:38.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:35:38.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_6 2026-02-18T08:35:38.446 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:35:38.446 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:35:38.446 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:35:38.446 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:35:38.446 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:35:38.446 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:35:38.446 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:35:38.446 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:35:38.446 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:35:38.446 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:35:38.446 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:35:38.446 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:35:38.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:35:38.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:35:38.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:35:38.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:35:38.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:35:38.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:35:38.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:35:38.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:35:38.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:35:38.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:35:38.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:35:38.448 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:35:38.448 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:35:38.448 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:35:38.448 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:35:38.448 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:35:38.448 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:35:38.448 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:35:38.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:35:38.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:35:38.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:35:38.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_6 2026-02-18T08:35:38.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_6'] 2026-02-18T08:35:38.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:35:38.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:35:38.523 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:35:38.523 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_6 2026-02-18T08:35:38.523 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:35:38.524 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:35:38.524 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:35:38.524 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:35:38.524 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:35:38.524 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:35:38.524 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:35:38.524 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:35:38.524 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:35:38.524 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:35:38.524 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:35:38.524 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:35:38.525 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:35:38.525 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:35:38.525 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:35:38.525 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:35:38.525 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:35:38.525 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:35:38.525 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:35:38.525 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:35:38.525 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:35:38.525 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:35:38.525 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:35:38.525 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:35:38.526 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:35:38.526 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:35:38.526 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:35:38.526 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:35:38.526 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:35:38.526 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:35:38.534 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:35:38.535 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:35:38.535 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:35:38.535 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_6 2026-02-18T08:35:38.594 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:35:38.595 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_6 2026-02-18T08:35:38.595 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:35:38.595 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:35:38.596 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_6 2026-02-18T08:35:38.596 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_6 '' 2026-02-18T08:35:38.596 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:35:38.596 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:35:38.596 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_6 2026-02-18T08:35:38.596 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:35:38.596 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_6.status 2026-02-18T08:35:38.596 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_6.status 2026-02-18T08:35:38.596 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:35:38.597 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_6.status 2026-02-18T08:35:38.597 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_6 2026-02-18T08:35:38.597 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_6.status 2026-02-18T08:35:38.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 4 2026-02-18T08:35:38.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 4 2026-02-18T08:35:38.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:35:38.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=2, entry_tid=1], [object_number=16, tag_tid=2, entry_tid=0], [object_number=19, tag_tid=1, entry_tid=20019], [object_number=18, tag_tid=1, entry_tid=20018]]], state=connected] 2026-02-18T08:35:38.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=17, tag_tid=1, entry_tid=20021], [object_number=16, tag_tid=1, entry_tid=20020], [object_number=19, tag_tid=1, entry_tid=20019], [object_number=18, tag_tid=1, entry_tid=20018]]], state=connected] 2026-02-18T08:35:38.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_6.status 2026-02-18T08:35:38.629 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=17, tag_tid=2, entry_tid=1' 2026-02-18T08:35:38.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_6 2026-02-18T08:35:38.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:35:38.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:35:38.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_6 2026-02-18T08:35:38.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_6 '..*' 2026-02-18T08:35:38.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:35:38.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:35:38.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_6 2026-02-18T08:35:38.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:35:38.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_6.status 2026-02-18T08:35:38.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_6.status 2026-02-18T08:35:38.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:35:38.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_6.status 2026-02-18T08:35:38.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_6 2026-02-18T08:35:38.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_6.status 2026-02-18T08:35:38.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 4 2026-02-18T08:35:38.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 4 2026-02-18T08:35:38.667 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:35:38.667 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=2, entry_tid=1], [object_number=16, tag_tid=2, entry_tid=0], [object_number=19, tag_tid=1, entry_tid=20019], [object_number=18, tag_tid=1, entry_tid=20018]]], state=connected] 2026-02-18T08:35:38.667 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=17, tag_tid=1, entry_tid=20021], [object_number=16, tag_tid=1, entry_tid=20020], [object_number=19, tag_tid=1, entry_tid=20019], [object_number=18, tag_tid=1, entry_tid=20018]]], state=connected] 2026-02-18T08:35:38.670 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_6.status 2026-02-18T08:35:38.672 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=17, tag_tid=1, entry_tid=20021' 2026-02-18T08:35:38.672 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=17, tag_tid=2, entry_tid=1' -a 'object_number=17, tag_tid=2, entry_tid=1' = 'object_number=17, tag_tid=1, entry_tid=20021' 2026-02-18T08:35:38.672 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=17, tag_tid=1, entry_tid=20021' '!=' '' 2026-02-18T08:35:38.673 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:35:38.673 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=17, tag_tid=1, entry_tid=20021' = '' 2026-02-18T08:35:38.673 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=17, tag_tid=1, entry_tid=20021' 2026-02-18T08:35:38.673 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=17, tag_tid=2, entry_tid=1' 2026-02-18T08:35:38.673 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:35:38.674 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:35:38.674 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=2 2026-02-18T08:35:38.675 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=17, tag_tid=1, entry_tid=20021' 2026-02-18T08:35:38.675 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:35:38.675 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:35:38.676 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=1 2026-02-18T08:35:38.677 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=17, tag_tid=2, entry_tid=1' 2026-02-18T08:35:38.677 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:35:38.677 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:35:38.679 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:35:38.680 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=17, tag_tid=1, entry_tid=20021' 2026-02-18T08:35:38.680 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:35:38.680 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:35:38.681 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=20021 2026-02-18T08:35:38.681 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 2 = 1 -a 1 -le 20021 2026-02-18T08:35:38.681 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:35:38.681 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:35:38.682 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:35:38.883 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_6 2026-02-18T08:35:38.883 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:35:38.883 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:35:38.883 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_6 2026-02-18T08:35:38.883 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:35:38.883 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_6 ']' 2026-02-18T08:35:38.884 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_6' 2026-02-18T08:35:38.884 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_6 2026-02-18T08:35:38.884 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:35:38.884 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:35:38.884 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:35:38.884 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:35:38.884 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:35:38.884 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:35:38.884 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:35:38.887 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:35:38.887 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:35:38.887 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:35:38.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_6 2026-02-18T08:35:38.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:35:38.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:35:38.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:35:38.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:35:38.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:35:38.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:35:38.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:35:38.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:35:38.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:35:38.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:35:38.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:35:38.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:35:38.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:35:38.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:35:38.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:35:38.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:35:38.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:35:38.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:35:38.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:35:38.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:35:38.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:35:38.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:35:38.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:35:38.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:35:38.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:35:38.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:35:38.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:35:38.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:35:38.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:35:38.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:35:38.897 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:35:38.897 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:35:38.897 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:35:38.897 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_6 2026-02-18T08:35:38.958 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_6'] 2026-02-18T08:35:38.958 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:35:38.958 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:35:38.961 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:35:38.961 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_6 2026-02-18T08:35:38.961 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:35:38.961 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:35:38.961 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:35:38.961 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:35:38.962 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:35:38.962 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:35:38.962 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:35:38.962 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:35:38.962 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:35:38.962 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:35:38.962 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:35:38.962 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:35:38.962 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:35:38.962 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:35:38.962 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:35:38.962 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:35:38.963 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:35:38.963 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:35:38.963 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:35:38.963 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:35:38.963 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:35:38.963 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:35:38.963 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:35:38.963 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:35:38.963 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:35:38.963 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:35:38.963 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:35:38.963 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:35:38.963 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:35:38.964 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:35:38.971 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:35:38.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:35:38.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:35:38.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_6 2026-02-18T08:35:39.039 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:35:39.040 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_6 2026-02-18T08:35:39.040 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:35:39.040 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:35:39.040 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_6 2026-02-18T08:35:39.040 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_6 '' 2026-02-18T08:35:39.041 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:35:39.041 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:35:39.041 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_6 2026-02-18T08:35:39.041 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:35:39.041 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_6.status 2026-02-18T08:35:39.041 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_6.status 2026-02-18T08:35:39.041 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:35:39.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_6.status 2026-02-18T08:35:39.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_6 2026-02-18T08:35:39.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_6.status 2026-02-18T08:35:39.069 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 4 2026-02-18T08:35:39.069 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 4 2026-02-18T08:35:39.069 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:35:39.069 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=2, entry_tid=1], [object_number=16, tag_tid=2, entry_tid=0], [object_number=19, tag_tid=1, entry_tid=20019], [object_number=18, tag_tid=1, entry_tid=20018]]], state=connected] 2026-02-18T08:35:39.069 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=17, tag_tid=1, entry_tid=20021], [object_number=16, tag_tid=1, entry_tid=20020], [object_number=19, tag_tid=1, entry_tid=20019], [object_number=18, tag_tid=1, entry_tid=20018]]], state=connected] 2026-02-18T08:35:39.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_6.status 2026-02-18T08:35:39.074 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=17, tag_tid=2, entry_tid=1' 2026-02-18T08:35:39.075 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_6 2026-02-18T08:35:39.075 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:35:39.075 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:35:39.075 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_6 2026-02-18T08:35:39.075 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_6 '..*' 2026-02-18T08:35:39.075 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:35:39.075 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:35:39.075 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_6 2026-02-18T08:35:39.075 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:35:39.075 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_6.status 2026-02-18T08:35:39.076 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_6.status 2026-02-18T08:35:39.076 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:35:39.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_6.status 2026-02-18T08:35:39.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_6 2026-02-18T08:35:39.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_6.status 2026-02-18T08:35:39.317 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 4 2026-02-18T08:35:39.317 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 4 2026-02-18T08:35:39.317 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:35:39.317 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=2, entry_tid=1], [object_number=16, tag_tid=2, entry_tid=0], [object_number=19, tag_tid=1, entry_tid=20019], [object_number=18, tag_tid=1, entry_tid=20018]]], state=connected] 2026-02-18T08:35:39.318 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=17, tag_tid=1, entry_tid=20021], [object_number=16, tag_tid=1, entry_tid=20020], [object_number=19, tag_tid=1, entry_tid=20019], [object_number=18, tag_tid=1, entry_tid=20018]]], state=connected] 2026-02-18T08:35:39.321 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_6.status 2026-02-18T08:35:39.323 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=17, tag_tid=1, entry_tid=20021' 2026-02-18T08:35:39.323 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=17, tag_tid=2, entry_tid=1' -a 'object_number=17, tag_tid=2, entry_tid=1' = 'object_number=17, tag_tid=1, entry_tid=20021' 2026-02-18T08:35:39.323 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=17, tag_tid=1, entry_tid=20021' '!=' 'object_number=17, tag_tid=1, entry_tid=20021' 2026-02-18T08:35:39.323 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:35:39.323 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.4 2026-02-18T08:35:39.725 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_6 2026-02-18T08:35:39.725 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:35:39.725 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:35:39.725 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_6 2026-02-18T08:35:39.725 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:35:39.725 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_6 ']' 2026-02-18T08:35:39.725 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_6' 2026-02-18T08:35:39.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_6 2026-02-18T08:35:39.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:35:39.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:35:39.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:35:39.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:35:39.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:35:39.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:35:39.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:35:39.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:35:39.729 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:35:39.729 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:35:39.729 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_6 2026-02-18T08:35:39.729 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:35:39.729 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:35:39.729 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:35:39.729 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:35:39.729 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:35:39.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:35:39.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:35:39.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:35:39.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:35:39.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:35:39.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:35:39.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:35:39.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:35:39.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:35:39.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:35:39.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:35:39.731 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:35:39.731 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:35:39.731 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:35:39.731 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:35:39.731 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:35:39.731 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:35:39.731 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:35:39.731 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:35:39.731 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:35:39.731 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:35:39.731 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:35:39.731 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:35:39.731 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:35:39.732 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:35:39.740 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:35:39.740 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:35:39.740 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:35:39.740 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_6 2026-02-18T08:35:39.801 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_6'] 2026-02-18T08:35:39.801 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:35:39.801 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:35:39.804 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:35:39.804 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_6 2026-02-18T08:35:39.804 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:35:39.804 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:35:39.805 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:35:39.805 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:35:39.805 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:35:39.805 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:35:39.805 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:35:39.805 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:35:39.805 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:35:39.805 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:35:39.805 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:35:39.805 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:35:39.806 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:35:39.806 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:35:39.806 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:35:39.806 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:35:39.806 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:35:39.806 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:35:39.806 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:35:39.806 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:35:39.806 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:35:39.806 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:35:39.807 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:35:39.807 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:35:39.807 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:35:39.807 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:35:39.807 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:35:39.807 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:35:39.807 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:35:39.807 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:35:39.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:35:39.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:35:39.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:35:39.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_6 2026-02-18T08:35:39.881 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:35:39.882 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_6 2026-02-18T08:35:39.883 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:35:39.883 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:35:39.883 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_6 2026-02-18T08:35:39.883 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_6 '' 2026-02-18T08:35:39.883 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:35:39.883 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:35:39.883 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_6 2026-02-18T08:35:39.883 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:35:39.883 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_6.status 2026-02-18T08:35:39.883 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_6.status 2026-02-18T08:35:39.883 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:35:39.884 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_6.status 2026-02-18T08:35:39.884 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_6 2026-02-18T08:35:39.885 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_6.status 2026-02-18T08:35:39.922 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 4 2026-02-18T08:35:39.922 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 4 2026-02-18T08:35:39.922 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:35:39.922 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=2, entry_tid=1], [object_number=16, tag_tid=2, entry_tid=0], [object_number=19, tag_tid=1, entry_tid=20019], [object_number=18, tag_tid=1, entry_tid=20018]]], state=connected] 2026-02-18T08:35:39.922 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=17, tag_tid=1, entry_tid=20021], [object_number=16, tag_tid=1, entry_tid=20020], [object_number=19, tag_tid=1, entry_tid=20019], [object_number=18, tag_tid=1, entry_tid=20018]]], state=connected] 2026-02-18T08:35:39.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_6.status 2026-02-18T08:35:39.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=17, tag_tid=2, entry_tid=1' 2026-02-18T08:35:39.928 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_6 2026-02-18T08:35:39.928 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:35:39.928 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:35:39.928 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_6 2026-02-18T08:35:39.928 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_6 '..*' 2026-02-18T08:35:39.928 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:35:39.928 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:35:39.929 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_6 2026-02-18T08:35:39.929 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:35:39.929 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_6.status 2026-02-18T08:35:39.929 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_6.status 2026-02-18T08:35:39.929 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:35:39.930 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_6.status 2026-02-18T08:35:39.930 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_6 2026-02-18T08:35:39.930 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_6.status 2026-02-18T08:35:39.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 4 2026-02-18T08:35:39.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 4 2026-02-18T08:35:39.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:35:39.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=2, entry_tid=1], [object_number=16, tag_tid=2, entry_tid=0], [object_number=19, tag_tid=1, entry_tid=20019], [object_number=18, tag_tid=1, entry_tid=20018]]], state=connected] 2026-02-18T08:35:39.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=17, tag_tid=1, entry_tid=20021], [object_number=16, tag_tid=1, entry_tid=20020], [object_number=19, tag_tid=1, entry_tid=20019], [object_number=18, tag_tid=1, entry_tid=20018]]], state=connected] 2026-02-18T08:35:39.959 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_6.status 2026-02-18T08:35:39.961 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=17, tag_tid=1, entry_tid=20021' 2026-02-18T08:35:39.961 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=17, tag_tid=2, entry_tid=1' -a 'object_number=17, tag_tid=2, entry_tid=1' = 'object_number=17, tag_tid=1, entry_tid=20021' 2026-02-18T08:35:39.961 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=17, tag_tid=1, entry_tid=20021' '!=' 'object_number=17, tag_tid=1, entry_tid=20021' 2026-02-18T08:35:39.961 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:35:39.961 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.8 2026-02-18T08:35:40.763 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_6 2026-02-18T08:35:40.763 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:35:40.763 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:35:40.763 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_6 2026-02-18T08:35:40.763 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:35:40.763 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_6 ']' 2026-02-18T08:35:40.763 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_6' 2026-02-18T08:35:40.763 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_6 2026-02-18T08:35:40.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:35:40.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:35:40.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:35:40.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:35:40.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:35:40.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:35:40.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:35:40.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:35:40.767 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:35:40.767 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:35:40.767 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_6 2026-02-18T08:35:40.767 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:35:40.767 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:35:40.767 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:35:40.767 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:35:40.767 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:35:40.767 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:35:40.768 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:35:40.768 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:35:40.768 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:35:40.768 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:35:40.768 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:35:40.768 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:35:40.768 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:35:40.768 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:35:40.768 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:35:40.768 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:35:40.768 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:35:40.768 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:35:40.769 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:35:40.769 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:35:40.769 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:35:40.769 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:35:40.769 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:35:40.769 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:35:40.769 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:35:40.769 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:35:40.769 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:35:40.769 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:35:40.769 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:35:40.769 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:35:40.777 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:35:40.777 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:35:40.778 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:35:40.778 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_6 2026-02-18T08:35:40.841 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_6'] 2026-02-18T08:35:40.841 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:35:40.841 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:35:40.844 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:35:40.844 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_6 2026-02-18T08:35:40.844 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:35:40.844 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:35:40.844 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:35:40.844 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:35:40.844 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:35:40.844 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:35:40.844 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:35:40.844 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:35:40.845 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:35:40.845 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:35:40.845 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:35:40.845 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:35:40.845 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:35:40.845 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:35:40.845 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:35:40.845 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:35:40.845 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:35:40.845 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:35:40.845 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:35:40.845 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:35:40.846 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:35:40.846 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:35:40.846 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:35:40.846 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:35:40.846 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:35:40.846 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:35:40.846 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:35:40.846 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:35:40.846 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:35:40.846 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:35:40.855 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:35:40.856 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:35:40.856 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:35:40.856 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_6 2026-02-18T08:35:40.922 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:35:40.924 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_6 2026-02-18T08:35:40.924 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:35:40.924 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:35:40.924 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_6 2026-02-18T08:35:40.924 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_6 '' 2026-02-18T08:35:40.924 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:35:40.924 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:35:40.924 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_6 2026-02-18T08:35:40.924 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:35:40.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_6.status 2026-02-18T08:35:40.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_6.status 2026-02-18T08:35:40.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:35:40.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_6.status 2026-02-18T08:35:40.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_6 2026-02-18T08:35:40.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_6.status 2026-02-18T08:35:40.961 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 4 2026-02-18T08:35:40.962 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 4 2026-02-18T08:35:40.962 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:35:40.962 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=2, entry_tid=1], [object_number=16, tag_tid=2, entry_tid=0], [object_number=19, tag_tid=1, entry_tid=20019], [object_number=18, tag_tid=1, entry_tid=20018]]], state=connected] 2026-02-18T08:35:40.962 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=17, tag_tid=1, entry_tid=20021], [object_number=16, tag_tid=1, entry_tid=20020], [object_number=19, tag_tid=1, entry_tid=20019], [object_number=18, tag_tid=1, entry_tid=20018]]], state=connected] 2026-02-18T08:35:40.965 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_6.status 2026-02-18T08:35:40.967 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=17, tag_tid=2, entry_tid=1' 2026-02-18T08:35:40.968 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_6 2026-02-18T08:35:40.968 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:35:40.968 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:35:40.969 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_6 2026-02-18T08:35:40.969 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_6 '..*' 2026-02-18T08:35:40.969 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:35:40.969 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:35:40.969 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_6 2026-02-18T08:35:40.969 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:35:40.969 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_6.status 2026-02-18T08:35:40.970 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_6.status 2026-02-18T08:35:40.970 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:35:40.970 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_6.status 2026-02-18T08:35:40.971 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_6 2026-02-18T08:35:40.971 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_6.status 2026-02-18T08:35:40.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 4 2026-02-18T08:35:40.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 4 2026-02-18T08:35:40.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:35:40.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=2, entry_tid=1], [object_number=16, tag_tid=2, entry_tid=0], [object_number=19, tag_tid=1, entry_tid=20019], [object_number=18, tag_tid=1, entry_tid=20018]]], state=connected] 2026-02-18T08:35:40.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=17, tag_tid=1, entry_tid=20021], [object_number=16, tag_tid=1, entry_tid=20020], [object_number=19, tag_tid=1, entry_tid=20019], [object_number=18, tag_tid=1, entry_tid=20018]]], state=connected] 2026-02-18T08:35:40.999 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_6.status 2026-02-18T08:35:41.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=17, tag_tid=1, entry_tid=20021' 2026-02-18T08:35:41.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=17, tag_tid=2, entry_tid=1' -a 'object_number=17, tag_tid=2, entry_tid=1' = 'object_number=17, tag_tid=1, entry_tid=20021' 2026-02-18T08:35:41.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=17, tag_tid=1, entry_tid=20021' '!=' 'object_number=17, tag_tid=1, entry_tid=20021' 2026-02-18T08:35:41.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:35:41.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1.6 2026-02-18T08:35:42.602 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_6 2026-02-18T08:35:42.603 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:35:42.603 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:35:42.603 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_6 2026-02-18T08:35:42.603 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:35:42.603 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_6 ']' 2026-02-18T08:35:42.603 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_6' 2026-02-18T08:35:42.603 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_6 2026-02-18T08:35:42.603 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:35:42.603 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:35:42.604 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:35:42.604 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:35:42.604 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:35:42.604 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:35:42.604 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:35:42.606 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:35:42.607 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:35:42.607 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:35:42.607 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_6 2026-02-18T08:35:42.607 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:35:42.607 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:35:42.607 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:35:42.607 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:35:42.607 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:35:42.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:35:42.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:35:42.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:35:42.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:35:42.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:35:42.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:35:42.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:35:42.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:35:42.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:35:42.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:35:42.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:35:42.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:35:42.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:35:42.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:35:42.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:35:42.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:35:42.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:35:42.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:35:42.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:35:42.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:35:42.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:35:42.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:35:42.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:35:42.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:35:42.610 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:35:42.618 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:35:42.618 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:35:42.618 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:35:42.618 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_6 2026-02-18T08:35:42.671 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_6'] 2026-02-18T08:35:42.671 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:35:42.671 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:35:42.674 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:35:42.674 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_6 2026-02-18T08:35:42.674 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:35:42.674 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:35:42.674 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:35:42.674 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:35:42.674 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:35:42.674 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:35:42.674 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:35:42.674 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:35:42.675 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:35:42.675 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:35:42.675 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:35:42.675 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:35:42.675 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:35:42.675 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:35:42.675 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:35:42.675 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:35:42.675 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:35:42.675 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:35:42.675 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:35:42.675 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:35:42.676 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:35:42.676 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:35:42.676 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:35:42.676 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:35:42.676 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:35:42.676 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:35:42.676 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:35:42.676 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:35:42.676 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:35:42.676 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:35:42.685 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:35:42.685 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:35:42.686 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:35:42.686 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_6 2026-02-18T08:35:42.755 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:35:42.756 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_6 2026-02-18T08:35:42.756 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:35:42.756 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:35:42.756 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_6 2026-02-18T08:35:42.756 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_6 '' 2026-02-18T08:35:42.756 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:35:42.756 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:35:42.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_6 2026-02-18T08:35:42.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:35:42.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_6.status 2026-02-18T08:35:42.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_6.status 2026-02-18T08:35:42.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:35:42.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_6.status 2026-02-18T08:35:42.758 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_6 2026-02-18T08:35:42.758 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_6.status 2026-02-18T08:35:42.796 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 4 2026-02-18T08:35:42.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 4 2026-02-18T08:35:42.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:35:42.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=2, entry_tid=1], [object_number=16, tag_tid=2, entry_tid=0], [object_number=19, tag_tid=1, entry_tid=20019], [object_number=18, tag_tid=1, entry_tid=20018]]], state=connected] 2026-02-18T08:35:42.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=17, tag_tid=1, entry_tid=20021], [object_number=16, tag_tid=1, entry_tid=20020], [object_number=19, tag_tid=1, entry_tid=20019], [object_number=18, tag_tid=1, entry_tid=20018]]], state=connected] 2026-02-18T08:35:42.800 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_6.status 2026-02-18T08:35:42.802 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=17, tag_tid=2, entry_tid=1' 2026-02-18T08:35:42.803 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_6 2026-02-18T08:35:42.803 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:35:42.803 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:35:42.803 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_6 2026-02-18T08:35:42.803 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_6 '..*' 2026-02-18T08:35:42.803 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:35:42.804 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:35:42.804 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_6 2026-02-18T08:35:42.804 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:35:42.804 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_6.status 2026-02-18T08:35:42.804 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_6.status 2026-02-18T08:35:42.804 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:35:42.805 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_6.status 2026-02-18T08:35:42.805 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_6 2026-02-18T08:35:42.805 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_6.status 2026-02-18T08:35:42.831 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 4 2026-02-18T08:35:42.831 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 4 2026-02-18T08:35:42.831 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:35:42.831 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=2, entry_tid=1], [object_number=16, tag_tid=2, entry_tid=0], [object_number=19, tag_tid=1, entry_tid=20019], [object_number=18, tag_tid=1, entry_tid=20018]]], state=connected] 2026-02-18T08:35:42.831 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=17, tag_tid=1, entry_tid=20021], [object_number=16, tag_tid=1, entry_tid=20020], [object_number=19, tag_tid=1, entry_tid=20019], [object_number=18, tag_tid=1, entry_tid=20018]]], state=connected] 2026-02-18T08:35:42.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_6.status 2026-02-18T08:35:42.835 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=17, tag_tid=1, entry_tid=20021' 2026-02-18T08:35:42.835 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=17, tag_tid=2, entry_tid=1' -a 'object_number=17, tag_tid=2, entry_tid=1' = 'object_number=17, tag_tid=1, entry_tid=20021' 2026-02-18T08:35:42.835 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=17, tag_tid=1, entry_tid=20021' '!=' 'object_number=17, tag_tid=1, entry_tid=20021' 2026-02-18T08:35:42.836 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:35:42.836 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 2 2026-02-18T08:35:43.686 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.0 with skip (0.30566456246048135) > weight (0.25) 2026-02-18T08:35:43.686 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.1 with skip (0.3205488589109975) > weight (0.25) 2026-02-18T08:35:43.686 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.2 with skip (0.6778750684072776) > weight (0.25) 2026-02-18T08:35:43.686 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.3 with skip (0.7027123082505806) > weight (0.25) 2026-02-18T08:35:43.686 INFO:tasks.rbd_mirror_thrash:waiting for 60.52581056310019 secs before thrashing 2026-02-18T08:35:44.837 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_6 2026-02-18T08:35:44.837 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:35:44.837 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:35:44.837 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_6 2026-02-18T08:35:44.837 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:35:44.837 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_6 ']' 2026-02-18T08:35:44.837 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_6' 2026-02-18T08:35:44.837 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_6 2026-02-18T08:35:44.837 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:35:44.838 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:35:44.838 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:35:44.838 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:35:44.838 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:35:44.838 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:35:44.838 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:35:44.840 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:35:44.840 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:35:44.840 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:35:44.840 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_6 2026-02-18T08:35:44.840 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:35:44.840 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:35:44.840 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:35:44.841 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:35:44.841 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:35:44.841 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:35:44.841 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:35:44.841 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:35:44.841 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:35:44.841 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:35:44.841 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:35:44.841 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:35:44.841 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:35:44.841 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:35:44.841 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:35:44.841 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:35:44.841 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:35:44.842 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:35:44.842 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:35:44.842 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:35:44.842 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:35:44.842 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:35:44.842 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:35:44.842 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:35:44.842 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:35:44.842 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:35:44.842 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:35:44.842 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:35:44.842 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:35:44.842 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:35:44.852 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:35:44.852 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:35:44.853 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:35:44.853 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_6 2026-02-18T08:35:44.917 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_6'] 2026-02-18T08:35:44.917 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:35:44.917 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:35:44.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:35:44.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_6 2026-02-18T08:35:44.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:35:44.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:35:44.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:35:44.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:35:44.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:35:44.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:35:44.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:35:44.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:35:44.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:35:44.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:35:44.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:35:44.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:35:44.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:35:44.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:35:44.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:35:44.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:35:44.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:35:44.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:35:44.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:35:44.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:35:44.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:35:44.922 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:35:44.922 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:35:44.922 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:35:44.922 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:35:44.922 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:35:44.922 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:35:44.922 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:35:44.922 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:35:44.922 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:35:44.931 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:35:44.932 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:35:44.932 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:35:44.932 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_6 2026-02-18T08:35:44.994 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:35:44.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_6 2026-02-18T08:35:44.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:35:44.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:35:44.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_6 2026-02-18T08:35:44.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_6 '' 2026-02-18T08:35:44.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:35:44.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:35:44.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_6 2026-02-18T08:35:44.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:35:44.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_6.status 2026-02-18T08:35:44.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_6.status 2026-02-18T08:35:44.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:35:44.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_6.status 2026-02-18T08:35:44.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_6 2026-02-18T08:35:44.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_6.status 2026-02-18T08:35:45.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 4 2026-02-18T08:35:45.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 4 2026-02-18T08:35:45.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:35:45.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=2, entry_tid=1], [object_number=16, tag_tid=2, entry_tid=0], [object_number=19, tag_tid=1, entry_tid=20019], [object_number=18, tag_tid=1, entry_tid=20018]]], state=connected] 2026-02-18T08:35:45.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=17, tag_tid=2, entry_tid=1], [object_number=16, tag_tid=2, entry_tid=0], [object_number=19, tag_tid=1, entry_tid=20019], [object_number=18, tag_tid=1, entry_tid=20018]]], state=connected] 2026-02-18T08:35:45.038 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_6.status 2026-02-18T08:35:45.040 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=17, tag_tid=2, entry_tid=1' 2026-02-18T08:35:45.041 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_6 2026-02-18T08:35:45.041 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:35:45.041 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:35:45.041 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_6 2026-02-18T08:35:45.041 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_6 '..*' 2026-02-18T08:35:45.041 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:35:45.041 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:35:45.041 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_6 2026-02-18T08:35:45.041 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:35:45.041 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_6.status 2026-02-18T08:35:45.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_6.status 2026-02-18T08:35:45.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:35:45.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_6.status 2026-02-18T08:35:45.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_6 2026-02-18T08:35:45.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_6.status 2026-02-18T08:35:45.076 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 4 2026-02-18T08:35:45.076 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 4 2026-02-18T08:35:45.076 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:35:45.076 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=2, entry_tid=1], [object_number=16, tag_tid=2, entry_tid=0], [object_number=19, tag_tid=1, entry_tid=20019], [object_number=18, tag_tid=1, entry_tid=20018]]], state=connected] 2026-02-18T08:35:45.076 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=17, tag_tid=2, entry_tid=1], [object_number=16, tag_tid=2, entry_tid=0], [object_number=19, tag_tid=1, entry_tid=20019], [object_number=18, tag_tid=1, entry_tid=20018]]], state=connected] 2026-02-18T08:35:45.080 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_6.status 2026-02-18T08:35:45.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=17, tag_tid=2, entry_tid=1' 2026-02-18T08:35:45.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=17, tag_tid=2, entry_tid=1' -a 'object_number=17, tag_tid=2, entry_tid=1' = 'object_number=17, tag_tid=2, entry_tid=1' 2026-02-18T08:35:45.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:35:45.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_snap_present cluster1 mirror image_6 snap 2026-02-18T08:35:45.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:35:45.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:35:45.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_6 2026-02-18T08:35:45.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap 2026-02-18T08:35:45.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-18T08:35:45.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-18T08:35:45.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1 2026-02-18T08:35:46.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster1 info mirror/image_6@snap 2026-02-18T08:35:46.112 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:rbd image 'image_6': 2026-02-18T08:35:46.112 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: size 128 MiB in 32 objects 2026-02-18T08:35:46.112 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: order 22 (4 MiB objects) 2026-02-18T08:35:46.113 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: snapshot_count: 1 2026-02-18T08:35:46.113 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: id: 10edd0980b4f 2026-02-18T08:35:46.113 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: block_name_prefix: rbd_data.10edd0980b4f 2026-02-18T08:35:46.113 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: format: 2 2026-02-18T08:35:46.113 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: features: layering, exclusive-lock, journaling 2026-02-18T08:35:46.113 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: op_features: 2026-02-18T08:35:46.113 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: flags: 2026-02-18T08:35:46.113 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: create_timestamp: Wed Feb 18 08:26:35 2026 2026-02-18T08:35:46.113 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: access_timestamp: Wed Feb 18 08:26:35 2026 2026-02-18T08:35:46.113 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: modify_timestamp: Wed Feb 18 08:26:35 2026 2026-02-18T08:35:46.113 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: protected: False 2026-02-18T08:35:46.114 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: journal: 10edd0980b4f 2026-02-18T08:35:46.114 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring state: enabled 2026-02-18T08:35:46.114 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring mode: journal 2026-02-18T08:35:46.114 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring global id: e6e8164a-7ca8-4842-ab30-95b27674490d 2026-02-18T08:35:46.114 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring primary: false 2026-02-18T08:35:46.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:35:46.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ compare_image_snaps mirror image_6 snap 2026-02-18T08:35:46.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:35:46.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_6 2026-02-18T08:35:46.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap 2026-02-18T08:35:46.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local ret=0 2026-02-18T08:35:46.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local rmt_export=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_6.export 2026-02-18T08:35:46.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loc_export=/tmp/tmp.vybh1m9yJF/cluster1-mirror-image_6.export 2026-02-18T08:35:46.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_6.export /tmp/tmp.vybh1m9yJF/cluster1-mirror-image_6.export 2026-02-18T08:35:46.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 -p mirror export image_6@snap /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_6.export 2026-02-18T08:35:46.260 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-18T08:35:46.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster1 -p mirror export image_6@snap /tmp/tmp.vybh1m9yJF/cluster1-mirror-image_6.export 2026-02-18T08:35:46.493 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-18T08:35:46.507 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmp /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_6.export /tmp/tmp.vybh1m9yJF/cluster1-mirror-image_6.export 2026-02-18T08:35:46.565 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_6.export /tmp/tmp.vybh1m9yJF/cluster1-mirror-image_6.export 2026-02-18T08:35:46.601 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:35:46.601 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T08:35:46.601 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_7 2026-02-18T08:35:46.602 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_snap cluster2 mirror image_7 snap 2026-02-18T08:35:46.602 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:35:46.602 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:35:46.602 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_7 2026-02-18T08:35:46.602 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap 2026-02-18T08:35:46.602 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 -p mirror snap create image_7@snap --debug-rbd=20 --debug-journaler=20 2026-02-18T08:35:49.843 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_replay_started cluster1 mirror image_7 2026-02-18T08:35:49.843 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:35:49.843 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:35:49.843 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_7 2026-02-18T08:35:49.844 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_replay_state cluster1 mirror image_7 started 2026-02-18T08:35:49.844 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:35:49.844 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:35:49.844 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_7 2026-02-18T08:35:49.844 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local state=started 2026-02-18T08:35:49.844 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-18T08:35:49.844 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:35:49.844 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1 2026-02-18T08:35:50.844 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test_image_replay_state cluster1 mirror image_7 started 2026-02-18T08:35:50.845 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:35:50.845 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:35:50.845 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_7 2026-02-18T08:35:50.845 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local test_state=started 2026-02-18T08:35:50.845 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local status_result 2026-02-18T08:35:50.845 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_state=stopped 2026-02-18T08:35:50.845 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_7 2026-02-18T08:35:50.846 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -i state 2026-02-18T08:35:50.846 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_instance=cluster1 2026-02-18T08:35:50.846 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-18T08:35:50.846 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:35:50.847 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance=cluster1 2026-02-18T08:35:50.847 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local loop_instance 2026-02-18T08:35:50.847 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:35:50.847 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sleep 0 2026-02-18T08:35:50.847 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:35:50.847 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ seq 0 3 2026-02-18T08:35:50.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:35:50.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_7 2026-02-18T08:35:50.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:0 2026-02-18T08:35:50.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-18T08:35:50.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:35:50.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:35:50.850 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:35:50.850 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-18T08:35:50.850 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:35:50.850 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:35:50.850 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:35:50.850 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:35:50.850 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-18T08:35:50.850 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:35:50.850 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:35:50.850 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:35:50.850 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:35:50.851 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:0 2026-02-18T08:35:50.851 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-18T08:35:50.851 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-18T08:35:50.851 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:35:50.851 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:0 2026-02-18T08:35:50.851 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T08:35:50.851 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-18T08:35:50.851 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:35:50.851 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:35:50.851 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-18T08:35:50.851 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-18T08:35:50.851 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-18T08:35:50.852 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=0 2026-02-18T08:35:50.852 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=0 2026-02-18T08:35:50.852 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:35:50.859 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:35:50.859 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:35:50.859 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:35:50.860 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_7 2026-02-18T08:35:50.922 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ return 0 2026-02-18T08:35:50.923 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ status_result=' "state": "Replaying"' 2026-02-18T08:35:50.924 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ echo ' "state": "Replaying"' 2026-02-18T08:35:50.924 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ grep -i Replaying 2026-02-18T08:35:50.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: "state": "Replaying" 2026-02-18T08:35:50.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ current_state=started 2026-02-18T08:35:50.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test started = started 2026-02-18T08:35:50.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:35:50.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror image_7 2026-02-18T08:35:50.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_cluster=cluster1 2026-02-18T08:35:50.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:35:50.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_pool=mirror 2026-02-18T08:35:50.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local remote_pool=mirror 2026-02-18T08:35:50.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_7 2026-02-18T08:35:50.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' journal = journal ']' 2026-02-18T08:35:50.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror mirror image_7 2026-02-18T08:35:50.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_cluster=cluster1 2026-02-18T08:35:50.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:35:50.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_pool=mirror 2026-02-18T08:35:50.928 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local remote_pool=mirror 2026-02-18T08:35:50.928 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_7 2026-02-18T08:35:50.928 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-02-18T08:35:50.928 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-02-18T08:35:50.928 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:35:50.928 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:35:50.928 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:35:51.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_7 2026-02-18T08:35:51.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:35:51.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:35:51.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_7 2026-02-18T08:35:51.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:35:51.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_7 ']' 2026-02-18T08:35:51.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_7' 2026-02-18T08:35:51.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_7 2026-02-18T08:35:51.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:35:51.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:35:51.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:35:51.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:35:51.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:35:51.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:35:51.129 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:35:51.131 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:35:51.131 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:35:51.131 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:35:51.132 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_7 2026-02-18T08:35:51.132 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:35:51.132 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:35:51.132 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:35:51.132 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:35:51.132 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:35:51.132 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:35:51.132 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:35:51.132 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:35:51.132 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:35:51.132 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:35:51.132 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:35:51.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:35:51.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:35:51.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:35:51.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:35:51.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:35:51.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:35:51.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:35:51.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:35:51.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:35:51.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:35:51.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:35:51.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:35:51.134 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:35:51.134 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:35:51.134 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:35:51.134 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:35:51.134 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:35:51.134 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:35:51.134 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:35:51.143 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:35:51.143 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:35:51.143 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:35:51.143 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_7 2026-02-18T08:35:51.207 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:35:51.208 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_7 2026-02-18T08:35:51.208 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:35:51.208 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:35:51.208 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_7 2026-02-18T08:35:51.208 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_7 '' 2026-02-18T08:35:51.208 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:35:51.208 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:35:51.208 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_7 2026-02-18T08:35:51.209 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:35:51.209 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_7.status 2026-02-18T08:35:51.209 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_7.status 2026-02-18T08:35:51.209 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:35:51.209 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_7.status 2026-02-18T08:35:51.210 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_7 2026-02-18T08:35:51.210 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_7.status 2026-02-18T08:35:51.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 5 2026-02-18T08:35:51.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 5 2026-02-18T08:35:51.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:35:51.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=21, tag_tid=2, entry_tid=1], [object_number=20, tag_tid=2, entry_tid=0], [object_number=23, tag_tid=1, entry_tid=25315], [object_number=22, tag_tid=1, entry_tid=25314]]], state=connected] 2026-02-18T08:35:51.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=21, tag_tid=1, entry_tid=25317], [object_number=20, tag_tid=1, entry_tid=25316], [object_number=23, tag_tid=1, entry_tid=25315], [object_number=22, tag_tid=1, entry_tid=25314]]], state=connected] 2026-02-18T08:35:51.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_7.status 2026-02-18T08:35:51.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=21, tag_tid=2, entry_tid=1' 2026-02-18T08:35:51.448 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_7 2026-02-18T08:35:51.448 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:35:51.448 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:35:51.448 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_7 2026-02-18T08:35:51.448 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_7 '..*' 2026-02-18T08:35:51.449 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:35:51.449 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:35:51.449 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_7 2026-02-18T08:35:51.449 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:35:51.449 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_7.status 2026-02-18T08:35:51.449 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_7.status 2026-02-18T08:35:51.450 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:35:51.450 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_7.status 2026-02-18T08:35:51.450 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_7 2026-02-18T08:35:51.451 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_7.status 2026-02-18T08:35:51.479 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 5 2026-02-18T08:35:51.479 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 5 2026-02-18T08:35:51.479 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:35:51.479 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=21, tag_tid=2, entry_tid=1], [object_number=20, tag_tid=2, entry_tid=0], [object_number=23, tag_tid=1, entry_tid=25315], [object_number=22, tag_tid=1, entry_tid=25314]]], state=connected] 2026-02-18T08:35:51.480 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=21, tag_tid=1, entry_tid=25317], [object_number=20, tag_tid=1, entry_tid=25316], [object_number=23, tag_tid=1, entry_tid=25315], [object_number=22, tag_tid=1, entry_tid=25314]]], state=connected] 2026-02-18T08:35:51.482 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_7.status 2026-02-18T08:35:51.484 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=21, tag_tid=1, entry_tid=25317' 2026-02-18T08:35:51.484 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=21, tag_tid=2, entry_tid=1' -a 'object_number=21, tag_tid=2, entry_tid=1' = 'object_number=21, tag_tid=1, entry_tid=25317' 2026-02-18T08:35:51.484 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=21, tag_tid=1, entry_tid=25317' '!=' '' 2026-02-18T08:35:51.484 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:35:51.485 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=21, tag_tid=1, entry_tid=25317' = '' 2026-02-18T08:35:51.485 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=21, tag_tid=1, entry_tid=25317' 2026-02-18T08:35:51.485 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=21, tag_tid=2, entry_tid=1' 2026-02-18T08:35:51.485 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:35:51.485 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:35:51.486 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=2 2026-02-18T08:35:51.487 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=21, tag_tid=1, entry_tid=25317' 2026-02-18T08:35:51.487 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:35:51.487 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:35:51.488 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=1 2026-02-18T08:35:51.489 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=21, tag_tid=2, entry_tid=1' 2026-02-18T08:35:51.489 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:35:51.489 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:35:51.490 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:35:51.491 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=21, tag_tid=1, entry_tid=25317' 2026-02-18T08:35:51.491 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:35:51.491 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:35:51.493 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=25317 2026-02-18T08:35:51.493 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 2 = 1 -a 1 -le 25317 2026-02-18T08:35:51.493 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:35:51.493 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:35:51.493 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:35:51.695 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_7 2026-02-18T08:35:51.695 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:35:51.695 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:35:51.695 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_7 2026-02-18T08:35:51.695 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:35:51.696 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_7 ']' 2026-02-18T08:35:51.696 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_7' 2026-02-18T08:35:51.696 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_7 2026-02-18T08:35:51.696 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:35:51.696 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:35:51.696 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:35:51.696 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:35:51.696 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:35:51.696 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:35:51.696 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:35:51.696 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:35:51.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:35:51.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:35:51.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_7 2026-02-18T08:35:51.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:35:51.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:35:51.700 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:35:51.700 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:35:51.700 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:35:51.700 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:35:51.700 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:35:51.700 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:35:51.700 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:35:51.700 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:35:51.700 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:35:51.700 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:35:51.700 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:35:51.700 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:35:51.701 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:35:51.701 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:35:51.701 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:35:51.701 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:35:51.701 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:35:51.701 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:35:51.701 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:35:51.701 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:35:51.701 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:35:51.701 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:35:51.701 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:35:51.701 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:35:51.702 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:35:51.702 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:35:51.702 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:35:51.702 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:35:51.709 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:35:51.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:35:51.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:35:51.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_7 2026-02-18T08:35:51.777 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:35:51.778 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_7 2026-02-18T08:35:51.778 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:35:51.779 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:35:51.779 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_7 2026-02-18T08:35:51.779 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_7 '' 2026-02-18T08:35:51.779 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:35:51.779 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:35:51.779 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_7 2026-02-18T08:35:51.779 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:35:51.779 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_7.status 2026-02-18T08:35:51.779 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_7.status 2026-02-18T08:35:51.779 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:35:51.780 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_7.status 2026-02-18T08:35:51.780 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_7 2026-02-18T08:35:51.780 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_7.status 2026-02-18T08:35:51.817 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 5 2026-02-18T08:35:51.817 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 5 2026-02-18T08:35:51.817 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:35:51.817 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=21, tag_tid=2, entry_tid=1], [object_number=20, tag_tid=2, entry_tid=0], [object_number=23, tag_tid=1, entry_tid=25315], [object_number=22, tag_tid=1, entry_tid=25314]]], state=connected] 2026-02-18T08:35:51.817 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=21, tag_tid=1, entry_tid=25317], [object_number=20, tag_tid=1, entry_tid=25316], [object_number=23, tag_tid=1, entry_tid=25315], [object_number=22, tag_tid=1, entry_tid=25314]]], state=connected] 2026-02-18T08:35:51.820 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_7.status 2026-02-18T08:35:51.821 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=21, tag_tid=2, entry_tid=1' 2026-02-18T08:35:51.822 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_7 2026-02-18T08:35:51.822 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:35:51.822 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:35:51.823 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_7 2026-02-18T08:35:51.823 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_7 '..*' 2026-02-18T08:35:51.823 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:35:51.823 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:35:51.823 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_7 2026-02-18T08:35:51.823 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:35:51.823 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_7.status 2026-02-18T08:35:51.824 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_7.status 2026-02-18T08:35:51.824 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:35:51.824 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_7.status 2026-02-18T08:35:51.825 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_7 2026-02-18T08:35:51.825 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_7.status 2026-02-18T08:35:51.859 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 5 2026-02-18T08:35:51.859 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 5 2026-02-18T08:35:51.860 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:35:51.860 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=21, tag_tid=2, entry_tid=1], [object_number=20, tag_tid=2, entry_tid=0], [object_number=23, tag_tid=1, entry_tid=25315], [object_number=22, tag_tid=1, entry_tid=25314]]], state=connected] 2026-02-18T08:35:51.860 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=21, tag_tid=1, entry_tid=25317], [object_number=20, tag_tid=1, entry_tid=25316], [object_number=23, tag_tid=1, entry_tid=25315], [object_number=22, tag_tid=1, entry_tid=25314]]], state=connected] 2026-02-18T08:35:51.862 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_7.status 2026-02-18T08:35:51.864 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=21, tag_tid=1, entry_tid=25317' 2026-02-18T08:35:51.864 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=21, tag_tid=2, entry_tid=1' -a 'object_number=21, tag_tid=2, entry_tid=1' = 'object_number=21, tag_tid=1, entry_tid=25317' 2026-02-18T08:35:51.864 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=21, tag_tid=1, entry_tid=25317' '!=' 'object_number=21, tag_tid=1, entry_tid=25317' 2026-02-18T08:35:51.864 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:35:51.864 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.4 2026-02-18T08:35:52.266 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_7 2026-02-18T08:35:52.266 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:35:52.266 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:35:52.266 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_7 2026-02-18T08:35:52.266 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:35:52.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_7 ']' 2026-02-18T08:35:52.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_7' 2026-02-18T08:35:52.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_7 2026-02-18T08:35:52.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:35:52.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:35:52.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:35:52.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:35:52.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:35:52.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:35:52.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:35:52.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:35:52.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:35:52.270 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:35:52.270 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_7 2026-02-18T08:35:52.270 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:35:52.270 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:35:52.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:35:52.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:35:52.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:35:52.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:35:52.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:35:52.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:35:52.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:35:52.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:35:52.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:35:52.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:35:52.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:35:52.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:35:52.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:35:52.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:35:52.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:35:52.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:35:52.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:35:52.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:35:52.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:35:52.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:35:52.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:35:52.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:35:52.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:35:52.273 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:35:52.273 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:35:52.273 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:35:52.273 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:35:52.273 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:35:52.279 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:35:52.280 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:35:52.280 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:35:52.280 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_7 2026-02-18T08:35:52.348 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:35:52.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_7 2026-02-18T08:35:52.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:35:52.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:35:52.350 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_7 2026-02-18T08:35:52.350 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_7 '' 2026-02-18T08:35:52.350 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:35:52.350 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:35:52.350 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_7 2026-02-18T08:35:52.350 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:35:52.350 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_7.status 2026-02-18T08:35:52.350 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_7.status 2026-02-18T08:35:52.350 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:35:52.351 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_7.status 2026-02-18T08:35:52.351 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_7 2026-02-18T08:35:52.351 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_7.status 2026-02-18T08:35:52.380 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 5 2026-02-18T08:35:52.380 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 5 2026-02-18T08:35:52.380 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:35:52.380 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=21, tag_tid=2, entry_tid=1], [object_number=20, tag_tid=2, entry_tid=0], [object_number=23, tag_tid=1, entry_tid=25315], [object_number=22, tag_tid=1, entry_tid=25314]]], state=connected] 2026-02-18T08:35:52.380 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=21, tag_tid=1, entry_tid=25317], [object_number=20, tag_tid=1, entry_tid=25316], [object_number=23, tag_tid=1, entry_tid=25315], [object_number=22, tag_tid=1, entry_tid=25314]]], state=connected] 2026-02-18T08:35:52.383 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_7.status 2026-02-18T08:35:52.384 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=21, tag_tid=2, entry_tid=1' 2026-02-18T08:35:52.385 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_7 2026-02-18T08:35:52.385 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:35:52.385 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:35:52.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_7 2026-02-18T08:35:52.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_7 '..*' 2026-02-18T08:35:52.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:35:52.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:35:52.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_7 2026-02-18T08:35:52.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:35:52.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_7.status 2026-02-18T08:35:52.387 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_7.status 2026-02-18T08:35:52.387 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:35:52.387 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_7.status 2026-02-18T08:35:52.387 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_7 2026-02-18T08:35:52.388 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_7.status 2026-02-18T08:35:52.413 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 5 2026-02-18T08:35:52.414 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 5 2026-02-18T08:35:52.414 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:35:52.414 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=21, tag_tid=2, entry_tid=1], [object_number=20, tag_tid=2, entry_tid=0], [object_number=23, tag_tid=1, entry_tid=25315], [object_number=22, tag_tid=1, entry_tid=25314]]], state=connected] 2026-02-18T08:35:52.414 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=21, tag_tid=1, entry_tid=25317], [object_number=20, tag_tid=1, entry_tid=25316], [object_number=23, tag_tid=1, entry_tid=25315], [object_number=22, tag_tid=1, entry_tid=25314]]], state=connected] 2026-02-18T08:35:52.416 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_7.status 2026-02-18T08:35:52.417 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=21, tag_tid=1, entry_tid=25317' 2026-02-18T08:35:52.418 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=21, tag_tid=2, entry_tid=1' -a 'object_number=21, tag_tid=2, entry_tid=1' = 'object_number=21, tag_tid=1, entry_tid=25317' 2026-02-18T08:35:52.418 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=21, tag_tid=1, entry_tid=25317' '!=' 'object_number=21, tag_tid=1, entry_tid=25317' 2026-02-18T08:35:52.418 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:35:52.418 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.8 2026-02-18T08:35:53.219 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_7 2026-02-18T08:35:53.220 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:35:53.220 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:35:53.220 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_7 2026-02-18T08:35:53.220 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:35:53.220 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_7 ']' 2026-02-18T08:35:53.220 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_7' 2026-02-18T08:35:53.220 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_7 2026-02-18T08:35:53.220 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:35:53.221 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:35:53.221 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:35:53.221 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:35:53.221 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:35:53.221 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:35:53.221 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:35:53.221 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:35:53.221 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:35:53.223 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:35:53.224 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_7 2026-02-18T08:35:53.224 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:35:53.224 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:35:53.224 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:35:53.224 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:35:53.224 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:35:53.224 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:35:53.224 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:35:53.224 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:35:53.224 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:35:53.225 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:35:53.225 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:35:53.225 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:35:53.225 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:35:53.225 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:35:53.225 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:35:53.225 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:35:53.225 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:35:53.225 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:35:53.225 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:35:53.225 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:35:53.225 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:35:53.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:35:53.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:35:53.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:35:53.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:35:53.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:35:53.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:35:53.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:35:53.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:35:53.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:35:53.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:35:53.234 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:35:53.234 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:35:53.234 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_7 2026-02-18T08:35:53.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:35:53.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_7 2026-02-18T08:35:53.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:35:53.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:35:53.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_7 2026-02-18T08:35:53.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_7 '' 2026-02-18T08:35:53.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:35:53.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:35:53.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_7 2026-02-18T08:35:53.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:35:53.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_7.status 2026-02-18T08:35:53.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_7.status 2026-02-18T08:35:53.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:35:53.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_7.status 2026-02-18T08:35:53.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_7 2026-02-18T08:35:53.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_7.status 2026-02-18T08:35:53.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 5 2026-02-18T08:35:53.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 5 2026-02-18T08:35:53.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:35:53.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=21, tag_tid=2, entry_tid=1], [object_number=20, tag_tid=2, entry_tid=0], [object_number=23, tag_tid=1, entry_tid=25315], [object_number=22, tag_tid=1, entry_tid=25314]]], state=connected] 2026-02-18T08:35:53.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=21, tag_tid=1, entry_tid=25317], [object_number=20, tag_tid=1, entry_tid=25316], [object_number=23, tag_tid=1, entry_tid=25315], [object_number=22, tag_tid=1, entry_tid=25314]]], state=connected] 2026-02-18T08:35:53.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_7.status 2026-02-18T08:35:53.350 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=21, tag_tid=2, entry_tid=1' 2026-02-18T08:35:53.351 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_7 2026-02-18T08:35:53.352 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:35:53.352 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:35:53.352 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_7 2026-02-18T08:35:53.352 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_7 '..*' 2026-02-18T08:35:53.352 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:35:53.352 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:35:53.352 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_7 2026-02-18T08:35:53.352 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:35:53.352 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_7.status 2026-02-18T08:35:53.353 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_7.status 2026-02-18T08:35:53.353 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:35:53.353 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_7.status 2026-02-18T08:35:53.354 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_7 2026-02-18T08:35:53.354 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_7.status 2026-02-18T08:35:53.381 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 5 2026-02-18T08:35:53.381 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 5 2026-02-18T08:35:53.381 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:35:53.381 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=21, tag_tid=2, entry_tid=1], [object_number=20, tag_tid=2, entry_tid=0], [object_number=23, tag_tid=1, entry_tid=25315], [object_number=22, tag_tid=1, entry_tid=25314]]], state=connected] 2026-02-18T08:35:53.381 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=21, tag_tid=1, entry_tid=25317], [object_number=20, tag_tid=1, entry_tid=25316], [object_number=23, tag_tid=1, entry_tid=25315], [object_number=22, tag_tid=1, entry_tid=25314]]], state=connected] 2026-02-18T08:35:53.384 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_7.status 2026-02-18T08:35:53.385 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=21, tag_tid=1, entry_tid=25317' 2026-02-18T08:35:53.385 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=21, tag_tid=2, entry_tid=1' -a 'object_number=21, tag_tid=2, entry_tid=1' = 'object_number=21, tag_tid=1, entry_tid=25317' 2026-02-18T08:35:53.385 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=21, tag_tid=1, entry_tid=25317' '!=' 'object_number=21, tag_tid=1, entry_tid=25317' 2026-02-18T08:35:53.385 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:35:53.385 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1.6 2026-02-18T08:35:54.986 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_7 2026-02-18T08:35:54.986 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:35:54.986 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:35:54.986 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_7 2026-02-18T08:35:54.986 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:35:54.987 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_7 ']' 2026-02-18T08:35:54.987 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_7' 2026-02-18T08:35:54.987 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_7 2026-02-18T08:35:54.987 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:35:54.987 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:35:54.987 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:35:54.987 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:35:54.987 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:35:54.987 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:35:54.987 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:35:54.987 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:35:54.988 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:35:54.989 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:35:54.989 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_7 2026-02-18T08:35:54.989 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:35:54.989 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:35:54.989 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:35:54.989 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:35:54.989 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:35:54.989 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:35:54.989 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:35:54.989 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:35:54.989 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:35:54.989 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:35:54.990 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:35:54.990 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:35:54.990 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:35:54.990 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:35:54.990 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:35:54.990 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:35:54.990 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:35:54.990 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:35:54.990 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:35:54.990 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:35:54.990 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:35:54.990 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:35:54.990 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:35:54.990 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:35:54.990 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:35:54.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:35:54.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:35:54.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:35:54.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:35:54.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:35:55.004 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:35:55.004 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:35:55.004 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:35:55.004 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_7 2026-02-18T08:35:55.062 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:35:55.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_7 2026-02-18T08:35:55.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:35:55.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:35:55.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_7 2026-02-18T08:35:55.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_7 '' 2026-02-18T08:35:55.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:35:55.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:35:55.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_7 2026-02-18T08:35:55.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:35:55.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_7.status 2026-02-18T08:35:55.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_7.status 2026-02-18T08:35:55.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:35:55.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_7.status 2026-02-18T08:35:55.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_7 2026-02-18T08:35:55.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_7.status 2026-02-18T08:35:55.101 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 5 2026-02-18T08:35:55.102 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 5 2026-02-18T08:35:55.102 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:35:55.102 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=21, tag_tid=2, entry_tid=1], [object_number=20, tag_tid=2, entry_tid=0], [object_number=23, tag_tid=1, entry_tid=25315], [object_number=22, tag_tid=1, entry_tid=25314]]], state=connected] 2026-02-18T08:35:55.102 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=21, tag_tid=1, entry_tid=25317], [object_number=20, tag_tid=1, entry_tid=25316], [object_number=23, tag_tid=1, entry_tid=25315], [object_number=22, tag_tid=1, entry_tid=25314]]], state=connected] 2026-02-18T08:35:55.105 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_7.status 2026-02-18T08:35:55.106 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=21, tag_tid=2, entry_tid=1' 2026-02-18T08:35:55.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_7 2026-02-18T08:35:55.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:35:55.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:35:55.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_7 2026-02-18T08:35:55.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_7 '..*' 2026-02-18T08:35:55.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:35:55.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:35:55.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_7 2026-02-18T08:35:55.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:35:55.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_7.status 2026-02-18T08:35:55.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_7.status 2026-02-18T08:35:55.110 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:35:55.110 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_7.status 2026-02-18T08:35:55.110 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_7 2026-02-18T08:35:55.110 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_7.status 2026-02-18T08:35:55.145 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 5 2026-02-18T08:35:55.145 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 5 2026-02-18T08:35:55.145 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:35:55.145 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=21, tag_tid=2, entry_tid=1], [object_number=20, tag_tid=2, entry_tid=0], [object_number=23, tag_tid=1, entry_tid=25315], [object_number=22, tag_tid=1, entry_tid=25314]]], state=connected] 2026-02-18T08:35:55.146 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=21, tag_tid=1, entry_tid=25317], [object_number=20, tag_tid=1, entry_tid=25316], [object_number=23, tag_tid=1, entry_tid=25315], [object_number=22, tag_tid=1, entry_tid=25314]]], state=connected] 2026-02-18T08:35:55.149 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_7.status 2026-02-18T08:35:55.150 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=21, tag_tid=1, entry_tid=25317' 2026-02-18T08:35:55.150 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=21, tag_tid=2, entry_tid=1' -a 'object_number=21, tag_tid=2, entry_tid=1' = 'object_number=21, tag_tid=1, entry_tid=25317' 2026-02-18T08:35:55.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=21, tag_tid=1, entry_tid=25317' '!=' 'object_number=21, tag_tid=1, entry_tid=25317' 2026-02-18T08:35:55.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:35:55.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 2 2026-02-18T08:35:57.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_7 2026-02-18T08:35:57.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:35:57.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:35:57.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_7 2026-02-18T08:35:57.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:35:57.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_7 ']' 2026-02-18T08:35:57.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_7' 2026-02-18T08:35:57.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_7 2026-02-18T08:35:57.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:35:57.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:35:57.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:35:57.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:35:57.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:35:57.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:35:57.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:35:57.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:35:57.157 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:35:57.157 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:35:57.157 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_7 2026-02-18T08:35:57.157 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:35:57.157 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:35:57.157 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:35:57.157 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:35:57.157 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:35:57.158 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:35:57.158 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:35:57.158 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:35:57.158 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:35:57.158 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:35:57.158 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:35:57.158 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:35:57.158 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:35:57.158 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:35:57.158 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:35:57.158 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:35:57.158 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:35:57.158 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:35:57.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:35:57.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:35:57.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:35:57.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:35:57.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:35:57.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:35:57.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:35:57.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:35:57.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:35:57.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:35:57.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:35:57.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:35:57.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:35:57.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:35:57.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:35:57.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_7 2026-02-18T08:35:57.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:35:57.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_7 2026-02-18T08:35:57.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:35:57.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:35:57.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_7 2026-02-18T08:35:57.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_7 '' 2026-02-18T08:35:57.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:35:57.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:35:57.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_7 2026-02-18T08:35:57.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:35:57.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_7.status 2026-02-18T08:35:57.238 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_7.status 2026-02-18T08:35:57.238 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:35:57.238 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_7.status 2026-02-18T08:35:57.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_7 2026-02-18T08:35:57.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_7.status 2026-02-18T08:35:57.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 5 2026-02-18T08:35:57.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 5 2026-02-18T08:35:57.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:35:57.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=21, tag_tid=2, entry_tid=1], [object_number=20, tag_tid=2, entry_tid=0], [object_number=23, tag_tid=1, entry_tid=25315], [object_number=22, tag_tid=1, entry_tid=25314]]], state=connected] 2026-02-18T08:35:57.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=21, tag_tid=2, entry_tid=1], [object_number=20, tag_tid=2, entry_tid=0], [object_number=23, tag_tid=1, entry_tid=25315], [object_number=22, tag_tid=1, entry_tid=25314]]], state=connected] 2026-02-18T08:35:57.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_7.status 2026-02-18T08:35:57.273 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=21, tag_tid=2, entry_tid=1' 2026-02-18T08:35:57.274 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_7 2026-02-18T08:35:57.274 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:35:57.274 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:35:57.274 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_7 2026-02-18T08:35:57.274 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_7 '..*' 2026-02-18T08:35:57.275 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:35:57.275 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:35:57.275 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_7 2026-02-18T08:35:57.275 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:35:57.275 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_7.status 2026-02-18T08:35:57.275 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_7.status 2026-02-18T08:35:57.275 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:35:57.276 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_7.status 2026-02-18T08:35:57.276 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_7 2026-02-18T08:35:57.276 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_7.status 2026-02-18T08:35:57.310 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 5 2026-02-18T08:35:57.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 5 2026-02-18T08:35:57.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:35:57.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=21, tag_tid=2, entry_tid=1], [object_number=20, tag_tid=2, entry_tid=0], [object_number=23, tag_tid=1, entry_tid=25315], [object_number=22, tag_tid=1, entry_tid=25314]]], state=connected] 2026-02-18T08:35:57.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=21, tag_tid=2, entry_tid=1], [object_number=20, tag_tid=2, entry_tid=0], [object_number=23, tag_tid=1, entry_tid=25315], [object_number=22, tag_tid=1, entry_tid=25314]]], state=connected] 2026-02-18T08:35:57.314 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_7.status 2026-02-18T08:35:57.316 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=21, tag_tid=2, entry_tid=1' 2026-02-18T08:35:57.317 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=21, tag_tid=2, entry_tid=1' -a 'object_number=21, tag_tid=2, entry_tid=1' = 'object_number=21, tag_tid=2, entry_tid=1' 2026-02-18T08:35:57.317 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:35:57.317 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_snap_present cluster1 mirror image_7 snap 2026-02-18T08:35:57.317 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:35:57.317 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:35:57.317 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_7 2026-02-18T08:35:57.317 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap 2026-02-18T08:35:57.317 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-18T08:35:57.317 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-18T08:35:57.317 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1 2026-02-18T08:35:58.317 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster1 info mirror/image_7@snap 2026-02-18T08:35:58.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:rbd image 'image_7': 2026-02-18T08:35:58.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: size 128 MiB in 32 objects 2026-02-18T08:35:58.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: order 22 (4 MiB objects) 2026-02-18T08:35:58.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: snapshot_count: 1 2026-02-18T08:35:58.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: id: 11041ee1a568 2026-02-18T08:35:58.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: block_name_prefix: rbd_data.11041ee1a568 2026-02-18T08:35:58.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: format: 2 2026-02-18T08:35:58.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: features: layering, exclusive-lock, journaling 2026-02-18T08:35:58.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: op_features: 2026-02-18T08:35:58.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: flags: 2026-02-18T08:35:58.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: create_timestamp: Wed Feb 18 08:26:41 2026 2026-02-18T08:35:58.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: access_timestamp: Wed Feb 18 08:26:41 2026 2026-02-18T08:35:58.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: modify_timestamp: Wed Feb 18 08:27:42 2026 2026-02-18T08:35:58.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: protected: False 2026-02-18T08:35:58.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: journal: 11041ee1a568 2026-02-18T08:35:58.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring state: enabled 2026-02-18T08:35:58.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring mode: journal 2026-02-18T08:35:58.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring global id: 0092d042-71d1-488e-887f-e0fa1fa53a73 2026-02-18T08:35:58.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring primary: false 2026-02-18T08:35:58.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:35:58.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ compare_image_snaps mirror image_7 snap 2026-02-18T08:35:58.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:35:58.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_7 2026-02-18T08:35:58.350 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap 2026-02-18T08:35:58.350 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local ret=0 2026-02-18T08:35:58.350 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local rmt_export=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_7.export 2026-02-18T08:35:58.350 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loc_export=/tmp/tmp.vybh1m9yJF/cluster1-mirror-image_7.export 2026-02-18T08:35:58.350 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_7.export /tmp/tmp.vybh1m9yJF/cluster1-mirror-image_7.export 2026-02-18T08:35:58.350 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 -p mirror export image_7@snap /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_7.export 2026-02-18T08:35:58.484 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-18T08:35:58.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster1 -p mirror export image_7@snap /tmp/tmp.vybh1m9yJF/cluster1-mirror-image_7.export 2026-02-18T08:35:58.687 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-18T08:35:58.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmp /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_7.export /tmp/tmp.vybh1m9yJF/cluster1-mirror-image_7.export 2026-02-18T08:35:58.756 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_7.export /tmp/tmp.vybh1m9yJF/cluster1-mirror-image_7.export 2026-02-18T08:35:58.803 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:35:58.803 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T08:35:58.803 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_8 2026-02-18T08:35:58.803 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_snap cluster2 mirror image_8 snap 2026-02-18T08:35:58.803 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:35:58.804 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:35:58.804 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_8 2026-02-18T08:35:58.804 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap 2026-02-18T08:35:58.804 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 -p mirror snap create image_8@snap --debug-rbd=20 --debug-journaler=20 2026-02-18T08:36:02.856 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_replay_started cluster1 mirror image_8 2026-02-18T08:36:02.856 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:36:02.857 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:36:02.857 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_8 2026-02-18T08:36:02.857 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_replay_state cluster1 mirror image_8 started 2026-02-18T08:36:02.857 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:36:02.857 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:36:02.857 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_8 2026-02-18T08:36:02.857 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local state=started 2026-02-18T08:36:02.857 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-18T08:36:02.857 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:36:02.857 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1 2026-02-18T08:36:03.857 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test_image_replay_state cluster1 mirror image_8 started 2026-02-18T08:36:03.858 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:36:03.858 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:36:03.858 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_8 2026-02-18T08:36:03.858 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local test_state=started 2026-02-18T08:36:03.858 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local status_result 2026-02-18T08:36:03.858 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_state=stopped 2026-02-18T08:36:03.859 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_8 2026-02-18T08:36:03.860 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -i state 2026-02-18T08:36:03.860 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_instance=cluster1 2026-02-18T08:36:03.860 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-18T08:36:03.860 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:36:03.860 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance=cluster1 2026-02-18T08:36:03.860 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local loop_instance 2026-02-18T08:36:03.860 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:36:03.860 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sleep 0 2026-02-18T08:36:03.860 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:36:03.860 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ seq 0 3 2026-02-18T08:36:03.862 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:36:03.863 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_8 2026-02-18T08:36:03.863 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:0 2026-02-18T08:36:03.863 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-18T08:36:03.863 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:36:03.863 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:36:03.863 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:36:03.863 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-18T08:36:03.863 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:36:03.863 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:36:03.863 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:36:03.863 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:36:03.864 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-18T08:36:03.864 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:36:03.864 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:36:03.864 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:36:03.864 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:36:03.864 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:0 2026-02-18T08:36:03.864 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-18T08:36:03.864 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-18T08:36:03.864 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:36:03.864 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:0 2026-02-18T08:36:03.864 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T08:36:03.864 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-18T08:36:03.865 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:36:03.865 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:36:03.865 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-18T08:36:03.865 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-18T08:36:03.865 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-18T08:36:03.865 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=0 2026-02-18T08:36:03.865 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=0 2026-02-18T08:36:03.865 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:36:03.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:36:03.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:36:03.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:36:03.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_8 2026-02-18T08:36:03.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ return 0 2026-02-18T08:36:03.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ status_result=' "state": "Replaying"' 2026-02-18T08:36:03.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ echo ' "state": "Replaying"' 2026-02-18T08:36:03.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ grep -i Replaying 2026-02-18T08:36:03.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: "state": "Replaying" 2026-02-18T08:36:03.944 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ current_state=started 2026-02-18T08:36:03.944 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test started = started 2026-02-18T08:36:03.944 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:36:03.944 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror image_8 2026-02-18T08:36:03.944 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_cluster=cluster1 2026-02-18T08:36:03.944 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:36:03.944 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_pool=mirror 2026-02-18T08:36:03.944 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local remote_pool=mirror 2026-02-18T08:36:03.944 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_8 2026-02-18T08:36:03.944 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' journal = journal ']' 2026-02-18T08:36:03.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror mirror image_8 2026-02-18T08:36:03.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_cluster=cluster1 2026-02-18T08:36:03.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:36:03.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_pool=mirror 2026-02-18T08:36:03.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local remote_pool=mirror 2026-02-18T08:36:03.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_8 2026-02-18T08:36:03.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-02-18T08:36:03.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-02-18T08:36:03.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:36:03.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:36:03.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:36:04.145 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_8 2026-02-18T08:36:04.145 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:36:04.145 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:36:04.145 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_8 2026-02-18T08:36:04.145 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:36:04.145 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_8 ']' 2026-02-18T08:36:04.145 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_8' 2026-02-18T08:36:04.146 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_8 2026-02-18T08:36:04.146 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:36:04.146 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:36:04.146 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:36:04.146 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:36:04.146 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:36:04.146 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:36:04.146 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:36:04.149 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:36:04.149 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:36:04.149 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:36:04.149 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_8 2026-02-18T08:36:04.149 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:36:04.149 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:36:04.149 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:36:04.149 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:36:04.150 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:36:04.150 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:36:04.150 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:36:04.150 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:36:04.150 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:36:04.150 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:36:04.150 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:36:04.150 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:36:04.150 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:36:04.150 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:36:04.150 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:36:04.150 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:36:04.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:36:04.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:36:04.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:36:04.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:36:04.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:36:04.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:36:04.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:36:04.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:36:04.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:36:04.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:36:04.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:36:04.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:36:04.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:36:04.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:36:04.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:36:04.161 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:36:04.161 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:36:04.161 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_8 2026-02-18T08:36:04.227 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:36:04.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_8 2026-02-18T08:36:04.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:36:04.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:36:04.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_8 2026-02-18T08:36:04.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_8 '' 2026-02-18T08:36:04.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:36:04.229 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:36:04.229 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_8 2026-02-18T08:36:04.229 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:36:04.229 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_8.status 2026-02-18T08:36:04.229 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_8.status 2026-02-18T08:36:04.229 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:36:04.230 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_8.status 2026-02-18T08:36:04.230 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_8 2026-02-18T08:36:04.230 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_8.status 2026-02-18T08:36:04.259 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 13 2026-02-18T08:36:04.259 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 13 2026-02-18T08:36:04.259 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:36:04.260 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=53, tag_tid=2, entry_tid=1], [object_number=52, tag_tid=2, entry_tid=0], [object_number=54, tag_tid=1, entry_tid=55706], [object_number=55, tag_tid=1, entry_tid=55703]]], state=connected] 2026-02-18T08:36:04.260 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=54, tag_tid=1, entry_tid=55706], [object_number=53, tag_tid=1, entry_tid=55705], [object_number=52, tag_tid=1, entry_tid=55704], [object_number=55, tag_tid=1, entry_tid=55703]]], state=connected] 2026-02-18T08:36:04.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_8.status 2026-02-18T08:36:04.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=53, tag_tid=2, entry_tid=1' 2026-02-18T08:36:04.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_8 2026-02-18T08:36:04.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:36:04.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:36:04.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_8 2026-02-18T08:36:04.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_8 '..*' 2026-02-18T08:36:04.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:36:04.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:36:04.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_8 2026-02-18T08:36:04.266 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:36:04.266 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_8.status 2026-02-18T08:36:04.266 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_8.status 2026-02-18T08:36:04.266 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:36:04.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_8.status 2026-02-18T08:36:04.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_8 2026-02-18T08:36:04.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_8.status 2026-02-18T08:36:04.300 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 13 2026-02-18T08:36:04.300 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 13 2026-02-18T08:36:04.300 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:36:04.300 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=53, tag_tid=2, entry_tid=1], [object_number=52, tag_tid=2, entry_tid=0], [object_number=54, tag_tid=1, entry_tid=55706], [object_number=55, tag_tid=1, entry_tid=55703]]], state=connected] 2026-02-18T08:36:04.300 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=54, tag_tid=1, entry_tid=55706], [object_number=53, tag_tid=1, entry_tid=55705], [object_number=52, tag_tid=1, entry_tid=55704], [object_number=55, tag_tid=1, entry_tid=55703]]], state=connected] 2026-02-18T08:36:04.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_8.status 2026-02-18T08:36:04.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=54, tag_tid=1, entry_tid=55706' 2026-02-18T08:36:04.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=53, tag_tid=2, entry_tid=1' -a 'object_number=53, tag_tid=2, entry_tid=1' = 'object_number=54, tag_tid=1, entry_tid=55706' 2026-02-18T08:36:04.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=54, tag_tid=1, entry_tid=55706' '!=' '' 2026-02-18T08:36:04.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:36:04.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=54, tag_tid=1, entry_tid=55706' = '' 2026-02-18T08:36:04.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=54, tag_tid=1, entry_tid=55706' 2026-02-18T08:36:04.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=53, tag_tid=2, entry_tid=1' 2026-02-18T08:36:04.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:36:04.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:36:04.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=2 2026-02-18T08:36:04.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=54, tag_tid=1, entry_tid=55706' 2026-02-18T08:36:04.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:36:04.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:36:04.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=1 2026-02-18T08:36:04.310 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=53, tag_tid=2, entry_tid=1' 2026-02-18T08:36:04.310 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:36:04.310 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:36:04.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:36:04.312 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=54, tag_tid=1, entry_tid=55706' 2026-02-18T08:36:04.312 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:36:04.312 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:36:04.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=55706 2026-02-18T08:36:04.314 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 2 = 1 -a 1 -le 55706 2026-02-18T08:36:04.314 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:36:04.314 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:36:04.314 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:36:04.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_8 2026-02-18T08:36:04.516 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:36:04.516 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:36:04.516 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_8 2026-02-18T08:36:04.516 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:36:04.516 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_8 ']' 2026-02-18T08:36:04.516 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_8' 2026-02-18T08:36:04.516 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_8 2026-02-18T08:36:04.516 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:36:04.516 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:36:04.516 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:36:04.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:36:04.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:36:04.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:36:04.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:36:04.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:36:04.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:36:04.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:36:04.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_8 2026-02-18T08:36:04.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:36:04.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:36:04.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:36:04.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:36:04.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:36:04.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:36:04.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:36:04.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:36:04.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:36:04.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:36:04.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:36:04.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:36:04.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:36:04.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:36:04.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:36:04.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:36:04.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:36:04.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:36:04.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:36:04.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:36:04.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:36:04.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:36:04.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:36:04.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:36:04.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:36:04.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:36:04.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:36:04.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:36:04.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:36:04.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:36:04.531 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:36:04.531 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:36:04.531 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:36:04.532 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_8 2026-02-18T08:36:04.602 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:36:04.603 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_8 2026-02-18T08:36:04.603 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:36:04.603 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:36:04.604 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_8 2026-02-18T08:36:04.604 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_8 '' 2026-02-18T08:36:04.604 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:36:04.604 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:36:04.604 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_8 2026-02-18T08:36:04.604 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:36:04.604 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_8.status 2026-02-18T08:36:04.604 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_8.status 2026-02-18T08:36:04.605 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:36:04.605 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_8.status 2026-02-18T08:36:04.606 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_8 2026-02-18T08:36:04.606 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_8.status 2026-02-18T08:36:04.642 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 13 2026-02-18T08:36:04.642 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 13 2026-02-18T08:36:04.642 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:36:04.642 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=53, tag_tid=2, entry_tid=1], [object_number=52, tag_tid=2, entry_tid=0], [object_number=54, tag_tid=1, entry_tid=55706], [object_number=55, tag_tid=1, entry_tid=55703]]], state=connected] 2026-02-18T08:36:04.642 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=54, tag_tid=1, entry_tid=55706], [object_number=53, tag_tid=1, entry_tid=55705], [object_number=52, tag_tid=1, entry_tid=55704], [object_number=55, tag_tid=1, entry_tid=55703]]], state=connected] 2026-02-18T08:36:04.646 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_8.status 2026-02-18T08:36:04.647 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=53, tag_tid=2, entry_tid=1' 2026-02-18T08:36:04.648 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_8 2026-02-18T08:36:04.648 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:36:04.648 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:36:04.649 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_8 2026-02-18T08:36:04.649 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_8 '..*' 2026-02-18T08:36:04.649 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:36:04.649 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:36:04.649 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_8 2026-02-18T08:36:04.649 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:36:04.649 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_8.status 2026-02-18T08:36:04.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_8.status 2026-02-18T08:36:04.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:36:04.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_8.status 2026-02-18T08:36:04.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_8 2026-02-18T08:36:04.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_8.status 2026-02-18T08:36:04.677 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 13 2026-02-18T08:36:04.678 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 13 2026-02-18T08:36:04.678 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:36:04.678 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=53, tag_tid=2, entry_tid=1], [object_number=52, tag_tid=2, entry_tid=0], [object_number=54, tag_tid=1, entry_tid=55706], [object_number=55, tag_tid=1, entry_tid=55703]]], state=connected] 2026-02-18T08:36:04.678 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=54, tag_tid=1, entry_tid=55706], [object_number=53, tag_tid=1, entry_tid=55705], [object_number=52, tag_tid=1, entry_tid=55704], [object_number=55, tag_tid=1, entry_tid=55703]]], state=connected] 2026-02-18T08:36:04.680 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_8.status 2026-02-18T08:36:04.682 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=54, tag_tid=1, entry_tid=55706' 2026-02-18T08:36:04.682 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=53, tag_tid=2, entry_tid=1' -a 'object_number=53, tag_tid=2, entry_tid=1' = 'object_number=54, tag_tid=1, entry_tid=55706' 2026-02-18T08:36:04.682 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=54, tag_tid=1, entry_tid=55706' '!=' 'object_number=54, tag_tid=1, entry_tid=55706' 2026-02-18T08:36:04.682 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:36:04.682 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.4 2026-02-18T08:36:05.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_8 2026-02-18T08:36:05.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:36:05.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:36:05.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_8 2026-02-18T08:36:05.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:36:05.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_8 ']' 2026-02-18T08:36:05.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_8' 2026-02-18T08:36:05.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_8 2026-02-18T08:36:05.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:36:05.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:36:05.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:36:05.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:36:05.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:36:05.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:36:05.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:36:05.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:36:05.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:36:05.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:36:05.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_8 2026-02-18T08:36:05.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:36:05.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:36:05.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:36:05.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:36:05.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:36:05.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:36:05.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:36:05.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:36:05.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:36:05.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:36:05.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:36:05.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:36:05.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:36:05.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:36:05.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:36:05.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:36:05.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:36:05.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:36:05.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:36:05.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:36:05.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:36:05.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:36:05.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:36:05.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:36:05.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:36:05.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:36:05.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:36:05.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:36:05.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:36:05.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:36:05.101 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:36:05.101 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:36:05.101 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:36:05.101 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_8 2026-02-18T08:36:05.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:36:05.169 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_8 2026-02-18T08:36:05.169 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:36:05.169 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:36:05.169 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_8 2026-02-18T08:36:05.170 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_8 '' 2026-02-18T08:36:05.170 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:36:05.170 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:36:05.170 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_8 2026-02-18T08:36:05.170 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:36:05.170 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_8.status 2026-02-18T08:36:05.170 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_8.status 2026-02-18T08:36:05.170 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:36:05.171 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_8.status 2026-02-18T08:36:05.171 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_8 2026-02-18T08:36:05.171 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_8.status 2026-02-18T08:36:05.199 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 13 2026-02-18T08:36:05.200 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 13 2026-02-18T08:36:05.200 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:36:05.200 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=53, tag_tid=2, entry_tid=1], [object_number=52, tag_tid=2, entry_tid=0], [object_number=54, tag_tid=1, entry_tid=55706], [object_number=55, tag_tid=1, entry_tid=55703]]], state=connected] 2026-02-18T08:36:05.200 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=54, tag_tid=1, entry_tid=55706], [object_number=53, tag_tid=1, entry_tid=55705], [object_number=52, tag_tid=1, entry_tid=55704], [object_number=55, tag_tid=1, entry_tid=55703]]], state=connected] 2026-02-18T08:36:05.202 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_8.status 2026-02-18T08:36:05.204 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=53, tag_tid=2, entry_tid=1' 2026-02-18T08:36:05.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_8 2026-02-18T08:36:05.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:36:05.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:36:05.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_8 2026-02-18T08:36:05.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_8 '..*' 2026-02-18T08:36:05.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:36:05.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:36:05.206 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_8 2026-02-18T08:36:05.206 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:36:05.206 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_8.status 2026-02-18T08:36:05.206 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_8.status 2026-02-18T08:36:05.206 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:36:05.207 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_8.status 2026-02-18T08:36:05.207 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_8 2026-02-18T08:36:05.208 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_8.status 2026-02-18T08:36:05.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 13 2026-02-18T08:36:05.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 13 2026-02-18T08:36:05.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:36:05.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=53, tag_tid=2, entry_tid=1], [object_number=52, tag_tid=2, entry_tid=0], [object_number=54, tag_tid=1, entry_tid=55706], [object_number=55, tag_tid=1, entry_tid=55703]]], state=connected] 2026-02-18T08:36:05.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=54, tag_tid=1, entry_tid=55706], [object_number=53, tag_tid=1, entry_tid=55705], [object_number=52, tag_tid=1, entry_tid=55704], [object_number=55, tag_tid=1, entry_tid=55703]]], state=connected] 2026-02-18T08:36:05.245 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_8.status 2026-02-18T08:36:05.248 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=54, tag_tid=1, entry_tid=55706' 2026-02-18T08:36:05.248 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=53, tag_tid=2, entry_tid=1' -a 'object_number=53, tag_tid=2, entry_tid=1' = 'object_number=54, tag_tid=1, entry_tid=55706' 2026-02-18T08:36:05.248 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=54, tag_tid=1, entry_tid=55706' '!=' 'object_number=54, tag_tid=1, entry_tid=55706' 2026-02-18T08:36:05.248 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:36:05.248 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.8 2026-02-18T08:36:06.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_8 2026-02-18T08:36:06.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:36:06.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:36:06.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_8 2026-02-18T08:36:06.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:36:06.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_8 ']' 2026-02-18T08:36:06.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_8' 2026-02-18T08:36:06.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_8 2026-02-18T08:36:06.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:36:06.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:36:06.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:36:06.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:36:06.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:36:06.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:36:06.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:36:06.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:36:06.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:36:06.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:36:06.055 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_8 2026-02-18T08:36:06.055 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:36:06.055 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:36:06.055 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:36:06.055 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:36:06.055 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:36:06.055 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:36:06.055 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:36:06.055 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:36:06.055 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:36:06.055 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:36:06.056 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:36:06.056 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:36:06.056 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:36:06.056 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:36:06.056 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:36:06.056 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:36:06.056 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:36:06.056 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:36:06.056 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:36:06.056 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:36:06.056 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:36:06.056 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:36:06.056 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:36:06.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:36:06.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:36:06.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:36:06.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:36:06.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:36:06.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:36:06.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:36:06.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:36:06.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:36:06.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:36:06.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_8 2026-02-18T08:36:06.131 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:36:06.132 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_8 2026-02-18T08:36:06.132 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:36:06.132 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:36:06.132 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_8 2026-02-18T08:36:06.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_8 '' 2026-02-18T08:36:06.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:36:06.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:36:06.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_8 2026-02-18T08:36:06.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:36:06.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_8.status 2026-02-18T08:36:06.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_8.status 2026-02-18T08:36:06.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:36:06.134 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_8.status 2026-02-18T08:36:06.134 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_8 2026-02-18T08:36:06.134 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_8.status 2026-02-18T08:36:06.162 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 13 2026-02-18T08:36:06.162 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 13 2026-02-18T08:36:06.162 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:36:06.162 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=53, tag_tid=2, entry_tid=1], [object_number=52, tag_tid=2, entry_tid=0], [object_number=54, tag_tid=1, entry_tid=55706], [object_number=55, tag_tid=1, entry_tid=55703]]], state=connected] 2026-02-18T08:36:06.163 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=54, tag_tid=1, entry_tid=55706], [object_number=53, tag_tid=1, entry_tid=55705], [object_number=52, tag_tid=1, entry_tid=55704], [object_number=55, tag_tid=1, entry_tid=55703]]], state=connected] 2026-02-18T08:36:06.165 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_8.status 2026-02-18T08:36:06.166 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=53, tag_tid=2, entry_tid=1' 2026-02-18T08:36:06.167 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_8 2026-02-18T08:36:06.167 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:36:06.167 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:36:06.167 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_8 2026-02-18T08:36:06.167 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_8 '..*' 2026-02-18T08:36:06.167 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:36:06.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:36:06.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_8 2026-02-18T08:36:06.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:36:06.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_8.status 2026-02-18T08:36:06.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_8.status 2026-02-18T08:36:06.169 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:36:06.169 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_8.status 2026-02-18T08:36:06.169 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_8 2026-02-18T08:36:06.170 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_8.status 2026-02-18T08:36:06.204 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 13 2026-02-18T08:36:06.204 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 13 2026-02-18T08:36:06.204 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:36:06.204 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=53, tag_tid=2, entry_tid=1], [object_number=52, tag_tid=2, entry_tid=0], [object_number=54, tag_tid=1, entry_tid=55706], [object_number=55, tag_tid=1, entry_tid=55703]]], state=connected] 2026-02-18T08:36:06.204 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=54, tag_tid=1, entry_tid=55706], [object_number=53, tag_tid=1, entry_tid=55705], [object_number=52, tag_tid=1, entry_tid=55704], [object_number=55, tag_tid=1, entry_tid=55703]]], state=connected] 2026-02-18T08:36:06.207 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_8.status 2026-02-18T08:36:06.209 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=54, tag_tid=1, entry_tid=55706' 2026-02-18T08:36:06.209 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=53, tag_tid=2, entry_tid=1' -a 'object_number=53, tag_tid=2, entry_tid=1' = 'object_number=54, tag_tid=1, entry_tid=55706' 2026-02-18T08:36:06.209 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=54, tag_tid=1, entry_tid=55706' '!=' 'object_number=54, tag_tid=1, entry_tid=55706' 2026-02-18T08:36:06.209 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:36:06.209 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1.6 2026-02-18T08:36:07.810 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_8 2026-02-18T08:36:07.811 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:36:07.811 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:36:07.811 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_8 2026-02-18T08:36:07.811 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:36:07.811 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_8 ']' 2026-02-18T08:36:07.811 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_8' 2026-02-18T08:36:07.811 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_8 2026-02-18T08:36:07.811 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:36:07.811 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:36:07.812 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:36:07.812 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:36:07.812 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:36:07.812 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:36:07.812 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:36:07.815 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:36:07.815 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:36:07.815 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:36:07.815 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_8 2026-02-18T08:36:07.815 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:36:07.815 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:36:07.815 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:36:07.815 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:36:07.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:36:07.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:36:07.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:36:07.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:36:07.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:36:07.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:36:07.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:36:07.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:36:07.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:36:07.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:36:07.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:36:07.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:36:07.817 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:36:07.817 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:36:07.817 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:36:07.817 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:36:07.817 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:36:07.817 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:36:07.817 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:36:07.817 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:36:07.817 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:36:07.817 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:36:07.817 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:36:07.817 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:36:07.817 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:36:07.818 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:36:07.825 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:36:07.826 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:36:07.826 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:36:07.826 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_8 2026-02-18T08:36:07.886 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:36:07.887 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_8 2026-02-18T08:36:07.887 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:36:07.887 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:36:07.887 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_8 2026-02-18T08:36:07.887 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_8 '' 2026-02-18T08:36:07.887 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:36:07.887 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:36:07.887 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_8 2026-02-18T08:36:07.887 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:36:07.887 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_8.status 2026-02-18T08:36:07.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_8.status 2026-02-18T08:36:07.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:36:07.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_8.status 2026-02-18T08:36:07.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_8 2026-02-18T08:36:07.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_8.status 2026-02-18T08:36:07.916 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 13 2026-02-18T08:36:07.916 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 13 2026-02-18T08:36:07.916 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:36:07.916 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=53, tag_tid=2, entry_tid=1], [object_number=52, tag_tid=2, entry_tid=0], [object_number=54, tag_tid=1, entry_tid=55706], [object_number=55, tag_tid=1, entry_tid=55703]]], state=connected] 2026-02-18T08:36:07.916 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=54, tag_tid=1, entry_tid=55706], [object_number=53, tag_tid=1, entry_tid=55705], [object_number=52, tag_tid=1, entry_tid=55704], [object_number=55, tag_tid=1, entry_tid=55703]]], state=connected] 2026-02-18T08:36:07.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_8.status 2026-02-18T08:36:07.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=53, tag_tid=2, entry_tid=1' 2026-02-18T08:36:07.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_8 2026-02-18T08:36:07.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:36:07.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:36:07.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_8 2026-02-18T08:36:07.922 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_8 '..*' 2026-02-18T08:36:07.922 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:36:07.922 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:36:07.922 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_8 2026-02-18T08:36:07.922 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:36:07.922 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_8.status 2026-02-18T08:36:07.922 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_8.status 2026-02-18T08:36:07.923 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:36:07.923 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_8.status 2026-02-18T08:36:07.923 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_8 2026-02-18T08:36:07.923 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_8.status 2026-02-18T08:36:07.958 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 13 2026-02-18T08:36:07.958 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 13 2026-02-18T08:36:07.958 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:36:07.958 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=53, tag_tid=2, entry_tid=1], [object_number=52, tag_tid=2, entry_tid=0], [object_number=54, tag_tid=1, entry_tid=55706], [object_number=55, tag_tid=1, entry_tid=55703]]], state=connected] 2026-02-18T08:36:07.958 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=54, tag_tid=1, entry_tid=55706], [object_number=53, tag_tid=1, entry_tid=55705], [object_number=52, tag_tid=1, entry_tid=55704], [object_number=55, tag_tid=1, entry_tid=55703]]], state=connected] 2026-02-18T08:36:07.962 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_8.status 2026-02-18T08:36:07.963 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=54, tag_tid=1, entry_tid=55706' 2026-02-18T08:36:07.963 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=53, tag_tid=2, entry_tid=1' -a 'object_number=53, tag_tid=2, entry_tid=1' = 'object_number=54, tag_tid=1, entry_tid=55706' 2026-02-18T08:36:07.963 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=54, tag_tid=1, entry_tid=55706' '!=' 'object_number=54, tag_tid=1, entry_tid=55706' 2026-02-18T08:36:07.963 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:36:07.963 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 2 2026-02-18T08:36:09.965 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_8 2026-02-18T08:36:09.965 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:36:09.965 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:36:09.965 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_8 2026-02-18T08:36:09.965 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:36:09.965 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_8 ']' 2026-02-18T08:36:09.965 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_8' 2026-02-18T08:36:09.966 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_8 2026-02-18T08:36:09.966 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:36:09.966 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:36:09.966 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:36:09.966 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:36:09.966 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:36:09.966 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:36:09.966 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:36:09.969 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:36:09.969 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:36:09.969 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:36:09.969 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_8 2026-02-18T08:36:09.969 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:36:09.969 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:36:09.969 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:36:09.969 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:36:09.969 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:36:09.969 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:36:09.969 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:36:09.970 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:36:09.970 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:36:09.970 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:36:09.970 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:36:09.970 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:36:09.970 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:36:09.970 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:36:09.970 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:36:09.970 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:36:09.970 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:36:09.970 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:36:09.970 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:36:09.971 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:36:09.971 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:36:09.971 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:36:09.971 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:36:09.971 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:36:09.971 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:36:09.971 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:36:09.971 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:36:09.971 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:36:09.971 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:36:09.971 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:36:09.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:36:09.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:36:09.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:36:09.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_8 2026-02-18T08:36:10.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:36:10.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_8 2026-02-18T08:36:10.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:36:10.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:36:10.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_8 2026-02-18T08:36:10.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_8 '' 2026-02-18T08:36:10.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:36:10.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:36:10.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_8 2026-02-18T08:36:10.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:36:10.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_8.status 2026-02-18T08:36:10.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_8.status 2026-02-18T08:36:10.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:36:10.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_8.status 2026-02-18T08:36:10.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_8 2026-02-18T08:36:10.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_8.status 2026-02-18T08:36:10.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 13 2026-02-18T08:36:10.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 13 2026-02-18T08:36:10.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:36:10.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=53, tag_tid=2, entry_tid=1], [object_number=52, tag_tid=2, entry_tid=0], [object_number=54, tag_tid=1, entry_tid=55706], [object_number=55, tag_tid=1, entry_tid=55703]]], state=connected] 2026-02-18T08:36:10.086 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=54, tag_tid=1, entry_tid=55706], [object_number=53, tag_tid=1, entry_tid=55705], [object_number=52, tag_tid=1, entry_tid=55704], [object_number=55, tag_tid=1, entry_tid=55703]]], state=connected] 2026-02-18T08:36:10.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_8.status 2026-02-18T08:36:10.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=53, tag_tid=2, entry_tid=1' 2026-02-18T08:36:10.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_8 2026-02-18T08:36:10.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:36:10.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:36:10.092 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_8 2026-02-18T08:36:10.092 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_8 '..*' 2026-02-18T08:36:10.092 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:36:10.092 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:36:10.092 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_8 2026-02-18T08:36:10.092 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:36:10.092 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_8.status 2026-02-18T08:36:10.093 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_8.status 2026-02-18T08:36:10.093 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:36:10.093 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_8.status 2026-02-18T08:36:10.094 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_8 2026-02-18T08:36:10.094 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_8.status 2026-02-18T08:36:10.129 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 13 2026-02-18T08:36:10.129 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 13 2026-02-18T08:36:10.129 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:36:10.129 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=53, tag_tid=2, entry_tid=1], [object_number=52, tag_tid=2, entry_tid=0], [object_number=54, tag_tid=1, entry_tid=55706], [object_number=55, tag_tid=1, entry_tid=55703]]], state=connected] 2026-02-18T08:36:10.129 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=54, tag_tid=1, entry_tid=55706], [object_number=53, tag_tid=1, entry_tid=55705], [object_number=52, tag_tid=1, entry_tid=55704], [object_number=55, tag_tid=1, entry_tid=55703]]], state=connected] 2026-02-18T08:36:10.132 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_8.status 2026-02-18T08:36:10.134 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=54, tag_tid=1, entry_tid=55706' 2026-02-18T08:36:10.134 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=53, tag_tid=2, entry_tid=1' -a 'object_number=53, tag_tid=2, entry_tid=1' = 'object_number=54, tag_tid=1, entry_tid=55706' 2026-02-18T08:36:10.134 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=54, tag_tid=1, entry_tid=55706' '!=' 'object_number=54, tag_tid=1, entry_tid=55706' 2026-02-18T08:36:10.134 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:36:10.134 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 2 2026-02-18T08:36:12.136 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_8 2026-02-18T08:36:12.137 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:36:12.137 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:36:12.137 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_8 2026-02-18T08:36:12.137 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:36:12.137 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_8 ']' 2026-02-18T08:36:12.137 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_8' 2026-02-18T08:36:12.137 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_8 2026-02-18T08:36:12.137 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:36:12.137 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:36:12.137 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:36:12.137 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:36:12.138 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:36:12.138 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:36:12.138 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:36:12.140 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:36:12.141 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:36:12.141 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:36:12.141 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_8 2026-02-18T08:36:12.141 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:36:12.141 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:36:12.141 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:36:12.141 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:36:12.141 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:36:12.141 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:36:12.141 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:36:12.141 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:36:12.142 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:36:12.142 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:36:12.142 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:36:12.142 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:36:12.142 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:36:12.142 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:36:12.142 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:36:12.142 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:36:12.142 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:36:12.142 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:36:12.142 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:36:12.142 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:36:12.143 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:36:12.143 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:36:12.143 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:36:12.143 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:36:12.143 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:36:12.143 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:36:12.143 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:36:12.143 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:36:12.143 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:36:12.143 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:36:12.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:36:12.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:36:12.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:36:12.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_8 2026-02-18T08:36:12.224 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:36:12.224 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_8 2026-02-18T08:36:12.225 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:36:12.225 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:36:12.225 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_8 2026-02-18T08:36:12.225 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_8 '' 2026-02-18T08:36:12.225 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:36:12.225 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:36:12.225 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_8 2026-02-18T08:36:12.225 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:36:12.225 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_8.status 2026-02-18T08:36:12.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_8.status 2026-02-18T08:36:12.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:36:12.227 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_8.status 2026-02-18T08:36:12.227 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_8 2026-02-18T08:36:12.227 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_8.status 2026-02-18T08:36:12.254 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 13 2026-02-18T08:36:12.254 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 13 2026-02-18T08:36:12.254 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:36:12.254 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=53, tag_tid=2, entry_tid=1], [object_number=52, tag_tid=2, entry_tid=0], [object_number=54, tag_tid=1, entry_tid=55706], [object_number=55, tag_tid=1, entry_tid=55703]]], state=connected] 2026-02-18T08:36:12.254 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=54, tag_tid=1, entry_tid=55706], [object_number=53, tag_tid=1, entry_tid=55705], [object_number=52, tag_tid=1, entry_tid=55704], [object_number=55, tag_tid=1, entry_tid=55703]]], state=connected] 2026-02-18T08:36:12.256 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_8.status 2026-02-18T08:36:12.257 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=53, tag_tid=2, entry_tid=1' 2026-02-18T08:36:12.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_8 2026-02-18T08:36:12.259 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:36:12.259 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:36:12.259 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_8 2026-02-18T08:36:12.259 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_8 '..*' 2026-02-18T08:36:12.259 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:36:12.259 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:36:12.259 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_8 2026-02-18T08:36:12.259 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:36:12.259 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_8.status 2026-02-18T08:36:12.260 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_8.status 2026-02-18T08:36:12.260 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:36:12.260 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_8.status 2026-02-18T08:36:12.261 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_8 2026-02-18T08:36:12.261 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_8.status 2026-02-18T08:36:12.287 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 13 2026-02-18T08:36:12.287 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 13 2026-02-18T08:36:12.287 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:36:12.287 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=53, tag_tid=2, entry_tid=1], [object_number=52, tag_tid=2, entry_tid=0], [object_number=54, tag_tid=1, entry_tid=55706], [object_number=55, tag_tid=1, entry_tid=55703]]], state=connected] 2026-02-18T08:36:12.288 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=54, tag_tid=1, entry_tid=55706], [object_number=53, tag_tid=1, entry_tid=55705], [object_number=52, tag_tid=1, entry_tid=55704], [object_number=55, tag_tid=1, entry_tid=55703]]], state=connected] 2026-02-18T08:36:12.289 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_8.status 2026-02-18T08:36:12.291 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=54, tag_tid=1, entry_tid=55706' 2026-02-18T08:36:12.292 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=53, tag_tid=2, entry_tid=1' -a 'object_number=53, tag_tid=2, entry_tid=1' = 'object_number=54, tag_tid=1, entry_tid=55706' 2026-02-18T08:36:12.292 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=54, tag_tid=1, entry_tid=55706' '!=' 'object_number=54, tag_tid=1, entry_tid=55706' 2026-02-18T08:36:12.292 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:36:12.292 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 4 2026-02-18T08:36:16.293 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_8 2026-02-18T08:36:16.293 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:36:16.293 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:36:16.294 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_8 2026-02-18T08:36:16.294 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:36:16.294 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_8 ']' 2026-02-18T08:36:16.294 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_8' 2026-02-18T08:36:16.294 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_8 2026-02-18T08:36:16.294 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:36:16.294 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:36:16.294 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:36:16.294 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:36:16.294 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:36:16.294 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:36:16.294 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:36:16.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:36:16.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:36:16.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:36:16.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_8 2026-02-18T08:36:16.297 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:36:16.297 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:36:16.297 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:36:16.297 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:36:16.297 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:36:16.297 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:36:16.297 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:36:16.297 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:36:16.297 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:36:16.297 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:36:16.297 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:36:16.297 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:36:16.297 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:36:16.297 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:36:16.297 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:36:16.297 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:36:16.297 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:36:16.298 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:36:16.298 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:36:16.298 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:36:16.298 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:36:16.298 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:36:16.298 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:36:16.298 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:36:16.298 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:36:16.298 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:36:16.298 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:36:16.298 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:36:16.298 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:36:16.298 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:36:16.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:36:16.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:36:16.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:36:16.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_8 2026-02-18T08:36:16.371 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:36:16.372 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_8 2026-02-18T08:36:16.372 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:36:16.372 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:36:16.372 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_8 2026-02-18T08:36:16.372 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_8 '' 2026-02-18T08:36:16.372 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:36:16.372 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:36:16.373 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_8 2026-02-18T08:36:16.373 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:36:16.373 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_8.status 2026-02-18T08:36:16.373 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_8.status 2026-02-18T08:36:16.373 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:36:16.374 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_8.status 2026-02-18T08:36:16.374 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_8 2026-02-18T08:36:16.375 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_8.status 2026-02-18T08:36:16.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 13 2026-02-18T08:36:16.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 13 2026-02-18T08:36:16.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:36:16.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=53, tag_tid=2, entry_tid=1], [object_number=52, tag_tid=2, entry_tid=0], [object_number=54, tag_tid=1, entry_tid=55706], [object_number=55, tag_tid=1, entry_tid=55703]]], state=connected] 2026-02-18T08:36:16.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=53, tag_tid=2, entry_tid=1], [object_number=52, tag_tid=2, entry_tid=0], [object_number=54, tag_tid=1, entry_tid=55706], [object_number=55, tag_tid=1, entry_tid=55703]]], state=connected] 2026-02-18T08:36:16.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_8.status 2026-02-18T08:36:16.406 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=53, tag_tid=2, entry_tid=1' 2026-02-18T08:36:16.407 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_8 2026-02-18T08:36:16.407 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:36:16.407 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:36:16.407 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_8 2026-02-18T08:36:16.407 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_8 '..*' 2026-02-18T08:36:16.407 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:36:16.407 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:36:16.407 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_8 2026-02-18T08:36:16.407 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:36:16.408 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_8.status 2026-02-18T08:36:16.408 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_8.status 2026-02-18T08:36:16.408 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:36:16.409 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_8.status 2026-02-18T08:36:16.409 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_8 2026-02-18T08:36:16.409 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_8.status 2026-02-18T08:36:16.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 13 2026-02-18T08:36:16.446 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 13 2026-02-18T08:36:16.446 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:36:16.446 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=53, tag_tid=2, entry_tid=1], [object_number=52, tag_tid=2, entry_tid=0], [object_number=54, tag_tid=1, entry_tid=55706], [object_number=55, tag_tid=1, entry_tid=55703]]], state=connected] 2026-02-18T08:36:16.446 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=53, tag_tid=2, entry_tid=1], [object_number=52, tag_tid=2, entry_tid=0], [object_number=54, tag_tid=1, entry_tid=55706], [object_number=55, tag_tid=1, entry_tid=55703]]], state=connected] 2026-02-18T08:36:16.449 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_8.status 2026-02-18T08:36:16.451 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=53, tag_tid=2, entry_tid=1' 2026-02-18T08:36:16.451 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=53, tag_tid=2, entry_tid=1' -a 'object_number=53, tag_tid=2, entry_tid=1' = 'object_number=53, tag_tid=2, entry_tid=1' 2026-02-18T08:36:16.451 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:36:16.451 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_snap_present cluster1 mirror image_8 snap 2026-02-18T08:36:16.451 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:36:16.451 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:36:16.451 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_8 2026-02-18T08:36:16.451 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap 2026-02-18T08:36:16.451 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-18T08:36:16.452 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-18T08:36:16.452 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1 2026-02-18T08:36:17.452 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster1 info mirror/image_8@snap 2026-02-18T08:36:17.480 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:rbd image 'image_8': 2026-02-18T08:36:17.480 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: size 128 MiB in 32 objects 2026-02-18T08:36:17.481 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: order 22 (4 MiB objects) 2026-02-18T08:36:17.481 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: snapshot_count: 1 2026-02-18T08:36:17.481 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: id: 11c07a0a44bb 2026-02-18T08:36:17.481 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: block_name_prefix: rbd_data.11c07a0a44bb 2026-02-18T08:36:17.481 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: format: 2 2026-02-18T08:36:17.481 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: features: layering, exclusive-lock, journaling 2026-02-18T08:36:17.481 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: op_features: 2026-02-18T08:36:17.481 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: flags: 2026-02-18T08:36:17.481 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: create_timestamp: Wed Feb 18 08:26:49 2026 2026-02-18T08:36:17.481 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: access_timestamp: Wed Feb 18 08:26:49 2026 2026-02-18T08:36:17.481 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: modify_timestamp: Wed Feb 18 08:28:51 2026 2026-02-18T08:36:17.481 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: protected: False 2026-02-18T08:36:17.482 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: journal: 11c07a0a44bb 2026-02-18T08:36:17.482 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring state: enabled 2026-02-18T08:36:17.482 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring mode: journal 2026-02-18T08:36:17.482 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring global id: 97d77571-dd3b-45f4-bfe5-e331977e0cd9 2026-02-18T08:36:17.482 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring primary: false 2026-02-18T08:36:17.484 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:36:17.484 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ compare_image_snaps mirror image_8 snap 2026-02-18T08:36:17.484 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:36:17.484 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_8 2026-02-18T08:36:17.484 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap 2026-02-18T08:36:17.484 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local ret=0 2026-02-18T08:36:17.484 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local rmt_export=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_8.export 2026-02-18T08:36:17.484 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loc_export=/tmp/tmp.vybh1m9yJF/cluster1-mirror-image_8.export 2026-02-18T08:36:17.484 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_8.export /tmp/tmp.vybh1m9yJF/cluster1-mirror-image_8.export 2026-02-18T08:36:17.485 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 -p mirror export image_8@snap /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_8.export 2026-02-18T08:36:17.619 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-18T08:36:17.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster1 -p mirror export image_8@snap /tmp/tmp.vybh1m9yJF/cluster1-mirror-image_8.export 2026-02-18T08:36:17.845 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-18T08:36:17.856 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmp /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_8.export /tmp/tmp.vybh1m9yJF/cluster1-mirror-image_8.export 2026-02-18T08:36:17.910 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_8.export /tmp/tmp.vybh1m9yJF/cluster1-mirror-image_8.export 2026-02-18T08:36:17.954 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:36:17.954 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T08:36:17.954 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_9 2026-02-18T08:36:17.954 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_snap cluster2 mirror image_9 snap 2026-02-18T08:36:17.954 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:36:17.954 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:36:17.954 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_9 2026-02-18T08:36:17.955 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap 2026-02-18T08:36:17.955 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 -p mirror snap create image_9@snap --debug-rbd=20 --debug-journaler=20 2026-02-18T08:36:22.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_replay_started cluster1 mirror image_9 2026-02-18T08:36:22.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:36:22.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:36:22.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_9 2026-02-18T08:36:22.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_replay_state cluster1 mirror image_9 started 2026-02-18T08:36:22.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:36:22.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:36:22.018 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_9 2026-02-18T08:36:22.018 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local state=started 2026-02-18T08:36:22.018 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-18T08:36:22.018 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:36:22.018 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1 2026-02-18T08:36:23.018 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test_image_replay_state cluster1 mirror image_9 started 2026-02-18T08:36:23.018 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:36:23.019 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:36:23.019 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_9 2026-02-18T08:36:23.019 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local test_state=started 2026-02-18T08:36:23.019 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local status_result 2026-02-18T08:36:23.019 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_state=stopped 2026-02-18T08:36:23.020 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_9 2026-02-18T08:36:23.020 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -i state 2026-02-18T08:36:23.020 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_instance=cluster1 2026-02-18T08:36:23.020 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-18T08:36:23.021 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:36:23.021 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance=cluster1 2026-02-18T08:36:23.021 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local loop_instance 2026-02-18T08:36:23.021 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:36:23.021 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sleep 0 2026-02-18T08:36:23.021 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:36:23.021 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ seq 0 3 2026-02-18T08:36:23.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:36:23.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_9 2026-02-18T08:36:23.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:0 2026-02-18T08:36:23.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-18T08:36:23.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:36:23.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:36:23.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:36:23.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-18T08:36:23.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:36:23.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:36:23.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:36:23.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:36:23.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-18T08:36:23.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:36:23.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:36:23.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:36:23.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:36:23.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:0 2026-02-18T08:36:23.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-18T08:36:23.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-18T08:36:23.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:36:23.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:0 2026-02-18T08:36:23.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T08:36:23.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-18T08:36:23.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:36:23.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:36:23.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-18T08:36:23.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-18T08:36:23.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-18T08:36:23.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=0 2026-02-18T08:36:23.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=0 2026-02-18T08:36:23.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:36:23.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:36:23.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:36:23.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:36:23.035 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_9 2026-02-18T08:36:23.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_9'] 2026-02-18T08:36:23.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status : get status for rbd mirror 2026-02-18T08:36:23.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:36:23.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:36:23.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_9 2026-02-18T08:36:23.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:1 2026-02-18T08:36:23.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-18T08:36:23.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:36:23.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:36:23.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:36:23.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-18T08:36:23.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:36:23.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:36:23.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:36:23.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:36:23.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-18T08:36:23.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:36:23.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:36:23.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:36:23.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:36:23.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:1 2026-02-18T08:36:23.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-18T08:36:23.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-18T08:36:23.092 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:36:23.092 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:1 2026-02-18T08:36:23.092 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T08:36:23.092 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-18T08:36:23.092 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:36:23.092 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:36:23.092 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-18T08:36:23.092 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-18T08:36:23.092 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-18T08:36:23.092 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=1 2026-02-18T08:36:23.092 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=1 2026-02-18T08:36:23.092 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:36:23.100 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:36:23.100 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:36:23.101 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:36:23.101 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_9 2026-02-18T08:36:23.167 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ return 0 2026-02-18T08:36:23.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ status_result=' "state": "Replaying"' 2026-02-18T08:36:23.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ echo ' "state": "Replaying"' 2026-02-18T08:36:23.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ grep -i Replaying 2026-02-18T08:36:23.169 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: "state": "Replaying" 2026-02-18T08:36:23.171 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ current_state=started 2026-02-18T08:36:23.171 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test started = started 2026-02-18T08:36:23.171 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:36:23.171 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror image_9 2026-02-18T08:36:23.171 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_cluster=cluster1 2026-02-18T08:36:23.171 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:36:23.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_pool=mirror 2026-02-18T08:36:23.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local remote_pool=mirror 2026-02-18T08:36:23.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_9 2026-02-18T08:36:23.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' journal = journal ']' 2026-02-18T08:36:23.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror mirror image_9 2026-02-18T08:36:23.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_cluster=cluster1 2026-02-18T08:36:23.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:36:23.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_pool=mirror 2026-02-18T08:36:23.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local remote_pool=mirror 2026-02-18T08:36:23.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_9 2026-02-18T08:36:23.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-02-18T08:36:23.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-02-18T08:36:23.173 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:36:23.173 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:36:23.173 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:36:23.372 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_9 2026-02-18T08:36:23.372 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:36:23.372 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:36:23.373 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_9 2026-02-18T08:36:23.373 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:36:23.373 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_9 ']' 2026-02-18T08:36:23.373 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_9' 2026-02-18T08:36:23.373 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_9 2026-02-18T08:36:23.373 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:36:23.373 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:36:23.373 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:36:23.373 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:36:23.373 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:36:23.373 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:36:23.373 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:36:23.374 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:36:23.376 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:36:23.376 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:36:23.376 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_9 2026-02-18T08:36:23.376 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:36:23.376 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:36:23.377 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:36:23.377 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:36:23.377 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:36:23.377 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:36:23.377 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:36:23.377 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:36:23.377 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:36:23.377 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:36:23.377 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:36:23.377 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:36:23.377 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:36:23.377 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:36:23.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:36:23.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:36:23.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:36:23.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:36:23.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:36:23.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:36:23.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:36:23.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:36:23.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:36:23.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:36:23.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:36:23.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:36:23.379 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:36:23.379 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:36:23.379 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:36:23.379 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:36:23.387 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:36:23.387 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:36:23.387 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:36:23.387 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_9 2026-02-18T08:36:23.446 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_9'] 2026-02-18T08:36:23.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:36:23.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:36:23.449 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:36:23.449 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_9 2026-02-18T08:36:23.449 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:36:23.450 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:36:23.450 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:36:23.450 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:36:23.450 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:36:23.450 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:36:23.450 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:36:23.450 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:36:23.450 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:36:23.450 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:36:23.450 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:36:23.450 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:36:23.451 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:36:23.451 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:36:23.451 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:36:23.451 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:36:23.451 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:36:23.451 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:36:23.451 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:36:23.451 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:36:23.451 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:36:23.451 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:36:23.451 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:36:23.451 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:36:23.451 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:36:23.452 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:36:23.452 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:36:23.452 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:36:23.452 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:36:23.452 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:36:23.460 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:36:23.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:36:23.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:36:23.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_9 2026-02-18T08:36:23.528 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:36:23.529 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_9 2026-02-18T08:36:23.529 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:36:23.530 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:36:23.530 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_9 2026-02-18T08:36:23.530 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_9 '' 2026-02-18T08:36:23.530 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:36:23.530 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:36:23.530 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_9 2026-02-18T08:36:23.530 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:36:23.530 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_9.status 2026-02-18T08:36:23.530 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_9.status 2026-02-18T08:36:23.530 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:36:23.531 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_9.status 2026-02-18T08:36:23.531 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_9 2026-02-18T08:36:23.531 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_9.status 2026-02-18T08:36:23.569 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 11 2026-02-18T08:36:23.569 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 11 2026-02-18T08:36:23.569 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:36:23.569 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=2, entry_tid=1], [object_number=44, tag_tid=2, entry_tid=0], [object_number=47, tag_tid=1, entry_tid=49299], [object_number=46, tag_tid=1, entry_tid=49298]]], state=connected] 2026-02-18T08:36:23.569 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=44, tag_tid=1, entry_tid=49300], [object_number=47, tag_tid=1, entry_tid=49299], [object_number=46, tag_tid=1, entry_tid=49298], [object_number=45, tag_tid=1, entry_tid=49297]]], state=connected] 2026-02-18T08:36:23.572 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_9.status 2026-02-18T08:36:23.574 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=45, tag_tid=2, entry_tid=1' 2026-02-18T08:36:23.575 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_9 2026-02-18T08:36:23.575 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:36:23.575 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:36:23.575 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_9 2026-02-18T08:36:23.575 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_9 '..*' 2026-02-18T08:36:23.575 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:36:23.576 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:36:23.576 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_9 2026-02-18T08:36:23.576 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:36:23.576 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_9.status 2026-02-18T08:36:23.576 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_9.status 2026-02-18T08:36:23.576 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:36:23.577 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_9.status 2026-02-18T08:36:23.577 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_9 2026-02-18T08:36:23.577 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_9.status 2026-02-18T08:36:23.604 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 11 2026-02-18T08:36:23.604 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 11 2026-02-18T08:36:23.604 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:36:23.604 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=2, entry_tid=1], [object_number=44, tag_tid=2, entry_tid=0], [object_number=47, tag_tid=1, entry_tid=49299], [object_number=46, tag_tid=1, entry_tid=49298]]], state=connected] 2026-02-18T08:36:23.604 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=44, tag_tid=1, entry_tid=49300], [object_number=47, tag_tid=1, entry_tid=49299], [object_number=46, tag_tid=1, entry_tid=49298], [object_number=45, tag_tid=1, entry_tid=49297]]], state=connected] 2026-02-18T08:36:23.606 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_9.status 2026-02-18T08:36:23.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=44, tag_tid=1, entry_tid=49300' 2026-02-18T08:36:23.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=45, tag_tid=2, entry_tid=1' -a 'object_number=45, tag_tid=2, entry_tid=1' = 'object_number=44, tag_tid=1, entry_tid=49300' 2026-02-18T08:36:23.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=44, tag_tid=1, entry_tid=49300' '!=' '' 2026-02-18T08:36:23.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:36:23.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=44, tag_tid=1, entry_tid=49300' = '' 2026-02-18T08:36:23.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=44, tag_tid=1, entry_tid=49300' 2026-02-18T08:36:23.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=45, tag_tid=2, entry_tid=1' 2026-02-18T08:36:23.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:36:23.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:36:23.610 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=2 2026-02-18T08:36:23.611 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=44, tag_tid=1, entry_tid=49300' 2026-02-18T08:36:23.611 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:36:23.611 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:36:23.612 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=1 2026-02-18T08:36:23.613 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=45, tag_tid=2, entry_tid=1' 2026-02-18T08:36:23.613 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:36:23.613 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:36:23.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:36:23.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=44, tag_tid=1, entry_tid=49300' 2026-02-18T08:36:23.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:36:23.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:36:23.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=49300 2026-02-18T08:36:23.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 2 = 1 -a 1 -le 49300 2026-02-18T08:36:23.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:36:23.618 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:36:23.618 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:36:23.819 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_9 2026-02-18T08:36:23.819 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:36:23.819 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:36:23.819 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_9 2026-02-18T08:36:23.819 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:36:23.819 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_9 ']' 2026-02-18T08:36:23.820 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_9' 2026-02-18T08:36:23.820 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_9 2026-02-18T08:36:23.820 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:36:23.820 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:36:23.820 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:36:23.820 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:36:23.820 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:36:23.820 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:36:23.820 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:36:23.820 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:36:23.823 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:36:23.823 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:36:23.823 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_9 2026-02-18T08:36:23.823 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:36:23.823 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:36:23.823 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:36:23.823 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:36:23.824 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:36:23.824 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:36:23.824 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:36:23.824 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:36:23.824 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:36:23.824 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:36:23.824 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:36:23.824 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:36:23.824 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:36:23.824 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:36:23.824 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:36:23.825 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:36:23.825 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:36:23.825 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:36:23.825 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:36:23.825 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:36:23.825 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:36:23.825 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:36:23.825 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:36:23.825 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:36:23.825 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:36:23.825 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:36:23.825 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:36:23.825 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:36:23.826 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:36:23.826 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:36:23.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:36:23.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:36:23.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:36:23.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_9 2026-02-18T08:36:23.884 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_9'] 2026-02-18T08:36:23.884 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:36:23.884 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:36:23.887 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:36:23.887 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_9 2026-02-18T08:36:23.887 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:36:23.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:36:23.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:36:23.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:36:23.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:36:23.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:36:23.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:36:23.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:36:23.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:36:23.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:36:23.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:36:23.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:36:23.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:36:23.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:36:23.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:36:23.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:36:23.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:36:23.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:36:23.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:36:23.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:36:23.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:36:23.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:36:23.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:36:23.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:36:23.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:36:23.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:36:23.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:36:23.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:36:23.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:36:23.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:36:23.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:36:23.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:36:23.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:36:23.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_9 2026-02-18T08:36:23.955 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:36:23.956 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_9 2026-02-18T08:36:23.956 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:36:23.956 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:36:23.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_9 2026-02-18T08:36:23.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_9 '' 2026-02-18T08:36:23.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:36:23.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:36:23.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_9 2026-02-18T08:36:23.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:36:23.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_9.status 2026-02-18T08:36:23.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_9.status 2026-02-18T08:36:23.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:36:23.958 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_9.status 2026-02-18T08:36:23.958 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_9 2026-02-18T08:36:23.958 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_9.status 2026-02-18T08:36:23.986 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 11 2026-02-18T08:36:23.986 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 11 2026-02-18T08:36:23.986 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:36:23.987 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=2, entry_tid=1], [object_number=44, tag_tid=2, entry_tid=0], [object_number=47, tag_tid=1, entry_tid=49299], [object_number=46, tag_tid=1, entry_tid=49298]]], state=connected] 2026-02-18T08:36:23.987 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=44, tag_tid=1, entry_tid=49300], [object_number=47, tag_tid=1, entry_tid=49299], [object_number=46, tag_tid=1, entry_tid=49298], [object_number=45, tag_tid=1, entry_tid=49297]]], state=connected] 2026-02-18T08:36:23.989 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_9.status 2026-02-18T08:36:23.990 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=45, tag_tid=2, entry_tid=1' 2026-02-18T08:36:23.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_9 2026-02-18T08:36:23.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:36:23.992 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:36:23.992 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_9 2026-02-18T08:36:23.992 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_9 '..*' 2026-02-18T08:36:23.992 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:36:23.992 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:36:23.992 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_9 2026-02-18T08:36:23.992 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:36:23.992 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_9.status 2026-02-18T08:36:23.993 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_9.status 2026-02-18T08:36:23.993 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:36:23.993 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_9.status 2026-02-18T08:36:23.994 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_9 2026-02-18T08:36:23.994 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_9.status 2026-02-18T08:36:24.020 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 11 2026-02-18T08:36:24.020 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 11 2026-02-18T08:36:24.020 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:36:24.020 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=2, entry_tid=1], [object_number=44, tag_tid=2, entry_tid=0], [object_number=47, tag_tid=1, entry_tid=49299], [object_number=46, tag_tid=1, entry_tid=49298]]], state=connected] 2026-02-18T08:36:24.020 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=44, tag_tid=1, entry_tid=49300], [object_number=47, tag_tid=1, entry_tid=49299], [object_number=46, tag_tid=1, entry_tid=49298], [object_number=45, tag_tid=1, entry_tid=49297]]], state=connected] 2026-02-18T08:36:24.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_9.status 2026-02-18T08:36:24.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=44, tag_tid=1, entry_tid=49300' 2026-02-18T08:36:24.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=45, tag_tid=2, entry_tid=1' -a 'object_number=45, tag_tid=2, entry_tid=1' = 'object_number=44, tag_tid=1, entry_tid=49300' 2026-02-18T08:36:24.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=44, tag_tid=1, entry_tid=49300' '!=' 'object_number=44, tag_tid=1, entry_tid=49300' 2026-02-18T08:36:24.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:36:24.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.4 2026-02-18T08:36:24.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_9 2026-02-18T08:36:24.427 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:36:24.427 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:36:24.427 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_9 2026-02-18T08:36:24.427 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:36:24.427 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_9 ']' 2026-02-18T08:36:24.427 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_9' 2026-02-18T08:36:24.427 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_9 2026-02-18T08:36:24.427 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:36:24.427 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:36:24.427 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:36:24.428 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:36:24.428 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:36:24.428 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:36:24.428 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:36:24.428 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:36:24.431 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:36:24.431 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:36:24.431 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_9 2026-02-18T08:36:24.431 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:36:24.431 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:36:24.431 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:36:24.431 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:36:24.431 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:36:24.431 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:36:24.432 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:36:24.432 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:36:24.432 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:36:24.432 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:36:24.432 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:36:24.432 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:36:24.432 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:36:24.432 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:36:24.432 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:36:24.432 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:36:24.432 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:36:24.432 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:36:24.433 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:36:24.433 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:36:24.433 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:36:24.433 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:36:24.433 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:36:24.433 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:36:24.433 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:36:24.433 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:36:24.433 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:36:24.433 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:36:24.433 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:36:24.433 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:36:24.440 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:36:24.440 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:36:24.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:36:24.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_9 2026-02-18T08:36:24.500 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_9'] 2026-02-18T08:36:24.500 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:36:24.501 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:36:24.503 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:36:24.503 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_9 2026-02-18T08:36:24.504 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:36:24.504 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:36:24.504 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:36:24.504 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:36:24.504 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:36:24.504 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:36:24.504 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:36:24.504 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:36:24.504 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:36:24.504 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:36:24.504 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:36:24.504 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:36:24.505 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:36:24.505 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:36:24.505 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:36:24.505 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:36:24.505 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:36:24.505 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:36:24.505 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:36:24.505 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:36:24.505 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:36:24.505 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:36:24.505 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:36:24.505 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:36:24.506 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:36:24.506 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:36:24.506 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:36:24.506 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:36:24.506 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:36:24.506 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:36:24.514 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:36:24.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:36:24.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:36:24.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_9 2026-02-18T08:36:24.587 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:36:24.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_9 2026-02-18T08:36:24.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:36:24.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:36:24.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_9 2026-02-18T08:36:24.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_9 '' 2026-02-18T08:36:24.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:36:24.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:36:24.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_9 2026-02-18T08:36:24.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:36:24.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_9.status 2026-02-18T08:36:24.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_9.status 2026-02-18T08:36:24.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:36:24.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_9.status 2026-02-18T08:36:24.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_9 2026-02-18T08:36:24.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_9.status 2026-02-18T08:36:24.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 11 2026-02-18T08:36:24.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 11 2026-02-18T08:36:24.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:36:24.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=2, entry_tid=1], [object_number=44, tag_tid=2, entry_tid=0], [object_number=47, tag_tid=1, entry_tid=49299], [object_number=46, tag_tid=1, entry_tid=49298]]], state=connected] 2026-02-18T08:36:24.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=45, tag_tid=2, entry_tid=1], [object_number=44, tag_tid=2, entry_tid=0], [object_number=47, tag_tid=1, entry_tid=49299], [object_number=46, tag_tid=1, entry_tid=49298]]], state=connected] 2026-02-18T08:36:24.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_9.status 2026-02-18T08:36:24.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=45, tag_tid=2, entry_tid=1' 2026-02-18T08:36:24.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_9 2026-02-18T08:36:24.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:36:24.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:36:24.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_9 2026-02-18T08:36:24.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_9 '..*' 2026-02-18T08:36:24.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:36:24.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:36:24.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_9 2026-02-18T08:36:24.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:36:24.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_9.status 2026-02-18T08:36:24.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_9.status 2026-02-18T08:36:24.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:36:24.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_9.status 2026-02-18T08:36:24.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_9 2026-02-18T08:36:24.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_9.status 2026-02-18T08:36:24.661 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 11 2026-02-18T08:36:24.661 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 11 2026-02-18T08:36:24.661 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:36:24.661 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=2, entry_tid=1], [object_number=44, tag_tid=2, entry_tid=0], [object_number=47, tag_tid=1, entry_tid=49299], [object_number=46, tag_tid=1, entry_tid=49298]]], state=connected] 2026-02-18T08:36:24.661 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=45, tag_tid=2, entry_tid=1], [object_number=44, tag_tid=2, entry_tid=0], [object_number=47, tag_tid=1, entry_tid=49299], [object_number=46, tag_tid=1, entry_tid=49298]]], state=connected] 2026-02-18T08:36:24.664 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_9.status 2026-02-18T08:36:24.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=45, tag_tid=2, entry_tid=1' 2026-02-18T08:36:24.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=45, tag_tid=2, entry_tid=1' -a 'object_number=45, tag_tid=2, entry_tid=1' = 'object_number=45, tag_tid=2, entry_tid=1' 2026-02-18T08:36:24.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:36:24.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_snap_present cluster1 mirror image_9 snap 2026-02-18T08:36:24.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:36:24.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:36:24.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_9 2026-02-18T08:36:24.667 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap 2026-02-18T08:36:24.667 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-18T08:36:24.667 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-18T08:36:24.667 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1 2026-02-18T08:36:25.667 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster1 info mirror/image_9@snap 2026-02-18T08:36:25.695 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:rbd image 'image_9': 2026-02-18T08:36:25.695 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: size 128 MiB in 32 objects 2026-02-18T08:36:25.695 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: order 22 (4 MiB objects) 2026-02-18T08:36:25.696 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: snapshot_count: 1 2026-02-18T08:36:25.696 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: id: 11f94cffaab1 2026-02-18T08:36:25.696 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: block_name_prefix: rbd_data.11f94cffaab1 2026-02-18T08:36:25.696 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: format: 2 2026-02-18T08:36:25.696 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: features: layering, exclusive-lock, journaling 2026-02-18T08:36:25.696 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: op_features: 2026-02-18T08:36:25.696 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: flags: 2026-02-18T08:36:25.696 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: create_timestamp: Wed Feb 18 08:27:03 2026 2026-02-18T08:36:25.696 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: access_timestamp: Wed Feb 18 08:27:03 2026 2026-02-18T08:36:25.696 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: modify_timestamp: Wed Feb 18 08:28:04 2026 2026-02-18T08:36:25.696 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: protected: False 2026-02-18T08:36:25.697 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: journal: 11f94cffaab1 2026-02-18T08:36:25.697 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring state: enabled 2026-02-18T08:36:25.697 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring mode: journal 2026-02-18T08:36:25.697 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring global id: bc830bc7-fb55-4a52-a57d-d302dda2a252 2026-02-18T08:36:25.697 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring primary: false 2026-02-18T08:36:25.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:36:25.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ compare_image_snaps mirror image_9 snap 2026-02-18T08:36:25.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:36:25.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_9 2026-02-18T08:36:25.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap 2026-02-18T08:36:25.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local ret=0 2026-02-18T08:36:25.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local rmt_export=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_9.export 2026-02-18T08:36:25.700 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loc_export=/tmp/tmp.vybh1m9yJF/cluster1-mirror-image_9.export 2026-02-18T08:36:25.700 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_9.export /tmp/tmp.vybh1m9yJF/cluster1-mirror-image_9.export 2026-02-18T08:36:25.700 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 -p mirror export image_9@snap /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_9.export 2026-02-18T08:36:25.851 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-18T08:36:25.861 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster1 -p mirror export image_9@snap /tmp/tmp.vybh1m9yJF/cluster1-mirror-image_9.export 2026-02-18T08:36:26.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-18T08:36:26.040 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmp /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_9.export /tmp/tmp.vybh1m9yJF/cluster1-mirror-image_9.export 2026-02-18T08:36:26.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_9.export /tmp/tmp.vybh1m9yJF/cluster1-mirror-image_9.export 2026-02-18T08:36:26.150 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:36:26.150 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T08:36:26.150 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_10 2026-02-18T08:36:26.150 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_snap cluster2 mirror image_10 snap 2026-02-18T08:36:26.150 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:36:26.150 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:36:26.150 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_10 2026-02-18T08:36:26.150 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap 2026-02-18T08:36:26.150 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 -p mirror snap create image_10@snap --debug-rbd=20 --debug-journaler=20 2026-02-18T08:36:29.028 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_replay_started cluster1 mirror image_10 2026-02-18T08:36:29.028 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:36:29.028 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:36:29.028 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_10 2026-02-18T08:36:29.029 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_replay_state cluster1 mirror image_10 started 2026-02-18T08:36:29.029 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:36:29.029 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:36:29.029 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_10 2026-02-18T08:36:29.029 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local state=started 2026-02-18T08:36:29.029 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-18T08:36:29.029 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:36:29.029 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1 2026-02-18T08:36:30.029 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test_image_replay_state cluster1 mirror image_10 started 2026-02-18T08:36:30.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:36:30.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:36:30.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_10 2026-02-18T08:36:30.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local test_state=started 2026-02-18T08:36:30.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local status_result 2026-02-18T08:36:30.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_state=stopped 2026-02-18T08:36:30.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_10 2026-02-18T08:36:30.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -i state 2026-02-18T08:36:30.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_instance=cluster1 2026-02-18T08:36:30.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-18T08:36:30.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:36:30.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance=cluster1 2026-02-18T08:36:30.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local loop_instance 2026-02-18T08:36:30.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:36:30.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sleep 0 2026-02-18T08:36:30.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:36:30.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ seq 0 3 2026-02-18T08:36:30.035 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:36:30.035 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_10 2026-02-18T08:36:30.035 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:0 2026-02-18T08:36:30.035 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-18T08:36:30.035 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:36:30.035 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:36:30.035 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:36:30.035 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-18T08:36:30.035 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:36:30.035 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:36:30.036 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:36:30.036 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:36:30.036 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-18T08:36:30.036 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:36:30.036 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:36:30.036 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:36:30.036 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:36:30.036 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:0 2026-02-18T08:36:30.036 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-18T08:36:30.036 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-18T08:36:30.036 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:36:30.036 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:0 2026-02-18T08:36:30.037 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T08:36:30.037 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-18T08:36:30.037 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:36:30.037 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:36:30.037 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-18T08:36:30.037 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-18T08:36:30.037 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-18T08:36:30.037 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=0 2026-02-18T08:36:30.037 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=0 2026-02-18T08:36:30.037 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:36:30.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:36:30.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:36:30.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:36:30.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_10 2026-02-18T08:36:30.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_10'] 2026-02-18T08:36:30.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status : get status for rbd mirror 2026-02-18T08:36:30.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:36:30.112 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:36:30.112 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_10 2026-02-18T08:36:30.112 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:1 2026-02-18T08:36:30.112 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-18T08:36:30.112 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:36:30.112 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:36:30.112 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:36:30.112 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-18T08:36:30.112 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:36:30.113 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:36:30.113 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:36:30.113 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:36:30.113 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-18T08:36:30.113 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:36:30.113 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:36:30.113 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:36:30.113 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:36:30.113 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:1 2026-02-18T08:36:30.113 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-18T08:36:30.113 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-18T08:36:30.113 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:36:30.113 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:1 2026-02-18T08:36:30.114 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T08:36:30.114 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-18T08:36:30.114 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:36:30.114 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:36:30.114 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-18T08:36:30.114 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-18T08:36:30.114 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-18T08:36:30.114 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=1 2026-02-18T08:36:30.114 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=1 2026-02-18T08:36:30.114 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:36:30.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:36:30.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:36:30.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:36:30.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_10 2026-02-18T08:36:30.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ return 0 2026-02-18T08:36:30.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ status_result=' "state": "Replaying"' 2026-02-18T08:36:30.183 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ echo ' "state": "Replaying"' 2026-02-18T08:36:30.183 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ grep -i Replaying 2026-02-18T08:36:30.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: "state": "Replaying" 2026-02-18T08:36:30.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ current_state=started 2026-02-18T08:36:30.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test started = started 2026-02-18T08:36:30.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:36:30.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror image_10 2026-02-18T08:36:30.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_cluster=cluster1 2026-02-18T08:36:30.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:36:30.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_pool=mirror 2026-02-18T08:36:30.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local remote_pool=mirror 2026-02-18T08:36:30.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_10 2026-02-18T08:36:30.187 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' journal = journal ']' 2026-02-18T08:36:30.187 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror mirror image_10 2026-02-18T08:36:30.187 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_cluster=cluster1 2026-02-18T08:36:30.187 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:36:30.187 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_pool=mirror 2026-02-18T08:36:30.187 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local remote_pool=mirror 2026-02-18T08:36:30.187 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_10 2026-02-18T08:36:30.187 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-02-18T08:36:30.187 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-02-18T08:36:30.187 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:36:30.187 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:36:30.188 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:36:30.387 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_10 2026-02-18T08:36:30.387 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:36:30.387 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:36:30.387 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_10 2026-02-18T08:36:30.387 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:36:30.387 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_10 ']' 2026-02-18T08:36:30.387 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_10' 2026-02-18T08:36:30.387 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_10 2026-02-18T08:36:30.388 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:36:30.388 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:36:30.388 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:36:30.388 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:36:30.388 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:36:30.388 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:36:30.388 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:36:30.388 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:36:30.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:36:30.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:36:30.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_10 2026-02-18T08:36:30.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:36:30.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:36:30.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:36:30.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:36:30.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:36:30.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:36:30.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:36:30.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:36:30.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:36:30.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:36:30.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:36:30.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:36:30.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:36:30.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:36:30.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:36:30.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:36:30.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:36:30.393 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:36:30.393 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:36:30.393 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:36:30.393 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:36:30.393 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:36:30.393 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:36:30.393 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:36:30.393 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:36:30.393 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:36:30.393 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:36:30.393 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:36:30.393 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:36:30.394 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:36:30.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:36:30.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:36:30.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:36:30.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_10 2026-02-18T08:36:30.433 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.0 with skip (0.4346605029263165) > weight (0.25) 2026-02-18T08:36:30.434 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.1 with skip (0.8627464480068506) > weight (0.25) 2026-02-18T08:36:30.434 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.2 with skip (0.7938762895514415) > weight (0.25) 2026-02-18T08:36:30.434 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.3 with skip (0.4914105319791251) > weight (0.25) 2026-02-18T08:36:30.434 INFO:tasks.rbd_mirror_thrash:waiting for 73.72335033141711 secs before thrashing 2026-02-18T08:36:30.454 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_10'] 2026-02-18T08:36:30.455 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:36:30.455 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:36:30.458 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:36:30.458 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_10 2026-02-18T08:36:30.458 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:36:30.458 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:36:30.458 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:36:30.458 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:36:30.458 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:36:30.458 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:36:30.458 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:36:30.458 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:36:30.458 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:36:30.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:36:30.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:36:30.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:36:30.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:36:30.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:36:30.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:36:30.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:36:30.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:36:30.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:36:30.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:36:30.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:36:30.460 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:36:30.460 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:36:30.460 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:36:30.460 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:36:30.460 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:36:30.460 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:36:30.460 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:36:30.460 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:36:30.460 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:36:30.460 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:36:30.468 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:36:30.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:36:30.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:36:30.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_10 2026-02-18T08:36:30.534 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:36:30.535 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_10 2026-02-18T08:36:30.535 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:36:30.535 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:36:30.535 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_10 2026-02-18T08:36:30.535 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_10 '' 2026-02-18T08:36:30.535 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:36:30.535 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:36:30.535 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_10 2026-02-18T08:36:30.536 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:36:30.536 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_10.status 2026-02-18T08:36:30.536 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_10.status 2026-02-18T08:36:30.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:36:30.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_10.status 2026-02-18T08:36:30.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_10 2026-02-18T08:36:30.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_10.status 2026-02-18T08:36:30.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 4 2026-02-18T08:36:30.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 4 2026-02-18T08:36:30.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:36:30.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=2, entry_tid=1], [object_number=16, tag_tid=2, entry_tid=0], [object_number=19, tag_tid=1, entry_tid=20655], [object_number=18, tag_tid=1, entry_tid=20654]]], state=connected] 2026-02-18T08:36:30.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=17, tag_tid=1, entry_tid=20657], [object_number=16, tag_tid=1, entry_tid=20656], [object_number=19, tag_tid=1, entry_tid=20655], [object_number=18, tag_tid=1, entry_tid=20654]]], state=connected] 2026-02-18T08:36:30.566 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_10.status 2026-02-18T08:36:30.568 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=17, tag_tid=2, entry_tid=1' 2026-02-18T08:36:30.569 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_10 2026-02-18T08:36:30.569 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:36:30.569 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:36:30.569 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_10 2026-02-18T08:36:30.569 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_10 '..*' 2026-02-18T08:36:30.569 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:36:30.569 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:36:30.570 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_10 2026-02-18T08:36:30.570 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:36:30.570 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_10.status 2026-02-18T08:36:30.570 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_10.status 2026-02-18T08:36:30.570 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:36:30.571 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_10.status 2026-02-18T08:36:30.571 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_10 2026-02-18T08:36:30.571 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_10.status 2026-02-18T08:36:30.811 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 4 2026-02-18T08:36:30.811 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 4 2026-02-18T08:36:30.811 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:36:30.811 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=2, entry_tid=1], [object_number=16, tag_tid=2, entry_tid=0], [object_number=19, tag_tid=1, entry_tid=20655], [object_number=18, tag_tid=1, entry_tid=20654]]], state=connected] 2026-02-18T08:36:30.811 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=17, tag_tid=1, entry_tid=20657], [object_number=16, tag_tid=1, entry_tid=20656], [object_number=19, tag_tid=1, entry_tid=20655], [object_number=18, tag_tid=1, entry_tid=20654]]], state=connected] 2026-02-18T08:36:30.814 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_10.status 2026-02-18T08:36:30.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=17, tag_tid=1, entry_tid=20657' 2026-02-18T08:36:30.817 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=17, tag_tid=2, entry_tid=1' -a 'object_number=17, tag_tid=2, entry_tid=1' = 'object_number=17, tag_tid=1, entry_tid=20657' 2026-02-18T08:36:30.817 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=17, tag_tid=1, entry_tid=20657' '!=' '' 2026-02-18T08:36:30.817 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:36:30.817 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=17, tag_tid=1, entry_tid=20657' = '' 2026-02-18T08:36:30.817 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=17, tag_tid=1, entry_tid=20657' 2026-02-18T08:36:30.818 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=17, tag_tid=2, entry_tid=1' 2026-02-18T08:36:30.818 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:36:30.818 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:36:30.819 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=2 2026-02-18T08:36:30.819 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=17, tag_tid=1, entry_tid=20657' 2026-02-18T08:36:30.820 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:36:30.820 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:36:30.821 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=1 2026-02-18T08:36:30.822 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=17, tag_tid=2, entry_tid=1' 2026-02-18T08:36:30.822 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:36:30.822 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:36:30.823 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:36:30.824 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=17, tag_tid=1, entry_tid=20657' 2026-02-18T08:36:30.824 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:36:30.824 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:36:30.826 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=20657 2026-02-18T08:36:30.826 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 2 = 1 -a 1 -le 20657 2026-02-18T08:36:30.826 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:36:30.826 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:36:30.826 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:36:31.028 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_10 2026-02-18T08:36:31.028 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:36:31.028 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:36:31.028 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_10 2026-02-18T08:36:31.028 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:36:31.029 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_10 ']' 2026-02-18T08:36:31.029 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_10' 2026-02-18T08:36:31.029 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_10 2026-02-18T08:36:31.029 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:36:31.029 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:36:31.029 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:36:31.029 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:36:31.029 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:36:31.029 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:36:31.029 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:36:31.029 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:36:31.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:36:31.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:36:31.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_10 2026-02-18T08:36:31.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:36:31.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:36:31.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:36:31.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:36:31.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:36:31.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:36:31.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:36:31.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:36:31.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:36:31.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:36:31.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:36:31.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:36:31.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:36:31.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:36:31.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:36:31.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:36:31.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:36:31.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:36:31.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:36:31.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:36:31.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:36:31.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:36:31.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:36:31.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:36:31.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:36:31.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:36:31.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:36:31.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:36:31.035 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:36:31.035 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:36:31.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:36:31.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:36:31.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:36:31.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_10 2026-02-18T08:36:31.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_10'] 2026-02-18T08:36:31.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:36:31.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:36:31.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:36:31.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_10 2026-02-18T08:36:31.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:36:31.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:36:31.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:36:31.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:36:31.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:36:31.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:36:31.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:36:31.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:36:31.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:36:31.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:36:31.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:36:31.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:36:31.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:36:31.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:36:31.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:36:31.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:36:31.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:36:31.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:36:31.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:36:31.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:36:31.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:36:31.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:36:31.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:36:31.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:36:31.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:36:31.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:36:31.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:36:31.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:36:31.110 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:36:31.110 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:36:31.119 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:36:31.120 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:36:31.120 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:36:31.120 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_10 2026-02-18T08:36:31.179 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:36:31.180 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_10 2026-02-18T08:36:31.180 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:36:31.180 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:36:31.180 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_10 2026-02-18T08:36:31.180 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_10 '' 2026-02-18T08:36:31.180 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:36:31.180 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:36:31.180 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_10 2026-02-18T08:36:31.180 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:36:31.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_10.status 2026-02-18T08:36:31.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_10.status 2026-02-18T08:36:31.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:36:31.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_10.status 2026-02-18T08:36:31.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_10 2026-02-18T08:36:31.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_10.status 2026-02-18T08:36:31.210 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 4 2026-02-18T08:36:31.210 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 4 2026-02-18T08:36:31.210 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:36:31.210 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=2, entry_tid=1], [object_number=16, tag_tid=2, entry_tid=0], [object_number=19, tag_tid=1, entry_tid=20655], [object_number=18, tag_tid=1, entry_tid=20654]]], state=connected] 2026-02-18T08:36:31.210 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=17, tag_tid=1, entry_tid=20657], [object_number=16, tag_tid=1, entry_tid=20656], [object_number=19, tag_tid=1, entry_tid=20655], [object_number=18, tag_tid=1, entry_tid=20654]]], state=connected] 2026-02-18T08:36:31.213 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_10.status 2026-02-18T08:36:31.214 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=17, tag_tid=2, entry_tid=1' 2026-02-18T08:36:31.215 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_10 2026-02-18T08:36:31.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:36:31.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:36:31.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_10 2026-02-18T08:36:31.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_10 '..*' 2026-02-18T08:36:31.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:36:31.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:36:31.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_10 2026-02-18T08:36:31.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:36:31.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_10.status 2026-02-18T08:36:31.217 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_10.status 2026-02-18T08:36:31.217 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:36:31.217 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_10.status 2026-02-18T08:36:31.218 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_10 2026-02-18T08:36:31.218 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_10.status 2026-02-18T08:36:31.252 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 4 2026-02-18T08:36:31.252 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 4 2026-02-18T08:36:31.252 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:36:31.252 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=2, entry_tid=1], [object_number=16, tag_tid=2, entry_tid=0], [object_number=19, tag_tid=1, entry_tid=20655], [object_number=18, tag_tid=1, entry_tid=20654]]], state=connected] 2026-02-18T08:36:31.252 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=17, tag_tid=1, entry_tid=20657], [object_number=16, tag_tid=1, entry_tid=20656], [object_number=19, tag_tid=1, entry_tid=20655], [object_number=18, tag_tid=1, entry_tid=20654]]], state=connected] 2026-02-18T08:36:31.255 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_10.status 2026-02-18T08:36:31.257 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=17, tag_tid=1, entry_tid=20657' 2026-02-18T08:36:31.257 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=17, tag_tid=2, entry_tid=1' -a 'object_number=17, tag_tid=2, entry_tid=1' = 'object_number=17, tag_tid=1, entry_tid=20657' 2026-02-18T08:36:31.257 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=17, tag_tid=1, entry_tid=20657' '!=' 'object_number=17, tag_tid=1, entry_tid=20657' 2026-02-18T08:36:31.257 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:36:31.257 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.4 2026-02-18T08:36:31.658 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_10 2026-02-18T08:36:31.659 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:36:31.659 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:36:31.659 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_10 2026-02-18T08:36:31.659 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:36:31.659 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_10 ']' 2026-02-18T08:36:31.659 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_10' 2026-02-18T08:36:31.659 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_10 2026-02-18T08:36:31.659 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:36:31.659 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:36:31.659 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:36:31.660 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:36:31.660 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:36:31.660 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:36:31.660 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:36:31.660 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:36:31.660 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:36:31.663 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:36:31.663 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_10 2026-02-18T08:36:31.663 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:36:31.663 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:36:31.663 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:36:31.663 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:36:31.663 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:36:31.664 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:36:31.664 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:36:31.664 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:36:31.664 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:36:31.664 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:36:31.664 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:36:31.664 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:36:31.664 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:36:31.664 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:36:31.664 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:36:31.664 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:36:31.664 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:36:31.664 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:36:31.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:36:31.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:36:31.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:36:31.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:36:31.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:36:31.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:36:31.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:36:31.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:36:31.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:36:31.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:36:31.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:36:31.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:36:31.675 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:36:31.675 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:36:31.675 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:36:31.676 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_10 2026-02-18T08:36:31.736 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_10'] 2026-02-18T08:36:31.737 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:36:31.737 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:36:31.739 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:36:31.740 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_10 2026-02-18T08:36:31.740 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:36:31.740 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:36:31.740 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:36:31.740 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:36:31.740 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:36:31.740 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:36:31.740 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:36:31.740 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:36:31.740 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:36:31.740 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:36:31.741 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:36:31.741 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:36:31.741 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:36:31.741 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:36:31.741 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:36:31.741 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:36:31.741 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:36:31.741 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:36:31.741 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:36:31.741 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:36:31.741 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:36:31.741 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:36:31.742 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:36:31.742 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:36:31.742 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:36:31.742 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:36:31.742 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:36:31.742 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:36:31.742 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:36:31.742 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:36:31.751 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:36:31.752 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:36:31.752 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:36:31.752 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_10 2026-02-18T08:36:31.818 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:36:31.819 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_10 2026-02-18T08:36:31.819 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:36:31.819 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:36:31.819 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_10 2026-02-18T08:36:31.819 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_10 '' 2026-02-18T08:36:31.819 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:36:31.819 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:36:31.820 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_10 2026-02-18T08:36:31.820 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:36:31.820 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_10.status 2026-02-18T08:36:31.820 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_10.status 2026-02-18T08:36:31.820 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:36:31.821 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_10.status 2026-02-18T08:36:31.821 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_10 2026-02-18T08:36:31.821 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_10.status 2026-02-18T08:36:31.858 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 4 2026-02-18T08:36:31.858 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 4 2026-02-18T08:36:31.859 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:36:31.859 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=2, entry_tid=1], [object_number=16, tag_tid=2, entry_tid=0], [object_number=19, tag_tid=1, entry_tid=20655], [object_number=18, tag_tid=1, entry_tid=20654]]], state=connected] 2026-02-18T08:36:31.859 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=17, tag_tid=1, entry_tid=20657], [object_number=16, tag_tid=1, entry_tid=20656], [object_number=19, tag_tid=1, entry_tid=20655], [object_number=18, tag_tid=1, entry_tid=20654]]], state=connected] 2026-02-18T08:36:31.862 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_10.status 2026-02-18T08:36:31.864 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=17, tag_tid=2, entry_tid=1' 2026-02-18T08:36:31.865 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_10 2026-02-18T08:36:31.865 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:36:31.865 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:36:31.865 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_10 2026-02-18T08:36:31.865 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_10 '..*' 2026-02-18T08:36:31.865 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:36:31.865 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:36:31.865 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_10 2026-02-18T08:36:31.866 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:36:31.866 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_10.status 2026-02-18T08:36:31.866 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_10.status 2026-02-18T08:36:31.866 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:36:31.867 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_10.status 2026-02-18T08:36:31.867 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_10 2026-02-18T08:36:31.867 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_10.status 2026-02-18T08:36:31.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 4 2026-02-18T08:36:31.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 4 2026-02-18T08:36:31.894 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:36:31.894 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=2, entry_tid=1], [object_number=16, tag_tid=2, entry_tid=0], [object_number=19, tag_tid=1, entry_tid=20655], [object_number=18, tag_tid=1, entry_tid=20654]]], state=connected] 2026-02-18T08:36:31.894 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=17, tag_tid=1, entry_tid=20657], [object_number=16, tag_tid=1, entry_tid=20656], [object_number=19, tag_tid=1, entry_tid=20655], [object_number=18, tag_tid=1, entry_tid=20654]]], state=connected] 2026-02-18T08:36:31.896 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_10.status 2026-02-18T08:36:31.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=17, tag_tid=1, entry_tid=20657' 2026-02-18T08:36:31.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=17, tag_tid=2, entry_tid=1' -a 'object_number=17, tag_tid=2, entry_tid=1' = 'object_number=17, tag_tid=1, entry_tid=20657' 2026-02-18T08:36:31.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=17, tag_tid=1, entry_tid=20657' '!=' 'object_number=17, tag_tid=1, entry_tid=20657' 2026-02-18T08:36:31.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:36:31.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.8 2026-02-18T08:36:32.701 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_10 2026-02-18T08:36:32.701 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:36:32.701 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:36:32.701 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_10 2026-02-18T08:36:32.701 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:36:32.701 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_10 ']' 2026-02-18T08:36:32.701 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_10' 2026-02-18T08:36:32.701 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_10 2026-02-18T08:36:32.702 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:36:32.702 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:36:32.702 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:36:32.702 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:36:32.702 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:36:32.702 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:36:32.702 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:36:32.702 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:36:32.702 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:36:32.705 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:36:32.705 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_10 2026-02-18T08:36:32.705 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:36:32.705 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:36:32.705 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:36:32.705 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:36:32.705 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:36:32.705 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:36:32.706 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:36:32.706 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:36:32.706 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:36:32.706 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:36:32.706 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:36:32.706 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:36:32.706 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:36:32.706 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:36:32.706 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:36:32.706 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:36:32.706 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:36:32.706 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:36:32.707 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:36:32.707 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:36:32.707 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:36:32.707 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:36:32.707 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:36:32.707 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:36:32.707 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:36:32.707 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:36:32.707 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:36:32.707 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:36:32.707 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:36:32.707 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:36:32.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:36:32.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:36:32.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:36:32.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_10 2026-02-18T08:36:32.768 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_10'] 2026-02-18T08:36:32.768 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:36:32.768 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:36:32.771 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:36:32.771 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_10 2026-02-18T08:36:32.771 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:36:32.771 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:36:32.771 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:36:32.771 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:36:32.771 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:36:32.771 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:36:32.772 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:36:32.772 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:36:32.772 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:36:32.772 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:36:32.772 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:36:32.772 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:36:32.772 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:36:32.772 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:36:32.772 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:36:32.772 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:36:32.772 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:36:32.772 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:36:32.773 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:36:32.773 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:36:32.773 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:36:32.773 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:36:32.773 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:36:32.773 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:36:32.773 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:36:32.773 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:36:32.773 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:36:32.773 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:36:32.773 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:36:32.773 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:36:32.782 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:36:32.783 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:36:32.783 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:36:32.783 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_10 2026-02-18T08:36:32.838 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:36:32.839 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_10 2026-02-18T08:36:32.840 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:36:32.840 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:36:32.840 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_10 2026-02-18T08:36:32.840 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_10 '' 2026-02-18T08:36:32.840 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:36:32.840 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:36:32.840 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_10 2026-02-18T08:36:32.840 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:36:32.840 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_10.status 2026-02-18T08:36:32.840 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_10.status 2026-02-18T08:36:32.841 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:36:32.841 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_10.status 2026-02-18T08:36:32.842 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_10 2026-02-18T08:36:32.842 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_10.status 2026-02-18T08:36:32.878 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 4 2026-02-18T08:36:32.878 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 4 2026-02-18T08:36:32.878 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:36:32.878 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=2, entry_tid=1], [object_number=16, tag_tid=2, entry_tid=0], [object_number=19, tag_tid=1, entry_tid=20655], [object_number=18, tag_tid=1, entry_tid=20654]]], state=connected] 2026-02-18T08:36:32.878 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=17, tag_tid=1, entry_tid=20657], [object_number=16, tag_tid=1, entry_tid=20656], [object_number=19, tag_tid=1, entry_tid=20655], [object_number=18, tag_tid=1, entry_tid=20654]]], state=connected] 2026-02-18T08:36:32.881 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_10.status 2026-02-18T08:36:32.883 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=17, tag_tid=2, entry_tid=1' 2026-02-18T08:36:32.884 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_10 2026-02-18T08:36:32.884 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:36:32.884 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:36:32.884 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_10 2026-02-18T08:36:32.884 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_10 '..*' 2026-02-18T08:36:32.884 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:36:32.885 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:36:32.885 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_10 2026-02-18T08:36:32.885 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:36:32.885 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_10.status 2026-02-18T08:36:32.885 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_10.status 2026-02-18T08:36:32.886 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:36:32.886 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_10.status 2026-02-18T08:36:32.886 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_10 2026-02-18T08:36:32.887 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_10.status 2026-02-18T08:36:32.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 4 2026-02-18T08:36:32.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 4 2026-02-18T08:36:32.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:36:32.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=2, entry_tid=1], [object_number=16, tag_tid=2, entry_tid=0], [object_number=19, tag_tid=1, entry_tid=20655], [object_number=18, tag_tid=1, entry_tid=20654]]], state=connected] 2026-02-18T08:36:32.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=17, tag_tid=1, entry_tid=20657], [object_number=16, tag_tid=1, entry_tid=20656], [object_number=19, tag_tid=1, entry_tid=20655], [object_number=18, tag_tid=1, entry_tid=20654]]], state=connected] 2026-02-18T08:36:32.923 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_10.status 2026-02-18T08:36:32.924 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=17, tag_tid=1, entry_tid=20657' 2026-02-18T08:36:32.924 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=17, tag_tid=2, entry_tid=1' -a 'object_number=17, tag_tid=2, entry_tid=1' = 'object_number=17, tag_tid=1, entry_tid=20657' 2026-02-18T08:36:32.924 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=17, tag_tid=1, entry_tid=20657' '!=' 'object_number=17, tag_tid=1, entry_tid=20657' 2026-02-18T08:36:32.924 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:36:32.924 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1.6 2026-02-18T08:36:34.527 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_10 2026-02-18T08:36:34.527 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:36:34.527 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:36:34.527 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_10 2026-02-18T08:36:34.527 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:36:34.527 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_10 ']' 2026-02-18T08:36:34.527 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_10' 2026-02-18T08:36:34.527 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_10 2026-02-18T08:36:34.527 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:36:34.528 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:36:34.528 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:36:34.528 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:36:34.528 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:36:34.528 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:36:34.528 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:36:34.528 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:36:34.531 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:36:34.531 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:36:34.531 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_10 2026-02-18T08:36:34.531 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:36:34.531 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:36:34.531 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:36:34.531 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:36:34.531 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:36:34.532 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:36:34.532 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:36:34.532 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:36:34.532 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:36:34.532 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:36:34.532 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:36:34.532 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:36:34.532 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:36:34.532 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:36:34.532 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:36:34.532 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:36:34.532 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:36:34.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:36:34.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:36:34.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:36:34.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:36:34.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:36:34.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:36:34.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:36:34.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:36:34.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:36:34.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:36:34.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:36:34.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:36:34.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:36:34.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:36:34.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:36:34.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:36:34.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_10 2026-02-18T08:36:34.604 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_10'] 2026-02-18T08:36:34.604 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:36:34.605 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:36:34.607 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:36:34.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_10 2026-02-18T08:36:34.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:36:34.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:36:34.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:36:34.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:36:34.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:36:34.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:36:34.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:36:34.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:36:34.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:36:34.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:36:34.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:36:34.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:36:34.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:36:34.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:36:34.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:36:34.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:36:34.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:36:34.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:36:34.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:36:34.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:36:34.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:36:34.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:36:34.610 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:36:34.610 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:36:34.610 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:36:34.610 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:36:34.610 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:36:34.610 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:36:34.610 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:36:34.610 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:36:34.619 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:36:34.620 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:36:34.620 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:36:34.620 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_10 2026-02-18T08:36:34.683 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:36:34.685 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_10 2026-02-18T08:36:34.685 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:36:34.685 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:36:34.685 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_10 2026-02-18T08:36:34.685 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_10 '' 2026-02-18T08:36:34.685 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:36:34.685 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:36:34.685 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_10 2026-02-18T08:36:34.685 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:36:34.685 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_10.status 2026-02-18T08:36:34.686 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_10.status 2026-02-18T08:36:34.686 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:36:34.686 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_10.status 2026-02-18T08:36:34.687 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_10 2026-02-18T08:36:34.687 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_10.status 2026-02-18T08:36:34.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 4 2026-02-18T08:36:34.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 4 2026-02-18T08:36:34.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:36:34.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=2, entry_tid=1], [object_number=16, tag_tid=2, entry_tid=0], [object_number=19, tag_tid=1, entry_tid=20655], [object_number=18, tag_tid=1, entry_tid=20654]]], state=connected] 2026-02-18T08:36:34.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=17, tag_tid=2, entry_tid=1], [object_number=16, tag_tid=2, entry_tid=0], [object_number=19, tag_tid=1, entry_tid=20655], [object_number=18, tag_tid=1, entry_tid=20654]]], state=connected] 2026-02-18T08:36:34.718 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_10.status 2026-02-18T08:36:34.719 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=17, tag_tid=2, entry_tid=1' 2026-02-18T08:36:34.720 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_10 2026-02-18T08:36:34.720 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:36:34.720 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:36:34.721 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_10 2026-02-18T08:36:34.721 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_10 '..*' 2026-02-18T08:36:34.721 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:36:34.721 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:36:34.721 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_10 2026-02-18T08:36:34.721 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:36:34.721 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_10.status 2026-02-18T08:36:34.722 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_10.status 2026-02-18T08:36:34.722 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:36:34.722 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_10.status 2026-02-18T08:36:34.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_10 2026-02-18T08:36:34.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_10.status 2026-02-18T08:36:34.752 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 4 2026-02-18T08:36:34.752 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 4 2026-02-18T08:36:34.752 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:36:34.752 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=2, entry_tid=1], [object_number=16, tag_tid=2, entry_tid=0], [object_number=19, tag_tid=1, entry_tid=20655], [object_number=18, tag_tid=1, entry_tid=20654]]], state=connected] 2026-02-18T08:36:34.752 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=17, tag_tid=2, entry_tid=1], [object_number=16, tag_tid=2, entry_tid=0], [object_number=19, tag_tid=1, entry_tid=20655], [object_number=18, tag_tid=1, entry_tid=20654]]], state=connected] 2026-02-18T08:36:34.755 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_10.status 2026-02-18T08:36:34.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=17, tag_tid=2, entry_tid=1' 2026-02-18T08:36:34.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=17, tag_tid=2, entry_tid=1' -a 'object_number=17, tag_tid=2, entry_tid=1' = 'object_number=17, tag_tid=2, entry_tid=1' 2026-02-18T08:36:34.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:36:34.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_snap_present cluster1 mirror image_10 snap 2026-02-18T08:36:34.758 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:36:34.758 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:36:34.758 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_10 2026-02-18T08:36:34.758 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap 2026-02-18T08:36:34.758 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-18T08:36:34.758 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-18T08:36:34.758 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1 2026-02-18T08:36:35.758 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster1 info mirror/image_10@snap 2026-02-18T08:36:35.788 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:rbd image 'image_10': 2026-02-18T08:36:35.788 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: size 128 MiB in 32 objects 2026-02-18T08:36:35.788 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: order 22 (4 MiB objects) 2026-02-18T08:36:35.788 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: snapshot_count: 1 2026-02-18T08:36:35.788 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: id: 11f997dcaa26 2026-02-18T08:36:35.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: block_name_prefix: rbd_data.11f997dcaa26 2026-02-18T08:36:35.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: format: 2 2026-02-18T08:36:35.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: features: layering, exclusive-lock, journaling 2026-02-18T08:36:35.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: op_features: 2026-02-18T08:36:35.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: flags: 2026-02-18T08:36:35.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: create_timestamp: Wed Feb 18 08:27:17 2026 2026-02-18T08:36:35.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: access_timestamp: Wed Feb 18 08:27:17 2026 2026-02-18T08:36:35.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: modify_timestamp: Wed Feb 18 08:27:17 2026 2026-02-18T08:36:35.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: protected: False 2026-02-18T08:36:35.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: journal: 11f997dcaa26 2026-02-18T08:36:35.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring state: enabled 2026-02-18T08:36:35.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring mode: journal 2026-02-18T08:36:35.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring global id: 67cf1f79-037c-4784-acb6-f3a0138f43db 2026-02-18T08:36:35.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring primary: false 2026-02-18T08:36:35.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:36:35.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ compare_image_snaps mirror image_10 snap 2026-02-18T08:36:35.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:36:35.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_10 2026-02-18T08:36:35.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap 2026-02-18T08:36:35.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local ret=0 2026-02-18T08:36:35.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local rmt_export=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_10.export 2026-02-18T08:36:35.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loc_export=/tmp/tmp.vybh1m9yJF/cluster1-mirror-image_10.export 2026-02-18T08:36:35.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_10.export /tmp/tmp.vybh1m9yJF/cluster1-mirror-image_10.export 2026-02-18T08:36:35.793 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 -p mirror export image_10@snap /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_10.export 2026-02-18T08:36:35.933 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-18T08:36:35.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster1 -p mirror export image_10@snap /tmp/tmp.vybh1m9yJF/cluster1-mirror-image_10.export 2026-02-18T08:36:36.106 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-18T08:36:36.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmp /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_10.export /tmp/tmp.vybh1m9yJF/cluster1-mirror-image_10.export 2026-02-18T08:36:36.170 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_10.export /tmp/tmp.vybh1m9yJF/cluster1-mirror-image_10.export 2026-02-18T08:36:36.206 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:36:36.206 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T08:36:36.207 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_11 2026-02-18T08:36:36.207 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_snap cluster2 mirror image_11 snap 2026-02-18T08:36:36.207 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:36:36.207 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:36:36.207 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_11 2026-02-18T08:36:36.207 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap 2026-02-18T08:36:36.207 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 -p mirror snap create image_11@snap --debug-rbd=20 --debug-journaler=20 2026-02-18T08:36:40.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_replay_started cluster1 mirror image_11 2026-02-18T08:36:40.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:36:40.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:36:40.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_11 2026-02-18T08:36:40.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_replay_state cluster1 mirror image_11 started 2026-02-18T08:36:40.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:36:40.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:36:40.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_11 2026-02-18T08:36:40.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local state=started 2026-02-18T08:36:40.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-18T08:36:40.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:36:40.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1 2026-02-18T08:36:41.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test_image_replay_state cluster1 mirror image_11 started 2026-02-18T08:36:41.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:36:41.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:36:41.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_11 2026-02-18T08:36:41.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local test_state=started 2026-02-18T08:36:41.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local status_result 2026-02-18T08:36:41.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_state=stopped 2026-02-18T08:36:41.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_11 2026-02-18T08:36:41.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -i state 2026-02-18T08:36:41.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_instance=cluster1 2026-02-18T08:36:41.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-18T08:36:41.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:36:41.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance=cluster1 2026-02-18T08:36:41.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local loop_instance 2026-02-18T08:36:41.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:36:41.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sleep 0 2026-02-18T08:36:41.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:36:41.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ seq 0 3 2026-02-18T08:36:41.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:36:41.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_11 2026-02-18T08:36:41.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:0 2026-02-18T08:36:41.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-18T08:36:41.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:36:41.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:36:41.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:36:41.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-18T08:36:41.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:36:41.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:36:41.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:36:41.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:36:41.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-18T08:36:41.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:36:41.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:36:41.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:36:41.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:36:41.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:0 2026-02-18T08:36:41.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-18T08:36:41.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-18T08:36:41.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:36:41.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:0 2026-02-18T08:36:41.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T08:36:41.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-18T08:36:41.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:36:41.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:36:41.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-18T08:36:41.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-18T08:36:41.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-18T08:36:41.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=0 2026-02-18T08:36:41.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=0 2026-02-18T08:36:41.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:36:41.058 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:36:41.059 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:36:41.059 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:36:41.059 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_11 2026-02-18T08:36:41.112 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_11'] 2026-02-18T08:36:41.112 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status : get status for rbd mirror 2026-02-18T08:36:41.112 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:36:41.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:36:41.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_11 2026-02-18T08:36:41.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:1 2026-02-18T08:36:41.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-18T08:36:41.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:36:41.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:36:41.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:36:41.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-18T08:36:41.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:36:41.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:36:41.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:36:41.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:36:41.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-18T08:36:41.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:36:41.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:36:41.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:36:41.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:36:41.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:1 2026-02-18T08:36:41.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-18T08:36:41.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-18T08:36:41.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:36:41.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:1 2026-02-18T08:36:41.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T08:36:41.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-18T08:36:41.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:36:41.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:36:41.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-18T08:36:41.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-18T08:36:41.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-18T08:36:41.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=1 2026-02-18T08:36:41.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=1 2026-02-18T08:36:41.118 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:36:41.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:36:41.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:36:41.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:36:41.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_11 2026-02-18T08:36:41.177 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_11'] 2026-02-18T08:36:41.177 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status : get status for rbd mirror 2026-02-18T08:36:41.177 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:36:41.180 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:36:41.180 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_11 2026-02-18T08:36:41.180 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:2 2026-02-18T08:36:41.180 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-18T08:36:41.180 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:36:41.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:36:41.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:36:41.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-18T08:36:41.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:36:41.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:36:41.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:36:41.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:36:41.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-18T08:36:41.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:36:41.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:36:41.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:36:41.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:36:41.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:2 2026-02-18T08:36:41.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-18T08:36:41.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-18T08:36:41.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:36:41.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:2 2026-02-18T08:36:41.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T08:36:41.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-18T08:36:41.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:36:41.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:36:41.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-18T08:36:41.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-18T08:36:41.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-18T08:36:41.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=2 2026-02-18T08:36:41.183 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=2 2026-02-18T08:36:41.183 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:36:41.190 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:36:41.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:36:41.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:36:41.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_11 2026-02-18T08:36:41.257 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ return 0 2026-02-18T08:36:41.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ status_result=' "state": "Replaying"' 2026-02-18T08:36:41.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ echo ' "state": "Replaying"' 2026-02-18T08:36:41.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ grep -i Replaying 2026-02-18T08:36:41.259 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: "state": "Replaying" 2026-02-18T08:36:41.261 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ current_state=started 2026-02-18T08:36:41.261 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test started = started 2026-02-18T08:36:41.261 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:36:41.261 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror image_11 2026-02-18T08:36:41.261 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_cluster=cluster1 2026-02-18T08:36:41.261 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:36:41.261 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_pool=mirror 2026-02-18T08:36:41.261 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local remote_pool=mirror 2026-02-18T08:36:41.261 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_11 2026-02-18T08:36:41.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' journal = journal ']' 2026-02-18T08:36:41.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror mirror image_11 2026-02-18T08:36:41.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_cluster=cluster1 2026-02-18T08:36:41.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:36:41.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_pool=mirror 2026-02-18T08:36:41.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local remote_pool=mirror 2026-02-18T08:36:41.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_11 2026-02-18T08:36:41.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-02-18T08:36:41.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-02-18T08:36:41.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:36:41.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:36:41.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:36:41.462 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_11 2026-02-18T08:36:41.462 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:36:41.462 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:36:41.462 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_11 2026-02-18T08:36:41.462 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:36:41.462 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_11 ']' 2026-02-18T08:36:41.462 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_11' 2026-02-18T08:36:41.463 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_11 2026-02-18T08:36:41.463 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:36:41.463 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:36:41.463 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:36:41.463 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:36:41.463 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:36:41.463 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:36:41.463 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:36:41.463 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:36:41.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:36:41.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:36:41.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_11 2026-02-18T08:36:41.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:36:41.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:36:41.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:36:41.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:36:41.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:36:41.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:36:41.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:36:41.467 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:36:41.467 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:36:41.467 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:36:41.467 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:36:41.467 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:36:41.467 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:36:41.467 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:36:41.467 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:36:41.467 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:36:41.467 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:36:41.467 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:36:41.467 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:36:41.468 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:36:41.468 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:36:41.468 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:36:41.468 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:36:41.468 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:36:41.468 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:36:41.468 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:36:41.468 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:36:41.468 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:36:41.468 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:36:41.468 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:36:41.476 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:36:41.484 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:36:41.484 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:36:41.484 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_11 2026-02-18T08:36:41.529 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_11'] 2026-02-18T08:36:41.529 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:36:41.529 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:36:41.532 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:36:41.532 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_11 2026-02-18T08:36:41.532 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:36:41.532 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:36:41.532 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:36:41.532 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:36:41.532 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:36:41.532 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:36:41.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:36:41.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:36:41.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:36:41.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:36:41.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:36:41.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:36:41.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:36:41.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:36:41.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:36:41.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:36:41.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:36:41.534 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:36:41.534 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:36:41.534 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:36:41.534 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:36:41.534 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:36:41.534 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:36:41.534 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:36:41.534 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:36:41.534 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:36:41.534 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:36:41.534 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:36:41.534 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:36:41.535 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:36:41.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:36:41.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:36:41.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:36:41.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_11 2026-02-18T08:36:41.602 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_11'] 2026-02-18T08:36:41.602 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:36:41.603 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:36:41.604 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:36:41.604 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_11 2026-02-18T08:36:41.604 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:36:41.604 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:36:41.604 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:36:41.604 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:36:41.605 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:36:41.605 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:36:41.605 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:36:41.605 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:36:41.605 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:36:41.605 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:36:41.605 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:36:41.605 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:36:41.605 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:36:41.605 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:36:41.605 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:36:41.605 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:36:41.605 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:36:41.605 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:36:41.605 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:36:41.605 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:36:41.605 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:36:41.605 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:36:41.605 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:36:41.605 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:36:41.605 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:36:41.605 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:36:41.605 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:36:41.605 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:36:41.605 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:36:41.605 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:36:41.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:36:41.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:36:41.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:36:41.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_11 2026-02-18T08:36:41.680 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:36:41.681 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_11 2026-02-18T08:36:41.681 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:36:41.681 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:36:41.681 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_11 2026-02-18T08:36:41.681 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_11 '' 2026-02-18T08:36:41.681 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:36:41.681 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:36:41.681 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_11 2026-02-18T08:36:41.681 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:36:41.681 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_11.status 2026-02-18T08:36:41.682 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_11.status 2026-02-18T08:36:41.682 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:36:41.683 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_11.status 2026-02-18T08:36:41.684 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_11 2026-02-18T08:36:41.684 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_11.status 2026-02-18T08:36:41.711 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 10 2026-02-18T08:36:41.711 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 10 2026-02-18T08:36:41.711 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:36:41.711 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=41, tag_tid=2, entry_tid=1], [object_number=40, tag_tid=2, entry_tid=0], [object_number=43, tag_tid=1, entry_tid=45523], [object_number=42, tag_tid=1, entry_tid=45522]]], state=connected] 2026-02-18T08:36:41.711 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=40, tag_tid=1, entry_tid=45524], [object_number=43, tag_tid=1, entry_tid=45523], [object_number=42, tag_tid=1, entry_tid=45522], [object_number=41, tag_tid=1, entry_tid=45521]]], state=connected] 2026-02-18T08:36:41.714 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_11.status 2026-02-18T08:36:41.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=41, tag_tid=2, entry_tid=1' 2026-02-18T08:36:41.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_11 2026-02-18T08:36:41.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:36:41.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:36:41.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_11 2026-02-18T08:36:41.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_11 '..*' 2026-02-18T08:36:41.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:36:41.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:36:41.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_11 2026-02-18T08:36:41.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:36:41.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_11.status 2026-02-18T08:36:41.717 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_11.status 2026-02-18T08:36:41.717 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:36:41.718 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_11.status 2026-02-18T08:36:41.718 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_11 2026-02-18T08:36:41.718 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_11.status 2026-02-18T08:36:41.751 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 10 2026-02-18T08:36:41.751 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 10 2026-02-18T08:36:41.751 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:36:41.751 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=41, tag_tid=2, entry_tid=1], [object_number=40, tag_tid=2, entry_tid=0], [object_number=43, tag_tid=1, entry_tid=45523], [object_number=42, tag_tid=1, entry_tid=45522]]], state=connected] 2026-02-18T08:36:41.751 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=40, tag_tid=1, entry_tid=45524], [object_number=43, tag_tid=1, entry_tid=45523], [object_number=42, tag_tid=1, entry_tid=45522], [object_number=41, tag_tid=1, entry_tid=45521]]], state=connected] 2026-02-18T08:36:41.755 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_11.status 2026-02-18T08:36:41.756 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=40, tag_tid=1, entry_tid=45524' 2026-02-18T08:36:41.756 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=41, tag_tid=2, entry_tid=1' -a 'object_number=41, tag_tid=2, entry_tid=1' = 'object_number=40, tag_tid=1, entry_tid=45524' 2026-02-18T08:36:41.756 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=40, tag_tid=1, entry_tid=45524' '!=' '' 2026-02-18T08:36:41.756 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:36:41.756 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=40, tag_tid=1, entry_tid=45524' = '' 2026-02-18T08:36:41.756 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=40, tag_tid=1, entry_tid=45524' 2026-02-18T08:36:41.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=41, tag_tid=2, entry_tid=1' 2026-02-18T08:36:41.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:36:41.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:36:41.759 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=2 2026-02-18T08:36:41.759 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=40, tag_tid=1, entry_tid=45524' 2026-02-18T08:36:41.759 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:36:41.759 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:36:41.761 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=1 2026-02-18T08:36:41.762 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=41, tag_tid=2, entry_tid=1' 2026-02-18T08:36:41.762 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:36:41.762 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:36:41.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:36:41.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=40, tag_tid=1, entry_tid=45524' 2026-02-18T08:36:41.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:36:41.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:36:41.766 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=45524 2026-02-18T08:36:41.766 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 2 = 1 -a 1 -le 45524 2026-02-18T08:36:41.766 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:36:41.766 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:36:41.766 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:36:41.967 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_11 2026-02-18T08:36:41.967 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:36:41.967 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:36:41.967 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_11 2026-02-18T08:36:41.967 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:36:41.967 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_11 ']' 2026-02-18T08:36:41.967 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_11' 2026-02-18T08:36:41.967 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_11 2026-02-18T08:36:41.967 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:36:41.967 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:36:41.967 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:36:41.967 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:36:41.967 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:36:41.967 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:36:41.967 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:36:41.968 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:36:41.968 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:36:41.969 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:36:41.969 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_11 2026-02-18T08:36:41.969 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:36:41.969 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:36:41.969 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:36:41.969 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:36:41.970 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:36:41.970 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:36:41.970 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:36:41.970 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:36:41.970 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:36:41.970 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:36:41.970 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:36:41.970 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:36:41.970 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:36:41.970 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:36:41.970 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:36:41.970 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:36:41.970 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:36:41.970 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:36:41.970 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:36:41.970 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:36:41.970 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:36:41.970 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:36:41.970 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:36:41.970 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:36:41.970 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:36:41.970 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:36:41.970 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:36:41.970 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:36:41.970 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:36:41.970 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:36:41.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:36:41.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:36:41.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:36:41.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_11 2026-02-18T08:36:42.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_11'] 2026-02-18T08:36:42.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:36:42.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:36:42.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:36:42.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_11 2026-02-18T08:36:42.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:36:42.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:36:42.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:36:42.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:36:42.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:36:42.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:36:42.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:36:42.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:36:42.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:36:42.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:36:42.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:36:42.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:36:42.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:36:42.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:36:42.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:36:42.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:36:42.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:36:42.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:36:42.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:36:42.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:36:42.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:36:42.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:36:42.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:36:42.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:36:42.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:36:42.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:36:42.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:36:42.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:36:42.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:36:42.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:36:42.062 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:36:42.062 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:36:42.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:36:42.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_11 2026-02-18T08:36:42.121 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_11'] 2026-02-18T08:36:42.121 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:36:42.121 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:36:42.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:36:42.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_11 2026-02-18T08:36:42.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:36:42.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:36:42.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:36:42.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:36:42.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:36:42.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:36:42.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:36:42.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:36:42.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:36:42.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:36:42.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:36:42.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:36:42.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:36:42.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:36:42.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:36:42.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:36:42.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:36:42.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:36:42.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:36:42.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:36:42.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:36:42.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:36:42.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:36:42.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:36:42.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:36:42.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:36:42.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:36:42.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:36:42.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:36:42.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:36:42.134 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:36:42.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:36:42.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:36:42.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_11 2026-02-18T08:36:42.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:36:42.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_11 2026-02-18T08:36:42.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:36:42.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:36:42.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_11 2026-02-18T08:36:42.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_11 '' 2026-02-18T08:36:42.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:36:42.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:36:42.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_11 2026-02-18T08:36:42.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:36:42.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_11.status 2026-02-18T08:36:42.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_11.status 2026-02-18T08:36:42.194 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:36:42.195 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_11.status 2026-02-18T08:36:42.195 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_11 2026-02-18T08:36:42.196 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_11.status 2026-02-18T08:36:42.222 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 10 2026-02-18T08:36:42.223 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 10 2026-02-18T08:36:42.223 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:36:42.223 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=41, tag_tid=2, entry_tid=1], [object_number=40, tag_tid=2, entry_tid=0], [object_number=43, tag_tid=1, entry_tid=45523], [object_number=42, tag_tid=1, entry_tid=45522]]], state=connected] 2026-02-18T08:36:42.223 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=40, tag_tid=1, entry_tid=45524], [object_number=43, tag_tid=1, entry_tid=45523], [object_number=42, tag_tid=1, entry_tid=45522], [object_number=41, tag_tid=1, entry_tid=45521]]], state=connected] 2026-02-18T08:36:42.225 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_11.status 2026-02-18T08:36:42.227 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=41, tag_tid=2, entry_tid=1' 2026-02-18T08:36:42.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_11 2026-02-18T08:36:42.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:36:42.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:36:42.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_11 2026-02-18T08:36:42.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_11 '..*' 2026-02-18T08:36:42.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:36:42.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:36:42.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_11 2026-02-18T08:36:42.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:36:42.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_11.status 2026-02-18T08:36:42.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_11.status 2026-02-18T08:36:42.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:36:42.230 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_11.status 2026-02-18T08:36:42.230 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_11 2026-02-18T08:36:42.230 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_11.status 2026-02-18T08:36:42.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 10 2026-02-18T08:36:42.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 10 2026-02-18T08:36:42.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:36:42.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=41, tag_tid=2, entry_tid=1], [object_number=40, tag_tid=2, entry_tid=0], [object_number=43, tag_tid=1, entry_tid=45523], [object_number=42, tag_tid=1, entry_tid=45522]]], state=connected] 2026-02-18T08:36:42.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=40, tag_tid=1, entry_tid=45524], [object_number=43, tag_tid=1, entry_tid=45523], [object_number=42, tag_tid=1, entry_tid=45522], [object_number=41, tag_tid=1, entry_tid=45521]]], state=connected] 2026-02-18T08:36:42.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_11.status 2026-02-18T08:36:42.269 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=40, tag_tid=1, entry_tid=45524' 2026-02-18T08:36:42.269 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=41, tag_tid=2, entry_tid=1' -a 'object_number=41, tag_tid=2, entry_tid=1' = 'object_number=40, tag_tid=1, entry_tid=45524' 2026-02-18T08:36:42.269 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=40, tag_tid=1, entry_tid=45524' '!=' 'object_number=40, tag_tid=1, entry_tid=45524' 2026-02-18T08:36:42.269 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:36:42.270 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.4 2026-02-18T08:36:42.671 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_11 2026-02-18T08:36:42.671 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:36:42.671 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:36:42.671 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_11 2026-02-18T08:36:42.671 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:36:42.671 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_11 ']' 2026-02-18T08:36:42.671 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_11' 2026-02-18T08:36:42.671 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_11 2026-02-18T08:36:42.671 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:36:42.671 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:36:42.671 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:36:42.671 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:36:42.671 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:36:42.671 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:36:42.671 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:36:42.672 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:36:42.672 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:36:42.673 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:36:42.673 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_11 2026-02-18T08:36:42.673 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:36:42.673 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:36:42.673 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:36:42.673 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:36:42.673 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:36:42.673 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:36:42.673 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:36:42.673 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:36:42.673 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:36:42.673 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:36:42.673 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:36:42.673 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:36:42.673 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:36:42.673 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:36:42.674 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:36:42.674 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:36:42.674 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:36:42.674 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:36:42.674 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:36:42.674 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:36:42.674 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:36:42.674 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:36:42.674 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:36:42.674 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:36:42.674 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:36:42.674 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:36:42.674 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:36:42.674 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:36:42.674 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:36:42.674 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:36:42.686 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:36:42.686 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:36:42.686 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:36:42.687 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_11 2026-02-18T08:36:42.740 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_11'] 2026-02-18T08:36:42.740 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:36:42.740 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:36:42.741 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:36:42.741 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_11 2026-02-18T08:36:42.742 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:36:42.742 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:36:42.742 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:36:42.742 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:36:42.742 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:36:42.742 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:36:42.742 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:36:42.742 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:36:42.742 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:36:42.742 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:36:42.742 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:36:42.742 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:36:42.742 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:36:42.742 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:36:42.742 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:36:42.742 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:36:42.742 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:36:42.742 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:36:42.742 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:36:42.742 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:36:42.742 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:36:42.742 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:36:42.743 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:36:42.743 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:36:42.743 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:36:42.743 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:36:42.743 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:36:42.743 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:36:42.743 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:36:42.743 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:36:42.754 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:36:42.754 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:36:42.754 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:36:42.754 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_11 2026-02-18T08:36:42.815 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_11'] 2026-02-18T08:36:42.815 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:36:42.815 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:36:42.817 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:36:42.817 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_11 2026-02-18T08:36:42.817 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:36:42.817 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:36:42.817 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:36:42.817 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:36:42.817 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:36:42.817 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:36:42.817 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:36:42.817 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:36:42.817 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:36:42.817 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:36:42.817 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:36:42.817 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:36:42.817 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:36:42.817 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:36:42.818 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:36:42.818 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:36:42.818 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:36:42.818 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:36:42.818 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:36:42.818 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:36:42.818 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:36:42.818 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:36:42.818 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:36:42.818 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:36:42.818 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:36:42.818 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:36:42.818 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:36:42.818 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:36:42.818 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:36:42.818 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:36:42.829 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:36:42.830 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:36:42.830 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:36:42.830 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_11 2026-02-18T08:36:42.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:36:42.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_11 2026-02-18T08:36:42.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:36:42.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:36:42.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_11 2026-02-18T08:36:42.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_11 '' 2026-02-18T08:36:42.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:36:42.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:36:42.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_11 2026-02-18T08:36:42.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:36:42.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_11.status 2026-02-18T08:36:42.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_11.status 2026-02-18T08:36:42.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:36:42.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_11.status 2026-02-18T08:36:42.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_11 2026-02-18T08:36:42.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_11.status 2026-02-18T08:36:42.922 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 10 2026-02-18T08:36:42.922 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 10 2026-02-18T08:36:42.922 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:36:42.922 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=41, tag_tid=2, entry_tid=1], [object_number=40, tag_tid=2, entry_tid=0], [object_number=43, tag_tid=1, entry_tid=45523], [object_number=42, tag_tid=1, entry_tid=45522]]], state=connected] 2026-02-18T08:36:42.922 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=40, tag_tid=1, entry_tid=45524], [object_number=43, tag_tid=1, entry_tid=45523], [object_number=42, tag_tid=1, entry_tid=45522], [object_number=41, tag_tid=1, entry_tid=45521]]], state=connected] 2026-02-18T08:36:42.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_11.status 2026-02-18T08:36:42.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=41, tag_tid=2, entry_tid=1' 2026-02-18T08:36:42.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_11 2026-02-18T08:36:42.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:36:42.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:36:42.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_11 2026-02-18T08:36:42.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_11 '..*' 2026-02-18T08:36:42.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:36:42.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:36:42.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_11 2026-02-18T08:36:42.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:36:42.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_11.status 2026-02-18T08:36:42.928 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_11.status 2026-02-18T08:36:42.928 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:36:42.929 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_11.status 2026-02-18T08:36:42.930 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_11 2026-02-18T08:36:42.930 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_11.status 2026-02-18T08:36:42.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 10 2026-02-18T08:36:42.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 10 2026-02-18T08:36:42.958 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:36:42.958 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=41, tag_tid=2, entry_tid=1], [object_number=40, tag_tid=2, entry_tid=0], [object_number=43, tag_tid=1, entry_tid=45523], [object_number=42, tag_tid=1, entry_tid=45522]]], state=connected] 2026-02-18T08:36:42.958 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=40, tag_tid=1, entry_tid=45524], [object_number=43, tag_tid=1, entry_tid=45523], [object_number=42, tag_tid=1, entry_tid=45522], [object_number=41, tag_tid=1, entry_tid=45521]]], state=connected] 2026-02-18T08:36:42.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_11.status 2026-02-18T08:36:42.961 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=40, tag_tid=1, entry_tid=45524' 2026-02-18T08:36:42.961 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=41, tag_tid=2, entry_tid=1' -a 'object_number=41, tag_tid=2, entry_tid=1' = 'object_number=40, tag_tid=1, entry_tid=45524' 2026-02-18T08:36:42.962 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=40, tag_tid=1, entry_tid=45524' '!=' 'object_number=40, tag_tid=1, entry_tid=45524' 2026-02-18T08:36:42.962 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:36:42.962 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.8 2026-02-18T08:36:43.763 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_11 2026-02-18T08:36:43.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:36:43.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:36:43.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_11 2026-02-18T08:36:43.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:36:43.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_11 ']' 2026-02-18T08:36:43.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_11' 2026-02-18T08:36:43.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_11 2026-02-18T08:36:43.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:36:43.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:36:43.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:36:43.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:36:43.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:36:43.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:36:43.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:36:43.768 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:36:43.768 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:36:43.768 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:36:43.768 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_11 2026-02-18T08:36:43.768 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:36:43.768 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:36:43.768 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:36:43.768 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:36:43.768 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:36:43.768 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:36:43.768 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:36:43.769 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:36:43.769 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:36:43.769 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:36:43.769 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:36:43.769 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:36:43.769 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:36:43.769 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:36:43.769 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:36:43.769 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:36:43.769 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:36:43.769 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:36:43.769 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:36:43.769 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:36:43.770 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:36:43.770 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:36:43.770 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:36:43.770 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:36:43.770 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:36:43.770 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:36:43.770 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:36:43.770 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:36:43.770 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:36:43.770 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:36:43.779 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:36:43.779 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:36:43.779 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:36:43.780 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_11 2026-02-18T08:36:43.839 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_11'] 2026-02-18T08:36:43.840 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:36:43.840 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:36:43.843 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:36:43.843 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_11 2026-02-18T08:36:43.843 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:36:43.843 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:36:43.843 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:36:43.843 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:36:43.843 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:36:43.843 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:36:43.843 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:36:43.844 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:36:43.844 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:36:43.844 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:36:43.844 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:36:43.844 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:36:43.844 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:36:43.844 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:36:43.844 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:36:43.844 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:36:43.844 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:36:43.844 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:36:43.844 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:36:43.844 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:36:43.845 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:36:43.845 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:36:43.845 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:36:43.845 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:36:43.845 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:36:43.845 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:36:43.845 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:36:43.845 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:36:43.845 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:36:43.845 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:36:43.855 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:36:43.855 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:36:43.856 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:36:43.856 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_11 2026-02-18T08:36:43.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_11'] 2026-02-18T08:36:43.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:36:43.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:36:43.924 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:36:43.924 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_11 2026-02-18T08:36:43.924 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:36:43.924 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:36:43.924 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:36:43.924 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:36:43.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:36:43.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:36:43.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:36:43.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:36:43.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:36:43.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:36:43.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:36:43.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:36:43.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:36:43.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:36:43.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:36:43.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:36:43.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:36:43.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:36:43.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:36:43.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:36:43.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:36:43.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:36:43.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:36:43.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:36:43.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:36:43.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:36:43.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:36:43.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:36:43.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:36:43.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:36:43.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:36:43.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:36:43.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:36:43.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_11 2026-02-18T08:36:43.999 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:36:44.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_11 2026-02-18T08:36:44.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:36:44.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:36:44.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_11 2026-02-18T08:36:44.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_11 '' 2026-02-18T08:36:44.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:36:44.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:36:44.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_11 2026-02-18T08:36:44.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:36:44.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_11.status 2026-02-18T08:36:44.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_11.status 2026-02-18T08:36:44.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:36:44.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_11.status 2026-02-18T08:36:44.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_11 2026-02-18T08:36:44.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_11.status 2026-02-18T08:36:44.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 10 2026-02-18T08:36:44.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 10 2026-02-18T08:36:44.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:36:44.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=41, tag_tid=2, entry_tid=1], [object_number=40, tag_tid=2, entry_tid=0], [object_number=43, tag_tid=1, entry_tid=45523], [object_number=42, tag_tid=1, entry_tid=45522]]], state=connected] 2026-02-18T08:36:44.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=41, tag_tid=2, entry_tid=1], [object_number=40, tag_tid=2, entry_tid=0], [object_number=43, tag_tid=1, entry_tid=45523], [object_number=42, tag_tid=1, entry_tid=45522]]], state=connected] 2026-02-18T08:36:44.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_11.status 2026-02-18T08:36:44.035 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=41, tag_tid=2, entry_tid=1' 2026-02-18T08:36:44.036 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_11 2026-02-18T08:36:44.036 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:36:44.036 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:36:44.037 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_11 2026-02-18T08:36:44.037 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_11 '..*' 2026-02-18T08:36:44.037 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:36:44.037 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:36:44.037 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_11 2026-02-18T08:36:44.037 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:36:44.037 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_11.status 2026-02-18T08:36:44.038 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_11.status 2026-02-18T08:36:44.038 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:36:44.038 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_11.status 2026-02-18T08:36:44.039 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_11 2026-02-18T08:36:44.039 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_11.status 2026-02-18T08:36:44.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 10 2026-02-18T08:36:44.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 10 2026-02-18T08:36:44.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:36:44.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=41, tag_tid=2, entry_tid=1], [object_number=40, tag_tid=2, entry_tid=0], [object_number=43, tag_tid=1, entry_tid=45523], [object_number=42, tag_tid=1, entry_tid=45522]]], state=connected] 2026-02-18T08:36:44.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=41, tag_tid=2, entry_tid=1], [object_number=40, tag_tid=2, entry_tid=0], [object_number=43, tag_tid=1, entry_tid=45523], [object_number=42, tag_tid=1, entry_tid=45522]]], state=connected] 2026-02-18T08:36:44.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_11.status 2026-02-18T08:36:44.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=41, tag_tid=2, entry_tid=1' 2026-02-18T08:36:44.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=41, tag_tid=2, entry_tid=1' -a 'object_number=41, tag_tid=2, entry_tid=1' = 'object_number=41, tag_tid=2, entry_tid=1' 2026-02-18T08:36:44.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:36:44.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_snap_present cluster1 mirror image_11 snap 2026-02-18T08:36:44.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:36:44.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:36:44.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_11 2026-02-18T08:36:44.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap 2026-02-18T08:36:44.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-18T08:36:44.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-18T08:36:44.080 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1 2026-02-18T08:36:44.212 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.0 with skip (0.9340174767835807) > weight (0.25) 2026-02-18T08:36:44.213 INFO:tasks.rbd_mirror_thrash:kill cluster1.client.mirror.1 2026-02-18T08:36:44.213 INFO:tasks.rbd_mirror.cluster1.client.mirror.1:Sent signal 15 2026-02-18T08:36:44.213 INFO:tasks.rbd_mirror_thrash:waiting for 2.5172394990774984 secs before reviving daemons 2026-02-18T08:36:44.214 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.trial163.stderr:2026-02-18T08:36:44.207+0000 7f460e4cb640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.1 (PID: 91376) UID: 1000 2026-02-18T08:36:45.080 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster1 info mirror/image_11@snap 2026-02-18T08:36:45.111 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:rbd image 'image_11': 2026-02-18T08:36:45.111 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: size 128 MiB in 32 objects 2026-02-18T08:36:45.111 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: order 22 (4 MiB objects) 2026-02-18T08:36:45.111 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: snapshot_count: 1 2026-02-18T08:36:45.111 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: id: 110444865539 2026-02-18T08:36:45.112 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: block_name_prefix: rbd_data.110444865539 2026-02-18T08:36:45.112 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: format: 2 2026-02-18T08:36:45.112 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: features: layering, exclusive-lock, journaling 2026-02-18T08:36:45.112 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: op_features: 2026-02-18T08:36:45.112 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: flags: 2026-02-18T08:36:45.112 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: create_timestamp: Wed Feb 18 08:27:23 2026 2026-02-18T08:36:45.112 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: access_timestamp: Wed Feb 18 08:27:23 2026 2026-02-18T08:36:45.112 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: modify_timestamp: Wed Feb 18 08:28:24 2026 2026-02-18T08:36:45.112 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: protected: False 2026-02-18T08:36:45.112 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: journal: 110444865539 2026-02-18T08:36:45.112 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring state: enabled 2026-02-18T08:36:45.112 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring mode: journal 2026-02-18T08:36:45.113 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring global id: c8c8e387-82b1-4ff7-8ac2-cfa349404a37 2026-02-18T08:36:45.113 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring primary: false 2026-02-18T08:36:45.114 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:36:45.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ compare_image_snaps mirror image_11 snap 2026-02-18T08:36:45.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:36:45.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_11 2026-02-18T08:36:45.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap 2026-02-18T08:36:45.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local ret=0 2026-02-18T08:36:45.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local rmt_export=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_11.export 2026-02-18T08:36:45.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loc_export=/tmp/tmp.vybh1m9yJF/cluster1-mirror-image_11.export 2026-02-18T08:36:45.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_11.export /tmp/tmp.vybh1m9yJF/cluster1-mirror-image_11.export 2026-02-18T08:36:45.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 -p mirror export image_11@snap /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_11.export 2026-02-18T08:36:45.276 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-18T08:36:45.288 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster1 -p mirror export image_11@snap /tmp/tmp.vybh1m9yJF/cluster1-mirror-image_11.export 2026-02-18T08:36:45.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-18T08:36:45.476 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmp /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_11.export /tmp/tmp.vybh1m9yJF/cluster1-mirror-image_11.export 2026-02-18T08:36:45.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_11.export /tmp/tmp.vybh1m9yJF/cluster1-mirror-image_11.export 2026-02-18T08:36:45.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:36:45.584 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T08:36:45.584 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_12 2026-02-18T08:36:45.584 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_snap cluster2 mirror image_12 snap 2026-02-18T08:36:45.584 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:36:45.584 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:36:45.584 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_12 2026-02-18T08:36:45.584 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap 2026-02-18T08:36:45.584 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 -p mirror snap create image_12@snap --debug-rbd=20 --debug-journaler=20 2026-02-18T08:36:46.731 INFO:tasks.rbd_mirror_thrash:waiting for cluster1.client.mirror.1 2026-02-18T08:36:46.731 INFO:teuthology.orchestra.run:waiting for 600 2026-02-18T08:36:46.732 INFO:tasks.rbd_mirror_thrash:reviving cluster1.client.mirror.1 2026-02-18T08:36:46.732 INFO:tasks.rbd_mirror.cluster1.client.mirror.1:Restarting daemon 2026-02-18T08:36:46.732 DEBUG:teuthology.orchestra.run.trial163:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.1 2026-02-18T08:36:46.734 INFO:tasks.rbd_mirror.cluster1.client.mirror.1:Started 2026-02-18T08:36:46.734 INFO:tasks.rbd_mirror_thrash:waiting for 97.01095994033022 secs before thrashing 2026-02-18T08:36:49.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_replay_started cluster1 mirror image_12 2026-02-18T08:36:49.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:36:49.058 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:36:49.058 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_12 2026-02-18T08:36:49.058 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_replay_state cluster1 mirror image_12 started 2026-02-18T08:36:49.058 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:36:49.058 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:36:49.058 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_12 2026-02-18T08:36:49.058 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local state=started 2026-02-18T08:36:49.058 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-18T08:36:49.058 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:36:49.058 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1 2026-02-18T08:36:50.058 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test_image_replay_state cluster1 mirror image_12 started 2026-02-18T08:36:50.059 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:36:50.059 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:36:50.059 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_12 2026-02-18T08:36:50.059 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local test_state=started 2026-02-18T08:36:50.059 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local status_result 2026-02-18T08:36:50.059 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_state=stopped 2026-02-18T08:36:50.060 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_12 2026-02-18T08:36:50.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -i state 2026-02-18T08:36:50.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_instance=cluster1 2026-02-18T08:36:50.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-18T08:36:50.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:36:50.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance=cluster1 2026-02-18T08:36:50.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local loop_instance 2026-02-18T08:36:50.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:36:50.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sleep 0 2026-02-18T08:36:50.062 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:36:50.062 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ seq 0 3 2026-02-18T08:36:50.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:36:50.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_12 2026-02-18T08:36:50.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:0 2026-02-18T08:36:50.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-18T08:36:50.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:36:50.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:36:50.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:36:50.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-18T08:36:50.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:36:50.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:36:50.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:36:50.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:36:50.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-18T08:36:50.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:36:50.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:36:50.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:36:50.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:36:50.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:0 2026-02-18T08:36:50.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-18T08:36:50.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-18T08:36:50.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:36:50.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:0 2026-02-18T08:36:50.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T08:36:50.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-18T08:36:50.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:36:50.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:36:50.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-18T08:36:50.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-18T08:36:50.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-18T08:36:50.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=0 2026-02-18T08:36:50.067 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=0 2026-02-18T08:36:50.067 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:36:50.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:36:50.074 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:36:50.074 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:36:50.074 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_12 2026-02-18T08:36:50.129 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_12'] 2026-02-18T08:36:50.130 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status : get status for rbd mirror 2026-02-18T08:36:50.130 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:36:50.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:36:50.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_12 2026-02-18T08:36:50.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:1 2026-02-18T08:36:50.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-18T08:36:50.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:36:50.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:36:50.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:36:50.134 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-18T08:36:50.134 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:36:50.134 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:36:50.134 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:36:50.134 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:36:50.134 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-18T08:36:50.134 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:36:50.134 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:36:50.134 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:36:50.134 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:36:50.134 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:1 2026-02-18T08:36:50.134 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-18T08:36:50.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-18T08:36:50.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:36:50.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:1 2026-02-18T08:36:50.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T08:36:50.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-18T08:36:50.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:36:50.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:36:50.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-18T08:36:50.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-18T08:36:50.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-18T08:36:50.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=1 2026-02-18T08:36:50.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=1 2026-02-18T08:36:50.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:36:50.142 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:36:50.143 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:36:50.143 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:36:50.143 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_12 2026-02-18T08:36:50.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_12'] 2026-02-18T08:36:50.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status : get status for rbd mirror 2026-02-18T08:36:50.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:36:50.196 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:36:50.196 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_12 2026-02-18T08:36:50.196 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:2 2026-02-18T08:36:50.196 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-18T08:36:50.197 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:36:50.197 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:36:50.197 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:36:50.197 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-18T08:36:50.197 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:36:50.197 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:36:50.197 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:36:50.197 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:36:50.197 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-18T08:36:50.197 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:36:50.197 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:36:50.197 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:36:50.198 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:36:50.198 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:2 2026-02-18T08:36:50.198 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-18T08:36:50.198 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-18T08:36:50.198 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:36:50.198 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:2 2026-02-18T08:36:50.198 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T08:36:50.198 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-18T08:36:50.198 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:36:50.198 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:36:50.198 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-18T08:36:50.198 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-18T08:36:50.198 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-18T08:36:50.199 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=2 2026-02-18T08:36:50.199 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=2 2026-02-18T08:36:50.199 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:36:50.206 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:36:50.207 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:36:50.207 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:36:50.207 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_12 2026-02-18T08:36:50.269 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_12'] 2026-02-18T08:36:50.269 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status : get status for rbd mirror 2026-02-18T08:36:50.269 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:36:50.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:36:50.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/image_12 2026-02-18T08:36:50.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:3 2026-02-18T08:36:50.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-18T08:36:50.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:36:50.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:36:50.273 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:36:50.273 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-18T08:36:50.273 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:36:50.273 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:36:50.273 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:36:50.273 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:36:50.273 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-18T08:36:50.273 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:36:50.273 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:36:50.273 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:36:50.273 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:36:50.273 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:3 2026-02-18T08:36:50.274 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-18T08:36:50.274 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-18T08:36:50.274 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:36:50.274 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:3 2026-02-18T08:36:50.274 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T08:36:50.274 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-18T08:36:50.274 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:36:50.274 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:36:50.274 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-18T08:36:50.274 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-18T08:36:50.274 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-18T08:36:50.274 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=3 2026-02-18T08:36:50.275 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=3 2026-02-18T08:36:50.275 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:36:50.282 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:36:50.283 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:36:50.283 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:36:50.283 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/image_12 2026-02-18T08:36:50.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_12'] 2026-02-18T08:36:50.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status : get status for rbd mirror 2026-02-18T08:36:50.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:36:50.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:36:50.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sleep 1 2026-02-18T08:36:51.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:36:51.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ seq 0 3 2026-02-18T08:36:51.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:36:51.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_12 2026-02-18T08:36:51.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:0 2026-02-18T08:36:51.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-18T08:36:51.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:36:51.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:36:51.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:36:51.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-18T08:36:51.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:36:51.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:36:51.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:36:51.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:36:51.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-18T08:36:51.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:36:51.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:36:51.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:36:51.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:36:51.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:0 2026-02-18T08:36:51.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-18T08:36:51.348 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-18T08:36:51.348 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:36:51.348 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:0 2026-02-18T08:36:51.348 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T08:36:51.348 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-18T08:36:51.348 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:36:51.348 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:36:51.348 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-18T08:36:51.348 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-18T08:36:51.348 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-18T08:36:51.348 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=0 2026-02-18T08:36:51.348 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=0 2026-02-18T08:36:51.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:36:51.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:36:51.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:36:51.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:36:51.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_12 2026-02-18T08:36:51.412 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_12'] 2026-02-18T08:36:51.412 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status : get status for rbd mirror 2026-02-18T08:36:51.412 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:36:51.415 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:36:51.415 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_12 2026-02-18T08:36:51.415 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:1 2026-02-18T08:36:51.415 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-18T08:36:51.415 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:36:51.415 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:36:51.416 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:36:51.416 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-18T08:36:51.416 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:36:51.416 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:36:51.416 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:36:51.416 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:36:51.416 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-18T08:36:51.416 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:36:51.416 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:36:51.416 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:36:51.416 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:36:51.417 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:1 2026-02-18T08:36:51.417 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-18T08:36:51.417 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-18T08:36:51.417 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:36:51.417 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:1 2026-02-18T08:36:51.417 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T08:36:51.417 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-18T08:36:51.417 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:36:51.417 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:36:51.417 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-18T08:36:51.417 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-18T08:36:51.417 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-18T08:36:51.418 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=1 2026-02-18T08:36:51.418 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=1 2026-02-18T08:36:51.418 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:36:51.427 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:36:51.427 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:36:51.427 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:36:51.427 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_12 2026-02-18T08:36:51.484 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_12'] 2026-02-18T08:36:51.484 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status : get status for rbd mirror 2026-02-18T08:36:51.484 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:36:51.486 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:36:51.486 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_12 2026-02-18T08:36:51.486 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:2 2026-02-18T08:36:51.487 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-18T08:36:51.487 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:36:51.487 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:36:51.487 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:36:51.487 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-18T08:36:51.487 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:36:51.487 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:36:51.487 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:36:51.487 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:36:51.487 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-18T08:36:51.487 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:36:51.488 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:36:51.488 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:36:51.488 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:36:51.488 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:2 2026-02-18T08:36:51.488 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-18T08:36:51.488 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-18T08:36:51.488 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:36:51.488 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:2 2026-02-18T08:36:51.488 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T08:36:51.488 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-18T08:36:51.488 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:36:51.488 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:36:51.489 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-18T08:36:51.489 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-18T08:36:51.489 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-18T08:36:51.489 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=2 2026-02-18T08:36:51.489 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=2 2026-02-18T08:36:51.489 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:36:51.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:36:51.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:36:51.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:36:51.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_12 2026-02-18T08:36:51.561 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_12'] 2026-02-18T08:36:51.561 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status : get status for rbd mirror 2026-02-18T08:36:51.561 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:36:51.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:36:51.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/image_12 2026-02-18T08:36:51.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:3 2026-02-18T08:36:51.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-18T08:36:51.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:36:51.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:36:51.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:36:51.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-18T08:36:51.565 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:36:51.565 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:36:51.565 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:36:51.565 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:36:51.565 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-18T08:36:51.565 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:36:51.565 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:36:51.565 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:36:51.565 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:36:51.565 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:3 2026-02-18T08:36:51.565 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-18T08:36:51.565 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-18T08:36:51.565 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:36:51.566 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:3 2026-02-18T08:36:51.566 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T08:36:51.566 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-18T08:36:51.566 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:36:51.566 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:36:51.566 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-18T08:36:51.566 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-18T08:36:51.566 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-18T08:36:51.566 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=3 2026-02-18T08:36:51.566 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=3 2026-02-18T08:36:51.566 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:36:51.575 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:36:51.575 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:36:51.575 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:36:51.575 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/image_12 2026-02-18T08:36:51.640 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_12'] 2026-02-18T08:36:51.640 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status : get status for rbd mirror 2026-02-18T08:36:51.640 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:36:51.641 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:36:51.642 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sleep 2 2026-02-18T08:36:53.643 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:36:53.643 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ seq 0 3 2026-02-18T08:36:53.647 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:36:53.647 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_12 2026-02-18T08:36:53.647 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:0 2026-02-18T08:36:53.647 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-18T08:36:53.647 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:36:53.647 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:36:53.647 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:36:53.647 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-18T08:36:53.647 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:36:53.648 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:36:53.648 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:36:53.648 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:36:53.648 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-18T08:36:53.648 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:36:53.648 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:36:53.648 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:36:53.648 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:36:53.648 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:0 2026-02-18T08:36:53.648 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-18T08:36:53.648 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-18T08:36:53.648 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:36:53.649 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:0 2026-02-18T08:36:53.649 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T08:36:53.649 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-18T08:36:53.649 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:36:53.649 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:36:53.649 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-18T08:36:53.649 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-18T08:36:53.649 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-18T08:36:53.649 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=0 2026-02-18T08:36:53.649 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=0 2026-02-18T08:36:53.649 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:36:53.659 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:36:53.660 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:36:53.660 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:36:53.660 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_12 2026-02-18T08:36:53.722 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_12'] 2026-02-18T08:36:53.722 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status : get status for rbd mirror 2026-02-18T08:36:53.722 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:36:53.725 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:36:53.725 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_12 2026-02-18T08:36:53.725 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:1 2026-02-18T08:36:53.725 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-18T08:36:53.725 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:36:53.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:36:53.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:36:53.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-18T08:36:53.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:36:53.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:36:53.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:36:53.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:36:53.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-18T08:36:53.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:36:53.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:36:53.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:36:53.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:36:53.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:1 2026-02-18T08:36:53.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-18T08:36:53.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-18T08:36:53.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:36:53.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:1 2026-02-18T08:36:53.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T08:36:53.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-18T08:36:53.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:36:53.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:36:53.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-18T08:36:53.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-18T08:36:53.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-18T08:36:53.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=1 2026-02-18T08:36:53.728 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=1 2026-02-18T08:36:53.728 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:36:53.735 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:36:53.736 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:36:53.736 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:36:53.736 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_12 2026-02-18T08:36:53.796 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_12'] 2026-02-18T08:36:53.796 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status : get status for rbd mirror 2026-02-18T08:36:53.796 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:36:53.798 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:36:53.798 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_12 2026-02-18T08:36:53.799 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:2 2026-02-18T08:36:53.799 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-18T08:36:53.799 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:36:53.799 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:36:53.799 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:36:53.799 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-18T08:36:53.799 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:36:53.799 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:36:53.799 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:36:53.799 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:36:53.799 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-18T08:36:53.800 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:36:53.800 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:36:53.800 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:36:53.800 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:36:53.800 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:2 2026-02-18T08:36:53.800 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-18T08:36:53.800 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-18T08:36:53.800 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:36:53.800 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:2 2026-02-18T08:36:53.800 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T08:36:53.800 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-18T08:36:53.800 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:36:53.800 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:36:53.801 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-18T08:36:53.801 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-18T08:36:53.801 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-18T08:36:53.801 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=2 2026-02-18T08:36:53.801 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=2 2026-02-18T08:36:53.801 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:36:53.809 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:36:53.810 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:36:53.810 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:36:53.810 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_12 2026-02-18T08:36:53.864 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_12'] 2026-02-18T08:36:53.864 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status : get status for rbd mirror 2026-02-18T08:36:53.864 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:36:53.867 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:36:53.867 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/image_12 2026-02-18T08:36:53.867 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:3 2026-02-18T08:36:53.867 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-18T08:36:53.867 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:36:53.867 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:36:53.867 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:36:53.867 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-18T08:36:53.868 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:36:53.868 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:36:53.868 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:36:53.868 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:36:53.868 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-18T08:36:53.868 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:36:53.868 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:36:53.868 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:36:53.868 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:36:53.868 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:3 2026-02-18T08:36:53.868 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-18T08:36:53.868 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-18T08:36:53.869 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:36:53.869 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:3 2026-02-18T08:36:53.869 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T08:36:53.869 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-18T08:36:53.869 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:36:53.869 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:36:53.869 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-18T08:36:53.869 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-18T08:36:53.869 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-18T08:36:53.869 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=3 2026-02-18T08:36:53.869 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=3 2026-02-18T08:36:53.869 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:36:53.878 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:36:53.878 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:36:53.878 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:36:53.878 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/image_12 2026-02-18T08:36:53.932 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_12'] 2026-02-18T08:36:53.932 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status : get status for rbd mirror 2026-02-18T08:36:53.932 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:36:53.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:36:53.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sleep 4 2026-02-18T08:36:57.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:36:57.936 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ seq 0 3 2026-02-18T08:36:57.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:36:57.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_12 2026-02-18T08:36:57.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:0 2026-02-18T08:36:57.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-18T08:36:57.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:36:57.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:36:57.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:36:57.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-18T08:36:57.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:36:57.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:36:57.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:36:57.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:36:57.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-18T08:36:57.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:36:57.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:36:57.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:36:57.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:36:57.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:0 2026-02-18T08:36:57.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-18T08:36:57.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-18T08:36:57.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:36:57.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:0 2026-02-18T08:36:57.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T08:36:57.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-18T08:36:57.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:36:57.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:36:57.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-18T08:36:57.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-18T08:36:57.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-18T08:36:57.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=0 2026-02-18T08:36:57.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=0 2026-02-18T08:36:57.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:36:57.950 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:36:57.950 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:36:57.950 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:36:57.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_12 2026-02-18T08:36:58.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_12'] 2026-02-18T08:36:58.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status : get status for rbd mirror 2026-02-18T08:36:58.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:36:58.005 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:36:58.005 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_12 2026-02-18T08:36:58.005 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:1 2026-02-18T08:36:58.006 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-18T08:36:58.006 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:36:58.006 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:36:58.006 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:36:58.006 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-18T08:36:58.006 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:36:58.006 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:36:58.006 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:36:58.006 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:36:58.006 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-18T08:36:58.006 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:36:58.006 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:36:58.006 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:36:58.007 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:36:58.007 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:1 2026-02-18T08:36:58.007 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-18T08:36:58.007 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-18T08:36:58.007 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:36:58.007 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:1 2026-02-18T08:36:58.007 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T08:36:58.007 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-18T08:36:58.007 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:36:58.007 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:36:58.007 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-18T08:36:58.007 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-18T08:36:58.008 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-18T08:36:58.008 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=1 2026-02-18T08:36:58.008 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=1 2026-02-18T08:36:58.008 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:36:58.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:36:58.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:36:58.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:36:58.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_12 2026-02-18T08:36:58.075 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_12'] 2026-02-18T08:36:58.076 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status : get status for rbd mirror 2026-02-18T08:36:58.076 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:36:58.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:36:58.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_12 2026-02-18T08:36:58.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:2 2026-02-18T08:36:58.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-18T08:36:58.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:36:58.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:36:58.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:36:58.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-18T08:36:58.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:36:58.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:36:58.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:36:58.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:36:58.080 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-18T08:36:58.080 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:36:58.080 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:36:58.080 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:36:58.080 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:36:58.080 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:2 2026-02-18T08:36:58.080 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-18T08:36:58.080 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-18T08:36:58.080 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:36:58.080 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:2 2026-02-18T08:36:58.080 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T08:36:58.080 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-18T08:36:58.080 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:36:58.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:36:58.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-18T08:36:58.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-18T08:36:58.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-18T08:36:58.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=2 2026-02-18T08:36:58.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=2 2026-02-18T08:36:58.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:36:58.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:36:58.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:36:58.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:36:58.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_12 2026-02-18T08:36:58.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_12'] 2026-02-18T08:36:58.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status : get status for rbd mirror 2026-02-18T08:36:58.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:36:58.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:36:58.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/image_12 2026-02-18T08:36:58.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:3 2026-02-18T08:36:58.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-18T08:36:58.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:36:58.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:36:58.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:36:58.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-18T08:36:58.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:36:58.155 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:36:58.155 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:36:58.155 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:36:58.155 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-18T08:36:58.155 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:36:58.155 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:36:58.155 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:36:58.155 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:36:58.155 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:3 2026-02-18T08:36:58.155 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-18T08:36:58.155 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-18T08:36:58.155 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:36:58.156 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:3 2026-02-18T08:36:58.156 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T08:36:58.156 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-18T08:36:58.156 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:36:58.156 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:36:58.156 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-18T08:36:58.156 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-18T08:36:58.156 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-18T08:36:58.156 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=3 2026-02-18T08:36:58.156 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=3 2026-02-18T08:36:58.156 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:36:58.164 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:36:58.165 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:36:58.165 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:36:58.165 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/image_12 2026-02-18T08:36:58.224 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_12'] 2026-02-18T08:36:58.225 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status : get status for rbd mirror 2026-02-18T08:36:58.225 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:36:58.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:36:58.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sleep 8 2026-02-18T08:37:06.227 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:37:06.227 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ seq 0 3 2026-02-18T08:37:06.230 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:37:06.230 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_12 2026-02-18T08:37:06.231 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:0 2026-02-18T08:37:06.231 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-18T08:37:06.231 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:37:06.231 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:37:06.231 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:37:06.231 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-18T08:37:06.231 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:37:06.231 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:37:06.231 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:37:06.231 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:37:06.231 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-18T08:37:06.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:37:06.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:37:06.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:37:06.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:37:06.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:0 2026-02-18T08:37:06.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-18T08:37:06.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-18T08:37:06.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:37:06.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:0 2026-02-18T08:37:06.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T08:37:06.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-18T08:37:06.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:37:06.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:37:06.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-18T08:37:06.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-18T08:37:06.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-18T08:37:06.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=0 2026-02-18T08:37:06.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=0 2026-02-18T08:37:06.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:37:06.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:37:06.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:37:06.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:37:06.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_12 2026-02-18T08:37:06.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_12'] 2026-02-18T08:37:06.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status : get status for rbd mirror 2026-02-18T08:37:06.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:37:06.298 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:37:06.298 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_12 2026-02-18T08:37:06.298 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:1 2026-02-18T08:37:06.298 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-18T08:37:06.298 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:37:06.298 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:37:06.298 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:37:06.298 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-18T08:37:06.298 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:37:06.299 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:37:06.299 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:37:06.299 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:37:06.299 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-18T08:37:06.299 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:37:06.299 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:37:06.299 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:37:06.299 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:37:06.299 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:1 2026-02-18T08:37:06.299 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-18T08:37:06.299 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-18T08:37:06.299 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:37:06.300 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:1 2026-02-18T08:37:06.300 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T08:37:06.300 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-18T08:37:06.300 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:37:06.300 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:37:06.300 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-18T08:37:06.300 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-18T08:37:06.300 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-18T08:37:06.300 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=1 2026-02-18T08:37:06.300 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=1 2026-02-18T08:37:06.300 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:37:06.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:37:06.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:37:06.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:37:06.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_12 2026-02-18T08:37:06.360 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_12'] 2026-02-18T08:37:06.360 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status : get status for rbd mirror 2026-02-18T08:37:06.360 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:37:06.363 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:37:06.363 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_12 2026-02-18T08:37:06.363 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:2 2026-02-18T08:37:06.363 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-18T08:37:06.363 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:37:06.363 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:37:06.363 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:37:06.363 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-18T08:37:06.363 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:37:06.364 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:37:06.364 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:37:06.364 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:37:06.364 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-18T08:37:06.364 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:37:06.364 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:37:06.364 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:37:06.364 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:37:06.364 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:2 2026-02-18T08:37:06.364 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-18T08:37:06.364 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-18T08:37:06.364 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:37:06.364 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:2 2026-02-18T08:37:06.365 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T08:37:06.365 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-18T08:37:06.365 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:37:06.365 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:37:06.365 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-18T08:37:06.365 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-18T08:37:06.365 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-18T08:37:06.365 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=2 2026-02-18T08:37:06.365 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=2 2026-02-18T08:37:06.365 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:37:06.373 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:37:06.374 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:37:06.374 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:37:06.374 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_12 2026-02-18T08:37:06.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_12'] 2026-02-18T08:37:06.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status : get status for rbd mirror 2026-02-18T08:37:06.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:37:06.429 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:37:06.430 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/image_12 2026-02-18T08:37:06.430 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:3 2026-02-18T08:37:06.430 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-18T08:37:06.430 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:37:06.430 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:37:06.430 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:37:06.430 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-18T08:37:06.430 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:37:06.430 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:37:06.430 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:37:06.430 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:37:06.431 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-18T08:37:06.431 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:37:06.431 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:37:06.431 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:37:06.431 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:37:06.431 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:3 2026-02-18T08:37:06.431 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-18T08:37:06.431 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-18T08:37:06.431 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:37:06.431 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:3 2026-02-18T08:37:06.431 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T08:37:06.431 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-18T08:37:06.432 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:37:06.432 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:37:06.432 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-18T08:37:06.432 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-18T08:37:06.432 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-18T08:37:06.432 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=3 2026-02-18T08:37:06.432 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=3 2026-02-18T08:37:06.432 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:37:06.439 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:37:06.440 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:37:06.440 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:37:06.440 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/image_12 2026-02-18T08:37:06.501 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_12'] 2026-02-18T08:37:06.501 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status : get status for rbd mirror 2026-02-18T08:37:06.501 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:37:06.502 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:37:06.502 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sleep 8 2026-02-18T08:37:14.503 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:37:14.504 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ seq 0 3 2026-02-18T08:37:14.507 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:37:14.507 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_12 2026-02-18T08:37:14.507 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:0 2026-02-18T08:37:14.507 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-18T08:37:14.507 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:37:14.507 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:37:14.508 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:37:14.508 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-18T08:37:14.508 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:37:14.508 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:37:14.508 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:37:14.508 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:37:14.508 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-18T08:37:14.508 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:37:14.508 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:37:14.508 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:37:14.508 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:37:14.509 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:0 2026-02-18T08:37:14.509 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-18T08:37:14.509 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-18T08:37:14.509 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:37:14.509 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:0 2026-02-18T08:37:14.509 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T08:37:14.509 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-18T08:37:14.509 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:37:14.509 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:37:14.509 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-18T08:37:14.509 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-18T08:37:14.509 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-18T08:37:14.510 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=0 2026-02-18T08:37:14.510 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=0 2026-02-18T08:37:14.510 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:37:14.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:37:14.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:37:14.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:37:14.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_12 2026-02-18T08:37:14.576 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ return 0 2026-02-18T08:37:14.577 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ status_result=' "state": "Replaying"' 2026-02-18T08:37:14.578 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ echo ' "state": "Replaying"' 2026-02-18T08:37:14.578 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ grep -i Replaying 2026-02-18T08:37:14.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: "state": "Replaying" 2026-02-18T08:37:14.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ current_state=started 2026-02-18T08:37:14.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test started = started 2026-02-18T08:37:14.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:37:14.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror image_12 2026-02-18T08:37:14.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_cluster=cluster1 2026-02-18T08:37:14.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:37:14.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_pool=mirror 2026-02-18T08:37:14.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local remote_pool=mirror 2026-02-18T08:37:14.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_12 2026-02-18T08:37:14.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' journal = journal ']' 2026-02-18T08:37:14.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror mirror image_12 2026-02-18T08:37:14.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_cluster=cluster1 2026-02-18T08:37:14.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:37:14.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_pool=mirror 2026-02-18T08:37:14.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local remote_pool=mirror 2026-02-18T08:37:14.582 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_12 2026-02-18T08:37:14.582 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-02-18T08:37:14.582 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-02-18T08:37:14.582 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:37:14.582 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:37:14.582 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:37:14.781 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_12 2026-02-18T08:37:14.782 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:37:14.782 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:37:14.782 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_12 2026-02-18T08:37:14.782 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:37:14.782 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_12 ']' 2026-02-18T08:37:14.782 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_12' 2026-02-18T08:37:14.782 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_12 2026-02-18T08:37:14.782 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:37:14.782 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:37:14.782 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:37:14.782 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:37:14.783 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:37:14.783 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:37:14.783 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:37:14.783 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:37:14.785 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:37:14.786 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:37:14.786 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_12 2026-02-18T08:37:14.786 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:37:14.786 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:37:14.786 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:37:14.786 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:37:14.786 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:37:14.786 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:37:14.786 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:37:14.786 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:37:14.786 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:37:14.787 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:37:14.787 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:37:14.787 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:37:14.787 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:37:14.787 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:37:14.787 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:37:14.787 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:37:14.787 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:37:14.787 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:37:14.787 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:37:14.787 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:37:14.787 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:37:14.788 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:37:14.788 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:37:14.788 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:37:14.788 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:37:14.788 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:37:14.788 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:37:14.788 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:37:14.788 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:37:14.788 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:37:14.796 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:37:14.796 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:37:14.796 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:37:14.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_12 2026-02-18T08:37:14.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:37:14.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_12 2026-02-18T08:37:14.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:37:14.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:37:14.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_12 2026-02-18T08:37:14.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_12 '' 2026-02-18T08:37:14.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:37:14.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:37:14.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_12 2026-02-18T08:37:14.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:37:14.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_12.status 2026-02-18T08:37:14.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_12.status 2026-02-18T08:37:14.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:37:14.874 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_12.status 2026-02-18T08:37:14.874 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_12 2026-02-18T08:37:14.874 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_12.status 2026-02-18T08:37:14.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 12 2026-02-18T08:37:14.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 12 2026-02-18T08:37:14.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:37:14.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=49, tag_tid=2, entry_tid=1], [object_number=48, tag_tid=2, entry_tid=0], [object_number=51, tag_tid=1, entry_tid=51467], [object_number=50, tag_tid=1, entry_tid=51466]]], state=connected] 2026-02-18T08:37:14.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=49, tag_tid=2, entry_tid=1], [object_number=48, tag_tid=2, entry_tid=0], [object_number=51, tag_tid=1, entry_tid=51467], [object_number=50, tag_tid=1, entry_tid=51466]]], state=connected] 2026-02-18T08:37:14.904 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_12.status 2026-02-18T08:37:14.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=49, tag_tid=2, entry_tid=1' 2026-02-18T08:37:14.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_12 2026-02-18T08:37:14.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:37:14.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:37:14.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_12 2026-02-18T08:37:14.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_12 '..*' 2026-02-18T08:37:14.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:37:14.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:37:14.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_12 2026-02-18T08:37:14.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:37:14.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_12.status 2026-02-18T08:37:14.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_12.status 2026-02-18T08:37:14.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:37:14.909 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_12.status 2026-02-18T08:37:14.909 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_12 2026-02-18T08:37:14.909 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_12.status 2026-02-18T08:37:14.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 12 2026-02-18T08:37:14.946 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 12 2026-02-18T08:37:14.946 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:37:14.946 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=49, tag_tid=2, entry_tid=1], [object_number=48, tag_tid=2, entry_tid=0], [object_number=51, tag_tid=1, entry_tid=51467], [object_number=50, tag_tid=1, entry_tid=51466]]], state=connected] 2026-02-18T08:37:14.946 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=49, tag_tid=2, entry_tid=1], [object_number=48, tag_tid=2, entry_tid=0], [object_number=51, tag_tid=1, entry_tid=51467], [object_number=50, tag_tid=1, entry_tid=51466]]], state=connected] 2026-02-18T08:37:14.949 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_12.status 2026-02-18T08:37:14.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=49, tag_tid=2, entry_tid=1' 2026-02-18T08:37:14.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=49, tag_tid=2, entry_tid=1' -a 'object_number=49, tag_tid=2, entry_tid=1' = 'object_number=49, tag_tid=2, entry_tid=1' 2026-02-18T08:37:14.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:37:14.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_snap_present cluster1 mirror image_12 snap 2026-02-18T08:37:14.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:37:14.952 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:37:14.952 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_12 2026-02-18T08:37:14.952 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap 2026-02-18T08:37:14.952 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-18T08:37:14.952 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-18T08:37:14.952 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1 2026-02-18T08:37:15.952 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster1 info mirror/image_12@snap 2026-02-18T08:37:15.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:rbd image 'image_12': 2026-02-18T08:37:15.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: size 128 MiB in 32 objects 2026-02-18T08:37:15.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: order 22 (4 MiB objects) 2026-02-18T08:37:15.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: snapshot_count: 1 2026-02-18T08:37:15.982 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: id: 11f97973ac51 2026-02-18T08:37:15.982 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: block_name_prefix: rbd_data.11f97973ac51 2026-02-18T08:37:15.982 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: format: 2 2026-02-18T08:37:15.982 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: features: layering, exclusive-lock, journaling 2026-02-18T08:37:15.982 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: op_features: 2026-02-18T08:37:15.982 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: flags: 2026-02-18T08:37:15.982 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: create_timestamp: Wed Feb 18 08:27:36 2026 2026-02-18T08:37:15.982 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: access_timestamp: Wed Feb 18 08:27:36 2026 2026-02-18T08:37:15.982 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: modify_timestamp: Wed Feb 18 08:28:37 2026 2026-02-18T08:37:15.982 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: protected: False 2026-02-18T08:37:15.982 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: journal: 11f97973ac51 2026-02-18T08:37:15.982 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring state: enabled 2026-02-18T08:37:15.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring mode: journal 2026-02-18T08:37:15.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring global id: c0c8847a-63a7-431b-978d-c476c32d6e1f 2026-02-18T08:37:15.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring primary: false 2026-02-18T08:37:15.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:37:15.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ compare_image_snaps mirror image_12 snap 2026-02-18T08:37:15.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:37:15.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_12 2026-02-18T08:37:15.985 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap 2026-02-18T08:37:15.985 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local ret=0 2026-02-18T08:37:15.985 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local rmt_export=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_12.export 2026-02-18T08:37:15.985 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loc_export=/tmp/tmp.vybh1m9yJF/cluster1-mirror-image_12.export 2026-02-18T08:37:15.985 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_12.export /tmp/tmp.vybh1m9yJF/cluster1-mirror-image_12.export 2026-02-18T08:37:15.985 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 -p mirror export image_12@snap /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_12.export 2026-02-18T08:37:16.129 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-18T08:37:16.139 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster1 -p mirror export image_12@snap /tmp/tmp.vybh1m9yJF/cluster1-mirror-image_12.export 2026-02-18T08:37:16.366 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-18T08:37:16.376 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmp /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_12.export /tmp/tmp.vybh1m9yJF/cluster1-mirror-image_12.export 2026-02-18T08:37:16.430 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_12.export /tmp/tmp.vybh1m9yJF/cluster1-mirror-image_12.export 2026-02-18T08:37:16.474 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:37:16.474 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T08:37:16.474 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_13 2026-02-18T08:37:16.474 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_snap cluster2 mirror image_13 snap 2026-02-18T08:37:16.475 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:37:16.475 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:37:16.475 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_13 2026-02-18T08:37:16.475 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap 2026-02-18T08:37:16.475 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 -p mirror snap create image_13@snap --debug-rbd=20 --debug-journaler=20 2026-02-18T08:37:20.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_replay_started cluster1 mirror image_13 2026-02-18T08:37:20.074 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:37:20.074 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:37:20.074 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_13 2026-02-18T08:37:20.074 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_replay_state cluster1 mirror image_13 started 2026-02-18T08:37:20.074 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:37:20.074 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:37:20.074 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_13 2026-02-18T08:37:20.074 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local state=started 2026-02-18T08:37:20.074 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-18T08:37:20.074 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:37:20.074 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1 2026-02-18T08:37:21.074 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test_image_replay_state cluster1 mirror image_13 started 2026-02-18T08:37:21.075 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:37:21.075 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:37:21.075 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_13 2026-02-18T08:37:21.075 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local test_state=started 2026-02-18T08:37:21.075 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local status_result 2026-02-18T08:37:21.075 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_state=stopped 2026-02-18T08:37:21.076 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_13 2026-02-18T08:37:21.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -i state 2026-02-18T08:37:21.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_instance=cluster1 2026-02-18T08:37:21.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-18T08:37:21.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:37:21.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance=cluster1 2026-02-18T08:37:21.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local loop_instance 2026-02-18T08:37:21.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:37:21.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sleep 0 2026-02-18T08:37:21.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:37:21.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ seq 0 3 2026-02-18T08:37:21.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:37:21.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_13 2026-02-18T08:37:21.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:0 2026-02-18T08:37:21.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-18T08:37:21.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:37:21.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:37:21.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:37:21.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-18T08:37:21.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:37:21.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:37:21.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:37:21.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:37:21.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-18T08:37:21.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:37:21.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:37:21.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:37:21.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:37:21.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:0 2026-02-18T08:37:21.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-18T08:37:21.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-18T08:37:21.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:37:21.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:0 2026-02-18T08:37:21.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T08:37:21.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-18T08:37:21.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:37:21.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:37:21.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-18T08:37:21.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-18T08:37:21.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-18T08:37:21.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=0 2026-02-18T08:37:21.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=0 2026-02-18T08:37:21.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:37:21.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:37:21.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:37:21.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:37:21.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_13 2026-02-18T08:37:21.146 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_13'] 2026-02-18T08:37:21.146 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status : get status for rbd mirror 2026-02-18T08:37:21.146 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:37:21.149 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:37:21.149 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_13 2026-02-18T08:37:21.149 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:1 2026-02-18T08:37:21.149 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-18T08:37:21.149 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:37:21.149 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:37:21.149 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:37:21.149 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-18T08:37:21.149 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:37:21.150 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:37:21.150 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:37:21.150 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:37:21.150 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-18T08:37:21.150 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:37:21.150 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:37:21.150 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:37:21.150 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:37:21.150 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:1 2026-02-18T08:37:21.150 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-18T08:37:21.150 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-18T08:37:21.150 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:37:21.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:1 2026-02-18T08:37:21.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T08:37:21.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-18T08:37:21.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:37:21.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:37:21.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-18T08:37:21.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-18T08:37:21.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-18T08:37:21.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=1 2026-02-18T08:37:21.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=1 2026-02-18T08:37:21.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:37:21.158 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:37:21.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:37:21.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:37:21.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_13 2026-02-18T08:37:21.225 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ return 0 2026-02-18T08:37:21.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ status_result=' "state": "Replaying"' 2026-02-18T08:37:21.227 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ echo ' "state": "Replaying"' 2026-02-18T08:37:21.227 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ grep -i Replaying 2026-02-18T08:37:21.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: "state": "Replaying" 2026-02-18T08:37:21.229 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ current_state=started 2026-02-18T08:37:21.230 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test started = started 2026-02-18T08:37:21.230 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:37:21.230 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror image_13 2026-02-18T08:37:21.230 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_cluster=cluster1 2026-02-18T08:37:21.230 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:37:21.230 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_pool=mirror 2026-02-18T08:37:21.230 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local remote_pool=mirror 2026-02-18T08:37:21.230 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_13 2026-02-18T08:37:21.230 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' journal = journal ']' 2026-02-18T08:37:21.230 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror mirror image_13 2026-02-18T08:37:21.230 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_cluster=cluster1 2026-02-18T08:37:21.231 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:37:21.231 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_pool=mirror 2026-02-18T08:37:21.231 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local remote_pool=mirror 2026-02-18T08:37:21.231 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_13 2026-02-18T08:37:21.231 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-02-18T08:37:21.231 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-02-18T08:37:21.231 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:37:21.231 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:37:21.231 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:37:21.430 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_13 2026-02-18T08:37:21.431 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:37:21.431 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:37:21.431 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_13 2026-02-18T08:37:21.431 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:37:21.431 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_13 ']' 2026-02-18T08:37:21.431 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_13' 2026-02-18T08:37:21.431 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_13 2026-02-18T08:37:21.431 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:37:21.431 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:37:21.432 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:37:21.432 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:37:21.432 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:37:21.432 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:37:21.432 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:37:21.432 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:37:21.434 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:37:21.435 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:37:21.435 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_13 2026-02-18T08:37:21.435 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:37:21.435 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:37:21.435 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:37:21.435 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:37:21.435 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:37:21.435 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:37:21.435 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:37:21.435 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:37:21.435 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:37:21.435 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:37:21.436 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:37:21.436 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:37:21.436 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:37:21.436 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:37:21.436 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:37:21.436 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:37:21.436 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:37:21.436 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:37:21.436 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:37:21.436 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:37:21.436 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:37:21.436 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:37:21.437 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:37:21.437 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:37:21.437 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:37:21.437 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:37:21.437 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:37:21.437 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:37:21.437 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:37:21.437 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:37:21.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:37:21.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:37:21.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:37:21.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_13 2026-02-18T08:37:21.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_13'] 2026-02-18T08:37:21.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:37:21.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:37:21.500 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:37:21.501 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_13 2026-02-18T08:37:21.501 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:37:21.501 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:37:21.501 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:37:21.501 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:37:21.501 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:37:21.501 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:37:21.501 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:37:21.501 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:37:21.501 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:37:21.501 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:37:21.502 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:37:21.502 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:37:21.502 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:37:21.502 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:37:21.502 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:37:21.502 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:37:21.502 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:37:21.502 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:37:21.502 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:37:21.502 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:37:21.502 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:37:21.502 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:37:21.503 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:37:21.503 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:37:21.503 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:37:21.503 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:37:21.503 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:37:21.503 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:37:21.503 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:37:21.503 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:37:21.511 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:37:21.511 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:37:21.511 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:37:21.511 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_13 2026-02-18T08:37:21.569 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:37:21.570 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_13 2026-02-18T08:37:21.571 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:37:21.571 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:37:21.571 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_13 2026-02-18T08:37:21.571 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_13 '' 2026-02-18T08:37:21.571 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:37:21.571 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:37:21.571 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_13 2026-02-18T08:37:21.571 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:37:21.571 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_13.status 2026-02-18T08:37:21.572 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_13.status 2026-02-18T08:37:21.572 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:37:21.573 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_13.status 2026-02-18T08:37:21.573 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_13 2026-02-18T08:37:21.573 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_13.status 2026-02-18T08:37:21.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 10 2026-02-18T08:37:21.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 10 2026-02-18T08:37:21.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:37:21.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=41, tag_tid=2, entry_tid=1], [object_number=40, tag_tid=2, entry_tid=0], [object_number=42, tag_tid=1, entry_tid=46046], [object_number=43, tag_tid=1, entry_tid=46043]]], state=connected] 2026-02-18T08:37:21.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=42, tag_tid=1, entry_tid=46046], [object_number=41, tag_tid=1, entry_tid=46045], [object_number=40, tag_tid=1, entry_tid=46044], [object_number=43, tag_tid=1, entry_tid=46043]]], state=connected] 2026-02-18T08:37:21.612 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_13.status 2026-02-18T08:37:21.613 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=41, tag_tid=2, entry_tid=1' 2026-02-18T08:37:21.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_13 2026-02-18T08:37:21.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:37:21.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:37:21.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_13 2026-02-18T08:37:21.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_13 '..*' 2026-02-18T08:37:21.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:37:21.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:37:21.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_13 2026-02-18T08:37:21.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:37:21.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_13.status 2026-02-18T08:37:21.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_13.status 2026-02-18T08:37:21.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:37:21.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_13.status 2026-02-18T08:37:21.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_13 2026-02-18T08:37:21.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_13.status 2026-02-18T08:37:21.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 10 2026-02-18T08:37:21.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 10 2026-02-18T08:37:21.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:37:21.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=41, tag_tid=2, entry_tid=1], [object_number=40, tag_tid=2, entry_tid=0], [object_number=42, tag_tid=1, entry_tid=46046], [object_number=43, tag_tid=1, entry_tid=46043]]], state=connected] 2026-02-18T08:37:21.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=42, tag_tid=1, entry_tid=46046], [object_number=41, tag_tid=1, entry_tid=46045], [object_number=40, tag_tid=1, entry_tid=46044], [object_number=43, tag_tid=1, entry_tid=46043]]], state=connected] 2026-02-18T08:37:21.654 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_13.status 2026-02-18T08:37:21.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=42, tag_tid=1, entry_tid=46046' 2026-02-18T08:37:21.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=41, tag_tid=2, entry_tid=1' -a 'object_number=41, tag_tid=2, entry_tid=1' = 'object_number=42, tag_tid=1, entry_tid=46046' 2026-02-18T08:37:21.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=42, tag_tid=1, entry_tid=46046' '!=' '' 2026-02-18T08:37:21.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:37:21.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=42, tag_tid=1, entry_tid=46046' = '' 2026-02-18T08:37:21.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=42, tag_tid=1, entry_tid=46046' 2026-02-18T08:37:21.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=41, tag_tid=2, entry_tid=1' 2026-02-18T08:37:21.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:37:21.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:37:21.658 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=2 2026-02-18T08:37:21.659 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=42, tag_tid=1, entry_tid=46046' 2026-02-18T08:37:21.659 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:37:21.659 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:37:21.660 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=1 2026-02-18T08:37:21.661 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=41, tag_tid=2, entry_tid=1' 2026-02-18T08:37:21.661 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:37:21.662 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:37:21.662 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:37:21.663 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=42, tag_tid=1, entry_tid=46046' 2026-02-18T08:37:21.663 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:37:21.663 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:37:21.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=46046 2026-02-18T08:37:21.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 2 = 1 -a 1 -le 46046 2026-02-18T08:37:21.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:37:21.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:37:21.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:37:21.867 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_13 2026-02-18T08:37:21.867 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:37:21.867 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:37:21.867 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_13 2026-02-18T08:37:21.867 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:37:21.868 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_13 ']' 2026-02-18T08:37:21.868 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_13' 2026-02-18T08:37:21.868 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_13 2026-02-18T08:37:21.868 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:37:21.868 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:37:21.868 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:37:21.868 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:37:21.868 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:37:21.868 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:37:21.868 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:37:21.868 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:37:21.869 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:37:21.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:37:21.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_13 2026-02-18T08:37:21.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:37:21.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:37:21.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:37:21.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:37:21.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:37:21.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:37:21.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:37:21.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:37:21.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:37:21.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:37:21.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:37:21.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:37:21.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:37:21.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:37:21.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:37:21.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:37:21.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:37:21.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:37:21.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:37:21.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:37:21.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:37:21.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:37:21.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:37:21.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:37:21.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:37:21.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:37:21.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:37:21.874 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:37:21.874 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:37:21.874 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:37:21.880 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:37:21.881 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:37:21.881 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:37:21.881 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_13 2026-02-18T08:37:21.944 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_13'] 2026-02-18T08:37:21.944 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:37:21.944 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:37:21.947 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:37:21.947 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_13 2026-02-18T08:37:21.947 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:37:21.947 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:37:21.947 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:37:21.947 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:37:21.947 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:37:21.948 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:37:21.948 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:37:21.948 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:37:21.948 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:37:21.948 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:37:21.948 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:37:21.948 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:37:21.948 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:37:21.948 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:37:21.948 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:37:21.948 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:37:21.948 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:37:21.949 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:37:21.949 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:37:21.949 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:37:21.949 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:37:21.949 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:37:21.949 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:37:21.949 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:37:21.949 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:37:21.949 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:37:21.949 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:37:21.949 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:37:21.949 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:37:21.950 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:37:21.958 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:37:21.959 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:37:21.959 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:37:21.959 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_13 2026-02-18T08:37:22.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:37:22.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_13 2026-02-18T08:37:22.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:37:22.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:37:22.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_13 2026-02-18T08:37:22.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_13 '' 2026-02-18T08:37:22.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:37:22.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:37:22.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_13 2026-02-18T08:37:22.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:37:22.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_13.status 2026-02-18T08:37:22.027 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_13.status 2026-02-18T08:37:22.027 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:37:22.027 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_13.status 2026-02-18T08:37:22.028 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_13 2026-02-18T08:37:22.028 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_13.status 2026-02-18T08:37:22.055 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 10 2026-02-18T08:37:22.056 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 10 2026-02-18T08:37:22.056 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:37:22.056 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=41, tag_tid=2, entry_tid=1], [object_number=40, tag_tid=2, entry_tid=0], [object_number=42, tag_tid=1, entry_tid=46046], [object_number=43, tag_tid=1, entry_tid=46043]]], state=connected] 2026-02-18T08:37:22.056 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=42, tag_tid=1, entry_tid=46046], [object_number=41, tag_tid=1, entry_tid=46045], [object_number=40, tag_tid=1, entry_tid=46044], [object_number=43, tag_tid=1, entry_tid=46043]]], state=connected] 2026-02-18T08:37:22.058 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_13.status 2026-02-18T08:37:22.060 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=41, tag_tid=2, entry_tid=1' 2026-02-18T08:37:22.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_13 2026-02-18T08:37:22.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:37:22.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:37:22.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_13 2026-02-18T08:37:22.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_13 '..*' 2026-02-18T08:37:22.062 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:37:22.062 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:37:22.062 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_13 2026-02-18T08:37:22.062 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:37:22.062 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_13.status 2026-02-18T08:37:22.062 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_13.status 2026-02-18T08:37:22.062 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:37:22.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_13.status 2026-02-18T08:37:22.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_13 2026-02-18T08:37:22.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_13.status 2026-02-18T08:37:22.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 10 2026-02-18T08:37:22.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 10 2026-02-18T08:37:22.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:37:22.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=41, tag_tid=2, entry_tid=1], [object_number=40, tag_tid=2, entry_tid=0], [object_number=42, tag_tid=1, entry_tid=46046], [object_number=43, tag_tid=1, entry_tid=46043]]], state=connected] 2026-02-18T08:37:22.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=42, tag_tid=1, entry_tid=46046], [object_number=41, tag_tid=1, entry_tid=46045], [object_number=40, tag_tid=1, entry_tid=46044], [object_number=43, tag_tid=1, entry_tid=46043]]], state=connected] 2026-02-18T08:37:22.102 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_13.status 2026-02-18T08:37:22.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=42, tag_tid=1, entry_tid=46046' 2026-02-18T08:37:22.105 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=41, tag_tid=2, entry_tid=1' -a 'object_number=41, tag_tid=2, entry_tid=1' = 'object_number=42, tag_tid=1, entry_tid=46046' 2026-02-18T08:37:22.105 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=42, tag_tid=1, entry_tid=46046' '!=' 'object_number=42, tag_tid=1, entry_tid=46046' 2026-02-18T08:37:22.105 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:37:22.105 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.4 2026-02-18T08:37:22.506 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_13 2026-02-18T08:37:22.507 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:37:22.507 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:37:22.507 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_13 2026-02-18T08:37:22.507 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:37:22.507 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_13 ']' 2026-02-18T08:37:22.507 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_13' 2026-02-18T08:37:22.507 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_13 2026-02-18T08:37:22.507 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:37:22.508 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:37:22.508 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:37:22.508 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:37:22.508 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:37:22.508 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:37:22.508 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:37:22.511 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:37:22.511 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:37:22.511 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:37:22.511 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_13 2026-02-18T08:37:22.512 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:37:22.512 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:37:22.512 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:37:22.512 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:37:22.512 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:37:22.512 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:37:22.512 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:37:22.512 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:37:22.512 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:37:22.512 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:37:22.512 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:37:22.512 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:37:22.513 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:37:22.513 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:37:22.513 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:37:22.513 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:37:22.513 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:37:22.513 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:37:22.513 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:37:22.513 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:37:22.513 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:37:22.513 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:37:22.513 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:37:22.513 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:37:22.513 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:37:22.514 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:37:22.514 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:37:22.514 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:37:22.514 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:37:22.514 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:37:22.523 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:37:22.523 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:37:22.523 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:37:22.523 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_13 2026-02-18T08:37:22.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_13'] 2026-02-18T08:37:22.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:37:22.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:37:22.586 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:37:22.586 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_13 2026-02-18T08:37:22.586 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:37:22.586 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:37:22.586 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:37:22.586 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:37:22.586 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:37:22.587 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:37:22.587 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:37:22.587 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:37:22.587 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:37:22.587 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:37:22.587 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:37:22.587 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:37:22.587 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:37:22.587 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:37:22.587 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:37:22.587 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:37:22.587 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:37:22.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:37:22.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:37:22.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:37:22.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:37:22.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:37:22.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:37:22.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:37:22.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:37:22.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:37:22.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:37:22.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:37:22.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:37:22.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:37:22.596 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:37:22.596 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:37:22.596 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:37:22.597 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_13 2026-02-18T08:37:22.662 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:37:22.663 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_13 2026-02-18T08:37:22.663 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:37:22.663 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:37:22.663 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_13 2026-02-18T08:37:22.664 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_13 '' 2026-02-18T08:37:22.664 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:37:22.664 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:37:22.664 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_13 2026-02-18T08:37:22.664 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:37:22.664 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_13.status 2026-02-18T08:37:22.664 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_13.status 2026-02-18T08:37:22.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:37:22.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_13.status 2026-02-18T08:37:22.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_13 2026-02-18T08:37:22.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_13.status 2026-02-18T08:37:22.694 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 10 2026-02-18T08:37:22.694 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 10 2026-02-18T08:37:22.694 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:37:22.694 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=41, tag_tid=2, entry_tid=1], [object_number=40, tag_tid=2, entry_tid=0], [object_number=42, tag_tid=1, entry_tid=46046], [object_number=43, tag_tid=1, entry_tid=46043]]], state=connected] 2026-02-18T08:37:22.694 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=42, tag_tid=1, entry_tid=46046], [object_number=41, tag_tid=1, entry_tid=46045], [object_number=40, tag_tid=1, entry_tid=46044], [object_number=43, tag_tid=1, entry_tid=46043]]], state=connected] 2026-02-18T08:37:22.696 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_13.status 2026-02-18T08:37:22.697 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=41, tag_tid=2, entry_tid=1' 2026-02-18T08:37:22.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_13 2026-02-18T08:37:22.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:37:22.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:37:22.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_13 2026-02-18T08:37:22.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_13 '..*' 2026-02-18T08:37:22.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:37:22.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:37:22.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_13 2026-02-18T08:37:22.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:37:22.700 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_13.status 2026-02-18T08:37:22.700 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_13.status 2026-02-18T08:37:22.700 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:37:22.700 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_13.status 2026-02-18T08:37:22.701 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_13 2026-02-18T08:37:22.701 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_13.status 2026-02-18T08:37:22.736 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 10 2026-02-18T08:37:22.736 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 10 2026-02-18T08:37:22.736 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:37:22.736 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=41, tag_tid=2, entry_tid=1], [object_number=40, tag_tid=2, entry_tid=0], [object_number=42, tag_tid=1, entry_tid=46046], [object_number=43, tag_tid=1, entry_tid=46043]]], state=connected] 2026-02-18T08:37:22.736 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=42, tag_tid=1, entry_tid=46046], [object_number=41, tag_tid=1, entry_tid=46045], [object_number=40, tag_tid=1, entry_tid=46044], [object_number=43, tag_tid=1, entry_tid=46043]]], state=connected] 2026-02-18T08:37:22.740 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_13.status 2026-02-18T08:37:22.741 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=42, tag_tid=1, entry_tid=46046' 2026-02-18T08:37:22.742 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=41, tag_tid=2, entry_tid=1' -a 'object_number=41, tag_tid=2, entry_tid=1' = 'object_number=42, tag_tid=1, entry_tid=46046' 2026-02-18T08:37:22.742 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=42, tag_tid=1, entry_tid=46046' '!=' 'object_number=42, tag_tid=1, entry_tid=46046' 2026-02-18T08:37:22.742 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:37:22.742 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.8 2026-02-18T08:37:23.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_13 2026-02-18T08:37:23.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:37:23.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:37:23.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_13 2026-02-18T08:37:23.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:37:23.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_13 ']' 2026-02-18T08:37:23.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_13' 2026-02-18T08:37:23.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_13 2026-02-18T08:37:23.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:37:23.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:37:23.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:37:23.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:37:23.545 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:37:23.545 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:37:23.545 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:37:23.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:37:23.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:37:23.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:37:23.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_13 2026-02-18T08:37:23.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:37:23.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:37:23.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:37:23.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:37:23.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:37:23.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:37:23.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:37:23.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:37:23.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:37:23.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:37:23.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:37:23.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:37:23.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:37:23.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:37:23.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:37:23.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:37:23.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:37:23.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:37:23.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:37:23.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:37:23.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:37:23.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:37:23.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:37:23.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:37:23.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:37:23.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:37:23.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:37:23.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:37:23.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:37:23.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:37:23.558 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:37:23.559 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:37:23.559 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:37:23.559 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_13 2026-02-18T08:37:23.619 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_13'] 2026-02-18T08:37:23.619 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:37:23.620 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:37:23.622 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:37:23.622 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_13 2026-02-18T08:37:23.622 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:37:23.623 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:37:23.623 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:37:23.623 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:37:23.623 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:37:23.623 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:37:23.623 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:37:23.623 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:37:23.623 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:37:23.623 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:37:23.623 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:37:23.623 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:37:23.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:37:23.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:37:23.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:37:23.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:37:23.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:37:23.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:37:23.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:37:23.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:37:23.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:37:23.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:37:23.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:37:23.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:37:23.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:37:23.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:37:23.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:37:23.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:37:23.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:37:23.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:37:23.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:37:23.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:37:23.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:37:23.635 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_13 2026-02-18T08:37:23.691 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:37:23.692 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_13 2026-02-18T08:37:23.692 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:37:23.692 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:37:23.692 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_13 2026-02-18T08:37:23.692 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_13 '' 2026-02-18T08:37:23.693 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:37:23.693 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:37:23.693 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_13 2026-02-18T08:37:23.693 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:37:23.693 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_13.status 2026-02-18T08:37:23.693 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_13.status 2026-02-18T08:37:23.693 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:37:23.694 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_13.status 2026-02-18T08:37:23.694 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_13 2026-02-18T08:37:23.694 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_13.status 2026-02-18T08:37:23.722 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 10 2026-02-18T08:37:23.722 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 10 2026-02-18T08:37:23.722 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:37:23.722 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=41, tag_tid=2, entry_tid=1], [object_number=40, tag_tid=2, entry_tid=0], [object_number=42, tag_tid=1, entry_tid=46046], [object_number=43, tag_tid=1, entry_tid=46043]]], state=connected] 2026-02-18T08:37:23.722 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=42, tag_tid=1, entry_tid=46046], [object_number=41, tag_tid=1, entry_tid=46045], [object_number=40, tag_tid=1, entry_tid=46044], [object_number=43, tag_tid=1, entry_tid=46043]]], state=connected] 2026-02-18T08:37:23.725 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_13.status 2026-02-18T08:37:23.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=41, tag_tid=2, entry_tid=1' 2026-02-18T08:37:23.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_13 2026-02-18T08:37:23.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:37:23.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:37:23.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_13 2026-02-18T08:37:23.728 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_13 '..*' 2026-02-18T08:37:23.728 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:37:23.728 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:37:23.728 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_13 2026-02-18T08:37:23.728 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:37:23.728 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_13.status 2026-02-18T08:37:23.728 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_13.status 2026-02-18T08:37:23.729 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:37:23.729 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_13.status 2026-02-18T08:37:23.729 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_13 2026-02-18T08:37:23.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_13.status 2026-02-18T08:37:23.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 10 2026-02-18T08:37:23.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 10 2026-02-18T08:37:23.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:37:23.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=41, tag_tid=2, entry_tid=1], [object_number=40, tag_tid=2, entry_tid=0], [object_number=42, tag_tid=1, entry_tid=46046], [object_number=43, tag_tid=1, entry_tid=46043]]], state=connected] 2026-02-18T08:37:23.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=42, tag_tid=1, entry_tid=46046], [object_number=41, tag_tid=1, entry_tid=46045], [object_number=40, tag_tid=1, entry_tid=46044], [object_number=43, tag_tid=1, entry_tid=46043]]], state=connected] 2026-02-18T08:37:23.760 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_13.status 2026-02-18T08:37:23.761 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=42, tag_tid=1, entry_tid=46046' 2026-02-18T08:37:23.761 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=41, tag_tid=2, entry_tid=1' -a 'object_number=41, tag_tid=2, entry_tid=1' = 'object_number=42, tag_tid=1, entry_tid=46046' 2026-02-18T08:37:23.762 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=42, tag_tid=1, entry_tid=46046' '!=' 'object_number=42, tag_tid=1, entry_tid=46046' 2026-02-18T08:37:23.762 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:37:23.762 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1.6 2026-02-18T08:37:25.363 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_13 2026-02-18T08:37:25.363 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:37:25.363 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:37:25.363 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_13 2026-02-18T08:37:25.364 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:37:25.364 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_13 ']' 2026-02-18T08:37:25.364 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_13' 2026-02-18T08:37:25.364 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_13 2026-02-18T08:37:25.364 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:37:25.364 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:37:25.364 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:37:25.364 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:37:25.364 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:37:25.364 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:37:25.364 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:37:25.367 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:37:25.367 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:37:25.367 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:37:25.368 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_13 2026-02-18T08:37:25.368 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:37:25.368 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:37:25.368 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:37:25.368 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:37:25.368 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:37:25.368 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:37:25.368 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:37:25.368 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:37:25.368 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:37:25.368 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:37:25.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:37:25.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:37:25.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:37:25.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:37:25.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:37:25.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:37:25.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:37:25.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:37:25.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:37:25.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:37:25.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:37:25.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:37:25.370 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:37:25.370 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:37:25.370 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:37:25.370 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:37:25.370 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:37:25.370 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:37:25.370 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:37:25.370 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:37:25.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:37:25.379 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:37:25.379 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:37:25.379 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_13 2026-02-18T08:37:25.431 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_13'] 2026-02-18T08:37:25.431 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:37:25.432 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:37:25.434 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:37:25.434 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_13 2026-02-18T08:37:25.434 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:37:25.434 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:37:25.435 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:37:25.435 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:37:25.435 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:37:25.435 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:37:25.435 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:37:25.435 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:37:25.435 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:37:25.435 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:37:25.435 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:37:25.435 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:37:25.435 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:37:25.435 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:37:25.436 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:37:25.436 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:37:25.436 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:37:25.436 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:37:25.436 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:37:25.436 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:37:25.436 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:37:25.436 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:37:25.436 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:37:25.436 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:37:25.436 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:37:25.436 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:37:25.436 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:37:25.437 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:37:25.437 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:37:25.437 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:37:25.446 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:37:25.446 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:37:25.446 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:37:25.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_13 2026-02-18T08:37:25.503 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:37:25.505 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_13 2026-02-18T08:37:25.505 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:37:25.505 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:37:25.505 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_13 2026-02-18T08:37:25.505 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_13 '' 2026-02-18T08:37:25.505 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:37:25.505 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:37:25.505 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_13 2026-02-18T08:37:25.505 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:37:25.505 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_13.status 2026-02-18T08:37:25.506 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_13.status 2026-02-18T08:37:25.506 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:37:25.506 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_13.status 2026-02-18T08:37:25.507 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_13 2026-02-18T08:37:25.507 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_13.status 2026-02-18T08:37:25.534 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 10 2026-02-18T08:37:25.534 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 10 2026-02-18T08:37:25.534 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:37:25.534 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=41, tag_tid=2, entry_tid=1], [object_number=40, tag_tid=2, entry_tid=0], [object_number=42, tag_tid=1, entry_tid=46046], [object_number=43, tag_tid=1, entry_tid=46043]]], state=connected] 2026-02-18T08:37:25.534 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=42, tag_tid=1, entry_tid=46046], [object_number=41, tag_tid=1, entry_tid=46045], [object_number=40, tag_tid=1, entry_tid=46044], [object_number=43, tag_tid=1, entry_tid=46043]]], state=connected] 2026-02-18T08:37:25.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_13.status 2026-02-18T08:37:25.539 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=41, tag_tid=2, entry_tid=1' 2026-02-18T08:37:25.540 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_13 2026-02-18T08:37:25.540 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:37:25.540 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:37:25.540 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_13 2026-02-18T08:37:25.540 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_13 '..*' 2026-02-18T08:37:25.540 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:37:25.540 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:37:25.540 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_13 2026-02-18T08:37:25.540 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:37:25.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_13.status 2026-02-18T08:37:25.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_13.status 2026-02-18T08:37:25.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:37:25.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_13.status 2026-02-18T08:37:25.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_13 2026-02-18T08:37:25.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_13.status 2026-02-18T08:37:25.568 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 10 2026-02-18T08:37:25.569 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 10 2026-02-18T08:37:25.569 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:37:25.569 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=41, tag_tid=2, entry_tid=1], [object_number=40, tag_tid=2, entry_tid=0], [object_number=42, tag_tid=1, entry_tid=46046], [object_number=43, tag_tid=1, entry_tid=46043]]], state=connected] 2026-02-18T08:37:25.569 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=42, tag_tid=1, entry_tid=46046], [object_number=41, tag_tid=1, entry_tid=46045], [object_number=40, tag_tid=1, entry_tid=46044], [object_number=43, tag_tid=1, entry_tid=46043]]], state=connected] 2026-02-18T08:37:25.571 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_13.status 2026-02-18T08:37:25.573 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=42, tag_tid=1, entry_tid=46046' 2026-02-18T08:37:25.573 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=41, tag_tid=2, entry_tid=1' -a 'object_number=41, tag_tid=2, entry_tid=1' = 'object_number=42, tag_tid=1, entry_tid=46046' 2026-02-18T08:37:25.573 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=42, tag_tid=1, entry_tid=46046' '!=' 'object_number=42, tag_tid=1, entry_tid=46046' 2026-02-18T08:37:25.573 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:37:25.573 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 2 2026-02-18T08:37:27.575 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_13 2026-02-18T08:37:27.575 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:37:27.576 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:37:27.576 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_13 2026-02-18T08:37:27.576 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:37:27.576 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_13 ']' 2026-02-18T08:37:27.576 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_13' 2026-02-18T08:37:27.576 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_13 2026-02-18T08:37:27.576 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:37:27.576 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:37:27.576 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:37:27.576 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:37:27.576 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:37:27.577 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:37:27.577 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:37:27.577 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:37:27.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:37:27.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:37:27.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_13 2026-02-18T08:37:27.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:37:27.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:37:27.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:37:27.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:37:27.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:37:27.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:37:27.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:37:27.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:37:27.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:37:27.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:37:27.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:37:27.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:37:27.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:37:27.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:37:27.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:37:27.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:37:27.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:37:27.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:37:27.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:37:27.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:37:27.582 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:37:27.582 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:37:27.582 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:37:27.582 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:37:27.582 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:37:27.582 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:37:27.582 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:37:27.582 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:37:27.582 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:37:27.582 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:37:27.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:37:27.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:37:27.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:37:27.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_13 2026-02-18T08:37:27.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_13'] 2026-02-18T08:37:27.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:37:27.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:37:27.655 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:37:27.655 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_13 2026-02-18T08:37:27.655 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:37:27.655 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:37:27.655 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:37:27.655 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:37:27.655 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:37:27.655 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:37:27.655 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:37:27.655 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:37:27.655 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:37:27.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:37:27.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:37:27.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:37:27.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:37:27.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:37:27.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:37:27.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:37:27.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:37:27.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:37:27.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:37:27.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:37:27.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:37:27.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:37:27.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:37:27.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:37:27.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:37:27.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:37:27.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:37:27.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:37:27.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:37:27.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:37:27.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:37:27.667 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:37:27.667 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:37:27.667 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_13 2026-02-18T08:37:27.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:37:27.728 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_13 2026-02-18T08:37:27.728 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:37:27.728 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:37:27.728 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_13 2026-02-18T08:37:27.728 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_13 '' 2026-02-18T08:37:27.728 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:37:27.728 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:37:27.729 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_13 2026-02-18T08:37:27.729 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:37:27.729 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_13.status 2026-02-18T08:37:27.729 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_13.status 2026-02-18T08:37:27.729 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:37:27.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_13.status 2026-02-18T08:37:27.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_13 2026-02-18T08:37:27.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_13.status 2026-02-18T08:37:27.758 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 10 2026-02-18T08:37:27.758 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 10 2026-02-18T08:37:27.758 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:37:27.758 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=41, tag_tid=2, entry_tid=1], [object_number=40, tag_tid=2, entry_tid=0], [object_number=42, tag_tid=1, entry_tid=46046], [object_number=43, tag_tid=1, entry_tid=46043]]], state=connected] 2026-02-18T08:37:27.758 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=42, tag_tid=1, entry_tid=46046], [object_number=41, tag_tid=1, entry_tid=46045], [object_number=40, tag_tid=1, entry_tid=46044], [object_number=43, tag_tid=1, entry_tid=46043]]], state=connected] 2026-02-18T08:37:27.760 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_13.status 2026-02-18T08:37:27.762 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=41, tag_tid=2, entry_tid=1' 2026-02-18T08:37:27.763 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_13 2026-02-18T08:37:27.763 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:37:27.763 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:37:27.763 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_13 2026-02-18T08:37:27.763 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_13 '..*' 2026-02-18T08:37:27.763 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:37:27.763 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:37:27.763 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_13 2026-02-18T08:37:27.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:37:27.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_13.status 2026-02-18T08:37:27.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_13.status 2026-02-18T08:37:27.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:37:27.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_13.status 2026-02-18T08:37:27.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_13 2026-02-18T08:37:27.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_13.status 2026-02-18T08:37:27.802 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 10 2026-02-18T08:37:27.802 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 10 2026-02-18T08:37:27.802 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:37:27.802 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=41, tag_tid=2, entry_tid=1], [object_number=40, tag_tid=2, entry_tid=0], [object_number=42, tag_tid=1, entry_tid=46046], [object_number=43, tag_tid=1, entry_tid=46043]]], state=connected] 2026-02-18T08:37:27.802 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=42, tag_tid=1, entry_tid=46046], [object_number=41, tag_tid=1, entry_tid=46045], [object_number=40, tag_tid=1, entry_tid=46044], [object_number=43, tag_tid=1, entry_tid=46043]]], state=connected] 2026-02-18T08:37:27.804 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_13.status 2026-02-18T08:37:27.806 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=42, tag_tid=1, entry_tid=46046' 2026-02-18T08:37:27.806 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=41, tag_tid=2, entry_tid=1' -a 'object_number=41, tag_tid=2, entry_tid=1' = 'object_number=42, tag_tid=1, entry_tid=46046' 2026-02-18T08:37:27.806 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=42, tag_tid=1, entry_tid=46046' '!=' 'object_number=42, tag_tid=1, entry_tid=46046' 2026-02-18T08:37:27.806 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:37:27.807 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 2 2026-02-18T08:37:29.808 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_13 2026-02-18T08:37:29.809 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:37:29.809 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:37:29.809 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_13 2026-02-18T08:37:29.809 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:37:29.809 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_13 ']' 2026-02-18T08:37:29.809 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_13' 2026-02-18T08:37:29.809 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_13 2026-02-18T08:37:29.809 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:37:29.809 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:37:29.809 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:37:29.810 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:37:29.810 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:37:29.810 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:37:29.810 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:37:29.810 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:37:29.810 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:37:29.812 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:37:29.813 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_13 2026-02-18T08:37:29.813 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:37:29.813 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:37:29.813 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:37:29.813 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:37:29.813 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:37:29.813 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:37:29.813 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:37:29.813 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:37:29.813 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:37:29.813 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:37:29.814 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:37:29.814 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:37:29.814 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:37:29.814 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:37:29.814 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:37:29.814 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:37:29.814 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:37:29.814 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:37:29.814 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:37:29.814 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:37:29.815 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:37:29.815 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:37:29.815 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:37:29.815 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:37:29.815 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:37:29.815 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:37:29.815 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:37:29.815 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:37:29.815 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:37:29.815 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:37:29.823 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:37:29.824 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:37:29.824 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:37:29.824 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_13 2026-02-18T08:37:29.885 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_13'] 2026-02-18T08:37:29.885 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:37:29.885 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:37:29.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:37:29.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_13 2026-02-18T08:37:29.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:37:29.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:37:29.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:37:29.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:37:29.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:37:29.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:37:29.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:37:29.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:37:29.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:37:29.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:37:29.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:37:29.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:37:29.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:37:29.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:37:29.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:37:29.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:37:29.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:37:29.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:37:29.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:37:29.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:37:29.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:37:29.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:37:29.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:37:29.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:37:29.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:37:29.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:37:29.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:37:29.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:37:29.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:37:29.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:37:29.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:37:29.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:37:29.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:37:29.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_13 2026-02-18T08:37:29.969 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:37:29.970 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_13 2026-02-18T08:37:29.970 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:37:29.971 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:37:29.971 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_13 2026-02-18T08:37:29.971 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_13 '' 2026-02-18T08:37:29.971 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:37:29.971 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:37:29.971 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_13 2026-02-18T08:37:29.971 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:37:29.971 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_13.status 2026-02-18T08:37:29.971 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_13.status 2026-02-18T08:37:29.971 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:37:29.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_13.status 2026-02-18T08:37:29.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_13 2026-02-18T08:37:29.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_13.status 2026-02-18T08:37:30.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 10 2026-02-18T08:37:30.010 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 10 2026-02-18T08:37:30.010 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:37:30.010 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=41, tag_tid=2, entry_tid=1], [object_number=40, tag_tid=2, entry_tid=0], [object_number=42, tag_tid=1, entry_tid=46046], [object_number=43, tag_tid=1, entry_tid=46043]]], state=connected] 2026-02-18T08:37:30.010 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=41, tag_tid=2, entry_tid=1], [object_number=40, tag_tid=2, entry_tid=0], [object_number=42, tag_tid=1, entry_tid=46046], [object_number=43, tag_tid=1, entry_tid=46043]]], state=connected] 2026-02-18T08:37:30.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_13.status 2026-02-18T08:37:30.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=41, tag_tid=2, entry_tid=1' 2026-02-18T08:37:30.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_13 2026-02-18T08:37:30.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:37:30.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:37:30.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_13 2026-02-18T08:37:30.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_13 '..*' 2026-02-18T08:37:30.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:37:30.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:37:30.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_13 2026-02-18T08:37:30.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:37:30.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_13.status 2026-02-18T08:37:30.018 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_13.status 2026-02-18T08:37:30.018 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:37:30.018 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_13.status 2026-02-18T08:37:30.019 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_13 2026-02-18T08:37:30.019 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_13.status 2026-02-18T08:37:30.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 10 2026-02-18T08:37:30.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 10 2026-02-18T08:37:30.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:37:30.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=41, tag_tid=2, entry_tid=1], [object_number=40, tag_tid=2, entry_tid=0], [object_number=42, tag_tid=1, entry_tid=46046], [object_number=43, tag_tid=1, entry_tid=46043]]], state=connected] 2026-02-18T08:37:30.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=41, tag_tid=2, entry_tid=1], [object_number=40, tag_tid=2, entry_tid=0], [object_number=42, tag_tid=1, entry_tid=46046], [object_number=43, tag_tid=1, entry_tid=46043]]], state=connected] 2026-02-18T08:37:30.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_13.status 2026-02-18T08:37:30.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=41, tag_tid=2, entry_tid=1' 2026-02-18T08:37:30.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=41, tag_tid=2, entry_tid=1' -a 'object_number=41, tag_tid=2, entry_tid=1' = 'object_number=41, tag_tid=2, entry_tid=1' 2026-02-18T08:37:30.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:37:30.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_snap_present cluster1 mirror image_13 snap 2026-02-18T08:37:30.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:37:30.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:37:30.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_13 2026-02-18T08:37:30.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap 2026-02-18T08:37:30.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-18T08:37:30.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-18T08:37:30.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1 2026-02-18T08:37:31.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster1 info mirror/image_13@snap 2026-02-18T08:37:31.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:rbd image 'image_13': 2026-02-18T08:37:31.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: size 128 MiB in 32 objects 2026-02-18T08:37:31.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: order 22 (4 MiB objects) 2026-02-18T08:37:31.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: snapshot_count: 1 2026-02-18T08:37:31.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: id: 10ac72f9ea31 2026-02-18T08:37:31.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: block_name_prefix: rbd_data.10ac72f9ea31 2026-02-18T08:37:31.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: format: 2 2026-02-18T08:37:31.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: features: layering, exclusive-lock, journaling 2026-02-18T08:37:31.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: op_features: 2026-02-18T08:37:31.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: flags: 2026-02-18T08:37:31.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: create_timestamp: Wed Feb 18 08:27:50 2026 2026-02-18T08:37:31.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: access_timestamp: Wed Feb 18 08:27:50 2026 2026-02-18T08:37:31.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: modify_timestamp: Wed Feb 18 08:28:51 2026 2026-02-18T08:37:31.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: protected: False 2026-02-18T08:37:31.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: journal: 10ac72f9ea31 2026-02-18T08:37:31.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring state: enabled 2026-02-18T08:37:31.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring mode: journal 2026-02-18T08:37:31.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring global id: 3a7795be-f4ed-43a8-93aa-cbd239723451 2026-02-18T08:37:31.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring primary: false 2026-02-18T08:37:31.086 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:37:31.086 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ compare_image_snaps mirror image_13 snap 2026-02-18T08:37:31.086 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:37:31.086 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_13 2026-02-18T08:37:31.086 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap 2026-02-18T08:37:31.086 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local ret=0 2026-02-18T08:37:31.086 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local rmt_export=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_13.export 2026-02-18T08:37:31.086 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loc_export=/tmp/tmp.vybh1m9yJF/cluster1-mirror-image_13.export 2026-02-18T08:37:31.086 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_13.export /tmp/tmp.vybh1m9yJF/cluster1-mirror-image_13.export 2026-02-18T08:37:31.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 -p mirror export image_13@snap /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_13.export 2026-02-18T08:37:31.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-18T08:37:31.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster1 -p mirror export image_13@snap /tmp/tmp.vybh1m9yJF/cluster1-mirror-image_13.export 2026-02-18T08:37:31.436 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-18T08:37:31.446 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmp /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_13.export /tmp/tmp.vybh1m9yJF/cluster1-mirror-image_13.export 2026-02-18T08:37:31.503 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_13.export /tmp/tmp.vybh1m9yJF/cluster1-mirror-image_13.export 2026-02-18T08:37:31.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:37:31.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T08:37:31.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_14 2026-02-18T08:37:31.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_snap cluster2 mirror image_14 snap 2026-02-18T08:37:31.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:37:31.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:37:31.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_14 2026-02-18T08:37:31.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap 2026-02-18T08:37:31.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 -p mirror snap create image_14@snap --debug-rbd=20 --debug-journaler=20 2026-02-18T08:37:34.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_replay_started cluster1 mirror image_14 2026-02-18T08:37:34.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:37:34.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:37:34.086 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_14 2026-02-18T08:37:34.086 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_replay_state cluster1 mirror image_14 started 2026-02-18T08:37:34.086 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:37:34.086 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:37:34.086 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_14 2026-02-18T08:37:34.086 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local state=started 2026-02-18T08:37:34.086 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-18T08:37:34.086 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:37:34.086 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1 2026-02-18T08:37:35.086 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test_image_replay_state cluster1 mirror image_14 started 2026-02-18T08:37:35.086 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:37:35.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:37:35.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_14 2026-02-18T08:37:35.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local test_state=started 2026-02-18T08:37:35.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local status_result 2026-02-18T08:37:35.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_state=stopped 2026-02-18T08:37:35.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_14 2026-02-18T08:37:35.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -i state 2026-02-18T08:37:35.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_instance=cluster1 2026-02-18T08:37:35.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-18T08:37:35.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:37:35.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance=cluster1 2026-02-18T08:37:35.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local loop_instance 2026-02-18T08:37:35.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:37:35.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sleep 0 2026-02-18T08:37:35.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:37:35.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ seq 0 3 2026-02-18T08:37:35.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:37:35.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_14 2026-02-18T08:37:35.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:0 2026-02-18T08:37:35.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-18T08:37:35.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:37:35.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:37:35.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:37:35.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-18T08:37:35.092 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:37:35.092 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:37:35.092 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:37:35.092 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:37:35.092 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-18T08:37:35.092 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:37:35.092 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:37:35.092 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:37:35.092 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:37:35.092 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:0 2026-02-18T08:37:35.092 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-18T08:37:35.092 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-18T08:37:35.093 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:37:35.093 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:0 2026-02-18T08:37:35.093 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T08:37:35.093 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-18T08:37:35.093 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:37:35.093 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:37:35.093 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-18T08:37:35.093 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-18T08:37:35.093 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-18T08:37:35.093 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=0 2026-02-18T08:37:35.093 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=0 2026-02-18T08:37:35.093 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:37:35.101 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:37:35.102 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:37:35.102 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:37:35.102 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_14 2026-02-18T08:37:35.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_14'] 2026-02-18T08:37:35.155 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status : get status for rbd mirror 2026-02-18T08:37:35.155 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:37:35.157 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:37:35.158 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_14 2026-02-18T08:37:35.158 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:1 2026-02-18T08:37:35.158 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-18T08:37:35.158 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:37:35.158 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:37:35.158 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:37:35.158 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-18T08:37:35.158 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:37:35.158 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:37:35.158 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:37:35.158 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:37:35.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-18T08:37:35.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:37:35.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:37:35.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:37:35.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:37:35.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:1 2026-02-18T08:37:35.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-18T08:37:35.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-18T08:37:35.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:37:35.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:1 2026-02-18T08:37:35.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T08:37:35.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-18T08:37:35.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:37:35.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:37:35.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-18T08:37:35.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-18T08:37:35.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-18T08:37:35.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=1 2026-02-18T08:37:35.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=1 2026-02-18T08:37:35.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:37:35.167 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:37:35.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:37:35.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:37:35.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_14 2026-02-18T08:37:35.229 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_14'] 2026-02-18T08:37:35.229 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status : get status for rbd mirror 2026-02-18T08:37:35.229 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:37:35.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:37:35.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_14 2026-02-18T08:37:35.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:2 2026-02-18T08:37:35.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-18T08:37:35.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:37:35.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:37:35.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:37:35.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-18T08:37:35.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:37:35.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:37:35.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:37:35.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:37:35.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-18T08:37:35.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:37:35.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:37:35.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:37:35.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:37:35.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:2 2026-02-18T08:37:35.234 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-18T08:37:35.234 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-18T08:37:35.234 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:37:35.234 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:2 2026-02-18T08:37:35.234 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T08:37:35.234 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-18T08:37:35.234 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:37:35.234 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:37:35.234 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-18T08:37:35.234 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-18T08:37:35.234 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-18T08:37:35.234 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=2 2026-02-18T08:37:35.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=2 2026-02-18T08:37:35.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:37:35.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:37:35.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:37:35.244 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:37:35.244 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_14 2026-02-18T08:37:35.304 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_14'] 2026-02-18T08:37:35.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status : get status for rbd mirror 2026-02-18T08:37:35.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:37:35.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:37:35.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/image_14 2026-02-18T08:37:35.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:3 2026-02-18T08:37:35.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-18T08:37:35.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:37:35.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:37:35.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:37:35.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-18T08:37:35.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:37:35.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:37:35.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:37:35.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:37:35.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-18T08:37:35.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:37:35.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:37:35.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:37:35.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:37:35.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:3 2026-02-18T08:37:35.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-18T08:37:35.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-18T08:37:35.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:37:35.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:3 2026-02-18T08:37:35.310 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T08:37:35.310 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-18T08:37:35.310 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:37:35.310 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:37:35.310 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-18T08:37:35.310 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-18T08:37:35.310 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-18T08:37:35.310 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=3 2026-02-18T08:37:35.310 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=3 2026-02-18T08:37:35.310 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:37:35.318 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:37:35.319 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:37:35.319 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:37:35.319 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/image_14 2026-02-18T08:37:35.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ return 0 2026-02-18T08:37:35.379 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ status_result=' "state": "Replaying"' 2026-02-18T08:37:35.379 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ echo ' "state": "Replaying"' 2026-02-18T08:37:35.379 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ grep -i Replaying 2026-02-18T08:37:35.380 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: "state": "Replaying" 2026-02-18T08:37:35.382 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ current_state=started 2026-02-18T08:37:35.382 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test started = started 2026-02-18T08:37:35.382 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:37:35.382 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror image_14 2026-02-18T08:37:35.382 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_cluster=cluster1 2026-02-18T08:37:35.382 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:37:35.382 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_pool=mirror 2026-02-18T08:37:35.382 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local remote_pool=mirror 2026-02-18T08:37:35.383 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_14 2026-02-18T08:37:35.383 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' journal = journal ']' 2026-02-18T08:37:35.383 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror mirror image_14 2026-02-18T08:37:35.383 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_cluster=cluster1 2026-02-18T08:37:35.383 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:37:35.383 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_pool=mirror 2026-02-18T08:37:35.383 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local remote_pool=mirror 2026-02-18T08:37:35.383 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_14 2026-02-18T08:37:35.383 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-02-18T08:37:35.383 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-02-18T08:37:35.383 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:37:35.384 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:37:35.384 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:37:35.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_14 2026-02-18T08:37:35.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:37:35.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:37:35.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_14 2026-02-18T08:37:35.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:37:35.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_14 ']' 2026-02-18T08:37:35.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_14' 2026-02-18T08:37:35.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_14 2026-02-18T08:37:35.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:37:35.584 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:37:35.584 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:37:35.584 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:37:35.584 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:37:35.584 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:37:35.584 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:37:35.587 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:37:35.587 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:37:35.587 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:37:35.587 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_14 2026-02-18T08:37:35.587 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:37:35.587 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:37:35.587 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:37:35.587 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:37:35.587 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:37:35.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:37:35.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:37:35.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:37:35.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:37:35.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:37:35.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:37:35.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:37:35.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:37:35.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:37:35.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:37:35.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:37:35.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:37:35.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:37:35.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:37:35.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:37:35.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:37:35.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:37:35.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:37:35.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:37:35.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:37:35.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:37:35.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:37:35.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:37:35.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:37:35.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:37:35.598 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:37:35.599 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:37:35.599 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:37:35.599 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_14 2026-02-18T08:37:35.659 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_14'] 2026-02-18T08:37:35.659 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:37:35.659 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:37:35.661 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:37:35.662 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_14 2026-02-18T08:37:35.662 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:37:35.662 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:37:35.662 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:37:35.662 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:37:35.662 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:37:35.662 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:37:35.662 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:37:35.662 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:37:35.662 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:37:35.663 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:37:35.663 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:37:35.663 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:37:35.663 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:37:35.663 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:37:35.663 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:37:35.663 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:37:35.663 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:37:35.663 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:37:35.663 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:37:35.663 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:37:35.663 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:37:35.664 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:37:35.664 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:37:35.664 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:37:35.664 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:37:35.664 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:37:35.664 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:37:35.664 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:37:35.664 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:37:35.664 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:37:35.671 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:37:35.672 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:37:35.672 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:37:35.672 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_14 2026-02-18T08:37:35.722 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_14'] 2026-02-18T08:37:35.722 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:37:35.722 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:37:35.725 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:37:35.725 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_14 2026-02-18T08:37:35.725 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:37:35.725 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:37:35.725 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:37:35.725 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:37:35.725 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:37:35.725 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:37:35.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:37:35.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:37:35.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:37:35.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:37:35.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:37:35.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:37:35.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:37:35.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:37:35.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:37:35.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:37:35.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:37:35.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:37:35.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:37:35.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:37:35.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:37:35.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:37:35.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:37:35.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:37:35.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:37:35.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:37:35.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:37:35.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:37:35.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:37:35.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:37:35.735 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:37:35.735 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:37:35.735 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:37:35.736 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_14 2026-02-18T08:37:35.786 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_14'] 2026-02-18T08:37:35.786 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:37:35.786 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:37:35.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:37:35.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/image_14 2026-02-18T08:37:35.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:37:35.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:37:35.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:37:35.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:37:35.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:37:35.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:37:35.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:37:35.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:37:35.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:37:35.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:37:35.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:37:35.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:37:35.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:37:35.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:37:35.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:37:35.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:37:35.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:37:35.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:37:35.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:37:35.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:37:35.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:37:35.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:37:35.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:37:35.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:37:35.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:37:35.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:37:35.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:37:35.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:37:35.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:37:35.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:37:35.799 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:37:35.799 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:37:35.799 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:37:35.800 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/image_14 2026-02-18T08:37:35.869 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:37:35.870 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_14 2026-02-18T08:37:35.870 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:37:35.870 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:37:35.870 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_14 2026-02-18T08:37:35.870 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_14 '' 2026-02-18T08:37:35.870 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:37:35.870 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:37:35.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_14 2026-02-18T08:37:35.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:37:35.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_14.status 2026-02-18T08:37:35.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_14.status 2026-02-18T08:37:35.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:37:35.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_14.status 2026-02-18T08:37:35.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_14 2026-02-18T08:37:35.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_14.status 2026-02-18T08:37:35.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 6 2026-02-18T08:37:35.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 6 2026-02-18T08:37:35.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:37:35.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=25, tag_tid=2, entry_tid=1], [object_number=24, tag_tid=2, entry_tid=0], [object_number=27, tag_tid=1, entry_tid=25347], [object_number=26, tag_tid=1, entry_tid=25346]]], state=connected] 2026-02-18T08:37:35.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=27, tag_tid=1, entry_tid=25347], [object_number=26, tag_tid=1, entry_tid=25346], [object_number=25, tag_tid=1, entry_tid=25345], [object_number=24, tag_tid=1, entry_tid=25344]]], state=connected] 2026-02-18T08:37:35.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_14.status 2026-02-18T08:37:35.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=25, tag_tid=2, entry_tid=1' 2026-02-18T08:37:35.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_14 2026-02-18T08:37:35.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:37:35.904 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:37:35.904 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_14 2026-02-18T08:37:35.904 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_14 '..*' 2026-02-18T08:37:35.904 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:37:35.904 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:37:35.904 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_14 2026-02-18T08:37:35.904 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:37:35.904 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_14.status 2026-02-18T08:37:35.904 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_14.status 2026-02-18T08:37:35.904 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:37:35.905 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_14.status 2026-02-18T08:37:35.905 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_14 2026-02-18T08:37:35.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_14.status 2026-02-18T08:37:35.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 6 2026-02-18T08:37:35.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 6 2026-02-18T08:37:35.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:37:35.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=25, tag_tid=2, entry_tid=1], [object_number=24, tag_tid=2, entry_tid=0], [object_number=27, tag_tid=1, entry_tid=25347], [object_number=26, tag_tid=1, entry_tid=25346]]], state=connected] 2026-02-18T08:37:35.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=27, tag_tid=1, entry_tid=25347], [object_number=26, tag_tid=1, entry_tid=25346], [object_number=25, tag_tid=1, entry_tid=25345], [object_number=24, tag_tid=1, entry_tid=25344]]], state=connected] 2026-02-18T08:37:35.944 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_14.status 2026-02-18T08:37:35.946 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=27, tag_tid=1, entry_tid=25347' 2026-02-18T08:37:35.946 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=25, tag_tid=2, entry_tid=1' -a 'object_number=25, tag_tid=2, entry_tid=1' = 'object_number=27, tag_tid=1, entry_tid=25347' 2026-02-18T08:37:35.946 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=27, tag_tid=1, entry_tid=25347' '!=' '' 2026-02-18T08:37:35.946 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:37:35.946 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=27, tag_tid=1, entry_tid=25347' = '' 2026-02-18T08:37:35.946 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=27, tag_tid=1, entry_tid=25347' 2026-02-18T08:37:35.947 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=25, tag_tid=2, entry_tid=1' 2026-02-18T08:37:35.947 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:37:35.947 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:37:35.948 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=2 2026-02-18T08:37:35.948 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=27, tag_tid=1, entry_tid=25347' 2026-02-18T08:37:35.949 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:37:35.949 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:37:35.950 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=1 2026-02-18T08:37:35.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=25, tag_tid=2, entry_tid=1' 2026-02-18T08:37:35.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:37:35.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:37:35.952 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:37:35.953 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=27, tag_tid=1, entry_tid=25347' 2026-02-18T08:37:35.953 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:37:35.953 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:37:35.955 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=25347 2026-02-18T08:37:35.955 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 2 = 1 -a 1 -le 25347 2026-02-18T08:37:35.955 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:37:35.955 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:37:35.955 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:37:36.157 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_14 2026-02-18T08:37:36.157 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:37:36.157 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:37:36.157 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_14 2026-02-18T08:37:36.157 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:37:36.157 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_14 ']' 2026-02-18T08:37:36.157 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_14' 2026-02-18T08:37:36.157 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_14 2026-02-18T08:37:36.157 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:37:36.157 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:37:36.158 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:37:36.158 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:37:36.158 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:37:36.158 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:37:36.158 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:37:36.158 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:37:36.161 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:37:36.161 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:37:36.161 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_14 2026-02-18T08:37:36.161 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:37:36.161 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:37:36.161 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:37:36.161 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:37:36.161 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:37:36.161 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:37:36.162 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:37:36.162 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:37:36.162 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:37:36.162 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:37:36.162 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:37:36.162 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:37:36.162 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:37:36.162 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:37:36.162 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:37:36.162 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:37:36.162 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:37:36.162 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:37:36.163 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:37:36.163 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:37:36.163 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:37:36.163 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:37:36.163 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:37:36.163 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:37:36.163 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:37:36.163 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:37:36.163 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:37:36.163 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:37:36.163 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:37:36.163 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:37:36.171 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:37:36.171 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:37:36.171 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:37:36.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_14 2026-02-18T08:37:36.231 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_14'] 2026-02-18T08:37:36.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:37:36.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:37:36.234 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:37:36.234 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_14 2026-02-18T08:37:36.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:37:36.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:37:36.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:37:36.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:37:36.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:37:36.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:37:36.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:37:36.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:37:36.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:37:36.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:37:36.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:37:36.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:37:36.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:37:36.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:37:36.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:37:36.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:37:36.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:37:36.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:37:36.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:37:36.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:37:36.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:37:36.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:37:36.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:37:36.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:37:36.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:37:36.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:37:36.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:37:36.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:37:36.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:37:36.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:37:36.246 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:37:36.246 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:37:36.246 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:37:36.246 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_14 2026-02-18T08:37:36.298 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_14'] 2026-02-18T08:37:36.298 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:37:36.298 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:37:36.301 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:37:36.301 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_14 2026-02-18T08:37:36.301 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:37:36.301 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:37:36.301 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:37:36.301 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:37:36.301 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:37:36.302 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:37:36.302 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:37:36.302 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:37:36.302 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:37:36.302 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:37:36.302 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:37:36.302 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:37:36.302 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:37:36.302 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:37:36.302 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:37:36.302 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:37:36.302 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:37:36.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:37:36.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:37:36.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:37:36.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:37:36.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:37:36.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:37:36.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:37:36.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:37:36.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:37:36.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:37:36.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:37:36.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:37:36.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:37:36.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:37:36.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:37:36.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:37:36.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_14 2026-02-18T08:37:36.371 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_14'] 2026-02-18T08:37:36.371 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:37:36.372 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:37:36.374 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:37:36.374 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/image_14 2026-02-18T08:37:36.374 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:37:36.375 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:37:36.375 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:37:36.375 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:37:36.375 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:37:36.375 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:37:36.375 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:37:36.375 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:37:36.375 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:37:36.375 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:37:36.375 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:37:36.375 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:37:36.375 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:37:36.376 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:37:36.376 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:37:36.376 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:37:36.376 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:37:36.376 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:37:36.376 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:37:36.376 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:37:36.376 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:37:36.376 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:37:36.376 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:37:36.376 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:37:36.376 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:37:36.377 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:37:36.377 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:37:36.377 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:37:36.377 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:37:36.377 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:37:36.385 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:37:36.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:37:36.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:37:36.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/image_14 2026-02-18T08:37:36.443 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:37:36.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_14 2026-02-18T08:37:36.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:37:36.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:37:36.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_14 2026-02-18T08:37:36.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_14 '' 2026-02-18T08:37:36.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:37:36.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:37:36.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_14 2026-02-18T08:37:36.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:37:36.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_14.status 2026-02-18T08:37:36.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_14.status 2026-02-18T08:37:36.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:37:36.446 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_14.status 2026-02-18T08:37:36.446 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_14 2026-02-18T08:37:36.446 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_14.status 2026-02-18T08:37:36.475 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 6 2026-02-18T08:37:36.475 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 6 2026-02-18T08:37:36.475 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:37:36.475 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=25, tag_tid=2, entry_tid=1], [object_number=24, tag_tid=2, entry_tid=0], [object_number=27, tag_tid=1, entry_tid=25347], [object_number=26, tag_tid=1, entry_tid=25346]]], state=connected] 2026-02-18T08:37:36.475 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=27, tag_tid=1, entry_tid=25347], [object_number=26, tag_tid=1, entry_tid=25346], [object_number=25, tag_tid=1, entry_tid=25345], [object_number=24, tag_tid=1, entry_tid=25344]]], state=connected] 2026-02-18T08:37:36.478 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_14.status 2026-02-18T08:37:36.479 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=25, tag_tid=2, entry_tid=1' 2026-02-18T08:37:36.481 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_14 2026-02-18T08:37:36.481 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:37:36.481 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:37:36.481 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_14 2026-02-18T08:37:36.481 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_14 '..*' 2026-02-18T08:37:36.481 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:37:36.481 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:37:36.481 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_14 2026-02-18T08:37:36.481 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:37:36.482 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_14.status 2026-02-18T08:37:36.482 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_14.status 2026-02-18T08:37:36.482 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:37:36.482 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_14.status 2026-02-18T08:37:36.483 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_14 2026-02-18T08:37:36.483 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_14.status 2026-02-18T08:37:36.516 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 6 2026-02-18T08:37:36.516 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 6 2026-02-18T08:37:36.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:37:36.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=25, tag_tid=2, entry_tid=1], [object_number=24, tag_tid=2, entry_tid=0], [object_number=27, tag_tid=1, entry_tid=25347], [object_number=26, tag_tid=1, entry_tid=25346]]], state=connected] 2026-02-18T08:37:36.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=27, tag_tid=1, entry_tid=25347], [object_number=26, tag_tid=1, entry_tid=25346], [object_number=25, tag_tid=1, entry_tid=25345], [object_number=24, tag_tid=1, entry_tid=25344]]], state=connected] 2026-02-18T08:37:36.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_14.status 2026-02-18T08:37:36.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=27, tag_tid=1, entry_tid=25347' 2026-02-18T08:37:36.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=25, tag_tid=2, entry_tid=1' -a 'object_number=25, tag_tid=2, entry_tid=1' = 'object_number=27, tag_tid=1, entry_tid=25347' 2026-02-18T08:37:36.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=27, tag_tid=1, entry_tid=25347' '!=' 'object_number=27, tag_tid=1, entry_tid=25347' 2026-02-18T08:37:36.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:37:36.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.4 2026-02-18T08:37:36.924 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_14 2026-02-18T08:37:36.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:37:36.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:37:36.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_14 2026-02-18T08:37:36.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:37:36.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_14 ']' 2026-02-18T08:37:36.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_14' 2026-02-18T08:37:36.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_14 2026-02-18T08:37:36.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:37:36.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:37:36.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:37:36.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:37:36.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:37:36.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:37:36.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:37:36.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:37:36.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:37:36.929 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:37:36.929 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_14 2026-02-18T08:37:36.929 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:37:36.929 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:37:36.929 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:37:36.929 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:37:36.929 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:37:36.929 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:37:36.929 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:37:36.930 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:37:36.930 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:37:36.930 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:37:36.930 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:37:36.930 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:37:36.930 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:37:36.930 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:37:36.930 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:37:36.930 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:37:36.930 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:37:36.930 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:37:36.930 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:37:36.931 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:37:36.931 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:37:36.931 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:37:36.931 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:37:36.931 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:37:36.931 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:37:36.931 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:37:36.931 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:37:36.931 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:37:36.931 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:37:36.931 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:37:36.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:37:36.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:37:36.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:37:36.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_14 2026-02-18T08:37:37.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_14'] 2026-02-18T08:37:37.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:37:37.003 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:37:37.005 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:37:37.005 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_14 2026-02-18T08:37:37.006 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:37:37.006 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:37:37.006 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:37:37.006 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:37:37.006 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:37:37.006 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:37:37.006 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:37:37.006 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:37:37.006 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:37:37.006 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:37:37.006 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:37:37.007 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:37:37.007 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:37:37.007 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:37:37.007 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:37:37.007 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:37:37.007 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:37:37.007 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:37:37.007 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:37:37.007 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:37:37.007 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:37:37.007 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:37:37.007 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:37:37.008 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:37:37.008 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:37:37.008 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:37:37.008 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:37:37.008 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:37:37.008 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:37:37.008 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:37:37.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:37:37.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:37:37.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:37:37.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_14 2026-02-18T08:37:37.076 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_14'] 2026-02-18T08:37:37.076 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:37:37.076 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:37:37.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:37:37.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_14 2026-02-18T08:37:37.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:37:37.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:37:37.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:37:37.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:37:37.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:37:37.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:37:37.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:37:37.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:37:37.080 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:37:37.080 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:37:37.080 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:37:37.080 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:37:37.080 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:37:37.080 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:37:37.080 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:37:37.080 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:37:37.080 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:37:37.080 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:37:37.080 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:37:37.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:37:37.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:37:37.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:37:37.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:37:37.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:37:37.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:37:37.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:37:37.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:37:37.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:37:37.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:37:37.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:37:37.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:37:37.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:37:37.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:37:37.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_14 2026-02-18T08:37:37.149 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_14'] 2026-02-18T08:37:37.150 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:37:37.150 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:37:37.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:37:37.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/image_14 2026-02-18T08:37:37.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:37:37.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:37:37.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:37:37.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:37:37.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:37:37.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:37:37.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:37:37.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:37:37.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:37:37.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:37:37.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:37:37.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:37:37.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:37:37.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:37:37.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:37:37.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:37:37.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:37:37.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:37:37.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:37:37.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:37:37.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:37:37.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:37:37.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:37:37.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:37:37.155 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:37:37.155 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:37:37.155 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:37:37.155 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:37:37.155 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:37:37.155 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:37:37.163 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:37:37.163 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:37:37.164 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:37:37.164 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/image_14 2026-02-18T08:37:37.229 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:37:37.229 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_14 2026-02-18T08:37:37.230 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:37:37.230 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:37:37.230 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_14 2026-02-18T08:37:37.230 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_14 '' 2026-02-18T08:37:37.230 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:37:37.230 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:37:37.230 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_14 2026-02-18T08:37:37.230 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:37:37.230 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_14.status 2026-02-18T08:37:37.231 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_14.status 2026-02-18T08:37:37.231 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:37:37.231 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_14.status 2026-02-18T08:37:37.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_14 2026-02-18T08:37:37.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_14.status 2026-02-18T08:37:37.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 6 2026-02-18T08:37:37.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 6 2026-02-18T08:37:37.259 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:37:37.259 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=25, tag_tid=2, entry_tid=1], [object_number=24, tag_tid=2, entry_tid=0], [object_number=27, tag_tid=1, entry_tid=25347], [object_number=26, tag_tid=1, entry_tid=25346]]], state=connected] 2026-02-18T08:37:37.259 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=27, tag_tid=1, entry_tid=25347], [object_number=26, tag_tid=1, entry_tid=25346], [object_number=25, tag_tid=1, entry_tid=25345], [object_number=24, tag_tid=1, entry_tid=25344]]], state=connected] 2026-02-18T08:37:37.261 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_14.status 2026-02-18T08:37:37.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=25, tag_tid=2, entry_tid=1' 2026-02-18T08:37:37.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_14 2026-02-18T08:37:37.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:37:37.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:37:37.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_14 2026-02-18T08:37:37.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_14 '..*' 2026-02-18T08:37:37.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:37:37.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:37:37.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_14 2026-02-18T08:37:37.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:37:37.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_14.status 2026-02-18T08:37:37.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_14.status 2026-02-18T08:37:37.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:37:37.266 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_14.status 2026-02-18T08:37:37.266 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_14 2026-02-18T08:37:37.266 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_14.status 2026-02-18T08:37:37.294 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 6 2026-02-18T08:37:37.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 6 2026-02-18T08:37:37.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:37:37.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=25, tag_tid=2, entry_tid=1], [object_number=24, tag_tid=2, entry_tid=0], [object_number=27, tag_tid=1, entry_tid=25347], [object_number=26, tag_tid=1, entry_tid=25346]]], state=connected] 2026-02-18T08:37:37.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=27, tag_tid=1, entry_tid=25347], [object_number=26, tag_tid=1, entry_tid=25346], [object_number=25, tag_tid=1, entry_tid=25345], [object_number=24, tag_tid=1, entry_tid=25344]]], state=connected] 2026-02-18T08:37:37.297 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_14.status 2026-02-18T08:37:37.299 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=27, tag_tid=1, entry_tid=25347' 2026-02-18T08:37:37.299 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=25, tag_tid=2, entry_tid=1' -a 'object_number=25, tag_tid=2, entry_tid=1' = 'object_number=27, tag_tid=1, entry_tid=25347' 2026-02-18T08:37:37.299 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=27, tag_tid=1, entry_tid=25347' '!=' 'object_number=27, tag_tid=1, entry_tid=25347' 2026-02-18T08:37:37.299 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:37:37.299 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.8 2026-02-18T08:37:38.101 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_14 2026-02-18T08:37:38.101 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:37:38.101 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:37:38.102 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_14 2026-02-18T08:37:38.102 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:37:38.102 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_14 ']' 2026-02-18T08:37:38.102 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_14' 2026-02-18T08:37:38.102 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_14 2026-02-18T08:37:38.102 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:37:38.102 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:37:38.102 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:37:38.102 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:37:38.102 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:37:38.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:37:38.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:37:38.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:37:38.106 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:37:38.106 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:37:38.106 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_14 2026-02-18T08:37:38.106 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:37:38.106 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:37:38.106 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:37:38.106 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:37:38.106 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:37:38.106 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:37:38.106 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:37:38.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:37:38.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:37:38.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:37:38.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:37:38.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:37:38.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:37:38.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:37:38.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:37:38.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:37:38.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:37:38.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:37:38.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:37:38.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:37:38.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:37:38.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:37:38.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:37:38.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:37:38.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:37:38.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:37:38.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:37:38.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:37:38.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:37:38.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:37:38.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:37:38.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:37:38.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:37:38.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_14 2026-02-18T08:37:38.177 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_14'] 2026-02-18T08:37:38.177 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:37:38.177 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:37:38.180 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:37:38.180 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_14 2026-02-18T08:37:38.180 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:37:38.180 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:37:38.180 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:37:38.180 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:37:38.180 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:37:38.180 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:37:38.180 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:37:38.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:37:38.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:37:38.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:37:38.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:37:38.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:37:38.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:37:38.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:37:38.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:37:38.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:37:38.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:37:38.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:37:38.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:37:38.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:37:38.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:37:38.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:37:38.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:37:38.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:37:38.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:37:38.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:37:38.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:37:38.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:37:38.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:37:38.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:37:38.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:37:38.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:37:38.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:37:38.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_14 2026-02-18T08:37:38.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_14'] 2026-02-18T08:37:38.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:37:38.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:37:38.246 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:37:38.246 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_14 2026-02-18T08:37:38.246 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:37:38.246 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:37:38.246 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:37:38.246 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:37:38.246 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:37:38.246 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:37:38.246 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:37:38.246 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:37:38.247 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:37:38.247 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:37:38.247 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:37:38.247 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:37:38.247 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:37:38.247 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:37:38.247 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:37:38.247 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:37:38.247 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:37:38.247 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:37:38.247 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:37:38.247 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:37:38.248 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:37:38.248 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:37:38.248 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:37:38.248 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:37:38.248 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:37:38.248 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:37:38.248 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:37:38.248 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:37:38.248 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:37:38.248 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:37:38.256 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:37:38.257 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:37:38.257 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:37:38.257 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_14 2026-02-18T08:37:38.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_14'] 2026-02-18T08:37:38.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:37:38.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:37:38.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:37:38.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/image_14 2026-02-18T08:37:38.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:37:38.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:37:38.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:37:38.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:37:38.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:37:38.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:37:38.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:37:38.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:37:38.312 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:37:38.312 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:37:38.312 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:37:38.312 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:37:38.312 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:37:38.312 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:37:38.312 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:37:38.312 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:37:38.312 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:37:38.312 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:37:38.312 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:37:38.312 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:37:38.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:37:38.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:37:38.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:37:38.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:37:38.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:37:38.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:37:38.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:37:38.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:37:38.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:37:38.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:37:38.322 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:37:38.323 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:37:38.323 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:37:38.323 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/image_14 2026-02-18T08:37:38.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:37:38.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_14 2026-02-18T08:37:38.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:37:38.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:37:38.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_14 2026-02-18T08:37:38.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_14 '' 2026-02-18T08:37:38.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:37:38.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:37:38.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_14 2026-02-18T08:37:38.393 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:37:38.393 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_14.status 2026-02-18T08:37:38.393 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_14.status 2026-02-18T08:37:38.393 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:37:38.394 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_14.status 2026-02-18T08:37:38.394 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_14 2026-02-18T08:37:38.394 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_14.status 2026-02-18T08:37:38.421 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 6 2026-02-18T08:37:38.421 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 6 2026-02-18T08:37:38.421 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:37:38.421 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=25, tag_tid=2, entry_tid=1], [object_number=24, tag_tid=2, entry_tid=0], [object_number=27, tag_tid=1, entry_tid=25347], [object_number=26, tag_tid=1, entry_tid=25346]]], state=connected] 2026-02-18T08:37:38.421 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=25, tag_tid=2, entry_tid=1], [object_number=24, tag_tid=2, entry_tid=0], [object_number=27, tag_tid=1, entry_tid=25347], [object_number=26, tag_tid=1, entry_tid=25346]]], state=connected] 2026-02-18T08:37:38.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_14.status 2026-02-18T08:37:38.425 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=25, tag_tid=2, entry_tid=1' 2026-02-18T08:37:38.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_14 2026-02-18T08:37:38.427 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:37:38.427 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:37:38.427 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_14 2026-02-18T08:37:38.427 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_14 '..*' 2026-02-18T08:37:38.427 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:37:38.427 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:37:38.427 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_14 2026-02-18T08:37:38.427 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:37:38.427 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_14.status 2026-02-18T08:37:38.428 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_14.status 2026-02-18T08:37:38.428 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:37:38.428 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_14.status 2026-02-18T08:37:38.428 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_14 2026-02-18T08:37:38.429 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_14.status 2026-02-18T08:37:38.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 6 2026-02-18T08:37:38.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 6 2026-02-18T08:37:38.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:37:38.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=25, tag_tid=2, entry_tid=1], [object_number=24, tag_tid=2, entry_tid=0], [object_number=27, tag_tid=1, entry_tid=25347], [object_number=26, tag_tid=1, entry_tid=25346]]], state=connected] 2026-02-18T08:37:38.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=25, tag_tid=2, entry_tid=1], [object_number=24, tag_tid=2, entry_tid=0], [object_number=27, tag_tid=1, entry_tid=25347], [object_number=26, tag_tid=1, entry_tid=25346]]], state=connected] 2026-02-18T08:37:38.468 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_14.status 2026-02-18T08:37:38.470 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=25, tag_tid=2, entry_tid=1' 2026-02-18T08:37:38.470 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=25, tag_tid=2, entry_tid=1' -a 'object_number=25, tag_tid=2, entry_tid=1' = 'object_number=25, tag_tid=2, entry_tid=1' 2026-02-18T08:37:38.470 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:37:38.470 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_snap_present cluster1 mirror image_14 snap 2026-02-18T08:37:38.470 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:37:38.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:37:38.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_14 2026-02-18T08:37:38.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap 2026-02-18T08:37:38.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-18T08:37:38.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-18T08:37:38.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1 2026-02-18T08:37:39.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster1 info mirror/image_14@snap 2026-02-18T08:37:39.500 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:rbd image 'image_14': 2026-02-18T08:37:39.500 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: size 128 MiB in 32 objects 2026-02-18T08:37:39.500 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: order 22 (4 MiB objects) 2026-02-18T08:37:39.500 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: snapshot_count: 1 2026-02-18T08:37:39.500 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: id: 10ed6402154a 2026-02-18T08:37:39.500 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: block_name_prefix: rbd_data.10ed6402154a 2026-02-18T08:37:39.500 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: format: 2 2026-02-18T08:37:39.501 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: features: layering, exclusive-lock, journaling 2026-02-18T08:37:39.501 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: op_features: 2026-02-18T08:37:39.501 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: flags: 2026-02-18T08:37:39.501 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: create_timestamp: Wed Feb 18 08:28:04 2026 2026-02-18T08:37:39.501 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: access_timestamp: Wed Feb 18 08:28:04 2026 2026-02-18T08:37:39.501 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: modify_timestamp: Wed Feb 18 08:29:05 2026 2026-02-18T08:37:39.501 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: protected: False 2026-02-18T08:37:39.501 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: journal: 10ed6402154a 2026-02-18T08:37:39.501 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring state: enabled 2026-02-18T08:37:39.501 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring mode: journal 2026-02-18T08:37:39.501 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring global id: b57b68ac-5e29-4f1a-84fb-abc650cfc14a 2026-02-18T08:37:39.501 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring primary: false 2026-02-18T08:37:39.503 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:37:39.503 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ compare_image_snaps mirror image_14 snap 2026-02-18T08:37:39.503 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:37:39.504 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_14 2026-02-18T08:37:39.504 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap 2026-02-18T08:37:39.504 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local ret=0 2026-02-18T08:37:39.504 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local rmt_export=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_14.export 2026-02-18T08:37:39.504 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loc_export=/tmp/tmp.vybh1m9yJF/cluster1-mirror-image_14.export 2026-02-18T08:37:39.504 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_14.export /tmp/tmp.vybh1m9yJF/cluster1-mirror-image_14.export 2026-02-18T08:37:39.504 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 -p mirror export image_14@snap /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_14.export 2026-02-18T08:37:39.636 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-18T08:37:39.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster1 -p mirror export image_14@snap /tmp/tmp.vybh1m9yJF/cluster1-mirror-image_14.export 2026-02-18T08:37:39.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-18T08:37:39.798 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmp /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_14.export /tmp/tmp.vybh1m9yJF/cluster1-mirror-image_14.export 2026-02-18T08:37:39.845 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_14.export /tmp/tmp.vybh1m9yJF/cluster1-mirror-image_14.export 2026-02-18T08:37:39.879 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:37:39.879 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T08:37:39.879 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_15 2026-02-18T08:37:39.880 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_snap cluster2 mirror image_15 snap 2026-02-18T08:37:39.880 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:37:39.880 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:37:39.880 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_15 2026-02-18T08:37:39.880 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap 2026-02-18T08:37:39.880 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 -p mirror snap create image_15@snap --debug-rbd=20 --debug-journaler=20 2026-02-18T08:37:43.850 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_replay_started cluster1 mirror image_15 2026-02-18T08:37:43.851 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:37:43.851 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:37:43.851 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_15 2026-02-18T08:37:43.851 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_replay_state cluster1 mirror image_15 started 2026-02-18T08:37:43.851 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:37:43.851 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:37:43.851 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_15 2026-02-18T08:37:43.851 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local state=started 2026-02-18T08:37:43.851 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-18T08:37:43.852 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:37:43.852 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1 2026-02-18T08:37:44.158 INFO:tasks.rbd_mirror_thrash:kill cluster2.client.mirror.0 2026-02-18T08:37:44.158 INFO:tasks.rbd_mirror.cluster2.client.mirror.0:Sent signal 15 2026-02-18T08:37:44.158 INFO:tasks.rbd_mirror_thrash:waiting for 8.306752126381829 secs before reviving daemons 2026-02-18T08:37:44.159 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.trial163.stderr:2026-02-18T08:37:44.150+0000 7f20dac85640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.0 (PID: 87006) UID: 1000 2026-02-18T08:37:44.852 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test_image_replay_state cluster1 mirror image_15 started 2026-02-18T08:37:44.853 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:37:44.853 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:37:44.853 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_15 2026-02-18T08:37:44.853 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local test_state=started 2026-02-18T08:37:44.853 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local status_result 2026-02-18T08:37:44.853 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_state=stopped 2026-02-18T08:37:44.853 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_15 2026-02-18T08:37:44.853 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -i state 2026-02-18T08:37:44.853 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_instance=cluster1 2026-02-18T08:37:44.853 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-18T08:37:44.854 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:37:44.854 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance=cluster1 2026-02-18T08:37:44.854 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local loop_instance 2026-02-18T08:37:44.854 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:37:44.854 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sleep 0 2026-02-18T08:37:44.856 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:37:44.856 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ seq 0 3 2026-02-18T08:37:44.856 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:37:44.857 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_15 2026-02-18T08:37:44.857 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:0 2026-02-18T08:37:44.857 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-18T08:37:44.857 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:37:44.857 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:37:44.857 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:37:44.857 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-18T08:37:44.857 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:37:44.857 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:37:44.857 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:37:44.857 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:37:44.858 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-18T08:37:44.858 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:37:44.858 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:37:44.858 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:37:44.858 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:37:44.858 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:0 2026-02-18T08:37:44.858 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-18T08:37:44.858 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-18T08:37:44.858 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:37:44.858 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:0 2026-02-18T08:37:44.858 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T08:37:44.858 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-18T08:37:44.859 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:37:44.859 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:37:44.859 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-18T08:37:44.859 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-18T08:37:44.859 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-18T08:37:44.859 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=0 2026-02-18T08:37:44.859 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=0 2026-02-18T08:37:44.859 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:37:44.867 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:37:44.868 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:37:44.868 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:37:44.868 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_15 2026-02-18T08:37:44.930 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_15'] 2026-02-18T08:37:44.931 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status : get status for rbd mirror 2026-02-18T08:37:44.931 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:37:44.933 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:37:44.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_15 2026-02-18T08:37:44.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:1 2026-02-18T08:37:44.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-18T08:37:44.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:37:44.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:37:44.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:37:44.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-18T08:37:44.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:37:44.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:37:44.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:37:44.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:37:44.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-18T08:37:44.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:37:44.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:37:44.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:37:44.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:37:44.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:1 2026-02-18T08:37:44.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-18T08:37:44.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-18T08:37:44.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:37:44.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:1 2026-02-18T08:37:44.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T08:37:44.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-18T08:37:44.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:37:44.936 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:37:44.936 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-18T08:37:44.936 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-18T08:37:44.936 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-18T08:37:44.936 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=1 2026-02-18T08:37:44.936 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=1 2026-02-18T08:37:44.936 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:37:44.944 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:37:44.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:37:44.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:37:44.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_15 2026-02-18T08:37:45.008 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_15'] 2026-02-18T08:37:45.008 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status : get status for rbd mirror 2026-02-18T08:37:45.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:37:45.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:37:45.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_15 2026-02-18T08:37:45.012 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:2 2026-02-18T08:37:45.012 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-18T08:37:45.012 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:37:45.012 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:37:45.012 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:37:45.012 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-18T08:37:45.012 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:37:45.012 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:37:45.012 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:37:45.012 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:37:45.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-18T08:37:45.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:37:45.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:37:45.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:37:45.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:37:45.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:2 2026-02-18T08:37:45.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-18T08:37:45.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-18T08:37:45.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:37:45.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:2 2026-02-18T08:37:45.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T08:37:45.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-18T08:37:45.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:37:45.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:37:45.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-18T08:37:45.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-18T08:37:45.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-18T08:37:45.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=2 2026-02-18T08:37:45.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=2 2026-02-18T08:37:45.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:37:45.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:37:45.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:37:45.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:37:45.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_15 2026-02-18T08:37:45.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_15'] 2026-02-18T08:37:45.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status : get status for rbd mirror 2026-02-18T08:37:45.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:37:45.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:37:45.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/image_15 2026-02-18T08:37:45.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:3 2026-02-18T08:37:45.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-18T08:37:45.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:37:45.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:37:45.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:37:45.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-18T08:37:45.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:37:45.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:37:45.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:37:45.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:37:45.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-18T08:37:45.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:37:45.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:37:45.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:37:45.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:37:45.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:3 2026-02-18T08:37:45.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-18T08:37:45.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-18T08:37:45.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:37:45.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:3 2026-02-18T08:37:45.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T08:37:45.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-18T08:37:45.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:37:45.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:37:45.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-18T08:37:45.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-18T08:37:45.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-18T08:37:45.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=3 2026-02-18T08:37:45.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=3 2026-02-18T08:37:45.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:37:45.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:37:45.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:37:45.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:37:45.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/image_15 2026-02-18T08:37:45.166 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ return 0 2026-02-18T08:37:45.167 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ status_result=' "state": "Replaying"' 2026-02-18T08:37:45.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ echo ' "state": "Replaying"' 2026-02-18T08:37:45.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ grep -i Replaying 2026-02-18T08:37:45.169 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: "state": "Replaying" 2026-02-18T08:37:45.170 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ current_state=started 2026-02-18T08:37:45.170 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test started = started 2026-02-18T08:37:45.170 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:37:45.170 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror image_15 2026-02-18T08:37:45.170 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_cluster=cluster1 2026-02-18T08:37:45.171 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:37:45.171 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_pool=mirror 2026-02-18T08:37:45.171 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local remote_pool=mirror 2026-02-18T08:37:45.171 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_15 2026-02-18T08:37:45.171 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' journal = journal ']' 2026-02-18T08:37:45.171 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror mirror image_15 2026-02-18T08:37:45.171 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_cluster=cluster1 2026-02-18T08:37:45.171 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:37:45.171 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_pool=mirror 2026-02-18T08:37:45.171 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local remote_pool=mirror 2026-02-18T08:37:45.171 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_15 2026-02-18T08:37:45.171 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-02-18T08:37:45.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-02-18T08:37:45.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:37:45.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:37:45.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:37:45.371 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_15 2026-02-18T08:37:45.372 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:37:45.372 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:37:45.372 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_15 2026-02-18T08:37:45.372 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:37:45.372 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_15 ']' 2026-02-18T08:37:45.372 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_15' 2026-02-18T08:37:45.372 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_15 2026-02-18T08:37:45.372 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:37:45.373 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:37:45.373 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:37:45.373 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:37:45.373 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:37:45.373 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:37:45.373 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:37:45.373 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:37:45.373 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:37:45.376 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:37:45.376 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_15 2026-02-18T08:37:45.376 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:37:45.376 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:37:45.376 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:37:45.376 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:37:45.376 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:37:45.376 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:37:45.377 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:37:45.377 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:37:45.377 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:37:45.377 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:37:45.377 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:37:45.377 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:37:45.377 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:37:45.377 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:37:45.377 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:37:45.377 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:37:45.377 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:37:45.377 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:37:45.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:37:45.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:37:45.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:37:45.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:37:45.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:37:45.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:37:45.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:37:45.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:37:45.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:37:45.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:37:45.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:37:45.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:37:45.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:37:45.387 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:37:45.387 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:37:45.387 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_15 2026-02-18T08:37:45.448 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_15'] 2026-02-18T08:37:45.448 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:37:45.448 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:37:45.451 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:37:45.451 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_15 2026-02-18T08:37:45.451 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:37:45.452 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:37:45.452 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:37:45.452 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:37:45.452 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:37:45.452 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:37:45.452 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:37:45.452 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:37:45.452 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:37:45.452 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:37:45.452 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:37:45.452 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:37:45.453 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:37:45.453 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:37:45.453 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:37:45.453 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:37:45.453 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:37:45.453 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:37:45.453 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:37:45.453 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:37:45.453 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:37:45.453 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:37:45.453 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:37:45.453 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:37:45.453 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:37:45.454 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:37:45.454 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:37:45.454 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:37:45.454 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:37:45.454 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:37:45.463 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:37:45.463 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:37:45.463 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:37:45.463 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_15 2026-02-18T08:37:45.523 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_15'] 2026-02-18T08:37:45.524 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:37:45.524 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:37:45.526 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:37:45.527 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_15 2026-02-18T08:37:45.527 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:37:45.527 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:37:45.527 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:37:45.527 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:37:45.527 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:37:45.527 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:37:45.527 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:37:45.527 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:37:45.527 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:37:45.527 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:37:45.528 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:37:45.528 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:37:45.528 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:37:45.528 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:37:45.528 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:37:45.528 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:37:45.528 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:37:45.528 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:37:45.528 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:37:45.528 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:37:45.528 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:37:45.528 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:37:45.528 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:37:45.529 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:37:45.529 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:37:45.529 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:37:45.529 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:37:45.529 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:37:45.529 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:37:45.529 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:37:45.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:37:45.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:37:45.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:37:45.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_15 2026-02-18T08:37:45.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_15'] 2026-02-18T08:37:45.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:37:45.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:37:45.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:37:45.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/image_15 2026-02-18T08:37:45.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:37:45.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:37:45.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:37:45.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:37:45.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:37:45.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:37:45.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:37:45.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:37:45.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:37:45.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:37:45.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:37:45.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:37:45.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:37:45.594 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:37:45.594 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:37:45.594 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:37:45.594 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:37:45.594 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:37:45.594 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:37:45.594 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:37:45.594 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:37:45.594 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:37:45.594 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:37:45.594 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:37:45.594 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:37:45.594 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:37:45.595 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:37:45.595 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:37:45.595 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:37:45.595 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:37:45.603 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:37:45.603 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:37:45.603 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:37:45.604 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/image_15 2026-02-18T08:37:45.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:37:45.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_15 2026-02-18T08:37:45.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:37:45.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:37:45.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_15 2026-02-18T08:37:45.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_15 '' 2026-02-18T08:37:45.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:37:45.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:37:45.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_15 2026-02-18T08:37:45.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:37:45.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_15.status 2026-02-18T08:37:45.667 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_15.status 2026-02-18T08:37:45.667 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:37:45.667 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_15.status 2026-02-18T08:37:45.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_15 2026-02-18T08:37:45.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_15.status 2026-02-18T08:37:45.695 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 10 2026-02-18T08:37:45.695 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 10 2026-02-18T08:37:45.695 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:37:45.695 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=41, tag_tid=2, entry_tid=1], [object_number=40, tag_tid=2, entry_tid=0], [object_number=43, tag_tid=1, entry_tid=45019], [object_number=42, tag_tid=1, entry_tid=45018]]], state=connected] 2026-02-18T08:37:45.695 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=41, tag_tid=2, entry_tid=1], [object_number=40, tag_tid=2, entry_tid=0], [object_number=43, tag_tid=1, entry_tid=45019], [object_number=42, tag_tid=1, entry_tid=45018]]], state=connected] 2026-02-18T08:37:45.697 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_15.status 2026-02-18T08:37:45.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=41, tag_tid=2, entry_tid=1' 2026-02-18T08:37:45.700 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_15 2026-02-18T08:37:45.700 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:37:45.700 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:37:45.701 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_15 2026-02-18T08:37:45.701 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_15 '..*' 2026-02-18T08:37:45.701 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:37:45.701 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:37:45.701 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_15 2026-02-18T08:37:45.701 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:37:45.701 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_15.status 2026-02-18T08:37:45.701 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_15.status 2026-02-18T08:37:45.702 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:37:45.702 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_15.status 2026-02-18T08:37:45.702 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_15 2026-02-18T08:37:45.703 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_15.status 2026-02-18T08:37:45.735 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 10 2026-02-18T08:37:45.735 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 10 2026-02-18T08:37:45.735 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:37:45.736 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=41, tag_tid=2, entry_tid=1], [object_number=40, tag_tid=2, entry_tid=0], [object_number=43, tag_tid=1, entry_tid=45019], [object_number=42, tag_tid=1, entry_tid=45018]]], state=connected] 2026-02-18T08:37:45.736 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=41, tag_tid=2, entry_tid=1], [object_number=40, tag_tid=2, entry_tid=0], [object_number=43, tag_tid=1, entry_tid=45019], [object_number=42, tag_tid=1, entry_tid=45018]]], state=connected] 2026-02-18T08:37:45.739 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_15.status 2026-02-18T08:37:45.741 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=41, tag_tid=2, entry_tid=1' 2026-02-18T08:37:45.742 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=41, tag_tid=2, entry_tid=1' -a 'object_number=41, tag_tid=2, entry_tid=1' = 'object_number=41, tag_tid=2, entry_tid=1' 2026-02-18T08:37:45.742 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:37:45.742 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_snap_present cluster1 mirror image_15 snap 2026-02-18T08:37:45.742 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:37:45.742 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:37:45.742 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_15 2026-02-18T08:37:45.742 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap 2026-02-18T08:37:45.742 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-18T08:37:45.742 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-18T08:37:45.742 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1 2026-02-18T08:37:46.742 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster1 info mirror/image_15@snap 2026-02-18T08:37:46.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:rbd image 'image_15': 2026-02-18T08:37:46.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: size 128 MiB in 32 objects 2026-02-18T08:37:46.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: order 22 (4 MiB objects) 2026-02-18T08:37:46.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: snapshot_count: 1 2026-02-18T08:37:46.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: id: 11043fa5fdc5 2026-02-18T08:37:46.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: block_name_prefix: rbd_data.11043fa5fdc5 2026-02-18T08:37:46.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: format: 2 2026-02-18T08:37:46.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: features: layering, exclusive-lock, journaling 2026-02-18T08:37:46.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: op_features: 2026-02-18T08:37:46.776 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: flags: 2026-02-18T08:37:46.776 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: create_timestamp: Wed Feb 18 08:28:12 2026 2026-02-18T08:37:46.776 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: access_timestamp: Wed Feb 18 08:28:12 2026 2026-02-18T08:37:46.776 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: modify_timestamp: Wed Feb 18 08:29:13 2026 2026-02-18T08:37:46.776 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: protected: False 2026-02-18T08:37:46.776 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: journal: 11043fa5fdc5 2026-02-18T08:37:46.776 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring state: enabled 2026-02-18T08:37:46.776 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring mode: journal 2026-02-18T08:37:46.776 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring global id: e0b53d35-5ded-4325-943c-880b42f8ddf0 2026-02-18T08:37:46.776 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring primary: false 2026-02-18T08:37:46.778 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:37:46.778 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ compare_image_snaps mirror image_15 snap 2026-02-18T08:37:46.778 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:37:46.778 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_15 2026-02-18T08:37:46.779 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap 2026-02-18T08:37:46.779 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local ret=0 2026-02-18T08:37:46.779 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local rmt_export=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_15.export 2026-02-18T08:37:46.779 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loc_export=/tmp/tmp.vybh1m9yJF/cluster1-mirror-image_15.export 2026-02-18T08:37:46.779 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_15.export /tmp/tmp.vybh1m9yJF/cluster1-mirror-image_15.export 2026-02-18T08:37:46.779 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 -p mirror export image_15@snap /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_15.export 2026-02-18T08:37:46.922 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-18T08:37:46.930 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster1 -p mirror export image_15@snap /tmp/tmp.vybh1m9yJF/cluster1-mirror-image_15.export 2026-02-18T08:37:47.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-18T08:37:47.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmp /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_15.export /tmp/tmp.vybh1m9yJF/cluster1-mirror-image_15.export 2026-02-18T08:37:47.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_15.export /tmp/tmp.vybh1m9yJF/cluster1-mirror-image_15.export 2026-02-18T08:37:47.208 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:37:47.208 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T08:37:47.208 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_16 2026-02-18T08:37:47.208 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_snap cluster2 mirror image_16 snap 2026-02-18T08:37:47.208 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:37:47.209 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:37:47.209 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_16 2026-02-18T08:37:47.209 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap 2026-02-18T08:37:47.209 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 -p mirror snap create image_16@snap --debug-rbd=20 --debug-journaler=20 2026-02-18T08:37:49.855 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_replay_started cluster1 mirror image_16 2026-02-18T08:37:49.856 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:37:49.856 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:37:49.856 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_16 2026-02-18T08:37:49.856 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_replay_state cluster1 mirror image_16 started 2026-02-18T08:37:49.856 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:37:49.856 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:37:49.856 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_16 2026-02-18T08:37:49.856 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local state=started 2026-02-18T08:37:49.857 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-18T08:37:49.857 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:37:49.857 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1 2026-02-18T08:37:50.857 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test_image_replay_state cluster1 mirror image_16 started 2026-02-18T08:37:50.857 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:37:50.857 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:37:50.857 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_16 2026-02-18T08:37:50.857 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local test_state=started 2026-02-18T08:37:50.858 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local status_result 2026-02-18T08:37:50.858 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_state=stopped 2026-02-18T08:37:50.859 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_16 2026-02-18T08:37:50.859 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -i state 2026-02-18T08:37:50.859 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_instance=cluster1 2026-02-18T08:37:50.859 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-18T08:37:50.859 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:37:50.859 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance=cluster1 2026-02-18T08:37:50.859 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local loop_instance 2026-02-18T08:37:50.859 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:37:50.860 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sleep 0 2026-02-18T08:37:50.860 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:37:50.860 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ seq 0 3 2026-02-18T08:37:50.862 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:37:50.862 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_16 2026-02-18T08:37:50.862 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:0 2026-02-18T08:37:50.863 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-18T08:37:50.863 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:37:50.863 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:37:50.863 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:37:50.863 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-18T08:37:50.863 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:37:50.863 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:37:50.863 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:37:50.863 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:37:50.863 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-18T08:37:50.863 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:37:50.864 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:37:50.864 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:37:50.864 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:37:50.864 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:0 2026-02-18T08:37:50.864 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-18T08:37:50.864 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-18T08:37:50.864 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:37:50.864 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:0 2026-02-18T08:37:50.864 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T08:37:50.864 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-18T08:37:50.864 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:37:50.864 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:37:50.864 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-18T08:37:50.865 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-18T08:37:50.865 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-18T08:37:50.865 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=0 2026-02-18T08:37:50.865 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=0 2026-02-18T08:37:50.865 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:37:50.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:37:50.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:37:50.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:37:50.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_16 2026-02-18T08:37:50.936 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_16'] 2026-02-18T08:37:50.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status : get status for rbd mirror 2026-02-18T08:37:50.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:37:50.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:37:50.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_16 2026-02-18T08:37:50.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:1 2026-02-18T08:37:50.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-18T08:37:50.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:37:50.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:37:50.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:37:50.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-18T08:37:50.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:37:50.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:37:50.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:37:50.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:37:50.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-18T08:37:50.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:37:50.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:37:50.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:37:50.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:37:50.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:1 2026-02-18T08:37:50.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-18T08:37:50.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-18T08:37:50.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:37:50.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:1 2026-02-18T08:37:50.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T08:37:50.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-18T08:37:50.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:37:50.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:37:50.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-18T08:37:50.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-18T08:37:50.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-18T08:37:50.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=1 2026-02-18T08:37:50.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=1 2026-02-18T08:37:50.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:37:50.950 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:37:50.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:37:50.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:37:50.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_16 2026-02-18T08:37:51.010 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ return 0 2026-02-18T08:37:51.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ status_result=' "state": "Replaying"' 2026-02-18T08:37:51.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ echo ' "state": "Replaying"' 2026-02-18T08:37:51.012 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ grep -i Replaying 2026-02-18T08:37:51.012 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: "state": "Replaying" 2026-02-18T08:37:51.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ current_state=started 2026-02-18T08:37:51.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test started = started 2026-02-18T08:37:51.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:37:51.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror image_16 2026-02-18T08:37:51.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_cluster=cluster1 2026-02-18T08:37:51.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:37:51.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_pool=mirror 2026-02-18T08:37:51.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local remote_pool=mirror 2026-02-18T08:37:51.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_16 2026-02-18T08:37:51.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' journal = journal ']' 2026-02-18T08:37:51.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror mirror image_16 2026-02-18T08:37:51.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_cluster=cluster1 2026-02-18T08:37:51.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:37:51.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_pool=mirror 2026-02-18T08:37:51.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local remote_pool=mirror 2026-02-18T08:37:51.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_16 2026-02-18T08:37:51.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-02-18T08:37:51.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-02-18T08:37:51.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:37:51.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:37:51.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:37:51.215 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_16 2026-02-18T08:37:51.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:37:51.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:37:51.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_16 2026-02-18T08:37:51.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:37:51.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_16 ']' 2026-02-18T08:37:51.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_16' 2026-02-18T08:37:51.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_16 2026-02-18T08:37:51.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:37:51.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:37:51.217 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:37:51.217 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:37:51.217 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:37:51.217 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:37:51.217 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:37:51.217 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:37:51.217 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:37:51.220 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:37:51.220 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_16 2026-02-18T08:37:51.220 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:37:51.220 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:37:51.220 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:37:51.220 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:37:51.220 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:37:51.220 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:37:51.220 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:37:51.220 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:37:51.221 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:37:51.221 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:37:51.221 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:37:51.221 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:37:51.221 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:37:51.221 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:37:51.221 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:37:51.221 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:37:51.221 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:37:51.221 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:37:51.221 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:37:51.221 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:37:51.221 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:37:51.222 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:37:51.222 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:37:51.222 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:37:51.222 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:37:51.222 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:37:51.222 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:37:51.222 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:37:51.222 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:37:51.222 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:37:51.230 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:37:51.230 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:37:51.231 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:37:51.231 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_16 2026-02-18T08:37:51.290 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_16'] 2026-02-18T08:37:51.290 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:37:51.291 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:37:51.293 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:37:51.293 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_16 2026-02-18T08:37:51.293 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:37:51.293 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:37:51.294 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:37:51.294 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:37:51.294 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:37:51.294 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:37:51.294 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:37:51.294 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:37:51.294 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:37:51.294 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:37:51.294 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:37:51.294 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:37:51.294 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:37:51.294 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:37:51.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:37:51.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:37:51.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:37:51.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:37:51.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:37:51.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:37:51.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:37:51.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:37:51.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:37:51.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:37:51.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:37:51.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:37:51.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:37:51.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:37:51.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:37:51.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:37:51.304 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:37:51.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:37:51.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:37:51.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_16 2026-02-18T08:37:51.368 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:37:51.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_16 2026-02-18T08:37:51.370 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:37:51.370 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:37:51.370 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_16 2026-02-18T08:37:51.370 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_16 '' 2026-02-18T08:37:51.370 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:37:51.370 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:37:51.370 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_16 2026-02-18T08:37:51.370 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:37:51.370 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_16.status 2026-02-18T08:37:51.370 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_16.status 2026-02-18T08:37:51.371 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:37:51.371 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_16.status 2026-02-18T08:37:51.372 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_16 2026-02-18T08:37:51.372 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_16.status 2026-02-18T08:37:51.408 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 10 2026-02-18T08:37:51.408 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 10 2026-02-18T08:37:51.408 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:37:51.408 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=41, tag_tid=2, entry_tid=1], [object_number=40, tag_tid=2, entry_tid=0], [object_number=43, tag_tid=1, entry_tid=45507], [object_number=42, tag_tid=1, entry_tid=45506]]], state=connected] 2026-02-18T08:37:51.408 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=41, tag_tid=1, entry_tid=45509], [object_number=40, tag_tid=1, entry_tid=45508], [object_number=43, tag_tid=1, entry_tid=45507], [object_number=42, tag_tid=1, entry_tid=45506]]], state=connected] 2026-02-18T08:37:51.412 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_16.status 2026-02-18T08:37:51.413 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=41, tag_tid=2, entry_tid=1' 2026-02-18T08:37:51.414 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_16 2026-02-18T08:37:51.414 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:37:51.415 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:37:51.415 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_16 2026-02-18T08:37:51.415 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_16 '..*' 2026-02-18T08:37:51.415 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:37:51.415 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:37:51.415 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_16 2026-02-18T08:37:51.415 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:37:51.415 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_16.status 2026-02-18T08:37:51.416 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_16.status 2026-02-18T08:37:51.416 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:37:51.417 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_16.status 2026-02-18T08:37:51.417 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_16 2026-02-18T08:37:51.417 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_16.status 2026-02-18T08:37:51.452 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 10 2026-02-18T08:37:51.452 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 10 2026-02-18T08:37:51.452 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:37:51.452 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=41, tag_tid=2, entry_tid=1], [object_number=40, tag_tid=2, entry_tid=0], [object_number=43, tag_tid=1, entry_tid=45507], [object_number=42, tag_tid=1, entry_tid=45506]]], state=connected] 2026-02-18T08:37:51.452 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=41, tag_tid=1, entry_tid=45509], [object_number=40, tag_tid=1, entry_tid=45508], [object_number=43, tag_tid=1, entry_tid=45507], [object_number=42, tag_tid=1, entry_tid=45506]]], state=connected] 2026-02-18T08:37:51.455 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_16.status 2026-02-18T08:37:51.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=41, tag_tid=1, entry_tid=45509' 2026-02-18T08:37:51.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=41, tag_tid=2, entry_tid=1' -a 'object_number=41, tag_tid=2, entry_tid=1' = 'object_number=41, tag_tid=1, entry_tid=45509' 2026-02-18T08:37:51.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=41, tag_tid=1, entry_tid=45509' '!=' '' 2026-02-18T08:37:51.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:37:51.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=41, tag_tid=1, entry_tid=45509' = '' 2026-02-18T08:37:51.458 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=41, tag_tid=1, entry_tid=45509' 2026-02-18T08:37:51.458 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=41, tag_tid=2, entry_tid=1' 2026-02-18T08:37:51.458 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:37:51.458 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:37:51.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=2 2026-02-18T08:37:51.460 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=41, tag_tid=1, entry_tid=45509' 2026-02-18T08:37:51.460 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:37:51.460 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:37:51.462 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=1 2026-02-18T08:37:51.462 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=41, tag_tid=2, entry_tid=1' 2026-02-18T08:37:51.463 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:37:51.463 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:37:51.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:37:51.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=41, tag_tid=1, entry_tid=45509' 2026-02-18T08:37:51.465 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:37:51.465 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:37:51.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=45509 2026-02-18T08:37:51.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 2 = 1 -a 1 -le 45509 2026-02-18T08:37:51.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:37:51.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:37:51.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:37:51.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_16 2026-02-18T08:37:51.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:37:51.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:37:51.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_16 2026-02-18T08:37:51.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:37:51.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_16 ']' 2026-02-18T08:37:51.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_16' 2026-02-18T08:37:51.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_16 2026-02-18T08:37:51.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:37:51.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:37:51.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:37:51.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:37:51.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:37:51.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:37:51.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:37:51.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:37:51.672 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:37:51.672 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:37:51.672 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_16 2026-02-18T08:37:51.672 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:37:51.673 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:37:51.673 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:37:51.673 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:37:51.673 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:37:51.673 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:37:51.673 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:37:51.673 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:37:51.673 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:37:51.673 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:37:51.673 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:37:51.673 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:37:51.673 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:37:51.674 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:37:51.674 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:37:51.674 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:37:51.674 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:37:51.674 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:37:51.674 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:37:51.674 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:37:51.674 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:37:51.674 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:37:51.674 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:37:51.674 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:37:51.674 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:37:51.675 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:37:51.675 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:37:51.675 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:37:51.675 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:37:51.675 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:37:51.681 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:37:51.682 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:37:51.682 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:37:51.682 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_16 2026-02-18T08:37:51.735 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_16'] 2026-02-18T08:37:51.735 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:37:51.736 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:37:51.738 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:37:51.739 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_16 2026-02-18T08:37:51.739 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:37:51.739 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:37:51.739 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:37:51.739 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:37:51.739 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:37:51.739 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:37:51.739 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:37:51.739 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:37:51.739 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:37:51.739 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:37:51.740 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:37:51.740 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:37:51.740 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:37:51.740 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:37:51.740 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:37:51.740 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:37:51.740 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:37:51.740 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:37:51.740 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:37:51.740 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:37:51.740 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:37:51.740 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:37:51.741 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:37:51.741 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:37:51.741 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:37:51.741 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:37:51.741 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:37:51.741 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:37:51.741 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:37:51.741 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:37:51.750 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:37:51.751 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:37:51.751 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:37:51.751 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_16 2026-02-18T08:37:51.819 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:37:51.820 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_16 2026-02-18T08:37:51.821 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:37:51.821 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:37:51.821 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_16 2026-02-18T08:37:51.821 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_16 '' 2026-02-18T08:37:51.821 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:37:51.821 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:37:51.821 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_16 2026-02-18T08:37:51.821 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:37:51.821 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_16.status 2026-02-18T08:37:51.821 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_16.status 2026-02-18T08:37:51.821 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:37:51.822 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_16.status 2026-02-18T08:37:51.822 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_16.status 2026-02-18T08:37:51.823 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_16 2026-02-18T08:37:51.851 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 10 2026-02-18T08:37:51.851 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 10 2026-02-18T08:37:51.851 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:37:51.851 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=41, tag_tid=2, entry_tid=1], [object_number=40, tag_tid=2, entry_tid=0], [object_number=43, tag_tid=1, entry_tid=45507], [object_number=42, tag_tid=1, entry_tid=45506]]], state=connected] 2026-02-18T08:37:51.851 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=41, tag_tid=1, entry_tid=45509], [object_number=40, tag_tid=1, entry_tid=45508], [object_number=43, tag_tid=1, entry_tid=45507], [object_number=42, tag_tid=1, entry_tid=45506]]], state=connected] 2026-02-18T08:37:51.854 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_16.status 2026-02-18T08:37:51.855 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=41, tag_tid=2, entry_tid=1' 2026-02-18T08:37:51.856 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_16 2026-02-18T08:37:51.856 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:37:51.856 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:37:51.857 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_16 2026-02-18T08:37:51.857 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_16 '..*' 2026-02-18T08:37:51.857 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:37:51.857 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:37:51.857 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_16 2026-02-18T08:37:51.857 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:37:51.857 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_16.status 2026-02-18T08:37:51.858 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_16.status 2026-02-18T08:37:51.858 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:37:51.858 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_16.status 2026-02-18T08:37:51.859 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_16 2026-02-18T08:37:51.859 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_16.status 2026-02-18T08:37:51.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 10 2026-02-18T08:37:51.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 10 2026-02-18T08:37:51.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:37:51.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=41, tag_tid=2, entry_tid=1], [object_number=40, tag_tid=2, entry_tid=0], [object_number=43, tag_tid=1, entry_tid=45507], [object_number=42, tag_tid=1, entry_tid=45506]]], state=connected] 2026-02-18T08:37:51.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=41, tag_tid=1, entry_tid=45509], [object_number=40, tag_tid=1, entry_tid=45508], [object_number=43, tag_tid=1, entry_tid=45507], [object_number=42, tag_tid=1, entry_tid=45506]]], state=connected] 2026-02-18T08:37:51.896 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_16.status 2026-02-18T08:37:51.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=41, tag_tid=1, entry_tid=45509' 2026-02-18T08:37:51.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=41, tag_tid=2, entry_tid=1' -a 'object_number=41, tag_tid=2, entry_tid=1' = 'object_number=41, tag_tid=1, entry_tid=45509' 2026-02-18T08:37:51.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=41, tag_tid=1, entry_tid=45509' '!=' 'object_number=41, tag_tid=1, entry_tid=45509' 2026-02-18T08:37:51.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:37:51.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.4 2026-02-18T08:37:52.300 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_16 2026-02-18T08:37:52.300 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:37:52.300 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:37:52.300 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_16 2026-02-18T08:37:52.300 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:37:52.301 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_16 ']' 2026-02-18T08:37:52.301 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_16' 2026-02-18T08:37:52.301 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_16 2026-02-18T08:37:52.301 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:37:52.301 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:37:52.301 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:37:52.301 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:37:52.301 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:37:52.301 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:37:52.301 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:37:52.301 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:37:52.304 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:37:52.304 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:37:52.304 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_16 2026-02-18T08:37:52.304 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:37:52.304 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:37:52.304 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:37:52.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:37:52.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:37:52.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:37:52.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:37:52.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:37:52.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:37:52.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:37:52.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:37:52.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:37:52.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:37:52.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:37:52.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:37:52.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:37:52.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:37:52.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:37:52.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:37:52.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:37:52.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:37:52.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:37:52.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:37:52.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:37:52.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:37:52.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:37:52.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:37:52.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:37:52.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:37:52.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:37:52.315 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:37:52.315 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:37:52.315 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:37:52.316 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_16 2026-02-18T08:37:52.375 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_16'] 2026-02-18T08:37:52.375 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:37:52.375 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:37:52.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:37:52.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_16 2026-02-18T08:37:52.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:37:52.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:37:52.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:37:52.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:37:52.379 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:37:52.379 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:37:52.379 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:37:52.379 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:37:52.379 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:37:52.379 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:37:52.379 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:37:52.379 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:37:52.379 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:37:52.379 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:37:52.379 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:37:52.379 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:37:52.380 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:37:52.380 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:37:52.380 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:37:52.380 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:37:52.380 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:37:52.380 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:37:52.380 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:37:52.380 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:37:52.380 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:37:52.380 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:37:52.380 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:37:52.381 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:37:52.381 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:37:52.381 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:37:52.388 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:37:52.388 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:37:52.389 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:37:52.389 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_16 2026-02-18T08:37:52.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:37:52.446 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_16 2026-02-18T08:37:52.446 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:37:52.446 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:37:52.446 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_16 2026-02-18T08:37:52.446 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_16 '' 2026-02-18T08:37:52.446 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:37:52.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:37:52.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_16 2026-02-18T08:37:52.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:37:52.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_16.status 2026-02-18T08:37:52.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_16.status 2026-02-18T08:37:52.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:37:52.448 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_16.status 2026-02-18T08:37:52.448 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_16 2026-02-18T08:37:52.448 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_16.status 2026-02-18T08:37:52.465 INFO:tasks.rbd_mirror_thrash:waiting for cluster2.client.mirror.0 2026-02-18T08:37:52.466 INFO:teuthology.orchestra.run:waiting for 600 2026-02-18T08:37:52.466 INFO:tasks.rbd_mirror_thrash:reviving cluster2.client.mirror.0 2026-02-18T08:37:52.466 INFO:tasks.rbd_mirror.cluster2.client.mirror.0:Restarting daemon 2026-02-18T08:37:52.466 DEBUG:teuthology.orchestra.run.trial163:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.0 2026-02-18T08:37:52.468 INFO:tasks.rbd_mirror.cluster2.client.mirror.0:Started 2026-02-18T08:37:52.468 INFO:tasks.rbd_mirror_thrash:waiting for 105.08002879545391 secs before thrashing 2026-02-18T08:37:52.475 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 10 2026-02-18T08:37:52.476 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 10 2026-02-18T08:37:52.476 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:37:52.476 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=41, tag_tid=2, entry_tid=1], [object_number=40, tag_tid=2, entry_tid=0], [object_number=43, tag_tid=1, entry_tid=45507], [object_number=42, tag_tid=1, entry_tid=45506]]], state=connected] 2026-02-18T08:37:52.476 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=41, tag_tid=1, entry_tid=45509], [object_number=40, tag_tid=1, entry_tid=45508], [object_number=43, tag_tid=1, entry_tid=45507], [object_number=42, tag_tid=1, entry_tid=45506]]], state=connected] 2026-02-18T08:37:52.478 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_16.status 2026-02-18T08:37:52.479 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=41, tag_tid=2, entry_tid=1' 2026-02-18T08:37:52.480 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_16 2026-02-18T08:37:52.480 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:37:52.481 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:37:52.481 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_16 2026-02-18T08:37:52.481 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_16 '..*' 2026-02-18T08:37:52.481 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:37:52.481 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:37:52.481 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_16 2026-02-18T08:37:52.481 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:37:52.481 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_16.status 2026-02-18T08:37:52.481 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_16.status 2026-02-18T08:37:52.482 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:37:52.482 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_16.status 2026-02-18T08:37:52.482 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_16 2026-02-18T08:37:52.482 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_16.status 2026-02-18T08:37:52.508 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 10 2026-02-18T08:37:52.509 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 10 2026-02-18T08:37:52.509 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:37:52.509 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=41, tag_tid=2, entry_tid=1], [object_number=40, tag_tid=2, entry_tid=0], [object_number=43, tag_tid=1, entry_tid=45507], [object_number=42, tag_tid=1, entry_tid=45506]]], state=connected] 2026-02-18T08:37:52.509 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=41, tag_tid=1, entry_tid=45509], [object_number=40, tag_tid=1, entry_tid=45508], [object_number=43, tag_tid=1, entry_tid=45507], [object_number=42, tag_tid=1, entry_tid=45506]]], state=connected] 2026-02-18T08:37:52.511 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_16.status 2026-02-18T08:37:52.513 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=41, tag_tid=1, entry_tid=45509' 2026-02-18T08:37:52.513 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=41, tag_tid=2, entry_tid=1' -a 'object_number=41, tag_tid=2, entry_tid=1' = 'object_number=41, tag_tid=1, entry_tid=45509' 2026-02-18T08:37:52.513 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=41, tag_tid=1, entry_tid=45509' '!=' 'object_number=41, tag_tid=1, entry_tid=45509' 2026-02-18T08:37:52.513 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:37:52.513 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.8 2026-02-18T08:37:53.315 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_16 2026-02-18T08:37:53.315 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:37:53.315 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:37:53.315 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_16 2026-02-18T08:37:53.315 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:37:53.315 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_16 ']' 2026-02-18T08:37:53.316 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_16' 2026-02-18T08:37:53.316 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_16 2026-02-18T08:37:53.316 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:37:53.316 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:37:53.316 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:37:53.316 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:37:53.316 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:37:53.316 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:37:53.316 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:37:53.319 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:37:53.319 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:37:53.319 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:37:53.319 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_16 2026-02-18T08:37:53.320 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:37:53.320 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:37:53.320 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:37:53.320 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:37:53.320 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:37:53.320 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:37:53.320 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:37:53.320 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:37:53.320 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:37:53.320 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:37:53.320 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:37:53.321 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:37:53.321 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:37:53.321 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:37:53.321 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:37:53.321 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:37:53.321 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:37:53.321 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:37:53.321 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:37:53.321 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:37:53.321 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:37:53.321 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:37:53.321 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:37:53.321 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:37:53.322 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:37:53.322 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:37:53.322 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:37:53.322 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:37:53.322 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:37:53.322 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:37:53.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:37:53.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:37:53.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:37:53.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_16 2026-02-18T08:37:53.395 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_16'] 2026-02-18T08:37:53.395 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:37:53.396 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:37:53.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:37:53.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_16 2026-02-18T08:37:53.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:37:53.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:37:53.399 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:37:53.399 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:37:53.399 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:37:53.399 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:37:53.399 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:37:53.399 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:37:53.399 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:37:53.399 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:37:53.399 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:37:53.399 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:37:53.399 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:37:53.399 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:37:53.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:37:53.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:37:53.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:37:53.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:37:53.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:37:53.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:37:53.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:37:53.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:37:53.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:37:53.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:37:53.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:37:53.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:37:53.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:37:53.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:37:53.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:37:53.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:37:53.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:37:53.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:37:53.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:37:53.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_16 2026-02-18T08:37:53.474 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:37:53.475 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_16 2026-02-18T08:37:53.475 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:37:53.475 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:37:53.475 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_16 2026-02-18T08:37:53.475 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_16 '' 2026-02-18T08:37:53.475 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:37:53.476 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:37:53.476 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_16 2026-02-18T08:37:53.476 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:37:53.476 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_16.status 2026-02-18T08:37:53.476 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_16.status 2026-02-18T08:37:53.476 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:37:53.477 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_16.status 2026-02-18T08:37:53.477 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_16 2026-02-18T08:37:53.477 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_16.status 2026-02-18T08:37:53.506 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 10 2026-02-18T08:37:53.506 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 10 2026-02-18T08:37:53.506 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:37:53.507 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=41, tag_tid=2, entry_tid=1], [object_number=40, tag_tid=2, entry_tid=0], [object_number=43, tag_tid=1, entry_tid=45507], [object_number=42, tag_tid=1, entry_tid=45506]]], state=connected] 2026-02-18T08:37:53.507 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=41, tag_tid=1, entry_tid=45509], [object_number=40, tag_tid=1, entry_tid=45508], [object_number=43, tag_tid=1, entry_tid=45507], [object_number=42, tag_tid=1, entry_tid=45506]]], state=connected] 2026-02-18T08:37:53.509 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_16.status 2026-02-18T08:37:53.510 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=41, tag_tid=2, entry_tid=1' 2026-02-18T08:37:53.511 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_16 2026-02-18T08:37:53.512 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:37:53.512 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:37:53.512 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_16 2026-02-18T08:37:53.512 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_16 '..*' 2026-02-18T08:37:53.512 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:37:53.512 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:37:53.512 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_16 2026-02-18T08:37:53.512 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:37:53.512 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_16.status 2026-02-18T08:37:53.513 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_16.status 2026-02-18T08:37:53.513 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:37:53.513 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_16.status 2026-02-18T08:37:53.514 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_16 2026-02-18T08:37:53.514 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_16.status 2026-02-18T08:37:53.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 10 2026-02-18T08:37:53.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 10 2026-02-18T08:37:53.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:37:53.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=41, tag_tid=2, entry_tid=1], [object_number=40, tag_tid=2, entry_tid=0], [object_number=43, tag_tid=1, entry_tid=45507], [object_number=42, tag_tid=1, entry_tid=45506]]], state=connected] 2026-02-18T08:37:53.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=41, tag_tid=1, entry_tid=45509], [object_number=40, tag_tid=1, entry_tid=45508], [object_number=43, tag_tid=1, entry_tid=45507], [object_number=42, tag_tid=1, entry_tid=45506]]], state=connected] 2026-02-18T08:37:53.551 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_16.status 2026-02-18T08:37:53.553 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=41, tag_tid=1, entry_tid=45509' 2026-02-18T08:37:53.553 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=41, tag_tid=2, entry_tid=1' -a 'object_number=41, tag_tid=2, entry_tid=1' = 'object_number=41, tag_tid=1, entry_tid=45509' 2026-02-18T08:37:53.553 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=41, tag_tid=1, entry_tid=45509' '!=' 'object_number=41, tag_tid=1, entry_tid=45509' 2026-02-18T08:37:53.553 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:37:53.553 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1.6 2026-02-18T08:37:55.155 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_16 2026-02-18T08:37:55.155 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:37:55.155 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:37:55.155 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_16 2026-02-18T08:37:55.155 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:37:55.155 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_16 ']' 2026-02-18T08:37:55.155 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_16' 2026-02-18T08:37:55.155 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_16 2026-02-18T08:37:55.156 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:37:55.156 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:37:55.156 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:37:55.156 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:37:55.156 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:37:55.156 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:37:55.156 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:37:55.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:37:55.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:37:55.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:37:55.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_16 2026-02-18T08:37:55.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:37:55.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:37:55.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:37:55.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:37:55.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:37:55.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:37:55.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:37:55.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:37:55.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:37:55.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:37:55.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:37:55.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:37:55.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:37:55.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:37:55.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:37:55.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:37:55.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:37:55.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:37:55.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:37:55.161 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:37:55.161 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:37:55.161 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:37:55.161 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:37:55.161 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:37:55.161 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:37:55.161 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:37:55.161 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:37:55.161 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:37:55.161 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:37:55.161 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:37:55.170 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:37:55.171 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:37:55.171 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:37:55.171 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_16 2026-02-18T08:37:55.231 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_16'] 2026-02-18T08:37:55.231 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:37:55.231 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:37:55.234 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:37:55.234 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_16 2026-02-18T08:37:55.234 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:37:55.234 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:37:55.234 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:37:55.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:37:55.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:37:55.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:37:55.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:37:55.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:37:55.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:37:55.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:37:55.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:37:55.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:37:55.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:37:55.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:37:55.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:37:55.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:37:55.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:37:55.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:37:55.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:37:55.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:37:55.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:37:55.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:37:55.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:37:55.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:37:55.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:37:55.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:37:55.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:37:55.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:37:55.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:37:55.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:37:55.245 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:37:55.246 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:37:55.246 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:37:55.246 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_16 2026-02-18T08:37:55.304 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:37:55.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_16 2026-02-18T08:37:55.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:37:55.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:37:55.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_16 2026-02-18T08:37:55.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_16 '' 2026-02-18T08:37:55.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:37:55.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:37:55.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_16 2026-02-18T08:37:55.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:37:55.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_16.status 2026-02-18T08:37:55.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_16.status 2026-02-18T08:37:55.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:37:55.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_16.status 2026-02-18T08:37:55.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_16 2026-02-18T08:37:55.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_16.status 2026-02-18T08:37:55.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 10 2026-02-18T08:37:55.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 10 2026-02-18T08:37:55.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:37:55.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=41, tag_tid=2, entry_tid=1], [object_number=40, tag_tid=2, entry_tid=0], [object_number=43, tag_tid=1, entry_tid=45507], [object_number=42, tag_tid=1, entry_tid=45506]]], state=connected] 2026-02-18T08:37:55.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=41, tag_tid=1, entry_tid=45509], [object_number=40, tag_tid=1, entry_tid=45508], [object_number=43, tag_tid=1, entry_tid=45507], [object_number=42, tag_tid=1, entry_tid=45506]]], state=connected] 2026-02-18T08:37:55.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_16.status 2026-02-18T08:37:55.348 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=41, tag_tid=2, entry_tid=1' 2026-02-18T08:37:55.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_16 2026-02-18T08:37:55.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:37:55.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:37:55.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_16 2026-02-18T08:37:55.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_16 '..*' 2026-02-18T08:37:55.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:37:55.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:37:55.350 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_16 2026-02-18T08:37:55.350 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:37:55.350 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_16.status 2026-02-18T08:37:55.350 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_16.status 2026-02-18T08:37:55.350 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:37:55.351 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_16.status 2026-02-18T08:37:55.351 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_16 2026-02-18T08:37:55.351 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_16.status 2026-02-18T08:37:55.385 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 10 2026-02-18T08:37:55.385 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 10 2026-02-18T08:37:55.385 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:37:55.385 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=41, tag_tid=2, entry_tid=1], [object_number=40, tag_tid=2, entry_tid=0], [object_number=43, tag_tid=1, entry_tid=45507], [object_number=42, tag_tid=1, entry_tid=45506]]], state=connected] 2026-02-18T08:37:55.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=41, tag_tid=1, entry_tid=45509], [object_number=40, tag_tid=1, entry_tid=45508], [object_number=43, tag_tid=1, entry_tid=45507], [object_number=42, tag_tid=1, entry_tid=45506]]], state=connected] 2026-02-18T08:37:55.388 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_16.status 2026-02-18T08:37:55.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=41, tag_tid=1, entry_tid=45509' 2026-02-18T08:37:55.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=41, tag_tid=2, entry_tid=1' -a 'object_number=41, tag_tid=2, entry_tid=1' = 'object_number=41, tag_tid=1, entry_tid=45509' 2026-02-18T08:37:55.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=41, tag_tid=1, entry_tid=45509' '!=' 'object_number=41, tag_tid=1, entry_tid=45509' 2026-02-18T08:37:55.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:37:55.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 2 2026-02-18T08:37:57.393 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_16 2026-02-18T08:37:57.393 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:37:57.393 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:37:57.393 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_16 2026-02-18T08:37:57.393 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:37:57.393 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_16 ']' 2026-02-18T08:37:57.393 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_16' 2026-02-18T08:37:57.393 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_16 2026-02-18T08:37:57.394 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:37:57.394 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:37:57.394 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:37:57.394 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:37:57.394 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:37:57.394 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:37:57.394 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:37:57.397 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:37:57.397 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:37:57.397 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:37:57.397 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_16 2026-02-18T08:37:57.397 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:37:57.397 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:37:57.397 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:37:57.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:37:57.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:37:57.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:37:57.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:37:57.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:37:57.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:37:57.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:37:57.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:37:57.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:37:57.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:37:57.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:37:57.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:37:57.399 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:37:57.399 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:37:57.399 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:37:57.399 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:37:57.399 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:37:57.399 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:37:57.399 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:37:57.399 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:37:57.399 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:37:57.399 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:37:57.399 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:37:57.399 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:37:57.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:37:57.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:37:57.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:37:57.407 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:37:57.407 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:37:57.408 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:37:57.408 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_16 2026-02-18T08:37:57.460 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_16'] 2026-02-18T08:37:57.460 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:37:57.460 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:37:57.463 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:37:57.463 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_16 2026-02-18T08:37:57.463 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:37:57.463 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:37:57.463 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:37:57.463 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:37:57.463 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:37:57.463 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:37:57.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:37:57.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:37:57.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:37:57.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:37:57.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:37:57.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:37:57.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:37:57.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:37:57.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:37:57.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:37:57.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:37:57.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:37:57.465 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:37:57.465 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:37:57.465 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:37:57.465 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:37:57.465 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:37:57.465 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:37:57.465 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:37:57.465 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:37:57.465 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:37:57.465 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:37:57.465 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:37:57.465 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:37:57.474 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:37:57.475 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:37:57.475 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:37:57.475 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_16 2026-02-18T08:37:57.539 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:37:57.540 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_16 2026-02-18T08:37:57.540 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:37:57.540 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:37:57.540 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_16 2026-02-18T08:37:57.540 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_16 '' 2026-02-18T08:37:57.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:37:57.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:37:57.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_16 2026-02-18T08:37:57.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:37:57.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_16.status 2026-02-18T08:37:57.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_16.status 2026-02-18T08:37:57.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:37:57.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_16.status 2026-02-18T08:37:57.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_16 2026-02-18T08:37:57.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_16.status 2026-02-18T08:37:57.569 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 10 2026-02-18T08:37:57.569 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 10 2026-02-18T08:37:57.570 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:37:57.570 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=41, tag_tid=2, entry_tid=1], [object_number=40, tag_tid=2, entry_tid=0], [object_number=43, tag_tid=1, entry_tid=45507], [object_number=42, tag_tid=1, entry_tid=45506]]], state=connected] 2026-02-18T08:37:57.570 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=41, tag_tid=1, entry_tid=45509], [object_number=40, tag_tid=1, entry_tid=45508], [object_number=43, tag_tid=1, entry_tid=45507], [object_number=42, tag_tid=1, entry_tid=45506]]], state=connected] 2026-02-18T08:37:57.572 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_16.status 2026-02-18T08:37:57.573 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=41, tag_tid=2, entry_tid=1' 2026-02-18T08:37:57.574 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_16 2026-02-18T08:37:57.574 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:37:57.574 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:37:57.575 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_16 2026-02-18T08:37:57.575 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_16 '..*' 2026-02-18T08:37:57.575 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:37:57.575 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:37:57.575 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_16 2026-02-18T08:37:57.575 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:37:57.575 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_16.status 2026-02-18T08:37:57.576 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_16.status 2026-02-18T08:37:57.576 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:37:57.576 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_16.status 2026-02-18T08:37:57.577 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_16 2026-02-18T08:37:57.577 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_16.status 2026-02-18T08:37:57.611 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 10 2026-02-18T08:37:57.611 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 10 2026-02-18T08:37:57.611 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:37:57.611 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=41, tag_tid=2, entry_tid=1], [object_number=40, tag_tid=2, entry_tid=0], [object_number=43, tag_tid=1, entry_tid=45507], [object_number=42, tag_tid=1, entry_tid=45506]]], state=connected] 2026-02-18T08:37:57.612 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=41, tag_tid=1, entry_tid=45509], [object_number=40, tag_tid=1, entry_tid=45508], [object_number=43, tag_tid=1, entry_tid=45507], [object_number=42, tag_tid=1, entry_tid=45506]]], state=connected] 2026-02-18T08:37:57.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_16.status 2026-02-18T08:37:57.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=41, tag_tid=1, entry_tid=45509' 2026-02-18T08:37:57.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=41, tag_tid=2, entry_tid=1' -a 'object_number=41, tag_tid=2, entry_tid=1' = 'object_number=41, tag_tid=1, entry_tid=45509' 2026-02-18T08:37:57.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=41, tag_tid=1, entry_tid=45509' '!=' 'object_number=41, tag_tid=1, entry_tid=45509' 2026-02-18T08:37:57.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:37:57.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 2 2026-02-18T08:37:59.619 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_16 2026-02-18T08:37:59.619 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:37:59.619 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:37:59.619 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_16 2026-02-18T08:37:59.619 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:37:59.619 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_16 ']' 2026-02-18T08:37:59.619 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_16' 2026-02-18T08:37:59.619 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_16 2026-02-18T08:37:59.620 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:37:59.620 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:37:59.620 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:37:59.620 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:37:59.620 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:37:59.620 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:37:59.620 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:37:59.623 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:37:59.623 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:37:59.623 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:37:59.623 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_16 2026-02-18T08:37:59.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:37:59.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:37:59.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:37:59.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:37:59.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:37:59.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:37:59.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:37:59.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:37:59.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:37:59.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:37:59.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:37:59.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:37:59.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:37:59.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:37:59.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:37:59.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:37:59.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:37:59.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:37:59.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:37:59.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:37:59.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:37:59.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:37:59.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:37:59.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:37:59.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:37:59.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:37:59.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:37:59.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:37:59.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:37:59.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:37:59.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:37:59.635 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:37:59.635 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:37:59.635 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_16 2026-02-18T08:37:59.696 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_16'] 2026-02-18T08:37:59.696 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:37:59.696 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:37:59.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:37:59.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_16 2026-02-18T08:37:59.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:37:59.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:37:59.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:37:59.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:37:59.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:37:59.700 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:37:59.700 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:37:59.700 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:37:59.700 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:37:59.700 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:37:59.700 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:37:59.700 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:37:59.700 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:37:59.700 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:37:59.700 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:37:59.700 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:37:59.700 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:37:59.701 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:37:59.701 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:37:59.701 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:37:59.701 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:37:59.701 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:37:59.701 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:37:59.701 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:37:59.701 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:37:59.701 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:37:59.701 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:37:59.701 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:37:59.701 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:37:59.702 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:37:59.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:37:59.711 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:37:59.711 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:37:59.711 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_16 2026-02-18T08:37:59.780 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:37:59.781 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_16 2026-02-18T08:37:59.781 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:37:59.781 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:37:59.781 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_16 2026-02-18T08:37:59.781 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_16 '' 2026-02-18T08:37:59.781 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:37:59.782 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:37:59.782 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_16 2026-02-18T08:37:59.782 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:37:59.782 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_16.status 2026-02-18T08:37:59.782 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_16.status 2026-02-18T08:37:59.782 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:37:59.783 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_16.status 2026-02-18T08:37:59.783 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_16 2026-02-18T08:37:59.783 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_16.status 2026-02-18T08:37:59.820 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 10 2026-02-18T08:37:59.820 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 10 2026-02-18T08:37:59.820 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:37:59.820 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=41, tag_tid=2, entry_tid=1], [object_number=40, tag_tid=2, entry_tid=0], [object_number=43, tag_tid=1, entry_tid=45507], [object_number=42, tag_tid=1, entry_tid=45506]]], state=connected] 2026-02-18T08:37:59.820 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=41, tag_tid=2, entry_tid=1], [object_number=40, tag_tid=2, entry_tid=0], [object_number=43, tag_tid=1, entry_tid=45507], [object_number=42, tag_tid=1, entry_tid=45506]]], state=connected] 2026-02-18T08:37:59.823 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_16.status 2026-02-18T08:37:59.825 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=41, tag_tid=2, entry_tid=1' 2026-02-18T08:37:59.826 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_16 2026-02-18T08:37:59.826 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:37:59.827 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:37:59.827 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_16 2026-02-18T08:37:59.827 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_16 '..*' 2026-02-18T08:37:59.827 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:37:59.827 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:37:59.827 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_16 2026-02-18T08:37:59.827 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:37:59.827 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_16.status 2026-02-18T08:37:59.828 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_16.status 2026-02-18T08:37:59.828 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:37:59.828 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_16.status 2026-02-18T08:37:59.829 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_16 2026-02-18T08:37:59.829 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_16.status 2026-02-18T08:37:59.854 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 10 2026-02-18T08:37:59.854 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 10 2026-02-18T08:37:59.854 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:37:59.855 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=41, tag_tid=2, entry_tid=1], [object_number=40, tag_tid=2, entry_tid=0], [object_number=43, tag_tid=1, entry_tid=45507], [object_number=42, tag_tid=1, entry_tid=45506]]], state=connected] 2026-02-18T08:37:59.855 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=41, tag_tid=2, entry_tid=1], [object_number=40, tag_tid=2, entry_tid=0], [object_number=43, tag_tid=1, entry_tid=45507], [object_number=42, tag_tid=1, entry_tid=45506]]], state=connected] 2026-02-18T08:37:59.857 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_16.status 2026-02-18T08:37:59.859 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=41, tag_tid=2, entry_tid=1' 2026-02-18T08:37:59.859 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=41, tag_tid=2, entry_tid=1' -a 'object_number=41, tag_tid=2, entry_tid=1' = 'object_number=41, tag_tid=2, entry_tid=1' 2026-02-18T08:37:59.859 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:37:59.859 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_snap_present cluster1 mirror image_16 snap 2026-02-18T08:37:59.860 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:37:59.860 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:37:59.860 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_16 2026-02-18T08:37:59.860 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap 2026-02-18T08:37:59.860 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-18T08:37:59.860 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-18T08:37:59.860 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1 2026-02-18T08:38:00.860 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster1 info mirror/image_16@snap 2026-02-18T08:38:00.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:rbd image 'image_16': 2026-02-18T08:38:00.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: size 128 MiB in 32 objects 2026-02-18T08:38:00.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: order 22 (4 MiB objects) 2026-02-18T08:38:00.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: snapshot_count: 1 2026-02-18T08:38:00.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: id: 11f9b9652f3a 2026-02-18T08:38:00.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: block_name_prefix: rbd_data.11f9b9652f3a 2026-02-18T08:38:00.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: format: 2 2026-02-18T08:38:00.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: features: layering, exclusive-lock, journaling 2026-02-18T08:38:00.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: op_features: 2026-02-18T08:38:00.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: flags: 2026-02-18T08:38:00.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: create_timestamp: Wed Feb 18 08:28:25 2026 2026-02-18T08:38:00.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: access_timestamp: Wed Feb 18 08:28:25 2026 2026-02-18T08:38:00.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: modify_timestamp: Wed Feb 18 08:29:26 2026 2026-02-18T08:38:00.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: protected: False 2026-02-18T08:38:00.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: journal: 11f9b9652f3a 2026-02-18T08:38:00.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring state: enabled 2026-02-18T08:38:00.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring mode: journal 2026-02-18T08:38:00.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring global id: 410b204e-0f9a-46e4-bfaa-063c35aa2613 2026-02-18T08:38:00.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring primary: false 2026-02-18T08:38:00.894 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:38:00.895 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ compare_image_snaps mirror image_16 snap 2026-02-18T08:38:00.895 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:38:00.895 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_16 2026-02-18T08:38:00.895 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap 2026-02-18T08:38:00.895 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local ret=0 2026-02-18T08:38:00.895 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local rmt_export=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_16.export 2026-02-18T08:38:00.895 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loc_export=/tmp/tmp.vybh1m9yJF/cluster1-mirror-image_16.export 2026-02-18T08:38:00.895 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_16.export /tmp/tmp.vybh1m9yJF/cluster1-mirror-image_16.export 2026-02-18T08:38:00.896 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 -p mirror export image_16@snap /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_16.export 2026-02-18T08:38:01.037 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-18T08:38:01.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster1 -p mirror export image_16@snap /tmp/tmp.vybh1m9yJF/cluster1-mirror-image_16.export 2026-02-18T08:38:01.203 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-18T08:38:01.211 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmp /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_16.export /tmp/tmp.vybh1m9yJF/cluster1-mirror-image_16.export 2026-02-18T08:38:01.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_16.export /tmp/tmp.vybh1m9yJF/cluster1-mirror-image_16.export 2026-02-18T08:38:01.315 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:38:01.315 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T08:38:01.316 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_17 2026-02-18T08:38:01.316 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_snap cluster2 mirror image_17 snap 2026-02-18T08:38:01.316 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:38:01.316 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:38:01.316 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_17 2026-02-18T08:38:01.316 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap 2026-02-18T08:38:01.316 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 -p mirror snap create image_17@snap --debug-rbd=20 --debug-journaler=20 2026-02-18T08:38:04.879 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_replay_started cluster1 mirror image_17 2026-02-18T08:38:04.880 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:38:04.880 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:38:04.880 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_17 2026-02-18T08:38:04.880 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_replay_state cluster1 mirror image_17 started 2026-02-18T08:38:04.880 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:38:04.880 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:38:04.880 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_17 2026-02-18T08:38:04.880 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local state=started 2026-02-18T08:38:04.880 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-18T08:38:04.880 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:38:04.881 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1 2026-02-18T08:38:05.880 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test_image_replay_state cluster1 mirror image_17 started 2026-02-18T08:38:05.881 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:38:05.881 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:38:05.881 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_17 2026-02-18T08:38:05.881 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local test_state=started 2026-02-18T08:38:05.881 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local status_result 2026-02-18T08:38:05.881 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_state=stopped 2026-02-18T08:38:05.882 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_17 2026-02-18T08:38:05.883 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -i state 2026-02-18T08:38:05.883 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_instance=cluster1 2026-02-18T08:38:05.883 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-18T08:38:05.883 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:38:05.883 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance=cluster1 2026-02-18T08:38:05.883 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local loop_instance 2026-02-18T08:38:05.883 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:38:05.883 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sleep 0 2026-02-18T08:38:05.883 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:38:05.883 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ seq 0 3 2026-02-18T08:38:05.885 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:38:05.886 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_17 2026-02-18T08:38:05.886 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:0 2026-02-18T08:38:05.886 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-18T08:38:05.886 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:38:05.886 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:38:05.886 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:38:05.886 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-18T08:38:05.886 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:38:05.886 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:38:05.886 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:38:05.886 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:38:05.886 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-18T08:38:05.887 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:38:05.887 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:38:05.887 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:38:05.887 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:38:05.887 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:0 2026-02-18T08:38:05.887 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-18T08:38:05.887 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-18T08:38:05.887 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:38:05.887 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:0 2026-02-18T08:38:05.887 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T08:38:05.887 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-18T08:38:05.887 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:38:05.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:38:05.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-18T08:38:05.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-18T08:38:05.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-18T08:38:05.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=0 2026-02-18T08:38:05.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=0 2026-02-18T08:38:05.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:38:05.897 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:38:05.897 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:38:05.897 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:38:05.897 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_17 2026-02-18T08:38:05.952 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_17'] 2026-02-18T08:38:05.952 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status : get status for rbd mirror 2026-02-18T08:38:05.952 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:38:05.955 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:38:05.955 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_17 2026-02-18T08:38:05.955 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:1 2026-02-18T08:38:05.955 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-18T08:38:05.955 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:38:05.955 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:38:05.956 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:38:05.956 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-18T08:38:05.956 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:38:05.956 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:38:05.956 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:38:05.956 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:38:05.956 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-18T08:38:05.956 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:38:05.956 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:38:05.956 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:38:05.956 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:38:05.956 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:1 2026-02-18T08:38:05.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-18T08:38:05.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-18T08:38:05.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:38:05.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:1 2026-02-18T08:38:05.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T08:38:05.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-18T08:38:05.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:38:05.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:38:05.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-18T08:38:05.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-18T08:38:05.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-18T08:38:05.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=1 2026-02-18T08:38:05.958 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=1 2026-02-18T08:38:05.958 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:38:05.965 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:38:05.966 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:38:05.966 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:38:05.966 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_17 2026-02-18T08:38:06.019 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_17'] 2026-02-18T08:38:06.019 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status : get status for rbd mirror 2026-02-18T08:38:06.019 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:38:06.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:38:06.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_17 2026-02-18T08:38:06.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:2 2026-02-18T08:38:06.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-18T08:38:06.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:38:06.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:38:06.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:38:06.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-18T08:38:06.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:38:06.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:38:06.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:38:06.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:38:06.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-18T08:38:06.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:38:06.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:38:06.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:38:06.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:38:06.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:2 2026-02-18T08:38:06.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-18T08:38:06.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-18T08:38:06.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:38:06.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:2 2026-02-18T08:38:06.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T08:38:06.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-18T08:38:06.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:38:06.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:38:06.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-18T08:38:06.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-18T08:38:06.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-18T08:38:06.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=2 2026-02-18T08:38:06.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=2 2026-02-18T08:38:06.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:38:06.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:38:06.035 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:38:06.035 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:38:06.035 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_17 2026-02-18T08:38:06.092 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ return 0 2026-02-18T08:38:06.093 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ status_result=' "state": "Replaying"' 2026-02-18T08:38:06.093 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ echo ' "state": "Replaying"' 2026-02-18T08:38:06.094 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ grep -i Replaying 2026-02-18T08:38:06.094 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: "state": "Replaying" 2026-02-18T08:38:06.096 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ current_state=started 2026-02-18T08:38:06.096 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test started = started 2026-02-18T08:38:06.096 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:38:06.096 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror image_17 2026-02-18T08:38:06.096 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_cluster=cluster1 2026-02-18T08:38:06.096 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:38:06.096 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_pool=mirror 2026-02-18T08:38:06.097 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local remote_pool=mirror 2026-02-18T08:38:06.097 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_17 2026-02-18T08:38:06.097 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' journal = journal ']' 2026-02-18T08:38:06.097 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror mirror image_17 2026-02-18T08:38:06.097 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_cluster=cluster1 2026-02-18T08:38:06.097 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:38:06.097 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_pool=mirror 2026-02-18T08:38:06.097 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local remote_pool=mirror 2026-02-18T08:38:06.097 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_17 2026-02-18T08:38:06.097 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-02-18T08:38:06.097 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-02-18T08:38:06.097 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:38:06.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:38:06.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:38:06.297 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_17 2026-02-18T08:38:06.297 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:38:06.297 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:38:06.297 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_17 2026-02-18T08:38:06.298 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:38:06.298 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_17 ']' 2026-02-18T08:38:06.298 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_17' 2026-02-18T08:38:06.298 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_17 2026-02-18T08:38:06.298 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:38:06.298 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:38:06.298 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:38:06.298 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:38:06.298 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:38:06.298 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:38:06.298 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:38:06.301 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:38:06.301 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:38:06.301 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:38:06.301 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_17 2026-02-18T08:38:06.301 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:38:06.301 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:38:06.301 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:38:06.302 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:38:06.302 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:38:06.302 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:38:06.302 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:38:06.302 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:38:06.302 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:38:06.302 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:38:06.302 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:38:06.302 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:38:06.302 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:38:06.302 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:38:06.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:38:06.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:38:06.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:38:06.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:38:06.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:38:06.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:38:06.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:38:06.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:38:06.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:38:06.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:38:06.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:38:06.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:38:06.304 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:38:06.304 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:38:06.304 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:38:06.304 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:38:06.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:38:06.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:38:06.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:38:06.312 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_17 2026-02-18T08:38:06.374 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_17'] 2026-02-18T08:38:06.374 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:38:06.375 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:38:06.377 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:38:06.377 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_17 2026-02-18T08:38:06.377 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:38:06.377 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:38:06.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:38:06.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:38:06.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:38:06.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:38:06.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:38:06.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:38:06.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:38:06.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:38:06.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:38:06.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:38:06.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:38:06.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:38:06.379 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:38:06.379 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:38:06.379 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:38:06.379 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:38:06.379 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:38:06.379 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:38:06.379 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:38:06.379 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:38:06.379 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:38:06.379 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:38:06.379 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:38:06.379 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:38:06.379 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:38:06.380 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:38:06.380 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:38:06.380 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:38:06.387 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:38:06.388 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:38:06.388 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:38:06.388 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_17 2026-02-18T08:38:06.446 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_17'] 2026-02-18T08:38:06.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:38:06.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:38:06.449 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:38:06.450 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_17 2026-02-18T08:38:06.450 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:38:06.450 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:38:06.450 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:38:06.450 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:38:06.450 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:38:06.450 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:38:06.450 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:38:06.450 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:38:06.450 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:38:06.451 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:38:06.451 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:38:06.451 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:38:06.451 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:38:06.451 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:38:06.451 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:38:06.451 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:38:06.451 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:38:06.451 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:38:06.451 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:38:06.451 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:38:06.451 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:38:06.451 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:38:06.452 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:38:06.452 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:38:06.452 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:38:06.452 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:38:06.452 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:38:06.452 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:38:06.452 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:38:06.452 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:38:06.460 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:38:06.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:38:06.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:38:06.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_17 2026-02-18T08:38:06.526 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:38:06.528 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_17 2026-02-18T08:38:06.528 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:38:06.528 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:38:06.528 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_17 2026-02-18T08:38:06.528 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_17 '' 2026-02-18T08:38:06.528 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:38:06.528 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:38:06.528 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_17 2026-02-18T08:38:06.528 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:38:06.528 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_17.status 2026-02-18T08:38:06.529 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_17.status 2026-02-18T08:38:06.529 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:38:06.529 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_17.status 2026-02-18T08:38:06.530 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_17 2026-02-18T08:38:06.530 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_17.status 2026-02-18T08:38:06.557 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 10 2026-02-18T08:38:06.557 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 10 2026-02-18T08:38:06.557 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:38:06.557 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=41, tag_tid=2, entry_tid=1], [object_number=40, tag_tid=2, entry_tid=0], [object_number=43, tag_tid=1, entry_tid=46215], [object_number=42, tag_tid=1, entry_tid=46214]]], state=connected] 2026-02-18T08:38:06.557 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=43, tag_tid=1, entry_tid=46215], [object_number=42, tag_tid=1, entry_tid=46214], [object_number=41, tag_tid=1, entry_tid=46213], [object_number=40, tag_tid=1, entry_tid=46212]]], state=connected] 2026-02-18T08:38:06.559 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_17.status 2026-02-18T08:38:06.561 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=41, tag_tid=2, entry_tid=1' 2026-02-18T08:38:06.562 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_17 2026-02-18T08:38:06.562 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:38:06.562 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:38:06.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_17 2026-02-18T08:38:06.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_17 '..*' 2026-02-18T08:38:06.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:38:06.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:38:06.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_17 2026-02-18T08:38:06.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:38:06.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_17.status 2026-02-18T08:38:06.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_17.status 2026-02-18T08:38:06.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:38:06.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_17.status 2026-02-18T08:38:06.565 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_17 2026-02-18T08:38:06.565 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_17.status 2026-02-18T08:38:06.599 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 10 2026-02-18T08:38:06.599 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 10 2026-02-18T08:38:06.599 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:38:06.599 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=41, tag_tid=2, entry_tid=1], [object_number=40, tag_tid=2, entry_tid=0], [object_number=43, tag_tid=1, entry_tid=46215], [object_number=42, tag_tid=1, entry_tid=46214]]], state=connected] 2026-02-18T08:38:06.599 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=43, tag_tid=1, entry_tid=46215], [object_number=42, tag_tid=1, entry_tid=46214], [object_number=41, tag_tid=1, entry_tid=46213], [object_number=40, tag_tid=1, entry_tid=46212]]], state=connected] 2026-02-18T08:38:06.603 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_17.status 2026-02-18T08:38:06.604 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=43, tag_tid=1, entry_tid=46215' 2026-02-18T08:38:06.605 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=41, tag_tid=2, entry_tid=1' -a 'object_number=41, tag_tid=2, entry_tid=1' = 'object_number=43, tag_tid=1, entry_tid=46215' 2026-02-18T08:38:06.605 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=43, tag_tid=1, entry_tid=46215' '!=' '' 2026-02-18T08:38:06.605 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:38:06.605 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=43, tag_tid=1, entry_tid=46215' = '' 2026-02-18T08:38:06.605 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=43, tag_tid=1, entry_tid=46215' 2026-02-18T08:38:06.606 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=41, tag_tid=2, entry_tid=1' 2026-02-18T08:38:06.606 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:38:06.606 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:38:06.607 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=2 2026-02-18T08:38:06.607 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=43, tag_tid=1, entry_tid=46215' 2026-02-18T08:38:06.607 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:38:06.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:38:06.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=1 2026-02-18T08:38:06.610 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=41, tag_tid=2, entry_tid=1' 2026-02-18T08:38:06.610 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:38:06.610 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:38:06.611 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:38:06.612 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=43, tag_tid=1, entry_tid=46215' 2026-02-18T08:38:06.612 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:38:06.612 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:38:06.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=46215 2026-02-18T08:38:06.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 2 = 1 -a 1 -le 46215 2026-02-18T08:38:06.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:38:06.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:38:06.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:38:06.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_17 2026-02-18T08:38:06.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:38:06.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:38:06.817 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_17 2026-02-18T08:38:06.817 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:38:06.817 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_17 ']' 2026-02-18T08:38:06.817 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_17' 2026-02-18T08:38:06.817 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_17 2026-02-18T08:38:06.817 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:38:06.817 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:38:06.817 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:38:06.817 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:38:06.817 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:38:06.817 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:38:06.818 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:38:06.818 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:38:06.818 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:38:06.820 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:38:06.820 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_17 2026-02-18T08:38:06.820 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:38:06.820 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:38:06.821 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:38:06.821 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:38:06.821 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:38:06.821 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:38:06.821 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:38:06.821 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:38:06.821 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:38:06.821 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:38:06.821 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:38:06.821 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:38:06.821 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:38:06.822 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:38:06.822 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:38:06.822 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:38:06.822 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:38:06.822 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:38:06.822 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:38:06.822 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:38:06.822 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:38:06.822 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:38:06.822 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:38:06.822 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:38:06.822 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:38:06.823 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:38:06.823 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:38:06.823 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:38:06.823 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:38:06.823 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:38:06.830 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:38:06.831 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:38:06.831 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:38:06.831 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_17 2026-02-18T08:38:06.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_17'] 2026-02-18T08:38:06.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:38:06.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:38:06.894 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:38:06.895 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_17 2026-02-18T08:38:06.895 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:38:06.895 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:38:06.895 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:38:06.895 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:38:06.895 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:38:06.895 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:38:06.895 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:38:06.895 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:38:06.895 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:38:06.895 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:38:06.896 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:38:06.896 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:38:06.896 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:38:06.896 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:38:06.896 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:38:06.896 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:38:06.896 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:38:06.896 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:38:06.896 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:38:06.896 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:38:06.896 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:38:06.896 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:38:06.896 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:38:06.897 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:38:06.897 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:38:06.897 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:38:06.897 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:38:06.897 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:38:06.897 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:38:06.897 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:38:06.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:38:06.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:38:06.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:38:06.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_17 2026-02-18T08:38:06.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_17'] 2026-02-18T08:38:06.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:38:06.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:38:06.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:38:06.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_17 2026-02-18T08:38:06.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:38:06.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:38:06.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:38:06.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:38:06.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:38:06.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:38:06.961 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:38:06.961 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:38:06.961 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:38:06.961 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:38:06.961 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:38:06.961 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:38:06.961 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:38:06.961 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:38:06.961 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:38:06.961 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:38:06.961 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:38:06.961 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:38:06.961 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:38:06.962 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:38:06.962 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:38:06.962 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:38:06.962 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:38:06.962 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:38:06.962 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:38:06.962 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:38:06.962 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:38:06.962 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:38:06.962 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:38:06.962 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:38:06.970 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:38:06.971 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:38:06.971 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:38:06.971 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_17 2026-02-18T08:38:07.036 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:38:07.037 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_17 2026-02-18T08:38:07.037 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:38:07.037 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:38:07.037 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_17 2026-02-18T08:38:07.037 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_17 '' 2026-02-18T08:38:07.037 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:38:07.037 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:38:07.038 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_17 2026-02-18T08:38:07.038 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:38:07.038 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_17.status 2026-02-18T08:38:07.038 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_17.status 2026-02-18T08:38:07.038 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:38:07.039 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_17.status 2026-02-18T08:38:07.039 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_17 2026-02-18T08:38:07.039 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_17.status 2026-02-18T08:38:07.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 10 2026-02-18T08:38:07.067 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 10 2026-02-18T08:38:07.067 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:38:07.067 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=41, tag_tid=2, entry_tid=1], [object_number=40, tag_tid=2, entry_tid=0], [object_number=43, tag_tid=1, entry_tid=46215], [object_number=42, tag_tid=1, entry_tid=46214]]], state=connected] 2026-02-18T08:38:07.067 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=43, tag_tid=1, entry_tid=46215], [object_number=42, tag_tid=1, entry_tid=46214], [object_number=41, tag_tid=1, entry_tid=46213], [object_number=40, tag_tid=1, entry_tid=46212]]], state=connected] 2026-02-18T08:38:07.069 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_17.status 2026-02-18T08:38:07.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=41, tag_tid=2, entry_tid=1' 2026-02-18T08:38:07.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_17 2026-02-18T08:38:07.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:38:07.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:38:07.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_17 2026-02-18T08:38:07.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_17 '..*' 2026-02-18T08:38:07.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:38:07.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:38:07.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_17 2026-02-18T08:38:07.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:38:07.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_17.status 2026-02-18T08:38:07.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_17.status 2026-02-18T08:38:07.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:38:07.074 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_17.status 2026-02-18T08:38:07.074 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_17 2026-02-18T08:38:07.074 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_17.status 2026-02-18T08:38:07.102 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 10 2026-02-18T08:38:07.102 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 10 2026-02-18T08:38:07.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:38:07.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=41, tag_tid=2, entry_tid=1], [object_number=40, tag_tid=2, entry_tid=0], [object_number=43, tag_tid=1, entry_tid=46215], [object_number=42, tag_tid=1, entry_tid=46214]]], state=connected] 2026-02-18T08:38:07.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=43, tag_tid=1, entry_tid=46215], [object_number=42, tag_tid=1, entry_tid=46214], [object_number=41, tag_tid=1, entry_tid=46213], [object_number=40, tag_tid=1, entry_tid=46212]]], state=connected] 2026-02-18T08:38:07.106 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_17.status 2026-02-18T08:38:07.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=43, tag_tid=1, entry_tid=46215' 2026-02-18T08:38:07.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=41, tag_tid=2, entry_tid=1' -a 'object_number=41, tag_tid=2, entry_tid=1' = 'object_number=43, tag_tid=1, entry_tid=46215' 2026-02-18T08:38:07.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=43, tag_tid=1, entry_tid=46215' '!=' 'object_number=43, tag_tid=1, entry_tid=46215' 2026-02-18T08:38:07.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:38:07.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.4 2026-02-18T08:38:07.509 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_17 2026-02-18T08:38:07.510 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:38:07.510 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:38:07.510 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_17 2026-02-18T08:38:07.510 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:38:07.510 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_17 ']' 2026-02-18T08:38:07.510 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_17' 2026-02-18T08:38:07.510 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_17 2026-02-18T08:38:07.510 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:38:07.510 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:38:07.510 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:38:07.511 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:38:07.511 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:38:07.511 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:38:07.511 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:38:07.511 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:38:07.513 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:38:07.513 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:38:07.514 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_17 2026-02-18T08:38:07.514 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:38:07.514 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:38:07.514 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:38:07.514 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:38:07.514 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:38:07.514 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:38:07.514 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:38:07.514 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:38:07.514 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:38:07.514 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:38:07.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:38:07.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:38:07.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:38:07.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:38:07.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:38:07.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:38:07.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:38:07.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:38:07.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:38:07.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:38:07.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:38:07.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:38:07.516 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:38:07.516 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:38:07.516 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:38:07.516 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:38:07.516 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:38:07.516 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:38:07.516 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:38:07.516 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:38:07.523 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:38:07.524 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:38:07.524 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:38:07.524 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_17 2026-02-18T08:38:07.575 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_17'] 2026-02-18T08:38:07.575 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:38:07.575 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:38:07.578 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:38:07.578 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_17 2026-02-18T08:38:07.578 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:38:07.578 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:38:07.578 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:38:07.578 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:38:07.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:38:07.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:38:07.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:38:07.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:38:07.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:38:07.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:38:07.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:38:07.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:38:07.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:38:07.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:38:07.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:38:07.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:38:07.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:38:07.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:38:07.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:38:07.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:38:07.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:38:07.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:38:07.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:38:07.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:38:07.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:38:07.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:38:07.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:38:07.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:38:07.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:38:07.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:38:07.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:38:07.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:38:07.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:38:07.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_17 2026-02-18T08:38:07.641 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_17'] 2026-02-18T08:38:07.641 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:38:07.641 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:38:07.643 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:38:07.643 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_17 2026-02-18T08:38:07.643 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:38:07.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:38:07.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:38:07.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:38:07.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:38:07.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:38:07.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:38:07.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:38:07.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:38:07.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:38:07.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:38:07.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:38:07.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:38:07.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:38:07.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:38:07.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:38:07.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:38:07.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:38:07.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:38:07.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:38:07.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:38:07.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:38:07.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:38:07.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:38:07.646 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:38:07.646 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:38:07.646 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:38:07.646 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:38:07.646 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:38:07.646 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:38:07.654 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:38:07.655 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:38:07.655 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:38:07.655 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_17 2026-02-18T08:38:07.720 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:38:07.721 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_17 2026-02-18T08:38:07.721 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:38:07.721 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:38:07.721 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_17 2026-02-18T08:38:07.721 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_17 '' 2026-02-18T08:38:07.721 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:38:07.721 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:38:07.721 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_17 2026-02-18T08:38:07.721 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:38:07.721 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_17.status 2026-02-18T08:38:07.722 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_17.status 2026-02-18T08:38:07.722 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:38:07.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_17.status 2026-02-18T08:38:07.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_17 2026-02-18T08:38:07.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_17.status 2026-02-18T08:38:07.759 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 10 2026-02-18T08:38:07.759 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 10 2026-02-18T08:38:07.759 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:38:07.759 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=41, tag_tid=2, entry_tid=1], [object_number=40, tag_tid=2, entry_tid=0], [object_number=43, tag_tid=1, entry_tid=46215], [object_number=42, tag_tid=1, entry_tid=46214]]], state=connected] 2026-02-18T08:38:07.759 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=43, tag_tid=1, entry_tid=46215], [object_number=42, tag_tid=1, entry_tid=46214], [object_number=41, tag_tid=1, entry_tid=46213], [object_number=40, tag_tid=1, entry_tid=46212]]], state=connected] 2026-02-18T08:38:07.762 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_17.status 2026-02-18T08:38:07.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=41, tag_tid=2, entry_tid=1' 2026-02-18T08:38:07.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_17 2026-02-18T08:38:07.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:38:07.766 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:38:07.766 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_17 2026-02-18T08:38:07.766 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_17 '..*' 2026-02-18T08:38:07.766 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:38:07.766 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:38:07.766 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_17 2026-02-18T08:38:07.766 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:38:07.766 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_17.status 2026-02-18T08:38:07.767 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_17.status 2026-02-18T08:38:07.767 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:38:07.768 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_17.status 2026-02-18T08:38:07.768 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_17 2026-02-18T08:38:07.768 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_17.status 2026-02-18T08:38:07.802 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 10 2026-02-18T08:38:07.802 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 10 2026-02-18T08:38:07.802 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:38:07.802 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=41, tag_tid=2, entry_tid=1], [object_number=40, tag_tid=2, entry_tid=0], [object_number=43, tag_tid=1, entry_tid=46215], [object_number=42, tag_tid=1, entry_tid=46214]]], state=connected] 2026-02-18T08:38:07.802 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=43, tag_tid=1, entry_tid=46215], [object_number=42, tag_tid=1, entry_tid=46214], [object_number=41, tag_tid=1, entry_tid=46213], [object_number=40, tag_tid=1, entry_tid=46212]]], state=connected] 2026-02-18T08:38:07.806 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_17.status 2026-02-18T08:38:07.808 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=43, tag_tid=1, entry_tid=46215' 2026-02-18T08:38:07.808 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=41, tag_tid=2, entry_tid=1' -a 'object_number=41, tag_tid=2, entry_tid=1' = 'object_number=43, tag_tid=1, entry_tid=46215' 2026-02-18T08:38:07.808 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=43, tag_tid=1, entry_tid=46215' '!=' 'object_number=43, tag_tid=1, entry_tid=46215' 2026-02-18T08:38:07.808 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:38:07.808 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.8 2026-02-18T08:38:08.610 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_17 2026-02-18T08:38:08.611 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:38:08.611 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:38:08.611 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_17 2026-02-18T08:38:08.611 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:38:08.611 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_17 ']' 2026-02-18T08:38:08.611 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_17' 2026-02-18T08:38:08.611 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_17 2026-02-18T08:38:08.611 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:38:08.612 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:38:08.612 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:38:08.612 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:38:08.612 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:38:08.612 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:38:08.612 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:38:08.612 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:38:08.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:38:08.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:38:08.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_17 2026-02-18T08:38:08.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:38:08.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:38:08.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:38:08.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:38:08.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:38:08.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:38:08.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:38:08.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:38:08.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:38:08.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:38:08.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:38:08.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:38:08.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:38:08.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:38:08.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:38:08.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:38:08.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:38:08.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:38:08.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:38:08.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:38:08.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:38:08.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:38:08.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:38:08.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:38:08.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:38:08.618 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:38:08.618 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:38:08.618 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:38:08.618 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:38:08.618 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:38:08.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:38:08.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:38:08.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:38:08.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_17 2026-02-18T08:38:08.678 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_17'] 2026-02-18T08:38:08.678 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:38:08.678 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:38:08.680 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:38:08.681 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_17 2026-02-18T08:38:08.681 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:38:08.681 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:38:08.681 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:38:08.681 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:38:08.681 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:38:08.681 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:38:08.681 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:38:08.681 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:38:08.681 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:38:08.682 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:38:08.682 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:38:08.682 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:38:08.682 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:38:08.682 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:38:08.682 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:38:08.682 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:38:08.682 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:38:08.682 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:38:08.682 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:38:08.682 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:38:08.682 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:38:08.683 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:38:08.683 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:38:08.683 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:38:08.683 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:38:08.683 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:38:08.683 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:38:08.683 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:38:08.683 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:38:08.683 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:38:08.691 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:38:08.691 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:38:08.691 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:38:08.691 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_17 2026-02-18T08:38:08.743 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_17'] 2026-02-18T08:38:08.744 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:38:08.744 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:38:08.746 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:38:08.747 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_17 2026-02-18T08:38:08.747 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:38:08.747 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:38:08.747 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:38:08.747 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:38:08.747 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:38:08.747 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:38:08.747 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:38:08.747 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:38:08.747 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:38:08.748 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:38:08.748 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:38:08.748 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:38:08.748 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:38:08.748 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:38:08.748 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:38:08.748 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:38:08.748 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:38:08.748 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:38:08.748 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:38:08.748 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:38:08.749 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:38:08.749 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:38:08.749 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:38:08.749 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:38:08.749 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:38:08.749 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:38:08.749 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:38:08.749 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:38:08.749 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:38:08.749 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:38:08.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:38:08.758 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:38:08.758 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:38:08.758 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_17 2026-02-18T08:38:08.821 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:38:08.822 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_17 2026-02-18T08:38:08.823 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:38:08.823 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:38:08.823 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_17 2026-02-18T08:38:08.823 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_17 '' 2026-02-18T08:38:08.823 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:38:08.823 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:38:08.823 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_17 2026-02-18T08:38:08.823 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:38:08.823 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_17.status 2026-02-18T08:38:08.824 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_17.status 2026-02-18T08:38:08.824 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:38:08.824 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_17.status 2026-02-18T08:38:08.824 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_17 2026-02-18T08:38:08.825 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_17.status 2026-02-18T08:38:08.862 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 10 2026-02-18T08:38:08.862 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 10 2026-02-18T08:38:08.862 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:38:08.862 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=41, tag_tid=2, entry_tid=1], [object_number=40, tag_tid=2, entry_tid=0], [object_number=43, tag_tid=1, entry_tid=46215], [object_number=42, tag_tid=1, entry_tid=46214]]], state=connected] 2026-02-18T08:38:08.862 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=41, tag_tid=2, entry_tid=1], [object_number=40, tag_tid=2, entry_tid=0], [object_number=43, tag_tid=1, entry_tid=46215], [object_number=42, tag_tid=1, entry_tid=46214]]], state=connected] 2026-02-18T08:38:08.866 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_17.status 2026-02-18T08:38:08.867 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=41, tag_tid=2, entry_tid=1' 2026-02-18T08:38:08.869 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_17 2026-02-18T08:38:08.869 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:38:08.869 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:38:08.869 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_17 2026-02-18T08:38:08.869 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_17 '..*' 2026-02-18T08:38:08.869 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:38:08.869 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:38:08.869 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_17 2026-02-18T08:38:08.869 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:38:08.870 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_17.status 2026-02-18T08:38:08.870 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_17.status 2026-02-18T08:38:08.870 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:38:08.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_17.status 2026-02-18T08:38:08.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_17 2026-02-18T08:38:08.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_17.status 2026-02-18T08:38:08.904 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 10 2026-02-18T08:38:08.904 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 10 2026-02-18T08:38:08.904 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:38:08.904 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=41, tag_tid=2, entry_tid=1], [object_number=40, tag_tid=2, entry_tid=0], [object_number=43, tag_tid=1, entry_tid=46215], [object_number=42, tag_tid=1, entry_tid=46214]]], state=connected] 2026-02-18T08:38:08.904 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=41, tag_tid=2, entry_tid=1], [object_number=40, tag_tid=2, entry_tid=0], [object_number=43, tag_tid=1, entry_tid=46215], [object_number=42, tag_tid=1, entry_tid=46214]]], state=connected] 2026-02-18T08:38:08.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_17.status 2026-02-18T08:38:08.909 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=41, tag_tid=2, entry_tid=1' 2026-02-18T08:38:08.909 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=41, tag_tid=2, entry_tid=1' -a 'object_number=41, tag_tid=2, entry_tid=1' = 'object_number=41, tag_tid=2, entry_tid=1' 2026-02-18T08:38:08.909 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:38:08.910 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_snap_present cluster1 mirror image_17 snap 2026-02-18T08:38:08.910 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:38:08.910 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:38:08.910 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_17 2026-02-18T08:38:08.910 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap 2026-02-18T08:38:08.910 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-18T08:38:08.910 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-18T08:38:08.910 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1 2026-02-18T08:38:09.910 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster1 info mirror/image_17@snap 2026-02-18T08:38:09.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:rbd image 'image_17': 2026-02-18T08:38:09.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: size 128 MiB in 32 objects 2026-02-18T08:38:09.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: order 22 (4 MiB objects) 2026-02-18T08:38:09.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: snapshot_count: 1 2026-02-18T08:38:09.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: id: 10acd82f679c 2026-02-18T08:38:09.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: block_name_prefix: rbd_data.10acd82f679c 2026-02-18T08:38:09.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: format: 2 2026-02-18T08:38:09.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: features: layering, exclusive-lock, journaling 2026-02-18T08:38:09.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: op_features: 2026-02-18T08:38:09.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: flags: 2026-02-18T08:38:09.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: create_timestamp: Wed Feb 18 08:28:37 2026 2026-02-18T08:38:09.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: access_timestamp: Wed Feb 18 08:28:37 2026 2026-02-18T08:38:09.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: modify_timestamp: Wed Feb 18 08:30:39 2026 2026-02-18T08:38:09.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: protected: False 2026-02-18T08:38:09.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: journal: 10acd82f679c 2026-02-18T08:38:09.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring state: enabled 2026-02-18T08:38:09.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring mode: journal 2026-02-18T08:38:09.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring global id: 77915acb-cd5d-4f78-82aa-b6d6e424e3b3 2026-02-18T08:38:09.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring primary: false 2026-02-18T08:38:09.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:38:09.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ compare_image_snaps mirror image_17 snap 2026-02-18T08:38:09.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:38:09.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_17 2026-02-18T08:38:09.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap 2026-02-18T08:38:09.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local ret=0 2026-02-18T08:38:09.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local rmt_export=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_17.export 2026-02-18T08:38:09.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loc_export=/tmp/tmp.vybh1m9yJF/cluster1-mirror-image_17.export 2026-02-18T08:38:09.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_17.export /tmp/tmp.vybh1m9yJF/cluster1-mirror-image_17.export 2026-02-18T08:38:09.944 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 -p mirror export image_17@snap /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_17.export 2026-02-18T08:38:10.297 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-18T08:38:10.304 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster1 -p mirror export image_17@snap /tmp/tmp.vybh1m9yJF/cluster1-mirror-image_17.export 2026-02-18T08:38:10.483 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-18T08:38:10.492 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmp /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_17.export /tmp/tmp.vybh1m9yJF/cluster1-mirror-image_17.export 2026-02-18T08:38:10.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_17.export /tmp/tmp.vybh1m9yJF/cluster1-mirror-image_17.export 2026-02-18T08:38:10.582 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:38:10.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T08:38:10.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_18 2026-02-18T08:38:10.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_snap cluster2 mirror image_18 snap 2026-02-18T08:38:10.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:38:10.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:38:10.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_18 2026-02-18T08:38:10.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap 2026-02-18T08:38:10.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 -p mirror snap create image_18@snap --debug-rbd=20 --debug-journaler=20 2026-02-18T08:38:13.895 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_replay_started cluster1 mirror image_18 2026-02-18T08:38:13.896 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:38:13.896 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:38:13.896 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_18 2026-02-18T08:38:13.896 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_replay_state cluster1 mirror image_18 started 2026-02-18T08:38:13.896 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:38:13.896 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:38:13.896 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_18 2026-02-18T08:38:13.896 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local state=started 2026-02-18T08:38:13.896 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-18T08:38:13.896 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:38:13.897 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1 2026-02-18T08:38:14.897 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test_image_replay_state cluster1 mirror image_18 started 2026-02-18T08:38:14.897 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:38:14.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:38:14.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_18 2026-02-18T08:38:14.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local test_state=started 2026-02-18T08:38:14.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local status_result 2026-02-18T08:38:14.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_state=stopped 2026-02-18T08:38:14.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_18 2026-02-18T08:38:14.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -i state 2026-02-18T08:38:14.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_instance=cluster1 2026-02-18T08:38:14.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-18T08:38:14.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:38:14.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance=cluster1 2026-02-18T08:38:14.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local loop_instance 2026-02-18T08:38:14.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:38:14.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sleep 0 2026-02-18T08:38:14.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:38:14.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ seq 0 3 2026-02-18T08:38:14.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:38:14.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_18 2026-02-18T08:38:14.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:0 2026-02-18T08:38:14.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-18T08:38:14.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:38:14.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:38:14.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:38:14.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-18T08:38:14.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:38:14.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:38:14.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:38:14.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:38:14.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-18T08:38:14.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:38:14.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:38:14.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:38:14.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:38:14.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:0 2026-02-18T08:38:14.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-18T08:38:14.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-18T08:38:14.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:38:14.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:0 2026-02-18T08:38:14.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T08:38:14.904 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-18T08:38:14.904 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:38:14.904 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:38:14.904 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-18T08:38:14.904 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-18T08:38:14.904 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-18T08:38:14.904 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=0 2026-02-18T08:38:14.904 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=0 2026-02-18T08:38:14.904 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:38:14.911 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:38:14.912 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:38:14.912 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:38:14.912 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_18 2026-02-18T08:38:14.974 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_18'] 2026-02-18T08:38:14.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status : get status for rbd mirror 2026-02-18T08:38:14.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:38:14.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:38:14.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_18 2026-02-18T08:38:14.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:1 2026-02-18T08:38:14.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-18T08:38:14.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:38:14.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:38:14.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:38:14.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-18T08:38:14.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:38:14.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:38:14.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:38:14.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:38:14.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-18T08:38:14.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:38:14.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:38:14.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:38:14.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:38:14.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:1 2026-02-18T08:38:14.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-18T08:38:14.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-18T08:38:14.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:38:14.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:1 2026-02-18T08:38:14.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T08:38:14.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-18T08:38:14.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:38:14.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:38:14.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-18T08:38:14.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-18T08:38:14.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-18T08:38:14.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=1 2026-02-18T08:38:14.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=1 2026-02-18T08:38:14.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:38:14.987 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:38:14.987 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:38:14.987 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:38:14.988 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_18 2026-02-18T08:38:15.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_18'] 2026-02-18T08:38:15.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status : get status for rbd mirror 2026-02-18T08:38:15.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:38:15.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:38:15.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_18 2026-02-18T08:38:15.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:2 2026-02-18T08:38:15.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-18T08:38:15.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:38:15.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:38:15.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:38:15.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-18T08:38:15.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:38:15.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:38:15.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:38:15.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:38:15.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-18T08:38:15.055 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:38:15.055 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:38:15.055 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:38:15.055 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:38:15.055 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:2 2026-02-18T08:38:15.055 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-18T08:38:15.055 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-18T08:38:15.055 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:38:15.055 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:2 2026-02-18T08:38:15.055 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T08:38:15.056 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-18T08:38:15.056 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:38:15.056 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:38:15.056 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-18T08:38:15.056 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-18T08:38:15.056 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-18T08:38:15.056 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=2 2026-02-18T08:38:15.056 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=2 2026-02-18T08:38:15.056 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:38:15.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:38:15.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:38:15.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:38:15.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_18 2026-02-18T08:38:15.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_18'] 2026-02-18T08:38:15.118 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status : get status for rbd mirror 2026-02-18T08:38:15.118 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:38:15.121 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:38:15.121 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/image_18 2026-02-18T08:38:15.121 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:3 2026-02-18T08:38:15.121 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-18T08:38:15.121 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:38:15.121 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:38:15.121 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:38:15.121 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-18T08:38:15.121 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:38:15.122 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:38:15.122 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:38:15.122 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:38:15.122 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-18T08:38:15.122 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:38:15.122 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:38:15.122 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:38:15.122 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:38:15.122 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:3 2026-02-18T08:38:15.122 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-18T08:38:15.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-18T08:38:15.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:38:15.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:3 2026-02-18T08:38:15.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T08:38:15.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-18T08:38:15.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:38:15.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:38:15.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-18T08:38:15.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-18T08:38:15.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-18T08:38:15.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=3 2026-02-18T08:38:15.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=3 2026-02-18T08:38:15.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:38:15.130 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:38:15.131 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:38:15.131 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:38:15.131 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/image_18 2026-02-18T08:38:15.199 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ return 0 2026-02-18T08:38:15.199 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ status_result=' "state": "Replaying"' 2026-02-18T08:38:15.200 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ echo ' "state": "Replaying"' 2026-02-18T08:38:15.200 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ grep -i Replaying 2026-02-18T08:38:15.201 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: "state": "Replaying" 2026-02-18T08:38:15.203 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ current_state=started 2026-02-18T08:38:15.203 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test started = started 2026-02-18T08:38:15.203 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:38:15.203 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror image_18 2026-02-18T08:38:15.203 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_cluster=cluster1 2026-02-18T08:38:15.203 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:38:15.203 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_pool=mirror 2026-02-18T08:38:15.203 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local remote_pool=mirror 2026-02-18T08:38:15.204 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_18 2026-02-18T08:38:15.204 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' journal = journal ']' 2026-02-18T08:38:15.204 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror mirror image_18 2026-02-18T08:38:15.204 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_cluster=cluster1 2026-02-18T08:38:15.204 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:38:15.204 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_pool=mirror 2026-02-18T08:38:15.204 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local remote_pool=mirror 2026-02-18T08:38:15.204 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_18 2026-02-18T08:38:15.204 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-02-18T08:38:15.204 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-02-18T08:38:15.204 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:38:15.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:38:15.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:38:15.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_18 2026-02-18T08:38:15.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:38:15.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:38:15.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_18 2026-02-18T08:38:15.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:38:15.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_18 ']' 2026-02-18T08:38:15.405 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_18' 2026-02-18T08:38:15.405 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_18 2026-02-18T08:38:15.405 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:38:15.405 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:38:15.405 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:38:15.405 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:38:15.405 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:38:15.405 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:38:15.405 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:38:15.405 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:38:15.408 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:38:15.408 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:38:15.408 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_18 2026-02-18T08:38:15.408 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:38:15.408 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:38:15.408 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:38:15.408 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:38:15.409 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:38:15.409 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:38:15.409 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:38:15.409 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:38:15.409 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:38:15.409 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:38:15.409 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:38:15.409 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:38:15.409 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:38:15.409 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:38:15.409 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:38:15.409 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:38:15.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:38:15.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:38:15.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:38:15.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:38:15.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:38:15.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:38:15.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:38:15.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:38:15.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:38:15.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:38:15.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:38:15.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:38:15.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:38:15.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:38:15.418 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:38:15.418 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:38:15.418 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:38:15.418 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_18 2026-02-18T08:38:15.478 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_18'] 2026-02-18T08:38:15.478 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:38:15.478 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:38:15.481 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:38:15.481 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_18 2026-02-18T08:38:15.481 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:38:15.481 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:38:15.481 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:38:15.481 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:38:15.482 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:38:15.482 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:38:15.482 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:38:15.482 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:38:15.482 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:38:15.482 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:38:15.482 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:38:15.482 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:38:15.482 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:38:15.482 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:38:15.482 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:38:15.482 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:38:15.483 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:38:15.483 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:38:15.483 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:38:15.483 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:38:15.483 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:38:15.483 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:38:15.483 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:38:15.483 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:38:15.483 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:38:15.483 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:38:15.483 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:38:15.483 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:38:15.484 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:38:15.484 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:38:15.491 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:38:15.492 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:38:15.492 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:38:15.492 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_18 2026-02-18T08:38:15.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_18'] 2026-02-18T08:38:15.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:38:15.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:38:15.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:38:15.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_18 2026-02-18T08:38:15.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:38:15.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:38:15.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:38:15.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:38:15.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:38:15.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:38:15.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:38:15.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:38:15.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:38:15.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:38:15.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:38:15.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:38:15.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:38:15.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:38:15.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:38:15.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:38:15.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:38:15.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:38:15.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:38:15.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:38:15.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:38:15.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:38:15.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:38:15.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:38:15.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:38:15.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:38:15.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:38:15.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:38:15.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:38:15.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:38:15.558 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:38:15.559 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:38:15.559 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:38:15.559 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_18 2026-02-18T08:38:15.611 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_18'] 2026-02-18T08:38:15.611 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:38:15.612 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:38:15.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:38:15.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/image_18 2026-02-18T08:38:15.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:38:15.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:38:15.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:38:15.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:38:15.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:38:15.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:38:15.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:38:15.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:38:15.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:38:15.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:38:15.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:38:15.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:38:15.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:38:15.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:38:15.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:38:15.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:38:15.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:38:15.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:38:15.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:38:15.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:38:15.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:38:15.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:38:15.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:38:15.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:38:15.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:38:15.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:38:15.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:38:15.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:38:15.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:38:15.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:38:15.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:38:15.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:38:15.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:38:15.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/image_18 2026-02-18T08:38:15.695 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:38:15.696 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_18 2026-02-18T08:38:15.696 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:38:15.696 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:38:15.696 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_18 2026-02-18T08:38:15.696 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_18 '' 2026-02-18T08:38:15.696 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:38:15.696 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:38:15.697 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_18 2026-02-18T08:38:15.697 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:38:15.697 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_18.status 2026-02-18T08:38:15.697 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_18.status 2026-02-18T08:38:15.697 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:38:15.697 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_18.status 2026-02-18T08:38:15.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_18 2026-02-18T08:38:15.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_18.status 2026-02-18T08:38:15.724 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 6 2026-02-18T08:38:15.725 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 6 2026-02-18T08:38:15.725 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:38:15.725 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=25, tag_tid=2, entry_tid=1], [object_number=24, tag_tid=2, entry_tid=0], [object_number=26, tag_tid=1, entry_tid=28022], [object_number=27, tag_tid=1, entry_tid=28019]]], state=connected] 2026-02-18T08:38:15.725 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=26, tag_tid=1, entry_tid=28022], [object_number=25, tag_tid=1, entry_tid=28021], [object_number=24, tag_tid=1, entry_tid=28020], [object_number=27, tag_tid=1, entry_tid=28019]]], state=connected] 2026-02-18T08:38:15.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_18.status 2026-02-18T08:38:15.729 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=25, tag_tid=2, entry_tid=1' 2026-02-18T08:38:15.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_18 2026-02-18T08:38:15.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:38:15.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:38:15.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_18 2026-02-18T08:38:15.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_18 '..*' 2026-02-18T08:38:15.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:38:15.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:38:15.731 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_18 2026-02-18T08:38:15.731 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:38:15.731 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_18.status 2026-02-18T08:38:15.731 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_18.status 2026-02-18T08:38:15.731 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:38:15.732 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_18.status 2026-02-18T08:38:15.732 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_18 2026-02-18T08:38:15.732 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_18.status 2026-02-18T08:38:15.968 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 6 2026-02-18T08:38:15.968 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 6 2026-02-18T08:38:15.968 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:38:15.969 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=25, tag_tid=2, entry_tid=1], [object_number=24, tag_tid=2, entry_tid=0], [object_number=26, tag_tid=1, entry_tid=28022], [object_number=27, tag_tid=1, entry_tid=28019]]], state=connected] 2026-02-18T08:38:15.969 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=26, tag_tid=1, entry_tid=28022], [object_number=25, tag_tid=1, entry_tid=28021], [object_number=24, tag_tid=1, entry_tid=28020], [object_number=27, tag_tid=1, entry_tid=28019]]], state=connected] 2026-02-18T08:38:15.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_18.status 2026-02-18T08:38:15.974 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=26, tag_tid=1, entry_tid=28022' 2026-02-18T08:38:15.974 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=25, tag_tid=2, entry_tid=1' -a 'object_number=25, tag_tid=2, entry_tid=1' = 'object_number=26, tag_tid=1, entry_tid=28022' 2026-02-18T08:38:15.974 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=26, tag_tid=1, entry_tid=28022' '!=' '' 2026-02-18T08:38:15.974 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:38:15.974 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=26, tag_tid=1, entry_tid=28022' = '' 2026-02-18T08:38:15.974 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=26, tag_tid=1, entry_tid=28022' 2026-02-18T08:38:15.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=25, tag_tid=2, entry_tid=1' 2026-02-18T08:38:15.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:38:15.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:38:15.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=2 2026-02-18T08:38:15.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=26, tag_tid=1, entry_tid=28022' 2026-02-18T08:38:15.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:38:15.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:38:15.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=1 2026-02-18T08:38:15.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=25, tag_tid=2, entry_tid=1' 2026-02-18T08:38:15.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:38:15.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:38:15.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:38:15.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=26, tag_tid=1, entry_tid=28022' 2026-02-18T08:38:15.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:38:15.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:38:15.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=28022 2026-02-18T08:38:15.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 2 = 1 -a 1 -le 28022 2026-02-18T08:38:15.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:38:15.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:38:15.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:38:16.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_18 2026-02-18T08:38:16.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:38:16.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:38:16.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_18 2026-02-18T08:38:16.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:38:16.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_18 ']' 2026-02-18T08:38:16.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_18' 2026-02-18T08:38:16.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_18 2026-02-18T08:38:16.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:38:16.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:38:16.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:38:16.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:38:16.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:38:16.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:38:16.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:38:16.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:38:16.189 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:38:16.189 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:38:16.189 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_18 2026-02-18T08:38:16.189 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:38:16.189 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:38:16.189 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:38:16.189 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:38:16.189 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:38:16.190 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:38:16.190 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:38:16.190 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:38:16.190 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:38:16.190 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:38:16.190 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:38:16.190 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:38:16.190 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:38:16.190 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:38:16.190 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:38:16.190 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:38:16.190 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:38:16.190 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:38:16.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:38:16.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:38:16.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:38:16.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:38:16.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:38:16.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:38:16.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:38:16.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:38:16.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:38:16.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:38:16.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:38:16.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:38:16.200 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:38:16.200 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:38:16.200 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:38:16.200 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_18 2026-02-18T08:38:16.251 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_18'] 2026-02-18T08:38:16.251 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:38:16.251 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:38:16.254 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:38:16.254 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_18 2026-02-18T08:38:16.254 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:38:16.254 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:38:16.254 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:38:16.254 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:38:16.254 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:38:16.254 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:38:16.254 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:38:16.255 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:38:16.255 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:38:16.255 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:38:16.255 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:38:16.255 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:38:16.255 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:38:16.255 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:38:16.255 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:38:16.255 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:38:16.255 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:38:16.255 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:38:16.255 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:38:16.256 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:38:16.256 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:38:16.256 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:38:16.256 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:38:16.256 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:38:16.256 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:38:16.256 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:38:16.256 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:38:16.256 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:38:16.256 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:38:16.256 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:38:16.266 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:38:16.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:38:16.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:38:16.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_18 2026-02-18T08:38:16.317 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_18'] 2026-02-18T08:38:16.317 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:38:16.317 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:38:16.320 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:38:16.320 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_18 2026-02-18T08:38:16.320 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:38:16.320 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:38:16.320 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:38:16.320 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:38:16.320 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:38:16.320 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:38:16.320 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:38:16.321 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:38:16.321 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:38:16.321 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:38:16.321 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:38:16.321 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:38:16.321 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:38:16.321 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:38:16.321 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:38:16.321 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:38:16.321 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:38:16.321 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:38:16.321 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:38:16.322 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:38:16.322 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:38:16.322 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:38:16.322 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:38:16.322 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:38:16.322 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:38:16.322 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:38:16.322 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:38:16.322 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:38:16.322 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:38:16.322 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:38:16.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:38:16.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:38:16.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:38:16.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_18 2026-02-18T08:38:16.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_18'] 2026-02-18T08:38:16.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:38:16.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:38:16.395 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:38:16.395 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/image_18 2026-02-18T08:38:16.395 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:38:16.395 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:38:16.395 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:38:16.395 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:38:16.395 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:38:16.395 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:38:16.395 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:38:16.396 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:38:16.396 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:38:16.396 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:38:16.396 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:38:16.396 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:38:16.396 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:38:16.396 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:38:16.396 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:38:16.396 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:38:16.396 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:38:16.396 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:38:16.396 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:38:16.397 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:38:16.397 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:38:16.397 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:38:16.397 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:38:16.397 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:38:16.397 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:38:16.397 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:38:16.397 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:38:16.397 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:38:16.397 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:38:16.397 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:38:16.406 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:38:16.406 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:38:16.407 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:38:16.407 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/image_18 2026-02-18T08:38:16.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:38:16.473 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_18 2026-02-18T08:38:16.473 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:38:16.473 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:38:16.473 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_18 2026-02-18T08:38:16.473 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_18 '' 2026-02-18T08:38:16.473 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:38:16.473 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:38:16.473 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_18 2026-02-18T08:38:16.473 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:38:16.473 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_18.status 2026-02-18T08:38:16.474 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_18.status 2026-02-18T08:38:16.474 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:38:16.474 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_18.status 2026-02-18T08:38:16.475 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_18 2026-02-18T08:38:16.475 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_18.status 2026-02-18T08:38:16.510 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 6 2026-02-18T08:38:16.510 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 6 2026-02-18T08:38:16.510 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:38:16.511 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=25, tag_tid=2, entry_tid=1], [object_number=24, tag_tid=2, entry_tid=0], [object_number=26, tag_tid=1, entry_tid=28022], [object_number=27, tag_tid=1, entry_tid=28019]]], state=connected] 2026-02-18T08:38:16.511 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=26, tag_tid=1, entry_tid=28022], [object_number=25, tag_tid=1, entry_tid=28021], [object_number=24, tag_tid=1, entry_tid=28020], [object_number=27, tag_tid=1, entry_tid=28019]]], state=connected] 2026-02-18T08:38:16.514 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_18.status 2026-02-18T08:38:16.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=25, tag_tid=2, entry_tid=1' 2026-02-18T08:38:16.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_18 2026-02-18T08:38:16.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:38:16.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:38:16.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_18 2026-02-18T08:38:16.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_18 '..*' 2026-02-18T08:38:16.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:38:16.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:38:16.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_18 2026-02-18T08:38:16.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:38:16.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_18.status 2026-02-18T08:38:16.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_18.status 2026-02-18T08:38:16.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:38:16.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_18.status 2026-02-18T08:38:16.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_18 2026-02-18T08:38:16.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_18.status 2026-02-18T08:38:16.552 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 6 2026-02-18T08:38:16.552 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 6 2026-02-18T08:38:16.552 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:38:16.553 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=25, tag_tid=2, entry_tid=1], [object_number=24, tag_tid=2, entry_tid=0], [object_number=26, tag_tid=1, entry_tid=28022], [object_number=27, tag_tid=1, entry_tid=28019]]], state=connected] 2026-02-18T08:38:16.553 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=26, tag_tid=1, entry_tid=28022], [object_number=25, tag_tid=1, entry_tid=28021], [object_number=24, tag_tid=1, entry_tid=28020], [object_number=27, tag_tid=1, entry_tid=28019]]], state=connected] 2026-02-18T08:38:16.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_18.status 2026-02-18T08:38:16.558 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=26, tag_tid=1, entry_tid=28022' 2026-02-18T08:38:16.558 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=25, tag_tid=2, entry_tid=1' -a 'object_number=25, tag_tid=2, entry_tid=1' = 'object_number=26, tag_tid=1, entry_tid=28022' 2026-02-18T08:38:16.558 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=26, tag_tid=1, entry_tid=28022' '!=' 'object_number=26, tag_tid=1, entry_tid=28022' 2026-02-18T08:38:16.558 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:38:16.558 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.4 2026-02-18T08:38:16.959 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_18 2026-02-18T08:38:16.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:38:16.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:38:16.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_18 2026-02-18T08:38:16.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:38:16.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_18 ']' 2026-02-18T08:38:16.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_18' 2026-02-18T08:38:16.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_18 2026-02-18T08:38:16.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:38:16.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:38:16.961 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:38:16.961 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:38:16.961 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:38:16.961 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:38:16.961 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:38:16.963 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:38:16.963 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:38:16.964 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:38:16.964 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_18 2026-02-18T08:38:16.964 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:38:16.964 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:38:16.964 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:38:16.964 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:38:16.964 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:38:16.964 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:38:16.964 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:38:16.964 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:38:16.964 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:38:16.965 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:38:16.965 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:38:16.965 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:38:16.965 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:38:16.965 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:38:16.965 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:38:16.965 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:38:16.965 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:38:16.965 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:38:16.965 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:38:16.965 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:38:16.965 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:38:16.966 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:38:16.966 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:38:16.966 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:38:16.966 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:38:16.966 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:38:16.966 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:38:16.966 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:38:16.966 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:38:16.966 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:38:16.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:38:16.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:38:16.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:38:16.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_18 2026-02-18T08:38:17.029 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_18'] 2026-02-18T08:38:17.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:38:17.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:38:17.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:38:17.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_18 2026-02-18T08:38:17.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:38:17.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:38:17.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:38:17.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:38:17.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:38:17.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:38:17.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:38:17.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:38:17.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:38:17.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:38:17.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:38:17.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:38:17.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:38:17.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:38:17.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:38:17.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:38:17.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:38:17.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:38:17.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:38:17.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:38:17.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:38:17.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:38:17.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:38:17.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:38:17.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:38:17.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:38:17.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:38:17.035 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:38:17.035 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:38:17.035 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:38:17.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:38:17.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:38:17.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:38:17.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_18 2026-02-18T08:38:17.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_18'] 2026-02-18T08:38:17.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:38:17.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:38:17.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:38:17.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_18 2026-02-18T08:38:17.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:38:17.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:38:17.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:38:17.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:38:17.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:38:17.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:38:17.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:38:17.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:38:17.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:38:17.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:38:17.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:38:17.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:38:17.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:38:17.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:38:17.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:38:17.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:38:17.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:38:17.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:38:17.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:38:17.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:38:17.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:38:17.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:38:17.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:38:17.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:38:17.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:38:17.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:38:17.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:38:17.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:38:17.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:38:17.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:38:17.118 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:38:17.119 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:38:17.119 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:38:17.119 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_18 2026-02-18T08:38:17.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_18'] 2026-02-18T08:38:17.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:38:17.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:38:17.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:38:17.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/image_18 2026-02-18T08:38:17.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:38:17.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:38:17.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:38:17.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:38:17.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:38:17.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:38:17.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:38:17.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:38:17.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:38:17.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:38:17.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:38:17.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:38:17.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:38:17.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:38:17.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:38:17.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:38:17.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:38:17.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:38:17.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:38:17.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:38:17.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:38:17.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:38:17.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:38:17.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:38:17.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:38:17.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:38:17.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:38:17.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:38:17.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:38:17.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:38:17.194 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:38:17.195 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:38:17.195 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:38:17.195 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/image_18 2026-02-18T08:38:17.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:38:17.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_18 2026-02-18T08:38:17.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:38:17.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:38:17.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_18 2026-02-18T08:38:17.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_18 '' 2026-02-18T08:38:17.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:38:17.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:38:17.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_18 2026-02-18T08:38:17.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:38:17.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_18.status 2026-02-18T08:38:17.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_18.status 2026-02-18T08:38:17.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:38:17.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_18.status 2026-02-18T08:38:17.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_18 2026-02-18T08:38:17.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_18.status 2026-02-18T08:38:17.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 6 2026-02-18T08:38:17.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 6 2026-02-18T08:38:17.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:38:17.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=25, tag_tid=2, entry_tid=1], [object_number=24, tag_tid=2, entry_tid=0], [object_number=26, tag_tid=1, entry_tid=28022], [object_number=27, tag_tid=1, entry_tid=28019]]], state=connected] 2026-02-18T08:38:17.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=26, tag_tid=1, entry_tid=28022], [object_number=25, tag_tid=1, entry_tid=28021], [object_number=24, tag_tid=1, entry_tid=28020], [object_number=27, tag_tid=1, entry_tid=28019]]], state=connected] 2026-02-18T08:38:17.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_18.status 2026-02-18T08:38:17.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=25, tag_tid=2, entry_tid=1' 2026-02-18T08:38:17.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_18 2026-02-18T08:38:17.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:38:17.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:38:17.310 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_18 2026-02-18T08:38:17.310 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_18 '..*' 2026-02-18T08:38:17.310 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:38:17.310 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:38:17.310 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_18 2026-02-18T08:38:17.310 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:38:17.310 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_18.status 2026-02-18T08:38:17.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_18.status 2026-02-18T08:38:17.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:38:17.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_18.status 2026-02-18T08:38:17.312 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_18 2026-02-18T08:38:17.312 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_18.status 2026-02-18T08:38:17.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 6 2026-02-18T08:38:17.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 6 2026-02-18T08:38:17.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:38:17.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=25, tag_tid=2, entry_tid=1], [object_number=24, tag_tid=2, entry_tid=0], [object_number=26, tag_tid=1, entry_tid=28022], [object_number=27, tag_tid=1, entry_tid=28019]]], state=connected] 2026-02-18T08:38:17.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=26, tag_tid=1, entry_tid=28022], [object_number=25, tag_tid=1, entry_tid=28021], [object_number=24, tag_tid=1, entry_tid=28020], [object_number=27, tag_tid=1, entry_tid=28019]]], state=connected] 2026-02-18T08:38:17.350 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_18.status 2026-02-18T08:38:17.352 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=26, tag_tid=1, entry_tid=28022' 2026-02-18T08:38:17.352 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=25, tag_tid=2, entry_tid=1' -a 'object_number=25, tag_tid=2, entry_tid=1' = 'object_number=26, tag_tid=1, entry_tid=28022' 2026-02-18T08:38:17.352 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=26, tag_tid=1, entry_tid=28022' '!=' 'object_number=26, tag_tid=1, entry_tid=28022' 2026-02-18T08:38:17.352 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:38:17.352 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.8 2026-02-18T08:38:18.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_18 2026-02-18T08:38:18.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:38:18.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:38:18.155 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_18 2026-02-18T08:38:18.155 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:38:18.155 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_18 ']' 2026-02-18T08:38:18.155 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_18' 2026-02-18T08:38:18.155 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_18 2026-02-18T08:38:18.155 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:38:18.155 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:38:18.155 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:38:18.155 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:38:18.155 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:38:18.156 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:38:18.156 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:38:18.158 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:38:18.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:38:18.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:38:18.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_18 2026-02-18T08:38:18.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:38:18.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:38:18.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:38:18.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:38:18.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:38:18.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:38:18.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:38:18.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:38:18.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:38:18.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:38:18.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:38:18.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:38:18.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:38:18.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:38:18.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:38:18.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:38:18.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:38:18.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:38:18.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:38:18.161 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:38:18.161 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:38:18.161 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:38:18.161 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:38:18.161 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:38:18.161 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:38:18.161 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:38:18.161 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:38:18.161 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:38:18.161 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:38:18.161 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:38:18.170 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:38:18.171 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:38:18.171 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:38:18.171 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_18 2026-02-18T08:38:18.231 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_18'] 2026-02-18T08:38:18.231 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:38:18.231 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:38:18.234 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:38:18.234 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_18 2026-02-18T08:38:18.234 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:38:18.234 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:38:18.234 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:38:18.234 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:38:18.234 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:38:18.234 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:38:18.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:38:18.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:38:18.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:38:18.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:38:18.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:38:18.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:38:18.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:38:18.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:38:18.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:38:18.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:38:18.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:38:18.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:38:18.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:38:18.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:38:18.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:38:18.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:38:18.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:38:18.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:38:18.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:38:18.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:38:18.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:38:18.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:38:18.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:38:18.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:38:18.245 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:38:18.246 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:38:18.246 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:38:18.246 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_18 2026-02-18T08:38:18.299 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_18'] 2026-02-18T08:38:18.299 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:38:18.299 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:38:18.302 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:38:18.302 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_18 2026-02-18T08:38:18.302 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:38:18.302 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:38:18.302 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:38:18.302 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:38:18.302 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:38:18.302 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:38:18.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:38:18.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:38:18.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:38:18.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:38:18.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:38:18.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:38:18.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:38:18.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:38:18.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:38:18.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:38:18.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:38:18.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:38:18.304 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:38:18.304 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:38:18.304 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:38:18.304 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:38:18.304 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:38:18.304 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:38:18.304 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:38:18.304 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:38:18.304 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:38:18.304 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:38:18.304 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:38:18.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:38:18.314 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:38:18.314 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:38:18.314 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:38:18.315 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_18 2026-02-18T08:38:18.366 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_18'] 2026-02-18T08:38:18.366 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:38:18.366 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:38:18.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:38:18.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/image_18 2026-02-18T08:38:18.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:38:18.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:38:18.370 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:38:18.370 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:38:18.370 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:38:18.370 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:38:18.370 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:38:18.370 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:38:18.370 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:38:18.370 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:38:18.370 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:38:18.370 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:38:18.370 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:38:18.370 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:38:18.371 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:38:18.371 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:38:18.371 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:38:18.371 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:38:18.371 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:38:18.371 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:38:18.371 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:38:18.371 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:38:18.371 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:38:18.371 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:38:18.371 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:38:18.371 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:38:18.372 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:38:18.372 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:38:18.372 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:38:18.372 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:38:18.379 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:38:18.380 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:38:18.380 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:38:18.380 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/image_18 2026-02-18T08:38:18.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:38:18.448 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_18 2026-02-18T08:38:18.448 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:38:18.448 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:38:18.448 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_18 2026-02-18T08:38:18.449 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_18 '' 2026-02-18T08:38:18.449 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:38:18.449 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:38:18.449 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_18 2026-02-18T08:38:18.449 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:38:18.449 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_18.status 2026-02-18T08:38:18.449 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_18.status 2026-02-18T08:38:18.449 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:38:18.449 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_18.status 2026-02-18T08:38:18.450 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_18 2026-02-18T08:38:18.450 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_18.status 2026-02-18T08:38:18.478 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 6 2026-02-18T08:38:18.478 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 6 2026-02-18T08:38:18.478 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:38:18.478 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=25, tag_tid=2, entry_tid=1], [object_number=24, tag_tid=2, entry_tid=0], [object_number=26, tag_tid=1, entry_tid=28022], [object_number=27, tag_tid=1, entry_tid=28019]]], state=connected] 2026-02-18T08:38:18.479 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=26, tag_tid=1, entry_tid=28022], [object_number=25, tag_tid=1, entry_tid=28021], [object_number=24, tag_tid=1, entry_tid=28020], [object_number=27, tag_tid=1, entry_tid=28019]]], state=connected] 2026-02-18T08:38:18.481 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_18.status 2026-02-18T08:38:18.483 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=25, tag_tid=2, entry_tid=1' 2026-02-18T08:38:18.484 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_18 2026-02-18T08:38:18.484 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:38:18.484 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:38:18.484 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_18 2026-02-18T08:38:18.484 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_18 '..*' 2026-02-18T08:38:18.484 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:38:18.485 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:38:18.485 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_18 2026-02-18T08:38:18.485 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:38:18.485 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_18.status 2026-02-18T08:38:18.485 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_18.status 2026-02-18T08:38:18.485 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:38:18.485 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_18.status 2026-02-18T08:38:18.486 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_18 2026-02-18T08:38:18.486 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_18.status 2026-02-18T08:38:18.513 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 6 2026-02-18T08:38:18.513 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 6 2026-02-18T08:38:18.513 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:38:18.513 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=25, tag_tid=2, entry_tid=1], [object_number=24, tag_tid=2, entry_tid=0], [object_number=26, tag_tid=1, entry_tid=28022], [object_number=27, tag_tid=1, entry_tid=28019]]], state=connected] 2026-02-18T08:38:18.513 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=26, tag_tid=1, entry_tid=28022], [object_number=25, tag_tid=1, entry_tid=28021], [object_number=24, tag_tid=1, entry_tid=28020], [object_number=27, tag_tid=1, entry_tid=28019]]], state=connected] 2026-02-18T08:38:18.516 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_18.status 2026-02-18T08:38:18.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=26, tag_tid=1, entry_tid=28022' 2026-02-18T08:38:18.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=25, tag_tid=2, entry_tid=1' -a 'object_number=25, tag_tid=2, entry_tid=1' = 'object_number=26, tag_tid=1, entry_tid=28022' 2026-02-18T08:38:18.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=26, tag_tid=1, entry_tid=28022' '!=' 'object_number=26, tag_tid=1, entry_tid=28022' 2026-02-18T08:38:18.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:38:18.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1.6 2026-02-18T08:38:20.120 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_18 2026-02-18T08:38:20.120 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:38:20.120 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:38:20.120 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_18 2026-02-18T08:38:20.120 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:38:20.121 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_18 ']' 2026-02-18T08:38:20.121 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_18' 2026-02-18T08:38:20.121 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_18 2026-02-18T08:38:20.121 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:38:20.121 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:38:20.121 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:38:20.121 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:38:20.121 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:38:20.121 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:38:20.121 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:38:20.122 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:38:20.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:38:20.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:38:20.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_18 2026-02-18T08:38:20.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:38:20.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:38:20.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:38:20.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:38:20.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:38:20.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:38:20.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:38:20.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:38:20.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:38:20.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:38:20.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:38:20.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:38:20.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:38:20.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:38:20.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:38:20.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:38:20.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:38:20.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:38:20.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:38:20.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:38:20.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:38:20.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:38:20.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:38:20.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:38:20.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:38:20.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:38:20.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:38:20.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:38:20.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:38:20.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:38:20.134 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:38:20.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:38:20.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:38:20.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_18 2026-02-18T08:38:20.196 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_18'] 2026-02-18T08:38:20.196 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:38:20.196 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:38:20.199 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:38:20.199 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_18 2026-02-18T08:38:20.199 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:38:20.199 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:38:20.199 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:38:20.199 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:38:20.199 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:38:20.199 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:38:20.200 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:38:20.200 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:38:20.200 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:38:20.200 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:38:20.200 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:38:20.200 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:38:20.200 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:38:20.200 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:38:20.200 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:38:20.200 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:38:20.200 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:38:20.200 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:38:20.200 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:38:20.201 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:38:20.201 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:38:20.201 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:38:20.201 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:38:20.201 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:38:20.201 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:38:20.201 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:38:20.201 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:38:20.201 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:38:20.201 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:38:20.201 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:38:20.210 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:38:20.210 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:38:20.210 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:38:20.210 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_18 2026-02-18T08:38:20.274 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_18'] 2026-02-18T08:38:20.274 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:38:20.274 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:38:20.277 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:38:20.277 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_18 2026-02-18T08:38:20.277 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:38:20.277 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:38:20.277 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:38:20.277 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:38:20.277 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:38:20.278 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:38:20.278 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:38:20.278 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:38:20.278 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:38:20.278 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:38:20.278 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:38:20.278 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:38:20.278 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:38:20.278 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:38:20.278 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:38:20.278 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:38:20.278 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:38:20.279 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:38:20.279 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:38:20.279 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:38:20.279 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:38:20.279 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:38:20.279 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:38:20.279 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:38:20.279 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:38:20.279 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:38:20.279 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:38:20.279 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:38:20.279 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:38:20.280 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:38:20.287 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:38:20.288 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:38:20.288 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:38:20.288 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_18 2026-02-18T08:38:20.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_18'] 2026-02-18T08:38:20.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:38:20.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:38:20.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:38:20.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/image_18 2026-02-18T08:38:20.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:38:20.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:38:20.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:38:20.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:38:20.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:38:20.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:38:20.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:38:20.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:38:20.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:38:20.350 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:38:20.350 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:38:20.350 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:38:20.350 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:38:20.350 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:38:20.350 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:38:20.350 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:38:20.350 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:38:20.350 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:38:20.350 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:38:20.350 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:38:20.350 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:38:20.351 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:38:20.351 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:38:20.351 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:38:20.351 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:38:20.351 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:38:20.351 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:38:20.351 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:38:20.351 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:38:20.351 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:38:20.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:38:20.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:38:20.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:38:20.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/image_18 2026-02-18T08:38:20.415 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:38:20.416 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_18 2026-02-18T08:38:20.416 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:38:20.416 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:38:20.416 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_18 2026-02-18T08:38:20.416 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_18 '' 2026-02-18T08:38:20.416 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:38:20.416 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:38:20.417 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_18 2026-02-18T08:38:20.417 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:38:20.417 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_18.status 2026-02-18T08:38:20.417 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_18.status 2026-02-18T08:38:20.417 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:38:20.418 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_18.status 2026-02-18T08:38:20.418 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_18 2026-02-18T08:38:20.418 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_18.status 2026-02-18T08:38:20.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 6 2026-02-18T08:38:20.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 6 2026-02-18T08:38:20.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:38:20.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=25, tag_tid=2, entry_tid=1], [object_number=24, tag_tid=2, entry_tid=0], [object_number=26, tag_tid=1, entry_tid=28022], [object_number=27, tag_tid=1, entry_tid=28019]]], state=connected] 2026-02-18T08:38:20.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=26, tag_tid=1, entry_tid=28022], [object_number=25, tag_tid=1, entry_tid=28021], [object_number=24, tag_tid=1, entry_tid=28020], [object_number=27, tag_tid=1, entry_tid=28019]]], state=connected] 2026-02-18T08:38:20.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_18.status 2026-02-18T08:38:20.449 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=25, tag_tid=2, entry_tid=1' 2026-02-18T08:38:20.450 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_18 2026-02-18T08:38:20.450 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:38:20.450 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:38:20.450 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_18 2026-02-18T08:38:20.450 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_18 '..*' 2026-02-18T08:38:20.450 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:38:20.451 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:38:20.451 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_18 2026-02-18T08:38:20.451 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:38:20.451 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_18.status 2026-02-18T08:38:20.451 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_18.status 2026-02-18T08:38:20.451 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:38:20.452 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_18.status 2026-02-18T08:38:20.452 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_18 2026-02-18T08:38:20.452 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_18.status 2026-02-18T08:38:20.486 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 6 2026-02-18T08:38:20.486 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 6 2026-02-18T08:38:20.486 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:38:20.487 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=25, tag_tid=2, entry_tid=1], [object_number=24, tag_tid=2, entry_tid=0], [object_number=26, tag_tid=1, entry_tid=28022], [object_number=27, tag_tid=1, entry_tid=28019]]], state=connected] 2026-02-18T08:38:20.487 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=26, tag_tid=1, entry_tid=28022], [object_number=25, tag_tid=1, entry_tid=28021], [object_number=24, tag_tid=1, entry_tid=28020], [object_number=27, tag_tid=1, entry_tid=28019]]], state=connected] 2026-02-18T08:38:20.490 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_18.status 2026-02-18T08:38:20.491 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=26, tag_tid=1, entry_tid=28022' 2026-02-18T08:38:20.491 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=25, tag_tid=2, entry_tid=1' -a 'object_number=25, tag_tid=2, entry_tid=1' = 'object_number=26, tag_tid=1, entry_tid=28022' 2026-02-18T08:38:20.491 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=26, tag_tid=1, entry_tid=28022' '!=' 'object_number=26, tag_tid=1, entry_tid=28022' 2026-02-18T08:38:20.491 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:38:20.491 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 2 2026-02-18T08:38:22.493 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_18 2026-02-18T08:38:22.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:38:22.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:38:22.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_18 2026-02-18T08:38:22.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:38:22.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_18 ']' 2026-02-18T08:38:22.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_18' 2026-02-18T08:38:22.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_18 2026-02-18T08:38:22.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:38:22.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:38:22.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:38:22.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:38:22.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:38:22.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:38:22.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:38:22.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:38:22.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:38:22.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:38:22.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_18 2026-02-18T08:38:22.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:38:22.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:38:22.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:38:22.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:38:22.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:38:22.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:38:22.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:38:22.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:38:22.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:38:22.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:38:22.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:38:22.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:38:22.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:38:22.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:38:22.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:38:22.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:38:22.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:38:22.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:38:22.500 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:38:22.500 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:38:22.500 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:38:22.500 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:38:22.500 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:38:22.500 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:38:22.500 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:38:22.500 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:38:22.500 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:38:22.500 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:38:22.500 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:38:22.501 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:38:22.507 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:38:22.508 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:38:22.508 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:38:22.508 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_18 2026-02-18T08:38:22.561 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_18'] 2026-02-18T08:38:22.561 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:38:22.561 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:38:22.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:38:22.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_18 2026-02-18T08:38:22.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:38:22.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:38:22.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:38:22.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:38:22.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:38:22.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:38:22.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:38:22.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:38:22.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:38:22.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:38:22.565 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:38:22.565 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:38:22.565 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:38:22.565 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:38:22.565 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:38:22.565 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:38:22.565 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:38:22.565 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:38:22.565 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:38:22.565 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:38:22.565 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:38:22.566 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:38:22.566 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:38:22.566 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:38:22.566 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:38:22.566 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:38:22.566 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:38:22.566 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:38:22.566 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:38:22.566 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:38:22.574 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:38:22.575 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:38:22.575 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:38:22.575 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_18 2026-02-18T08:38:22.635 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_18'] 2026-02-18T08:38:22.635 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:38:22.635 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:38:22.638 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:38:22.638 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_18 2026-02-18T08:38:22.638 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:38:22.638 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:38:22.638 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:38:22.638 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:38:22.638 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:38:22.639 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:38:22.639 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:38:22.639 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:38:22.639 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:38:22.639 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:38:22.639 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:38:22.639 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:38:22.639 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:38:22.639 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:38:22.639 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:38:22.639 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:38:22.639 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:38:22.639 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:38:22.640 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:38:22.640 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:38:22.640 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:38:22.640 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:38:22.640 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:38:22.640 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:38:22.640 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:38:22.640 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:38:22.640 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:38:22.640 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:38:22.640 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:38:22.640 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:38:22.649 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:38:22.649 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:38:22.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:38:22.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_18 2026-02-18T08:38:22.701 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_18'] 2026-02-18T08:38:22.701 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:38:22.701 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:38:22.705 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:38:22.705 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/image_18 2026-02-18T08:38:22.705 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:38:22.705 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:38:22.705 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:38:22.705 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:38:22.705 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:38:22.705 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:38:22.705 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:38:22.705 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:38:22.706 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:38:22.706 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:38:22.706 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:38:22.706 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:38:22.706 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:38:22.706 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:38:22.706 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:38:22.706 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:38:22.706 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:38:22.706 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:38:22.706 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:38:22.706 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:38:22.707 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:38:22.707 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:38:22.707 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:38:22.707 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:38:22.707 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:38:22.707 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:38:22.707 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:38:22.707 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:38:22.707 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:38:22.707 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:38:22.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:38:22.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:38:22.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:38:22.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/image_18 2026-02-18T08:38:22.781 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:38:22.782 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_18 2026-02-18T08:38:22.782 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:38:22.783 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:38:22.783 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_18 2026-02-18T08:38:22.783 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_18 '' 2026-02-18T08:38:22.783 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:38:22.783 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:38:22.783 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_18 2026-02-18T08:38:22.783 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:38:22.783 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_18.status 2026-02-18T08:38:22.783 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_18.status 2026-02-18T08:38:22.783 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:38:22.784 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_18.status 2026-02-18T08:38:22.784 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_18 2026-02-18T08:38:22.784 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_18.status 2026-02-18T08:38:22.819 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 6 2026-02-18T08:38:22.819 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 6 2026-02-18T08:38:22.819 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:38:22.820 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=25, tag_tid=2, entry_tid=1], [object_number=24, tag_tid=2, entry_tid=0], [object_number=26, tag_tid=1, entry_tid=28022], [object_number=27, tag_tid=1, entry_tid=28019]]], state=connected] 2026-02-18T08:38:22.820 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=26, tag_tid=1, entry_tid=28022], [object_number=25, tag_tid=1, entry_tid=28021], [object_number=24, tag_tid=1, entry_tid=28020], [object_number=27, tag_tid=1, entry_tid=28019]]], state=connected] 2026-02-18T08:38:22.823 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_18.status 2026-02-18T08:38:22.825 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=25, tag_tid=2, entry_tid=1' 2026-02-18T08:38:22.826 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_18 2026-02-18T08:38:22.826 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:38:22.826 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:38:22.827 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_18 2026-02-18T08:38:22.827 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_18 '..*' 2026-02-18T08:38:22.827 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:38:22.827 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:38:22.827 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_18 2026-02-18T08:38:22.827 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:38:22.827 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_18.status 2026-02-18T08:38:22.828 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_18.status 2026-02-18T08:38:22.828 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:38:22.828 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_18.status 2026-02-18T08:38:22.829 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_18 2026-02-18T08:38:22.829 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_18.status 2026-02-18T08:38:22.864 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 6 2026-02-18T08:38:22.864 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 6 2026-02-18T08:38:22.864 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:38:22.864 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=25, tag_tid=2, entry_tid=1], [object_number=24, tag_tid=2, entry_tid=0], [object_number=26, tag_tid=1, entry_tid=28022], [object_number=27, tag_tid=1, entry_tid=28019]]], state=connected] 2026-02-18T08:38:22.864 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=26, tag_tid=1, entry_tid=28022], [object_number=25, tag_tid=1, entry_tid=28021], [object_number=24, tag_tid=1, entry_tid=28020], [object_number=27, tag_tid=1, entry_tid=28019]]], state=connected] 2026-02-18T08:38:22.868 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_18.status 2026-02-18T08:38:22.870 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=26, tag_tid=1, entry_tid=28022' 2026-02-18T08:38:22.870 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=25, tag_tid=2, entry_tid=1' -a 'object_number=25, tag_tid=2, entry_tid=1' = 'object_number=26, tag_tid=1, entry_tid=28022' 2026-02-18T08:38:22.870 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=26, tag_tid=1, entry_tid=28022' '!=' 'object_number=26, tag_tid=1, entry_tid=28022' 2026-02-18T08:38:22.870 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:38:22.870 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 2 2026-02-18T08:38:23.746 INFO:tasks.rbd_mirror_thrash:kill cluster1.client.mirror.0 2026-02-18T08:38:23.746 INFO:tasks.rbd_mirror.cluster1.client.mirror.0:Sent signal 15 2026-02-18T08:38:23.746 INFO:tasks.rbd_mirror_thrash:waiting for 7.252625132824613 secs before reviving daemons 2026-02-18T08:38:23.747 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.trial163.stderr:2026-02-18T08:38:23.738+0000 7f401c103640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.0 (PID: 86664) UID: 1000 2026-02-18T08:38:24.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_18 2026-02-18T08:38:24.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:38:24.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:38:24.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_18 2026-02-18T08:38:24.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:38:24.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_18 ']' 2026-02-18T08:38:24.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_18' 2026-02-18T08:38:24.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_18 2026-02-18T08:38:24.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:38:24.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:38:24.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:38:24.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:38:24.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:38:24.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:38:24.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:38:24.876 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:38:24.876 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:38:24.876 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:38:24.876 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_18 2026-02-18T08:38:24.876 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:38:24.876 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:38:24.876 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:38:24.877 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:38:24.877 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:38:24.877 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:38:24.877 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:38:24.877 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:38:24.877 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:38:24.877 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:38:24.877 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:38:24.877 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:38:24.877 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:38:24.877 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:38:24.878 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:38:24.878 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:38:24.878 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:38:24.878 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:38:24.878 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:38:24.878 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:38:24.878 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:38:24.878 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:38:24.878 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:38:24.878 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:38:24.879 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:38:24.879 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:38:24.879 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:38:24.879 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:38:24.879 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:38:24.879 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:38:24.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:38:24.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:38:24.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:38:24.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_18 2026-02-18T08:38:24.897 INFO:tasks.ceph.cluster2.osd.2.trial163.stderr:2026-02-18T08:38:24.890+0000 7f2b60c87640 -1 reset not still connected to 0x55fc298f5e10 2026-02-18T08:38:24.947 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: exception getting command descriptions: [Errno 2] No such file or directory 2026-02-18T08:38:24.950 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:38:24.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_18 2026-02-18T08:38:24.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:38:24.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:38:24.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:38:24.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:38:24.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:38:24.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:38:24.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:38:24.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:38:24.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:38:24.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:38:24.952 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:38:24.952 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:38:24.952 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:38:24.952 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:38:24.952 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:38:24.952 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:38:24.952 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:38:24.952 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:38:24.952 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:38:24.952 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:38:24.952 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:38:24.952 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:38:24.953 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:38:24.953 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:38:24.953 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:38:24.953 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:38:24.953 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:38:24.953 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:38:24.953 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:38:24.953 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:38:24.964 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:38:24.965 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:38:24.965 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:38:24.965 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_18 2026-02-18T08:38:25.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_18'] 2026-02-18T08:38:25.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:38:25.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:38:25.028 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:38:25.028 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_18 2026-02-18T08:38:25.028 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:38:25.028 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:38:25.028 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:38:25.029 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:38:25.029 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:38:25.029 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:38:25.029 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:38:25.029 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:38:25.029 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:38:25.029 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:38:25.029 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:38:25.029 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:38:25.029 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:38:25.029 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:38:25.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:38:25.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:38:25.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:38:25.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:38:25.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:38:25.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:38:25.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:38:25.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:38:25.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:38:25.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:38:25.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:38:25.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:38:25.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:38:25.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:38:25.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:38:25.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:38:25.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:38:25.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:38:25.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:38:25.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_18 2026-02-18T08:38:25.100 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_18'] 2026-02-18T08:38:25.101 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:38:25.101 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:38:25.105 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:38:25.105 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/image_18 2026-02-18T08:38:25.105 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:38:25.105 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:38:25.105 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:38:25.105 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:38:25.105 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:38:25.105 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:38:25.105 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:38:25.105 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:38:25.106 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:38:25.106 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:38:25.106 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:38:25.106 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:38:25.106 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:38:25.106 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:38:25.106 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:38:25.106 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:38:25.106 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:38:25.106 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:38:25.106 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:38:25.106 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:38:25.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:38:25.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:38:25.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:38:25.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:38:25.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:38:25.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:38:25.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:38:25.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:38:25.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:38:25.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:38:25.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:38:25.118 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:38:25.118 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:38:25.118 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/image_18 2026-02-18T08:38:25.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:38:25.187 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_18 2026-02-18T08:38:25.187 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:38:25.187 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:38:25.187 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_18 2026-02-18T08:38:25.187 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_18 '' 2026-02-18T08:38:25.188 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:38:25.188 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:38:25.188 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_18 2026-02-18T08:38:25.188 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:38:25.188 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_18.status 2026-02-18T08:38:25.188 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_18.status 2026-02-18T08:38:25.189 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:38:25.189 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_18.status 2026-02-18T08:38:25.190 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_18 2026-02-18T08:38:25.190 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_18.status 2026-02-18T08:38:25.219 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 6 2026-02-18T08:38:25.220 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 6 2026-02-18T08:38:25.220 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:38:25.220 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=25, tag_tid=2, entry_tid=1], [object_number=24, tag_tid=2, entry_tid=0], [object_number=26, tag_tid=1, entry_tid=28022], [object_number=27, tag_tid=1, entry_tid=28019]]], state=connected] 2026-02-18T08:38:25.220 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=25, tag_tid=2, entry_tid=1], [object_number=24, tag_tid=2, entry_tid=0], [object_number=26, tag_tid=1, entry_tid=28022], [object_number=27, tag_tid=1, entry_tid=28019]]], state=connected] 2026-02-18T08:38:25.222 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_18.status 2026-02-18T08:38:25.224 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=25, tag_tid=2, entry_tid=1' 2026-02-18T08:38:25.225 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_18 2026-02-18T08:38:25.225 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:38:25.225 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:38:25.225 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_18 2026-02-18T08:38:25.225 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_18 '..*' 2026-02-18T08:38:25.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:38:25.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:38:25.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_18 2026-02-18T08:38:25.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:38:25.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_18.status 2026-02-18T08:38:25.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_18.status 2026-02-18T08:38:25.227 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:38:25.227 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_18.status 2026-02-18T08:38:25.227 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_18 2026-02-18T08:38:25.227 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_18.status 2026-02-18T08:38:25.257 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 6 2026-02-18T08:38:25.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 6 2026-02-18T08:38:25.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:38:25.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=25, tag_tid=2, entry_tid=1], [object_number=24, tag_tid=2, entry_tid=0], [object_number=26, tag_tid=1, entry_tid=28022], [object_number=27, tag_tid=1, entry_tid=28019]]], state=connected] 2026-02-18T08:38:25.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=25, tag_tid=2, entry_tid=1], [object_number=24, tag_tid=2, entry_tid=0], [object_number=26, tag_tid=1, entry_tid=28022], [object_number=27, tag_tid=1, entry_tid=28019]]], state=connected] 2026-02-18T08:38:25.260 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_18.status 2026-02-18T08:38:25.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=25, tag_tid=2, entry_tid=1' 2026-02-18T08:38:25.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=25, tag_tid=2, entry_tid=1' -a 'object_number=25, tag_tid=2, entry_tid=1' = 'object_number=25, tag_tid=2, entry_tid=1' 2026-02-18T08:38:25.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:38:25.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_snap_present cluster1 mirror image_18 snap 2026-02-18T08:38:25.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:38:25.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:38:25.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_18 2026-02-18T08:38:25.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap 2026-02-18T08:38:25.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-18T08:38:25.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-18T08:38:25.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1 2026-02-18T08:38:26.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster1 info mirror/image_18@snap 2026-02-18T08:38:26.291 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:rbd image 'image_18': 2026-02-18T08:38:26.291 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: size 128 MiB in 32 objects 2026-02-18T08:38:26.291 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: order 22 (4 MiB objects) 2026-02-18T08:38:26.291 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: snapshot_count: 1 2026-02-18T08:38:26.292 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: id: 10ed2858e0ea 2026-02-18T08:38:26.292 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: block_name_prefix: rbd_data.10ed2858e0ea 2026-02-18T08:38:26.292 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: format: 2 2026-02-18T08:38:26.292 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: features: layering, exclusive-lock, journaling 2026-02-18T08:38:26.292 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: op_features: 2026-02-18T08:38:26.292 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: flags: 2026-02-18T08:38:26.292 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: create_timestamp: Wed Feb 18 08:28:51 2026 2026-02-18T08:38:26.292 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: access_timestamp: Wed Feb 18 08:28:51 2026 2026-02-18T08:38:26.292 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: modify_timestamp: Wed Feb 18 08:29:52 2026 2026-02-18T08:38:26.292 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: protected: False 2026-02-18T08:38:26.292 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: journal: 10ed2858e0ea 2026-02-18T08:38:26.292 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring state: enabled 2026-02-18T08:38:26.293 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring mode: journal 2026-02-18T08:38:26.293 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring global id: fc72cfd2-961d-4e40-9f89-ec0a5196953d 2026-02-18T08:38:26.293 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring primary: false 2026-02-18T08:38:26.294 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:38:26.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ compare_image_snaps mirror image_18 snap 2026-02-18T08:38:26.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:38:26.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_18 2026-02-18T08:38:26.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap 2026-02-18T08:38:26.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local ret=0 2026-02-18T08:38:26.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local rmt_export=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_18.export 2026-02-18T08:38:26.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loc_export=/tmp/tmp.vybh1m9yJF/cluster1-mirror-image_18.export 2026-02-18T08:38:26.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_18.export /tmp/tmp.vybh1m9yJF/cluster1-mirror-image_18.export 2026-02-18T08:38:26.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 -p mirror export image_18@snap /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_18.export 2026-02-18T08:38:26.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-18T08:38:26.451 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster1 -p mirror export image_18@snap /tmp/tmp.vybh1m9yJF/cluster1-mirror-image_18.export 2026-02-18T08:38:26.663 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-18T08:38:26.672 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmp /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_18.export /tmp/tmp.vybh1m9yJF/cluster1-mirror-image_18.export 2026-02-18T08:38:26.729 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_18.export /tmp/tmp.vybh1m9yJF/cluster1-mirror-image_18.export 2026-02-18T08:38:26.774 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:38:26.774 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T08:38:26.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_19 2026-02-18T08:38:26.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_snap cluster2 mirror image_19 snap 2026-02-18T08:38:26.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:38:26.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:38:26.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_19 2026-02-18T08:38:26.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap 2026-02-18T08:38:26.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 -p mirror snap create image_19@snap --debug-rbd=20 --debug-journaler=20 2026-02-18T08:38:27.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_replay_started cluster1 mirror image_19 2026-02-18T08:38:27.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:38:27.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:38:27.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_19 2026-02-18T08:38:27.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_replay_state cluster1 mirror image_19 started 2026-02-18T08:38:27.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:38:27.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:38:27.904 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_19 2026-02-18T08:38:27.904 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local state=started 2026-02-18T08:38:27.904 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-18T08:38:27.904 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:38:27.904 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1 2026-02-18T08:38:28.904 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test_image_replay_state cluster1 mirror image_19 started 2026-02-18T08:38:28.904 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:38:28.904 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:38:28.904 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_19 2026-02-18T08:38:28.905 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local test_state=started 2026-02-18T08:38:28.905 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local status_result 2026-02-18T08:38:28.905 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_state=stopped 2026-02-18T08:38:28.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_19 2026-02-18T08:38:28.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -i state 2026-02-18T08:38:28.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_instance=cluster1 2026-02-18T08:38:28.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-18T08:38:28.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:38:28.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance=cluster1 2026-02-18T08:38:28.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local loop_instance 2026-02-18T08:38:28.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:38:28.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sleep 0 2026-02-18T08:38:28.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:38:28.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ seq 0 3 2026-02-18T08:38:28.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:38:28.909 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_19 2026-02-18T08:38:28.909 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:0 2026-02-18T08:38:28.909 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-18T08:38:28.909 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:38:28.909 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:38:28.909 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:38:28.909 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-18T08:38:28.909 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:38:28.909 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:38:28.909 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:38:28.910 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:38:28.910 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-18T08:38:28.910 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:38:28.910 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:38:28.910 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:38:28.910 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:38:28.910 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:0 2026-02-18T08:38:28.910 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-18T08:38:28.910 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-18T08:38:28.910 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:38:28.910 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:0 2026-02-18T08:38:28.910 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T08:38:28.911 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-18T08:38:28.911 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:38:28.911 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:38:28.911 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-18T08:38:28.911 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-18T08:38:28.911 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-18T08:38:28.911 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=0 2026-02-18T08:38:28.911 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=0 2026-02-18T08:38:28.911 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:38:28.918 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:38:28.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:38:28.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:38:28.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_19 2026-02-18T08:38:28.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: exception getting command descriptions: [Errno 2] No such file or directory 2026-02-18T08:38:28.974 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:38:28.974 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_19 2026-02-18T08:38:28.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:1 2026-02-18T08:38:28.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-18T08:38:28.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:38:28.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:38:28.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:38:28.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-18T08:38:28.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:38:28.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:38:28.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:38:28.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:38:28.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-18T08:38:28.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:38:28.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:38:28.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:38:28.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:38:28.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:1 2026-02-18T08:38:28.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-18T08:38:28.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-18T08:38:28.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:38:28.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:1 2026-02-18T08:38:28.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T08:38:28.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-18T08:38:28.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:38:28.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:38:28.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-18T08:38:28.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-18T08:38:28.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-18T08:38:28.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=1 2026-02-18T08:38:28.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=1 2026-02-18T08:38:28.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:38:28.985 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:38:28.986 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:38:28.986 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:38:28.986 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_19 2026-02-18T08:38:29.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_19'] 2026-02-18T08:38:29.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status : get status for rbd mirror 2026-02-18T08:38:29.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:38:29.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:38:29.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_19 2026-02-18T08:38:29.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:2 2026-02-18T08:38:29.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-18T08:38:29.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:38:29.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:38:29.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:38:29.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-18T08:38:29.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:38:29.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:38:29.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:38:29.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:38:29.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-18T08:38:29.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:38:29.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:38:29.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:38:29.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:38:29.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:2 2026-02-18T08:38:29.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-18T08:38:29.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-18T08:38:29.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:38:29.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:2 2026-02-18T08:38:29.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T08:38:29.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-18T08:38:29.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:38:29.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:38:29.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-18T08:38:29.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-18T08:38:29.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-18T08:38:29.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=2 2026-02-18T08:38:29.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=2 2026-02-18T08:38:29.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:38:29.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:38:29.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:38:29.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:38:29.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_19 2026-02-18T08:38:29.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ return 0 2026-02-18T08:38:29.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ status_result=' "state": "Replaying"' 2026-02-18T08:38:29.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ echo ' "state": "Replaying"' 2026-02-18T08:38:29.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ grep -i Replaying 2026-02-18T08:38:29.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: "state": "Replaying" 2026-02-18T08:38:29.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ current_state=started 2026-02-18T08:38:29.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test started = started 2026-02-18T08:38:29.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:38:29.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror image_19 2026-02-18T08:38:29.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_cluster=cluster1 2026-02-18T08:38:29.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:38:29.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_pool=mirror 2026-02-18T08:38:29.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local remote_pool=mirror 2026-02-18T08:38:29.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_19 2026-02-18T08:38:29.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' journal = journal ']' 2026-02-18T08:38:29.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror mirror image_19 2026-02-18T08:38:29.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_cluster=cluster1 2026-02-18T08:38:29.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:38:29.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_pool=mirror 2026-02-18T08:38:29.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local remote_pool=mirror 2026-02-18T08:38:29.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_19 2026-02-18T08:38:29.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-02-18T08:38:29.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-02-18T08:38:29.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:38:29.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:38:29.129 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:38:29.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_19 2026-02-18T08:38:29.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:38:29.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:38:29.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_19 2026-02-18T08:38:29.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:38:29.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_19 ']' 2026-02-18T08:38:29.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_19' 2026-02-18T08:38:29.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_19 2026-02-18T08:38:29.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:38:29.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:38:29.329 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:38:29.329 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:38:29.329 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:38:29.329 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:38:29.329 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:38:29.329 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:38:29.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:38:29.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:38:29.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_19 2026-02-18T08:38:29.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:38:29.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:38:29.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:38:29.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:38:29.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:38:29.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:38:29.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:38:29.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:38:29.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:38:29.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:38:29.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:38:29.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:38:29.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:38:29.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:38:29.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:38:29.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:38:29.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:38:29.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:38:29.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:38:29.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:38:29.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:38:29.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:38:29.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:38:29.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:38:29.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:38:29.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:38:29.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:38:29.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:38:29.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:38:29.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:38:29.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:38:29.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:38:29.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:38:29.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_19 2026-02-18T08:38:29.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: exception getting command descriptions: [Errno 2] No such file or directory 2026-02-18T08:38:29.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:38:29.405 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_19 2026-02-18T08:38:29.405 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:38:29.405 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:38:29.405 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:38:29.405 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:38:29.405 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:38:29.405 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:38:29.405 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:38:29.405 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:38:29.405 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:38:29.406 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:38:29.406 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:38:29.406 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:38:29.406 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:38:29.406 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:38:29.406 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:38:29.406 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:38:29.406 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:38:29.406 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:38:29.406 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:38:29.406 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:38:29.407 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:38:29.407 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:38:29.407 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:38:29.407 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:38:29.407 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:38:29.407 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:38:29.407 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:38:29.407 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:38:29.407 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:38:29.407 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:38:29.415 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:38:29.415 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:38:29.415 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:38:29.415 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_19 2026-02-18T08:38:29.475 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_19'] 2026-02-18T08:38:29.475 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:38:29.475 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:38:29.478 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:38:29.478 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_19 2026-02-18T08:38:29.478 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:38:29.478 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:38:29.478 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:38:29.478 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:38:29.478 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:38:29.479 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:38:29.479 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:38:29.479 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:38:29.479 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:38:29.479 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:38:29.479 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:38:29.479 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:38:29.479 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:38:29.479 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:38:29.479 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:38:29.480 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:38:29.480 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:38:29.480 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:38:29.480 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:38:29.480 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:38:29.480 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:38:29.480 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:38:29.480 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:38:29.480 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:38:29.480 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:38:29.480 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:38:29.481 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:38:29.481 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:38:29.481 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:38:29.481 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:38:29.487 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:38:29.488 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:38:29.488 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:38:29.488 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_19 2026-02-18T08:38:29.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:38:29.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_19 2026-02-18T08:38:29.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:38:29.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:38:29.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_19 2026-02-18T08:38:29.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_19 '' 2026-02-18T08:38:29.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:38:29.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:38:29.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_19 2026-02-18T08:38:29.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:38:29.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_19.status 2026-02-18T08:38:29.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_19.status 2026-02-18T08:38:29.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:38:29.557 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_19.status 2026-02-18T08:38:29.557 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_19 2026-02-18T08:38:29.557 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_19.status 2026-02-18T08:38:29.586 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 12 2026-02-18T08:38:29.586 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 12 2026-02-18T08:38:29.586 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:38:29.586 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=49, tag_tid=2, entry_tid=1], [object_number=48, tag_tid=2, entry_tid=0], [object_number=51, tag_tid=1, entry_tid=51795], [object_number=50, tag_tid=1, entry_tid=51794]]], state=connected] 2026-02-18T08:38:29.586 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=48, tag_tid=1, entry_tid=51796], [object_number=51, tag_tid=1, entry_tid=51795], [object_number=50, tag_tid=1, entry_tid=51794], [object_number=49, tag_tid=1, entry_tid=51793]]], state=connected] 2026-02-18T08:38:29.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_19.status 2026-02-18T08:38:29.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=49, tag_tid=2, entry_tid=1' 2026-02-18T08:38:29.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_19 2026-02-18T08:38:29.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:38:29.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:38:29.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_19 2026-02-18T08:38:29.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_19 '..*' 2026-02-18T08:38:29.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:38:29.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:38:29.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_19 2026-02-18T08:38:29.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:38:29.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_19.status 2026-02-18T08:38:29.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_19.status 2026-02-18T08:38:29.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:38:29.594 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_19.status 2026-02-18T08:38:29.594 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_19 2026-02-18T08:38:29.594 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_19.status 2026-02-18T08:38:29.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 12 2026-02-18T08:38:29.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 12 2026-02-18T08:38:29.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:38:29.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=49, tag_tid=2, entry_tid=1], [object_number=48, tag_tid=2, entry_tid=0], [object_number=51, tag_tid=1, entry_tid=51795], [object_number=50, tag_tid=1, entry_tid=51794]]], state=connected] 2026-02-18T08:38:29.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=48, tag_tid=1, entry_tid=51796], [object_number=51, tag_tid=1, entry_tid=51795], [object_number=50, tag_tid=1, entry_tid=51794], [object_number=49, tag_tid=1, entry_tid=51793]]], state=connected] 2026-02-18T08:38:29.623 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_19.status 2026-02-18T08:38:29.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=48, tag_tid=1, entry_tid=51796' 2026-02-18T08:38:29.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=49, tag_tid=2, entry_tid=1' -a 'object_number=49, tag_tid=2, entry_tid=1' = 'object_number=48, tag_tid=1, entry_tid=51796' 2026-02-18T08:38:29.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=48, tag_tid=1, entry_tid=51796' '!=' '' 2026-02-18T08:38:29.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:38:29.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=48, tag_tid=1, entry_tid=51796' = '' 2026-02-18T08:38:29.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=48, tag_tid=1, entry_tid=51796' 2026-02-18T08:38:29.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=49, tag_tid=2, entry_tid=1' 2026-02-18T08:38:29.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:38:29.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:38:29.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=2 2026-02-18T08:38:29.628 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=48, tag_tid=1, entry_tid=51796' 2026-02-18T08:38:29.628 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:38:29.628 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:38:29.629 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=1 2026-02-18T08:38:29.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=49, tag_tid=2, entry_tid=1' 2026-02-18T08:38:29.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:38:29.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:38:29.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:38:29.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=48, tag_tid=1, entry_tid=51796' 2026-02-18T08:38:29.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:38:29.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:38:29.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=51796 2026-02-18T08:38:29.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 2 = 1 -a 1 -le 51796 2026-02-18T08:38:29.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:38:29.635 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:38:29.635 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:38:29.836 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_19 2026-02-18T08:38:29.836 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:38:29.837 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:38:29.837 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_19 2026-02-18T08:38:29.837 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:38:29.837 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_19 ']' 2026-02-18T08:38:29.837 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_19' 2026-02-18T08:38:29.837 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_19 2026-02-18T08:38:29.837 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:38:29.837 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:38:29.837 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:38:29.837 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:38:29.837 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:38:29.837 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:38:29.838 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:38:29.838 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:38:29.841 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:38:29.841 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:38:29.841 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_19 2026-02-18T08:38:29.841 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:38:29.841 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:38:29.841 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:38:29.841 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:38:29.841 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:38:29.841 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:38:29.841 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:38:29.841 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:38:29.842 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:38:29.842 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:38:29.842 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:38:29.842 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:38:29.842 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:38:29.842 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:38:29.842 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:38:29.842 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:38:29.842 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:38:29.842 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:38:29.842 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:38:29.843 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:38:29.843 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:38:29.843 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:38:29.843 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:38:29.843 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:38:29.843 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:38:29.843 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:38:29.843 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:38:29.843 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:38:29.843 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:38:29.843 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:38:29.850 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:38:29.851 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:38:29.851 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:38:29.851 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_19 2026-02-18T08:38:29.897 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: exception getting command descriptions: [Errno 2] No such file or directory 2026-02-18T08:38:29.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:38:29.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_19 2026-02-18T08:38:29.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:38:29.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:38:29.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:38:29.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:38:29.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:38:29.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:38:29.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:38:29.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:38:29.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:38:29.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:38:29.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:38:29.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:38:29.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:38:29.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:38:29.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:38:29.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:38:29.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:38:29.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:38:29.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:38:29.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:38:29.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:38:29.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:38:29.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:38:29.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:38:29.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:38:29.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:38:29.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:38:29.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:38:29.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:38:29.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:38:29.911 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:38:29.912 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:38:29.912 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:38:29.912 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_19 2026-02-18T08:38:29.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_19'] 2026-02-18T08:38:29.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:38:29.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:38:29.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:38:29.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_19 2026-02-18T08:38:29.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:38:29.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:38:29.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:38:29.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:38:29.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:38:29.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:38:29.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:38:29.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:38:29.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:38:29.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:38:29.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:38:29.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:38:29.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:38:29.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:38:29.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:38:29.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:38:29.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:38:29.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:38:29.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:38:29.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:38:29.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:38:29.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:38:29.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:38:29.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:38:29.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:38:29.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:38:29.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:38:29.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:38:29.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:38:29.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:38:29.987 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:38:29.987 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:38:29.987 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:38:29.988 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_19 2026-02-18T08:38:30.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:38:30.055 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_19 2026-02-18T08:38:30.056 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:38:30.056 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:38:30.056 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_19 2026-02-18T08:38:30.056 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_19 '' 2026-02-18T08:38:30.056 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:38:30.056 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:38:30.056 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_19 2026-02-18T08:38:30.056 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:38:30.056 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_19.status 2026-02-18T08:38:30.056 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_19.status 2026-02-18T08:38:30.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:38:30.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_19.status 2026-02-18T08:38:30.058 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_19 2026-02-18T08:38:30.058 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_19.status 2026-02-18T08:38:30.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 12 2026-02-18T08:38:30.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 12 2026-02-18T08:38:30.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:38:30.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=49, tag_tid=2, entry_tid=1], [object_number=48, tag_tid=2, entry_tid=0], [object_number=51, tag_tid=1, entry_tid=51795], [object_number=50, tag_tid=1, entry_tid=51794]]], state=connected] 2026-02-18T08:38:30.086 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=48, tag_tid=1, entry_tid=51796], [object_number=51, tag_tid=1, entry_tid=51795], [object_number=50, tag_tid=1, entry_tid=51794], [object_number=49, tag_tid=1, entry_tid=51793]]], state=connected] 2026-02-18T08:38:30.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_19.status 2026-02-18T08:38:30.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=49, tag_tid=2, entry_tid=1' 2026-02-18T08:38:30.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_19 2026-02-18T08:38:30.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:38:30.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:38:30.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_19 2026-02-18T08:38:30.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_19 '..*' 2026-02-18T08:38:30.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:38:30.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:38:30.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_19 2026-02-18T08:38:30.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:38:30.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_19.status 2026-02-18T08:38:30.092 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_19.status 2026-02-18T08:38:30.092 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:38:30.092 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_19.status 2026-02-18T08:38:30.093 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_19 2026-02-18T08:38:30.093 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_19.status 2026-02-18T08:38:30.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 12 2026-02-18T08:38:30.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 12 2026-02-18T08:38:30.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:38:30.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=49, tag_tid=2, entry_tid=1], [object_number=48, tag_tid=2, entry_tid=0], [object_number=51, tag_tid=1, entry_tid=51795], [object_number=50, tag_tid=1, entry_tid=51794]]], state=connected] 2026-02-18T08:38:30.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=48, tag_tid=1, entry_tid=51796], [object_number=51, tag_tid=1, entry_tid=51795], [object_number=50, tag_tid=1, entry_tid=51794], [object_number=49, tag_tid=1, entry_tid=51793]]], state=connected] 2026-02-18T08:38:30.131 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_19.status 2026-02-18T08:38:30.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=48, tag_tid=1, entry_tid=51796' 2026-02-18T08:38:30.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=49, tag_tid=2, entry_tid=1' -a 'object_number=49, tag_tid=2, entry_tid=1' = 'object_number=48, tag_tid=1, entry_tid=51796' 2026-02-18T08:38:30.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=48, tag_tid=1, entry_tid=51796' '!=' 'object_number=48, tag_tid=1, entry_tid=51796' 2026-02-18T08:38:30.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:38:30.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.4 2026-02-18T08:38:30.535 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_19 2026-02-18T08:38:30.535 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:38:30.535 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:38:30.535 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_19 2026-02-18T08:38:30.535 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:38:30.535 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_19 ']' 2026-02-18T08:38:30.535 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_19' 2026-02-18T08:38:30.535 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_19 2026-02-18T08:38:30.536 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:38:30.536 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:38:30.536 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:38:30.536 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:38:30.536 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:38:30.536 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:38:30.536 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:38:30.536 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:38:30.539 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:38:30.539 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:38:30.539 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_19 2026-02-18T08:38:30.539 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:38:30.539 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:38:30.539 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:38:30.539 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:38:30.539 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:38:30.539 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:38:30.540 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:38:30.540 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:38:30.540 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:38:30.540 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:38:30.540 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:38:30.540 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:38:30.540 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:38:30.540 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:38:30.540 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:38:30.540 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:38:30.540 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:38:30.540 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:38:30.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:38:30.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:38:30.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:38:30.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:38:30.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:38:30.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:38:30.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:38:30.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:38:30.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:38:30.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:38:30.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:38:30.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:38:30.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:38:30.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:38:30.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:38:30.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_19 2026-02-18T08:38:30.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: exception getting command descriptions: [Errno 2] No such file or directory 2026-02-18T08:38:30.596 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:38:30.596 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_19 2026-02-18T08:38:30.596 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:38:30.596 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:38:30.596 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:38:30.596 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:38:30.597 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:38:30.597 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:38:30.597 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:38:30.597 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:38:30.597 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:38:30.597 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:38:30.597 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:38:30.597 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:38:30.597 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:38:30.597 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:38:30.597 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:38:30.597 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:38:30.597 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:38:30.598 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:38:30.598 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:38:30.598 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:38:30.598 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:38:30.598 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:38:30.598 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:38:30.598 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:38:30.598 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:38:30.598 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:38:30.598 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:38:30.598 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:38:30.598 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:38:30.599 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:38:30.606 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:38:30.607 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:38:30.607 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:38:30.607 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_19 2026-02-18T08:38:30.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_19'] 2026-02-18T08:38:30.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:38:30.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:38:30.671 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:38:30.671 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_19 2026-02-18T08:38:30.671 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:38:30.671 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:38:30.671 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:38:30.671 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:38:30.671 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:38:30.671 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:38:30.671 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:38:30.672 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:38:30.672 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:38:30.672 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:38:30.672 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:38:30.672 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:38:30.672 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:38:30.672 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:38:30.672 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:38:30.672 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:38:30.672 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:38:30.672 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:38:30.672 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:38:30.673 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:38:30.673 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:38:30.673 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:38:30.673 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:38:30.673 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:38:30.673 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:38:30.673 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:38:30.673 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:38:30.673 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:38:30.673 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:38:30.673 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:38:30.682 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:38:30.682 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:38:30.682 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:38:30.682 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_19 2026-02-18T08:38:30.741 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:38:30.742 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_19 2026-02-18T08:38:30.742 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:38:30.742 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:38:30.742 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_19 2026-02-18T08:38:30.742 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_19 '' 2026-02-18T08:38:30.742 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:38:30.742 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:38:30.742 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_19 2026-02-18T08:38:30.742 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:38:30.743 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_19.status 2026-02-18T08:38:30.743 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_19.status 2026-02-18T08:38:30.743 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:38:30.743 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_19.status 2026-02-18T08:38:30.744 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_19 2026-02-18T08:38:30.744 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_19.status 2026-02-18T08:38:30.780 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 12 2026-02-18T08:38:30.780 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 12 2026-02-18T08:38:30.780 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:38:30.780 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=49, tag_tid=2, entry_tid=1], [object_number=48, tag_tid=2, entry_tid=0], [object_number=51, tag_tid=1, entry_tid=51795], [object_number=50, tag_tid=1, entry_tid=51794]]], state=connected] 2026-02-18T08:38:30.781 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=48, tag_tid=1, entry_tid=51796], [object_number=51, tag_tid=1, entry_tid=51795], [object_number=50, tag_tid=1, entry_tid=51794], [object_number=49, tag_tid=1, entry_tid=51793]]], state=connected] 2026-02-18T08:38:30.783 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_19.status 2026-02-18T08:38:30.785 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=49, tag_tid=2, entry_tid=1' 2026-02-18T08:38:30.786 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_19 2026-02-18T08:38:30.786 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:38:30.786 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:38:30.787 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_19 2026-02-18T08:38:30.787 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_19 '..*' 2026-02-18T08:38:30.787 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:38:30.787 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:38:30.787 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_19 2026-02-18T08:38:30.787 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:38:30.787 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_19.status 2026-02-18T08:38:30.787 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_19.status 2026-02-18T08:38:30.788 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:38:30.788 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_19.status 2026-02-18T08:38:30.788 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_19 2026-02-18T08:38:30.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_19.status 2026-02-18T08:38:30.815 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 12 2026-02-18T08:38:30.815 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 12 2026-02-18T08:38:30.815 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:38:30.815 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=49, tag_tid=2, entry_tid=1], [object_number=48, tag_tid=2, entry_tid=0], [object_number=51, tag_tid=1, entry_tid=51795], [object_number=50, tag_tid=1, entry_tid=51794]]], state=connected] 2026-02-18T08:38:30.815 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=48, tag_tid=1, entry_tid=51796], [object_number=51, tag_tid=1, entry_tid=51795], [object_number=50, tag_tid=1, entry_tid=51794], [object_number=49, tag_tid=1, entry_tid=51793]]], state=connected] 2026-02-18T08:38:30.818 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_19.status 2026-02-18T08:38:30.820 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=48, tag_tid=1, entry_tid=51796' 2026-02-18T08:38:30.820 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=49, tag_tid=2, entry_tid=1' -a 'object_number=49, tag_tid=2, entry_tid=1' = 'object_number=48, tag_tid=1, entry_tid=51796' 2026-02-18T08:38:30.820 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=48, tag_tid=1, entry_tid=51796' '!=' 'object_number=48, tag_tid=1, entry_tid=51796' 2026-02-18T08:38:30.820 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:38:30.820 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.8 2026-02-18T08:38:31.000 INFO:tasks.rbd_mirror_thrash:waiting for cluster1.client.mirror.0 2026-02-18T08:38:31.000 INFO:teuthology.orchestra.run:waiting for 600 2026-02-18T08:38:31.001 INFO:tasks.rbd_mirror_thrash:reviving cluster1.client.mirror.0 2026-02-18T08:38:31.001 INFO:tasks.rbd_mirror.cluster1.client.mirror.0:Restarting daemon 2026-02-18T08:38:31.001 DEBUG:teuthology.orchestra.run.trial163:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.0 2026-02-18T08:38:31.003 INFO:tasks.rbd_mirror.cluster1.client.mirror.0:Started 2026-02-18T08:38:31.003 INFO:tasks.rbd_mirror_thrash:waiting for 77.12024090183631 secs before thrashing 2026-02-18T08:38:31.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_19 2026-02-18T08:38:31.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:38:31.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:38:31.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_19 2026-02-18T08:38:31.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:38:31.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_19 ']' 2026-02-18T08:38:31.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_19' 2026-02-18T08:38:31.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_19 2026-02-18T08:38:31.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:38:31.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:38:31.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:38:31.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:38:31.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:38:31.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:38:31.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:38:31.622 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:38:31.622 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:38:31.623 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:38:31.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_19 2026-02-18T08:38:31.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:38:31.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:38:31.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:38:31.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:38:31.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:38:31.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:38:31.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:38:31.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:38:31.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:38:31.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:38:31.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:38:31.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:38:31.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:38:31.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:38:31.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:38:31.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:38:31.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:38:31.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:38:31.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:38:31.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:38:31.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:38:31.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:38:31.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:38:31.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:38:31.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:38:31.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:38:31.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:38:31.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:38:31.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:38:31.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:38:31.638 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:38:31.639 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:38:31.639 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:38:31.639 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_19 2026-02-18T08:38:31.696 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_19'] 2026-02-18T08:38:31.696 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:38:31.696 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:38:31.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:38:31.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_19 2026-02-18T08:38:31.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:38:31.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:38:31.700 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:38:31.700 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:38:31.700 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:38:31.700 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:38:31.700 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:38:31.700 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:38:31.700 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:38:31.700 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:38:31.700 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:38:31.700 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:38:31.700 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:38:31.700 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:38:31.701 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:38:31.701 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:38:31.701 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:38:31.701 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:38:31.701 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:38:31.701 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:38:31.701 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:38:31.701 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:38:31.701 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:38:31.701 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:38:31.701 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:38:31.701 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:38:31.701 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:38:31.702 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:38:31.702 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:38:31.702 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:38:31.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:38:31.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:38:31.711 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:38:31.711 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_19 2026-02-18T08:38:31.771 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_19'] 2026-02-18T08:38:31.771 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:38:31.771 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:38:31.774 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:38:31.774 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_19 2026-02-18T08:38:31.774 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:38:31.774 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:38:31.774 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:38:31.774 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:38:31.774 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:38:31.774 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:38:31.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:38:31.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:38:31.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:38:31.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:38:31.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:38:31.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:38:31.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:38:31.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:38:31.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:38:31.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:38:31.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:38:31.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:38:31.776 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:38:31.776 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:38:31.776 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:38:31.776 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:38:31.776 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:38:31.776 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:38:31.776 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:38:31.776 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:38:31.776 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:38:31.776 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:38:31.776 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:38:31.776 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:38:31.785 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:38:31.786 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:38:31.786 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:38:31.786 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_19 2026-02-18T08:38:31.845 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:38:31.846 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_19 2026-02-18T08:38:31.846 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:38:31.847 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:38:31.847 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_19 2026-02-18T08:38:31.847 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_19 '' 2026-02-18T08:38:31.847 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:38:31.847 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:38:31.847 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_19 2026-02-18T08:38:31.847 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:38:31.847 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_19.status 2026-02-18T08:38:31.847 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_19.status 2026-02-18T08:38:31.847 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:38:31.848 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_19.status 2026-02-18T08:38:31.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_19 2026-02-18T08:38:31.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_19.status 2026-02-18T08:38:31.885 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 12 2026-02-18T08:38:31.885 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 12 2026-02-18T08:38:31.885 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:38:31.885 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=49, tag_tid=2, entry_tid=1], [object_number=48, tag_tid=2, entry_tid=0], [object_number=51, tag_tid=1, entry_tid=51795], [object_number=50, tag_tid=1, entry_tid=51794]]], state=connected] 2026-02-18T08:38:31.885 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=48, tag_tid=1, entry_tid=51796], [object_number=51, tag_tid=1, entry_tid=51795], [object_number=50, tag_tid=1, entry_tid=51794], [object_number=49, tag_tid=1, entry_tid=51793]]], state=connected] 2026-02-18T08:38:31.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_19.status 2026-02-18T08:38:31.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=49, tag_tid=2, entry_tid=1' 2026-02-18T08:38:31.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_19 2026-02-18T08:38:31.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:38:31.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:38:31.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_19 2026-02-18T08:38:31.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_19 '..*' 2026-02-18T08:38:31.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:38:31.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:38:31.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_19 2026-02-18T08:38:31.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:38:31.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_19.status 2026-02-18T08:38:31.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_19.status 2026-02-18T08:38:31.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:38:31.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_19.status 2026-02-18T08:38:31.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_19 2026-02-18T08:38:31.894 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_19.status 2026-02-18T08:38:31.922 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 12 2026-02-18T08:38:31.922 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 12 2026-02-18T08:38:31.922 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:38:31.922 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=49, tag_tid=2, entry_tid=1], [object_number=48, tag_tid=2, entry_tid=0], [object_number=51, tag_tid=1, entry_tid=51795], [object_number=50, tag_tid=1, entry_tid=51794]]], state=connected] 2026-02-18T08:38:31.923 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=48, tag_tid=1, entry_tid=51796], [object_number=51, tag_tid=1, entry_tid=51795], [object_number=50, tag_tid=1, entry_tid=51794], [object_number=49, tag_tid=1, entry_tid=51793]]], state=connected] 2026-02-18T08:38:31.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_19.status 2026-02-18T08:38:31.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=48, tag_tid=1, entry_tid=51796' 2026-02-18T08:38:31.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=49, tag_tid=2, entry_tid=1' -a 'object_number=49, tag_tid=2, entry_tid=1' = 'object_number=48, tag_tid=1, entry_tid=51796' 2026-02-18T08:38:31.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=48, tag_tid=1, entry_tid=51796' '!=' 'object_number=48, tag_tid=1, entry_tid=51796' 2026-02-18T08:38:31.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:38:31.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1.6 2026-02-18T08:38:33.529 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_19 2026-02-18T08:38:33.529 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:38:33.530 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:38:33.530 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_19 2026-02-18T08:38:33.530 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:38:33.530 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_19 ']' 2026-02-18T08:38:33.530 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_19' 2026-02-18T08:38:33.530 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_19 2026-02-18T08:38:33.530 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:38:33.530 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:38:33.530 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:38:33.530 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:38:33.530 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:38:33.531 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:38:33.531 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:38:33.531 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:38:33.535 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:38:33.535 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:38:33.536 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_19 2026-02-18T08:38:33.536 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:38:33.536 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:38:33.536 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:38:33.536 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:38:33.536 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:38:33.536 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:38:33.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:38:33.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:38:33.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:38:33.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:38:33.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:38:33.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:38:33.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:38:33.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:38:33.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:38:33.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:38:33.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:38:33.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:38:33.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:38:33.539 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:38:33.539 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:38:33.539 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:38:33.539 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:38:33.539 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:38:33.539 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:38:33.539 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:38:33.539 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:38:33.539 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:38:33.539 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:38:33.539 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:38:33.545 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:38:33.545 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:38:33.545 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:38:33.545 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_19 2026-02-18T08:38:33.605 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_19'] 2026-02-18T08:38:33.605 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:38:33.605 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:38:33.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:38:33.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_19 2026-02-18T08:38:33.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:38:33.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:38:33.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:38:33.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:38:33.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:38:33.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:38:33.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:38:33.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:38:33.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:38:33.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:38:33.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:38:33.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:38:33.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:38:33.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:38:33.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:38:33.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:38:33.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:38:33.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:38:33.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:38:33.610 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:38:33.610 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:38:33.610 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:38:33.610 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:38:33.610 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:38:33.610 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:38:33.610 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:38:33.610 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:38:33.610 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:38:33.610 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:38:33.610 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:38:33.618 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:38:33.619 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:38:33.619 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:38:33.619 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_19 2026-02-18T08:38:33.680 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_19'] 2026-02-18T08:38:33.680 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:38:33.680 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:38:33.683 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:38:33.683 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_19 2026-02-18T08:38:33.683 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:38:33.683 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:38:33.683 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:38:33.684 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:38:33.684 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:38:33.684 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:38:33.684 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:38:33.684 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:38:33.684 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:38:33.684 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:38:33.684 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:38:33.684 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:38:33.684 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:38:33.684 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:38:33.685 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:38:33.685 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:38:33.685 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:38:33.685 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:38:33.685 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:38:33.685 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:38:33.685 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:38:33.685 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:38:33.685 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:38:33.685 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:38:33.685 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:38:33.686 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:38:33.686 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:38:33.686 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:38:33.686 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:38:33.686 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:38:33.694 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:38:33.694 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:38:33.694 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:38:33.694 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_19 2026-02-18T08:38:33.754 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:38:33.755 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_19 2026-02-18T08:38:33.755 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:38:33.755 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:38:33.755 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_19 2026-02-18T08:38:33.755 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_19 '' 2026-02-18T08:38:33.755 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:38:33.755 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:38:33.756 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_19 2026-02-18T08:38:33.756 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:38:33.756 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_19.status 2026-02-18T08:38:33.756 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_19.status 2026-02-18T08:38:33.756 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:38:33.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_19.status 2026-02-18T08:38:33.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_19 2026-02-18T08:38:33.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_19.status 2026-02-18T08:38:33.785 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 12 2026-02-18T08:38:33.786 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 12 2026-02-18T08:38:33.786 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:38:33.786 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=49, tag_tid=2, entry_tid=1], [object_number=48, tag_tid=2, entry_tid=0], [object_number=51, tag_tid=1, entry_tid=51795], [object_number=50, tag_tid=1, entry_tid=51794]]], state=connected] 2026-02-18T08:38:33.786 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=48, tag_tid=1, entry_tid=51796], [object_number=51, tag_tid=1, entry_tid=51795], [object_number=50, tag_tid=1, entry_tid=51794], [object_number=49, tag_tid=1, entry_tid=51793]]], state=connected] 2026-02-18T08:38:33.788 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_19.status 2026-02-18T08:38:33.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=49, tag_tid=2, entry_tid=1' 2026-02-18T08:38:33.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_19 2026-02-18T08:38:33.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:38:33.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:38:33.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_19 2026-02-18T08:38:33.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_19 '..*' 2026-02-18T08:38:33.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:38:33.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:38:33.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_19 2026-02-18T08:38:33.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:38:33.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_19.status 2026-02-18T08:38:33.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_19.status 2026-02-18T08:38:33.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:38:33.793 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_19.status 2026-02-18T08:38:33.793 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_19 2026-02-18T08:38:33.793 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_19.status 2026-02-18T08:38:33.818 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 12 2026-02-18T08:38:33.818 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 12 2026-02-18T08:38:33.819 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:38:33.819 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=49, tag_tid=2, entry_tid=1], [object_number=48, tag_tid=2, entry_tid=0], [object_number=51, tag_tid=1, entry_tid=51795], [object_number=50, tag_tid=1, entry_tid=51794]]], state=connected] 2026-02-18T08:38:33.819 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=48, tag_tid=1, entry_tid=51796], [object_number=51, tag_tid=1, entry_tid=51795], [object_number=50, tag_tid=1, entry_tid=51794], [object_number=49, tag_tid=1, entry_tid=51793]]], state=connected] 2026-02-18T08:38:33.821 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_19.status 2026-02-18T08:38:33.822 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=48, tag_tid=1, entry_tid=51796' 2026-02-18T08:38:33.823 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=49, tag_tid=2, entry_tid=1' -a 'object_number=49, tag_tid=2, entry_tid=1' = 'object_number=48, tag_tid=1, entry_tid=51796' 2026-02-18T08:38:33.823 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=48, tag_tid=1, entry_tid=51796' '!=' 'object_number=48, tag_tid=1, entry_tid=51796' 2026-02-18T08:38:33.823 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:38:33.823 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 2 2026-02-18T08:38:35.830 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_19 2026-02-18T08:38:35.831 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:38:35.831 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:38:35.831 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_19 2026-02-18T08:38:35.831 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:38:35.831 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_19 ']' 2026-02-18T08:38:35.832 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_19' 2026-02-18T08:38:35.832 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_19 2026-02-18T08:38:35.832 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:38:35.832 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:38:35.832 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:38:35.832 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:38:35.832 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:38:35.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:38:35.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:38:35.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:38:35.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:38:35.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:38:35.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_19 2026-02-18T08:38:35.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:38:35.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:38:35.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:38:35.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:38:35.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:38:35.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:38:35.835 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:38:35.835 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:38:35.835 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:38:35.835 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:38:35.835 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:38:35.835 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:38:35.835 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:38:35.836 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:38:35.836 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:38:35.836 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:38:35.836 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:38:35.836 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:38:35.836 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:38:35.837 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:38:35.837 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:38:35.837 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:38:35.837 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:38:35.837 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:38:35.837 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:38:35.837 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:38:35.838 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:38:35.838 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:38:35.838 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:38:35.838 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:38:35.840 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:38:35.841 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:38:35.841 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:38:35.841 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_19 2026-02-18T08:38:35.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_19'] 2026-02-18T08:38:35.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:38:35.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:38:35.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:38:35.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_19 2026-02-18T08:38:35.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:38:35.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:38:35.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:38:35.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:38:35.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:38:35.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:38:35.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:38:35.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:38:35.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:38:35.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:38:35.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:38:35.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:38:35.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:38:35.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:38:35.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:38:35.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:38:35.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:38:35.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:38:35.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:38:35.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:38:35.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:38:35.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:38:35.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:38:35.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:38:35.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:38:35.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:38:35.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:38:35.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:38:35.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:38:35.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:38:35.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:38:35.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:38:35.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:38:35.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_19 2026-02-18T08:38:35.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_19'] 2026-02-18T08:38:35.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:38:35.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:38:35.959 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:38:35.959 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_19 2026-02-18T08:38:35.959 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:38:35.959 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:38:35.959 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:38:35.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:38:35.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:38:35.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:38:35.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:38:35.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:38:35.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:38:35.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:38:35.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:38:35.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:38:35.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:38:35.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:38:35.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:38:35.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:38:35.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:38:35.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:38:35.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:38:35.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:38:35.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:38:35.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:38:35.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:38:35.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:38:35.961 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:38:35.961 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:38:35.961 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:38:35.961 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:38:35.961 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:38:35.961 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:38:35.971 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:38:35.971 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:38:35.971 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:38:35.971 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_19 2026-02-18T08:38:36.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:38:36.035 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_19 2026-02-18T08:38:36.035 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:38:36.035 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:38:36.035 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_19 2026-02-18T08:38:36.036 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_19 '' 2026-02-18T08:38:36.036 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:38:36.036 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:38:36.036 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_19 2026-02-18T08:38:36.036 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:38:36.036 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_19.status 2026-02-18T08:38:36.037 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_19.status 2026-02-18T08:38:36.037 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:38:36.038 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_19.status 2026-02-18T08:38:36.038 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_19 2026-02-18T08:38:36.038 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_19.status 2026-02-18T08:38:36.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 12 2026-02-18T08:38:36.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 12 2026-02-18T08:38:36.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:38:36.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=49, tag_tid=2, entry_tid=1], [object_number=48, tag_tid=2, entry_tid=0], [object_number=51, tag_tid=1, entry_tid=51795], [object_number=50, tag_tid=1, entry_tid=51794]]], state=connected] 2026-02-18T08:38:36.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=48, tag_tid=1, entry_tid=51796], [object_number=51, tag_tid=1, entry_tid=51795], [object_number=50, tag_tid=1, entry_tid=51794], [object_number=49, tag_tid=1, entry_tid=51793]]], state=connected] 2026-02-18T08:38:36.076 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_19.status 2026-02-18T08:38:36.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=49, tag_tid=2, entry_tid=1' 2026-02-18T08:38:36.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_19 2026-02-18T08:38:36.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:38:36.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:38:36.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_19 2026-02-18T08:38:36.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_19 '..*' 2026-02-18T08:38:36.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:38:36.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:38:36.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_19 2026-02-18T08:38:36.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:38:36.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_19.status 2026-02-18T08:38:36.080 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_19.status 2026-02-18T08:38:36.080 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:38:36.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_19.status 2026-02-18T08:38:36.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_19 2026-02-18T08:38:36.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_19.status 2026-02-18T08:38:36.114 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 12 2026-02-18T08:38:36.114 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 12 2026-02-18T08:38:36.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:38:36.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=49, tag_tid=2, entry_tid=1], [object_number=48, tag_tid=2, entry_tid=0], [object_number=51, tag_tid=1, entry_tid=51795], [object_number=50, tag_tid=1, entry_tid=51794]]], state=connected] 2026-02-18T08:38:36.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=48, tag_tid=1, entry_tid=51796], [object_number=51, tag_tid=1, entry_tid=51795], [object_number=50, tag_tid=1, entry_tid=51794], [object_number=49, tag_tid=1, entry_tid=51793]]], state=connected] 2026-02-18T08:38:36.118 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_19.status 2026-02-18T08:38:36.120 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=48, tag_tid=1, entry_tid=51796' 2026-02-18T08:38:36.120 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=49, tag_tid=2, entry_tid=1' -a 'object_number=49, tag_tid=2, entry_tid=1' = 'object_number=48, tag_tid=1, entry_tid=51796' 2026-02-18T08:38:36.120 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=48, tag_tid=1, entry_tid=51796' '!=' 'object_number=48, tag_tid=1, entry_tid=51796' 2026-02-18T08:38:36.121 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:38:36.121 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 2 2026-02-18T08:38:38.122 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_19 2026-02-18T08:38:38.122 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:38:38.122 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:38:38.122 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_19 2026-02-18T08:38:38.122 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:38:38.122 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_19 ']' 2026-02-18T08:38:38.122 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_19' 2026-02-18T08:38:38.122 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_19 2026-02-18T08:38:38.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:38:38.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:38:38.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:38:38.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:38:38.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:38:38.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:38:38.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:38:38.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:38:38.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:38:38.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:38:38.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_19 2026-02-18T08:38:38.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:38:38.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:38:38.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:38:38.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:38:38.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:38:38.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:38:38.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:38:38.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:38:38.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:38:38.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:38:38.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:38:38.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:38:38.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:38:38.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:38:38.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:38:38.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:38:38.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:38:38.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:38:38.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:38:38.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:38:38.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:38:38.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:38:38.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:38:38.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:38:38.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:38:38.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:38:38.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:38:38.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:38:38.129 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:38:38.129 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:38:38.136 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:38:38.137 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:38:38.137 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:38:38.137 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_19 2026-02-18T08:38:38.188 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_19'] 2026-02-18T08:38:38.188 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:38:38.188 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:38:38.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:38:38.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_19 2026-02-18T08:38:38.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:38:38.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:38:38.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:38:38.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:38:38.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:38:38.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:38:38.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:38:38.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:38:38.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:38:38.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:38:38.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:38:38.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:38:38.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:38:38.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:38:38.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:38:38.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:38:38.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:38:38.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:38:38.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:38:38.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:38:38.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:38:38.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:38:38.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:38:38.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:38:38.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:38:38.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:38:38.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:38:38.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:38:38.194 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:38:38.194 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:38:38.202 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:38:38.202 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:38:38.203 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:38:38.203 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_19 2026-02-18T08:38:38.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_19'] 2026-02-18T08:38:38.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:38:38.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:38:38.266 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:38:38.266 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_19 2026-02-18T08:38:38.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:38:38.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:38:38.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:38:38.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:38:38.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:38:38.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:38:38.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:38:38.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:38:38.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:38:38.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:38:38.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:38:38.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:38:38.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:38:38.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:38:38.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:38:38.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:38:38.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:38:38.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:38:38.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:38:38.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:38:38.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:38:38.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:38:38.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:38:38.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:38:38.269 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:38:38.269 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:38:38.269 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:38:38.269 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:38:38.269 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:38:38.269 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:38:38.277 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:38:38.278 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:38:38.278 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:38:38.278 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_19 2026-02-18T08:38:38.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:38:38.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_19 2026-02-18T08:38:38.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:38:38.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:38:38.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_19 2026-02-18T08:38:38.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_19 '' 2026-02-18T08:38:38.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:38:38.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:38:38.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_19 2026-02-18T08:38:38.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:38:38.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_19.status 2026-02-18T08:38:38.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_19.status 2026-02-18T08:38:38.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:38:38.344 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_19.status 2026-02-18T08:38:38.344 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_19 2026-02-18T08:38:38.344 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_19.status 2026-02-18T08:38:38.372 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 12 2026-02-18T08:38:38.373 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 12 2026-02-18T08:38:38.373 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:38:38.373 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=49, tag_tid=2, entry_tid=1], [object_number=48, tag_tid=2, entry_tid=0], [object_number=51, tag_tid=1, entry_tid=51795], [object_number=50, tag_tid=1, entry_tid=51794]]], state=connected] 2026-02-18T08:38:38.373 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=49, tag_tid=2, entry_tid=1], [object_number=48, tag_tid=2, entry_tid=0], [object_number=51, tag_tid=1, entry_tid=51795], [object_number=50, tag_tid=1, entry_tid=51794]]], state=connected] 2026-02-18T08:38:38.375 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_19.status 2026-02-18T08:38:38.377 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=49, tag_tid=2, entry_tid=1' 2026-02-18T08:38:38.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_19 2026-02-18T08:38:38.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:38:38.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:38:38.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_19 2026-02-18T08:38:38.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_19 '..*' 2026-02-18T08:38:38.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:38:38.379 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:38:38.379 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_19 2026-02-18T08:38:38.379 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:38:38.379 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_19.status 2026-02-18T08:38:38.379 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_19.status 2026-02-18T08:38:38.379 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:38:38.379 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_19.status 2026-02-18T08:38:38.380 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_19 2026-02-18T08:38:38.380 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_19.status 2026-02-18T08:38:38.415 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 12 2026-02-18T08:38:38.416 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 12 2026-02-18T08:38:38.416 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:38:38.416 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=49, tag_tid=2, entry_tid=1], [object_number=48, tag_tid=2, entry_tid=0], [object_number=51, tag_tid=1, entry_tid=51795], [object_number=50, tag_tid=1, entry_tid=51794]]], state=connected] 2026-02-18T08:38:38.416 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=49, tag_tid=2, entry_tid=1], [object_number=48, tag_tid=2, entry_tid=0], [object_number=51, tag_tid=1, entry_tid=51795], [object_number=50, tag_tid=1, entry_tid=51794]]], state=connected] 2026-02-18T08:38:38.419 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_19.status 2026-02-18T08:38:38.421 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=49, tag_tid=2, entry_tid=1' 2026-02-18T08:38:38.422 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=49, tag_tid=2, entry_tid=1' -a 'object_number=49, tag_tid=2, entry_tid=1' = 'object_number=49, tag_tid=2, entry_tid=1' 2026-02-18T08:38:38.422 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:38:38.422 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_snap_present cluster1 mirror image_19 snap 2026-02-18T08:38:38.422 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:38:38.422 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:38:38.422 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_19 2026-02-18T08:38:38.422 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap 2026-02-18T08:38:38.422 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-18T08:38:38.422 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-18T08:38:38.422 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1 2026-02-18T08:38:39.422 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster1 info mirror/image_19@snap 2026-02-18T08:38:39.452 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:rbd image 'image_19': 2026-02-18T08:38:39.452 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: size 128 MiB in 32 objects 2026-02-18T08:38:39.452 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: order 22 (4 MiB objects) 2026-02-18T08:38:39.452 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: snapshot_count: 1 2026-02-18T08:38:39.452 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: id: 110419269ade 2026-02-18T08:38:39.453 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: block_name_prefix: rbd_data.110419269ade 2026-02-18T08:38:39.453 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: format: 2 2026-02-18T08:38:39.453 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: features: layering, exclusive-lock, journaling 2026-02-18T08:38:39.453 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: op_features: 2026-02-18T08:38:39.453 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: flags: 2026-02-18T08:38:39.453 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: create_timestamp: Wed Feb 18 08:28:59 2026 2026-02-18T08:38:39.453 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: access_timestamp: Wed Feb 18 08:28:59 2026 2026-02-18T08:38:39.453 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: modify_timestamp: Wed Feb 18 08:31:01 2026 2026-02-18T08:38:39.453 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: protected: False 2026-02-18T08:38:39.453 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: journal: 110419269ade 2026-02-18T08:38:39.453 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring state: enabled 2026-02-18T08:38:39.454 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring mode: journal 2026-02-18T08:38:39.454 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring global id: e1d7b0fc-6af7-41b7-a6f2-b5022047f55f 2026-02-18T08:38:39.454 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring primary: false 2026-02-18T08:38:39.455 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:38:39.455 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ compare_image_snaps mirror image_19 snap 2026-02-18T08:38:39.455 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:38:39.455 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_19 2026-02-18T08:38:39.455 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap 2026-02-18T08:38:39.455 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local ret=0 2026-02-18T08:38:39.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local rmt_export=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_19.export 2026-02-18T08:38:39.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loc_export=/tmp/tmp.vybh1m9yJF/cluster1-mirror-image_19.export 2026-02-18T08:38:39.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_19.export /tmp/tmp.vybh1m9yJF/cluster1-mirror-image_19.export 2026-02-18T08:38:39.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 -p mirror export image_19@snap /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_19.export 2026-02-18T08:38:39.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-18T08:38:39.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster1 -p mirror export image_19@snap /tmp/tmp.vybh1m9yJF/cluster1-mirror-image_19.export 2026-02-18T08:38:39.803 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-18T08:38:39.813 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmp /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_19.export /tmp/tmp.vybh1m9yJF/cluster1-mirror-image_19.export 2026-02-18T08:38:39.867 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_19.export /tmp/tmp.vybh1m9yJF/cluster1-mirror-image_19.export 2026-02-18T08:38:39.913 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:38:39.913 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T08:38:39.913 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_20 2026-02-18T08:38:39.913 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_snap cluster2 mirror image_20 snap 2026-02-18T08:38:39.913 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:38:39.913 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:38:39.913 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_20 2026-02-18T08:38:39.913 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap 2026-02-18T08:38:39.913 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 -p mirror snap create image_20@snap --debug-rbd=20 --debug-journaler=20 2026-02-18T08:38:42.076 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_replay_started cluster1 mirror image_20 2026-02-18T08:38:42.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:38:42.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:38:42.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_20 2026-02-18T08:38:42.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_replay_state cluster1 mirror image_20 started 2026-02-18T08:38:42.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:38:42.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:38:42.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_20 2026-02-18T08:38:42.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local state=started 2026-02-18T08:38:42.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-18T08:38:42.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:38:42.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1 2026-02-18T08:38:43.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test_image_replay_state cluster1 mirror image_20 started 2026-02-18T08:38:43.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:38:43.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:38:43.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_20 2026-02-18T08:38:43.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local test_state=started 2026-02-18T08:38:43.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local status_result 2026-02-18T08:38:43.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_state=stopped 2026-02-18T08:38:43.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_20 2026-02-18T08:38:43.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -i state 2026-02-18T08:38:43.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_instance=cluster1 2026-02-18T08:38:43.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-18T08:38:43.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:38:43.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance=cluster1 2026-02-18T08:38:43.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local loop_instance 2026-02-18T08:38:43.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:38:43.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sleep 0 2026-02-18T08:38:43.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:38:43.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ seq 0 3 2026-02-18T08:38:43.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:38:43.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_20 2026-02-18T08:38:43.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:0 2026-02-18T08:38:43.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-18T08:38:43.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:38:43.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:38:43.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:38:43.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-18T08:38:43.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:38:43.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:38:43.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:38:43.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:38:43.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-18T08:38:43.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:38:43.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:38:43.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:38:43.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:38:43.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:0 2026-02-18T08:38:43.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-18T08:38:43.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-18T08:38:43.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:38:43.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:0 2026-02-18T08:38:43.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T08:38:43.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-18T08:38:43.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:38:43.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:38:43.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-18T08:38:43.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-18T08:38:43.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-18T08:38:43.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=0 2026-02-18T08:38:43.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=0 2026-02-18T08:38:43.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:38:43.092 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:38:43.092 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:38:43.093 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:38:43.093 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_20 2026-02-18T08:38:43.155 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_20'] 2026-02-18T08:38:43.155 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status : get status for rbd mirror 2026-02-18T08:38:43.155 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:38:43.158 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:38:43.158 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_20 2026-02-18T08:38:43.158 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:1 2026-02-18T08:38:43.158 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-18T08:38:43.158 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:38:43.158 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:38:43.158 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:38:43.158 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-18T08:38:43.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:38:43.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:38:43.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:38:43.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:38:43.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-18T08:38:43.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:38:43.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:38:43.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:38:43.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:38:43.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:1 2026-02-18T08:38:43.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-18T08:38:43.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-18T08:38:43.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:38:43.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:1 2026-02-18T08:38:43.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T08:38:43.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-18T08:38:43.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:38:43.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:38:43.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-18T08:38:43.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-18T08:38:43.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-18T08:38:43.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=1 2026-02-18T08:38:43.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=1 2026-02-18T08:38:43.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:38:43.167 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:38:43.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:38:43.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:38:43.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_20 2026-02-18T08:38:43.231 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_20'] 2026-02-18T08:38:43.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status : get status for rbd mirror 2026-02-18T08:38:43.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:38:43.234 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:38:43.234 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_20 2026-02-18T08:38:43.234 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:2 2026-02-18T08:38:43.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-18T08:38:43.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:38:43.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:38:43.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:38:43.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-18T08:38:43.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:38:43.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:38:43.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:38:43.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:38:43.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-18T08:38:43.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:38:43.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:38:43.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:38:43.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:38:43.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:2 2026-02-18T08:38:43.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-18T08:38:43.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-18T08:38:43.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:38:43.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:2 2026-02-18T08:38:43.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T08:38:43.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-18T08:38:43.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:38:43.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:38:43.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-18T08:38:43.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-18T08:38:43.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-18T08:38:43.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=2 2026-02-18T08:38:43.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=2 2026-02-18T08:38:43.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:38:43.245 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:38:43.246 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:38:43.246 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:38:43.246 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_20 2026-02-18T08:38:43.314 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ return 0 2026-02-18T08:38:43.315 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ status_result=' "state": "Replaying"' 2026-02-18T08:38:43.315 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ echo ' "state": "Replaying"' 2026-02-18T08:38:43.316 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ grep -i Replaying 2026-02-18T08:38:43.317 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: "state": "Replaying" 2026-02-18T08:38:43.319 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ current_state=started 2026-02-18T08:38:43.319 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test started = started 2026-02-18T08:38:43.319 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:38:43.319 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror image_20 2026-02-18T08:38:43.319 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_cluster=cluster1 2026-02-18T08:38:43.319 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:38:43.319 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_pool=mirror 2026-02-18T08:38:43.320 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local remote_pool=mirror 2026-02-18T08:38:43.320 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_20 2026-02-18T08:38:43.320 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' journal = journal ']' 2026-02-18T08:38:43.320 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror mirror image_20 2026-02-18T08:38:43.320 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_cluster=cluster1 2026-02-18T08:38:43.320 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:38:43.320 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_pool=mirror 2026-02-18T08:38:43.320 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local remote_pool=mirror 2026-02-18T08:38:43.320 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_20 2026-02-18T08:38:43.320 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-02-18T08:38:43.321 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-02-18T08:38:43.321 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:38:43.321 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:38:43.321 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:38:43.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_20 2026-02-18T08:38:43.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:38:43.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:38:43.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_20 2026-02-18T08:38:43.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:38:43.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_20 ']' 2026-02-18T08:38:43.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_20' 2026-02-18T08:38:43.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_20 2026-02-18T08:38:43.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:38:43.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:38:43.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:38:43.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:38:43.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:38:43.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:38:43.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:38:43.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:38:43.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:38:43.524 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:38:43.524 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_20 2026-02-18T08:38:43.524 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:38:43.524 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:38:43.524 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:38:43.524 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:38:43.524 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:38:43.525 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:38:43.525 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:38:43.525 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:38:43.525 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:38:43.525 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:38:43.525 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:38:43.525 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:38:43.525 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:38:43.525 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:38:43.525 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:38:43.526 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:38:43.526 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:38:43.526 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:38:43.526 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:38:43.526 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:38:43.526 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:38:43.526 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:38:43.526 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:38:43.526 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:38:43.526 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:38:43.526 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:38:43.527 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:38:43.527 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:38:43.527 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:38:43.527 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:38:43.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:38:43.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:38:43.534 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:38:43.534 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_20 2026-02-18T08:38:43.584 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_20'] 2026-02-18T08:38:43.584 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:38:43.585 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:38:43.587 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:38:43.587 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_20 2026-02-18T08:38:43.587 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:38:43.587 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:38:43.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:38:43.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:38:43.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:38:43.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:38:43.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:38:43.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:38:43.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:38:43.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:38:43.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:38:43.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:38:43.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:38:43.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:38:43.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:38:43.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:38:43.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:38:43.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:38:43.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:38:43.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:38:43.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:38:43.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:38:43.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:38:43.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:38:43.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:38:43.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:38:43.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:38:43.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:38:43.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:38:43.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:38:43.598 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:38:43.598 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:38:43.598 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:38:43.598 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_20 2026-02-18T08:38:43.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_20'] 2026-02-18T08:38:43.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:38:43.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:38:43.653 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:38:43.654 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_20 2026-02-18T08:38:43.654 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:38:43.654 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:38:43.654 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:38:43.654 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:38:43.654 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:38:43.654 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:38:43.654 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:38:43.654 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:38:43.654 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:38:43.655 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:38:43.655 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:38:43.655 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:38:43.655 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:38:43.655 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:38:43.655 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:38:43.655 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:38:43.655 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:38:43.655 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:38:43.655 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:38:43.655 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:38:43.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:38:43.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:38:43.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:38:43.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:38:43.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:38:43.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:38:43.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:38:43.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:38:43.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:38:43.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:38:43.663 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:38:43.664 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:38:43.664 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:38:43.664 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_20 2026-02-18T08:38:43.729 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:38:43.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_20 2026-02-18T08:38:43.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:38:43.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:38:43.731 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_20 2026-02-18T08:38:43.731 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_20 '' 2026-02-18T08:38:43.731 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:38:43.731 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:38:43.731 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_20 2026-02-18T08:38:43.731 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:38:43.731 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_20.status 2026-02-18T08:38:43.731 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_20.status 2026-02-18T08:38:43.731 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:38:43.732 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_20.status 2026-02-18T08:38:43.732 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_20 2026-02-18T08:38:43.732 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_20.status 2026-02-18T08:38:43.769 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 8 2026-02-18T08:38:43.769 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 8 2026-02-18T08:38:43.769 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:38:43.769 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=33, tag_tid=2, entry_tid=1], [object_number=32, tag_tid=2, entry_tid=0], [object_number=35, tag_tid=1, entry_tid=35979], [object_number=34, tag_tid=1, entry_tid=35978]]], state=connected] 2026-02-18T08:38:43.769 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=32, tag_tid=1, entry_tid=35980], [object_number=35, tag_tid=1, entry_tid=35979], [object_number=34, tag_tid=1, entry_tid=35978], [object_number=33, tag_tid=1, entry_tid=35977]]], state=connected] 2026-02-18T08:38:43.772 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_20.status 2026-02-18T08:38:43.774 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=33, tag_tid=2, entry_tid=1' 2026-02-18T08:38:43.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_20 2026-02-18T08:38:43.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:38:43.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:38:43.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_20 2026-02-18T08:38:43.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_20 '..*' 2026-02-18T08:38:43.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:38:43.776 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:38:43.776 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_20 2026-02-18T08:38:43.776 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:38:43.776 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_20.status 2026-02-18T08:38:43.777 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_20.status 2026-02-18T08:38:43.777 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:38:43.777 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_20.status 2026-02-18T08:38:43.778 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_20 2026-02-18T08:38:43.778 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_20.status 2026-02-18T08:38:43.811 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 8 2026-02-18T08:38:43.811 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 8 2026-02-18T08:38:43.811 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:38:43.811 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=33, tag_tid=2, entry_tid=1], [object_number=32, tag_tid=2, entry_tid=0], [object_number=35, tag_tid=1, entry_tid=35979], [object_number=34, tag_tid=1, entry_tid=35978]]], state=connected] 2026-02-18T08:38:43.811 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=32, tag_tid=1, entry_tid=35980], [object_number=35, tag_tid=1, entry_tid=35979], [object_number=34, tag_tid=1, entry_tid=35978], [object_number=33, tag_tid=1, entry_tid=35977]]], state=connected] 2026-02-18T08:38:43.814 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_20.status 2026-02-18T08:38:43.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=32, tag_tid=1, entry_tid=35980' 2026-02-18T08:38:43.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=33, tag_tid=2, entry_tid=1' -a 'object_number=33, tag_tid=2, entry_tid=1' = 'object_number=32, tag_tid=1, entry_tid=35980' 2026-02-18T08:38:43.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=32, tag_tid=1, entry_tid=35980' '!=' '' 2026-02-18T08:38:43.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:38:43.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=32, tag_tid=1, entry_tid=35980' = '' 2026-02-18T08:38:43.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=32, tag_tid=1, entry_tid=35980' 2026-02-18T08:38:43.817 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=33, tag_tid=2, entry_tid=1' 2026-02-18T08:38:43.817 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:38:43.817 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:38:43.818 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=2 2026-02-18T08:38:43.818 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=32, tag_tid=1, entry_tid=35980' 2026-02-18T08:38:43.819 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:38:43.819 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:38:43.820 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=1 2026-02-18T08:38:43.821 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=33, tag_tid=2, entry_tid=1' 2026-02-18T08:38:43.821 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:38:43.821 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:38:43.822 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:38:43.823 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=32, tag_tid=1, entry_tid=35980' 2026-02-18T08:38:43.823 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:38:43.823 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:38:43.825 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=35980 2026-02-18T08:38:43.825 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 2 = 1 -a 1 -le 35980 2026-02-18T08:38:43.825 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:38:43.825 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:38:43.825 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:38:44.027 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_20 2026-02-18T08:38:44.027 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:38:44.027 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:38:44.027 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_20 2026-02-18T08:38:44.027 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:38:44.027 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_20 ']' 2026-02-18T08:38:44.027 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_20' 2026-02-18T08:38:44.027 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_20 2026-02-18T08:38:44.027 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:38:44.028 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:38:44.028 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:38:44.028 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:38:44.028 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:38:44.028 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:38:44.028 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:38:44.028 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:38:44.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:38:44.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:38:44.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_20 2026-02-18T08:38:44.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:38:44.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:38:44.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:38:44.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:38:44.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:38:44.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:38:44.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:38:44.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:38:44.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:38:44.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:38:44.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:38:44.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:38:44.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:38:44.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:38:44.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:38:44.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:38:44.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:38:44.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:38:44.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:38:44.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:38:44.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:38:44.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:38:44.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:38:44.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:38:44.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:38:44.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:38:44.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:38:44.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:38:44.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:38:44.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:38:44.040 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:38:44.040 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:38:44.040 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:38:44.040 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_20 2026-02-18T08:38:44.100 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_20'] 2026-02-18T08:38:44.100 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:38:44.100 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:38:44.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:38:44.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_20 2026-02-18T08:38:44.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:38:44.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:38:44.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:38:44.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:38:44.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:38:44.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:38:44.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:38:44.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:38:44.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:38:44.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:38:44.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:38:44.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:38:44.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:38:44.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:38:44.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:38:44.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:38:44.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:38:44.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:38:44.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:38:44.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:38:44.105 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:38:44.105 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:38:44.105 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:38:44.105 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:38:44.105 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:38:44.105 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:38:44.105 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:38:44.105 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:38:44.105 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:38:44.105 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:38:44.114 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:38:44.114 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:38:44.114 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:38:44.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_20 2026-02-18T08:38:44.176 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_20'] 2026-02-18T08:38:44.176 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:38:44.176 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:38:44.178 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:38:44.178 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_20 2026-02-18T08:38:44.178 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:38:44.178 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:38:44.179 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:38:44.179 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:38:44.179 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:38:44.179 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:38:44.179 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:38:44.179 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:38:44.179 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:38:44.179 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:38:44.179 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:38:44.179 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:38:44.179 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:38:44.179 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:38:44.180 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:38:44.180 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:38:44.180 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:38:44.180 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:38:44.180 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:38:44.180 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:38:44.180 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:38:44.180 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:38:44.180 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:38:44.180 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:38:44.180 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:38:44.180 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:38:44.180 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:38:44.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:38:44.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:38:44.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:38:44.190 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:38:44.190 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:38:44.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:38:44.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_20 2026-02-18T08:38:44.255 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:38:44.257 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_20 2026-02-18T08:38:44.257 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:38:44.257 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:38:44.257 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_20 2026-02-18T08:38:44.257 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_20 '' 2026-02-18T08:38:44.257 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:38:44.257 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:38:44.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_20 2026-02-18T08:38:44.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:38:44.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_20.status 2026-02-18T08:38:44.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_20.status 2026-02-18T08:38:44.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:38:44.259 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_20.status 2026-02-18T08:38:44.259 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_20 2026-02-18T08:38:44.259 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_20.status 2026-02-18T08:38:44.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 8 2026-02-18T08:38:44.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 8 2026-02-18T08:38:44.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:38:44.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=33, tag_tid=2, entry_tid=1], [object_number=32, tag_tid=2, entry_tid=0], [object_number=35, tag_tid=1, entry_tid=35979], [object_number=34, tag_tid=1, entry_tid=35978]]], state=connected] 2026-02-18T08:38:44.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=32, tag_tid=1, entry_tid=35980], [object_number=35, tag_tid=1, entry_tid=35979], [object_number=34, tag_tid=1, entry_tid=35978], [object_number=33, tag_tid=1, entry_tid=35977]]], state=connected] 2026-02-18T08:38:44.299 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_20.status 2026-02-18T08:38:44.300 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=33, tag_tid=2, entry_tid=1' 2026-02-18T08:38:44.302 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_20 2026-02-18T08:38:44.302 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:38:44.302 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:38:44.302 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_20 2026-02-18T08:38:44.302 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_20 '..*' 2026-02-18T08:38:44.302 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:38:44.302 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:38:44.302 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_20 2026-02-18T08:38:44.302 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:38:44.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_20.status 2026-02-18T08:38:44.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_20.status 2026-02-18T08:38:44.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:38:44.304 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_20.status 2026-02-18T08:38:44.304 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_20 2026-02-18T08:38:44.304 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_20.status 2026-02-18T08:38:44.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 8 2026-02-18T08:38:44.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 8 2026-02-18T08:38:44.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:38:44.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=33, tag_tid=2, entry_tid=1], [object_number=32, tag_tid=2, entry_tid=0], [object_number=35, tag_tid=1, entry_tid=35979], [object_number=34, tag_tid=1, entry_tid=35978]]], state=connected] 2026-02-18T08:38:44.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=32, tag_tid=1, entry_tid=35980], [object_number=35, tag_tid=1, entry_tid=35979], [object_number=34, tag_tid=1, entry_tid=35978], [object_number=33, tag_tid=1, entry_tid=35977]]], state=connected] 2026-02-18T08:38:44.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_20.status 2026-02-18T08:38:44.344 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=32, tag_tid=1, entry_tid=35980' 2026-02-18T08:38:44.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=33, tag_tid=2, entry_tid=1' -a 'object_number=33, tag_tid=2, entry_tid=1' = 'object_number=32, tag_tid=1, entry_tid=35980' 2026-02-18T08:38:44.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=32, tag_tid=1, entry_tid=35980' '!=' 'object_number=32, tag_tid=1, entry_tid=35980' 2026-02-18T08:38:44.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:38:44.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.4 2026-02-18T08:38:44.747 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_20 2026-02-18T08:38:44.747 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:38:44.747 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:38:44.748 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_20 2026-02-18T08:38:44.748 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:38:44.748 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_20 ']' 2026-02-18T08:38:44.748 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_20' 2026-02-18T08:38:44.748 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_20 2026-02-18T08:38:44.748 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:38:44.748 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:38:44.748 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:38:44.748 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:38:44.748 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:38:44.749 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:38:44.749 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:38:44.749 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:38:44.752 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:38:44.752 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:38:44.752 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_20 2026-02-18T08:38:44.752 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:38:44.752 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:38:44.752 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:38:44.752 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:38:44.753 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:38:44.753 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:38:44.753 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:38:44.753 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:38:44.753 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:38:44.753 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:38:44.753 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:38:44.753 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:38:44.753 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:38:44.753 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:38:44.753 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:38:44.754 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:38:44.754 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:38:44.754 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:38:44.754 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:38:44.754 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:38:44.754 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:38:44.754 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:38:44.754 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:38:44.754 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:38:44.754 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:38:44.754 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:38:44.755 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:38:44.755 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:38:44.755 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:38:44.755 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:38:44.762 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:38:44.762 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:38:44.763 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:38:44.763 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_20 2026-02-18T08:38:44.821 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_20'] 2026-02-18T08:38:44.821 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:38:44.821 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:38:44.824 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:38:44.824 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_20 2026-02-18T08:38:44.824 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:38:44.824 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:38:44.824 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:38:44.825 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:38:44.825 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:38:44.825 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:38:44.825 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:38:44.825 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:38:44.825 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:38:44.825 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:38:44.825 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:38:44.825 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:38:44.825 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:38:44.826 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:38:44.826 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:38:44.826 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:38:44.826 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:38:44.826 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:38:44.826 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:38:44.826 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:38:44.826 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:38:44.826 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:38:44.826 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:38:44.826 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:38:44.827 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:38:44.827 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:38:44.827 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:38:44.827 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:38:44.827 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:38:44.827 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:38:44.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:38:44.835 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:38:44.835 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:38:44.835 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_20 2026-02-18T08:38:44.895 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_20'] 2026-02-18T08:38:44.896 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:38:44.896 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:38:44.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:38:44.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_20 2026-02-18T08:38:44.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:38:44.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:38:44.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:38:44.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:38:44.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:38:44.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:38:44.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:38:44.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:38:44.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:38:44.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:38:44.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:38:44.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:38:44.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:38:44.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:38:44.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:38:44.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:38:44.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:38:44.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:38:44.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:38:44.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:38:44.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:38:44.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:38:44.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:38:44.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:38:44.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:38:44.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:38:44.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:38:44.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:38:44.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:38:44.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:38:44.910 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:38:44.910 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:38:44.910 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:38:44.911 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_20 2026-02-18T08:38:44.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:38:44.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_20 2026-02-18T08:38:44.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:38:44.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:38:44.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_20 2026-02-18T08:38:44.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_20 '' 2026-02-18T08:38:44.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:38:44.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:38:44.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_20 2026-02-18T08:38:44.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:38:44.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_20.status 2026-02-18T08:38:44.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_20.status 2026-02-18T08:38:44.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:38:44.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_20.status 2026-02-18T08:38:44.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_20 2026-02-18T08:38:44.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_20.status 2026-02-18T08:38:45.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 8 2026-02-18T08:38:45.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 8 2026-02-18T08:38:45.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:38:45.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=33, tag_tid=2, entry_tid=1], [object_number=32, tag_tid=2, entry_tid=0], [object_number=35, tag_tid=1, entry_tid=35979], [object_number=34, tag_tid=1, entry_tid=35978]]], state=connected] 2026-02-18T08:38:45.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=32, tag_tid=1, entry_tid=35980], [object_number=35, tag_tid=1, entry_tid=35979], [object_number=34, tag_tid=1, entry_tid=35978], [object_number=33, tag_tid=1, entry_tid=35977]]], state=connected] 2026-02-18T08:38:45.019 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_20.status 2026-02-18T08:38:45.021 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=33, tag_tid=2, entry_tid=1' 2026-02-18T08:38:45.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_20 2026-02-18T08:38:45.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:38:45.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:38:45.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_20 2026-02-18T08:38:45.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_20 '..*' 2026-02-18T08:38:45.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:38:45.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:38:45.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_20 2026-02-18T08:38:45.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:38:45.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_20.status 2026-02-18T08:38:45.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_20.status 2026-02-18T08:38:45.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:38:45.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_20.status 2026-02-18T08:38:45.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_20 2026-02-18T08:38:45.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_20.status 2026-02-18T08:38:45.060 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 8 2026-02-18T08:38:45.060 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 8 2026-02-18T08:38:45.060 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:38:45.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=33, tag_tid=2, entry_tid=1], [object_number=32, tag_tid=2, entry_tid=0], [object_number=35, tag_tid=1, entry_tid=35979], [object_number=34, tag_tid=1, entry_tid=35978]]], state=connected] 2026-02-18T08:38:45.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=32, tag_tid=1, entry_tid=35980], [object_number=35, tag_tid=1, entry_tid=35979], [object_number=34, tag_tid=1, entry_tid=35978], [object_number=33, tag_tid=1, entry_tid=35977]]], state=connected] 2026-02-18T08:38:45.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_20.status 2026-02-18T08:38:45.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=32, tag_tid=1, entry_tid=35980' 2026-02-18T08:38:45.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=33, tag_tid=2, entry_tid=1' -a 'object_number=33, tag_tid=2, entry_tid=1' = 'object_number=32, tag_tid=1, entry_tid=35980' 2026-02-18T08:38:45.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=32, tag_tid=1, entry_tid=35980' '!=' 'object_number=32, tag_tid=1, entry_tid=35980' 2026-02-18T08:38:45.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:38:45.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.8 2026-02-18T08:38:45.867 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_20 2026-02-18T08:38:45.868 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:38:45.868 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:38:45.868 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_20 2026-02-18T08:38:45.868 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:38:45.868 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_20 ']' 2026-02-18T08:38:45.868 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_20' 2026-02-18T08:38:45.868 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_20 2026-02-18T08:38:45.868 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:38:45.868 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:38:45.869 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:38:45.869 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:38:45.869 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:38:45.869 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:38:45.869 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:38:45.869 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:38:45.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:38:45.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:38:45.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_20 2026-02-18T08:38:45.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:38:45.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:38:45.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:38:45.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:38:45.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:38:45.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:38:45.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:38:45.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:38:45.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:38:45.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:38:45.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:38:45.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:38:45.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:38:45.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:38:45.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:38:45.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:38:45.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:38:45.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:38:45.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:38:45.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:38:45.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:38:45.874 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:38:45.874 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:38:45.874 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:38:45.874 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:38:45.874 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:38:45.874 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:38:45.874 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:38:45.874 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:38:45.874 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:38:45.882 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:38:45.883 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:38:45.883 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:38:45.883 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_20 2026-02-18T08:38:45.933 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_20'] 2026-02-18T08:38:45.933 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:38:45.933 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:38:45.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:38:45.936 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_20 2026-02-18T08:38:45.936 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:38:45.936 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:38:45.936 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:38:45.936 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:38:45.936 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:38:45.936 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:38:45.936 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:38:45.936 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:38:45.936 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:38:45.936 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:38:45.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:38:45.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:38:45.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:38:45.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:38:45.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:38:45.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:38:45.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:38:45.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:38:45.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:38:45.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:38:45.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:38:45.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:38:45.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:38:45.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:38:45.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:38:45.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:38:45.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:38:45.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:38:45.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:38:45.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:38:45.947 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:38:45.947 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:38:45.948 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:38:45.948 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_20 2026-02-18T08:38:46.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_20'] 2026-02-18T08:38:46.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:38:46.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:38:46.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:38:46.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_20 2026-02-18T08:38:46.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:38:46.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:38:46.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:38:46.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:38:46.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:38:46.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:38:46.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:38:46.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:38:46.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:38:46.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:38:46.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:38:46.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:38:46.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:38:46.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:38:46.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:38:46.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:38:46.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:38:46.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:38:46.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:38:46.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:38:46.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:38:46.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:38:46.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:38:46.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:38:46.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:38:46.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:38:46.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:38:46.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:38:46.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:38:46.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:38:46.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:38:46.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:38:46.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:38:46.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_20 2026-02-18T08:38:46.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:38:46.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_20 2026-02-18T08:38:46.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:38:46.092 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:38:46.092 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_20 2026-02-18T08:38:46.092 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_20 '' 2026-02-18T08:38:46.092 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:38:46.092 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:38:46.092 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_20 2026-02-18T08:38:46.092 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:38:46.092 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_20.status 2026-02-18T08:38:46.093 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_20.status 2026-02-18T08:38:46.093 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:38:46.093 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_20.status 2026-02-18T08:38:46.094 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_20 2026-02-18T08:38:46.094 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_20.status 2026-02-18T08:38:46.122 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 8 2026-02-18T08:38:46.122 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 8 2026-02-18T08:38:46.122 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:38:46.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=33, tag_tid=2, entry_tid=1], [object_number=32, tag_tid=2, entry_tid=0], [object_number=35, tag_tid=1, entry_tid=35979], [object_number=34, tag_tid=1, entry_tid=35978]]], state=connected] 2026-02-18T08:38:46.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=32, tag_tid=1, entry_tid=35980], [object_number=35, tag_tid=1, entry_tid=35979], [object_number=34, tag_tid=1, entry_tid=35978], [object_number=33, tag_tid=1, entry_tid=35977]]], state=connected] 2026-02-18T08:38:46.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_20.status 2026-02-18T08:38:46.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=33, tag_tid=2, entry_tid=1' 2026-02-18T08:38:46.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_20 2026-02-18T08:38:46.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:38:46.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:38:46.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_20 2026-02-18T08:38:46.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_20 '..*' 2026-02-18T08:38:46.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:38:46.129 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:38:46.129 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_20 2026-02-18T08:38:46.129 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:38:46.129 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_20.status 2026-02-18T08:38:46.129 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_20.status 2026-02-18T08:38:46.130 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:38:46.130 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_20.status 2026-02-18T08:38:46.130 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_20 2026-02-18T08:38:46.130 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_20.status 2026-02-18T08:38:46.156 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 8 2026-02-18T08:38:46.156 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 8 2026-02-18T08:38:46.156 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:38:46.156 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=33, tag_tid=2, entry_tid=1], [object_number=32, tag_tid=2, entry_tid=0], [object_number=35, tag_tid=1, entry_tid=35979], [object_number=34, tag_tid=1, entry_tid=35978]]], state=connected] 2026-02-18T08:38:46.156 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=32, tag_tid=1, entry_tid=35980], [object_number=35, tag_tid=1, entry_tid=35979], [object_number=34, tag_tid=1, entry_tid=35978], [object_number=33, tag_tid=1, entry_tid=35977]]], state=connected] 2026-02-18T08:38:46.158 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_20.status 2026-02-18T08:38:46.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=32, tag_tid=1, entry_tid=35980' 2026-02-18T08:38:46.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=33, tag_tid=2, entry_tid=1' -a 'object_number=33, tag_tid=2, entry_tid=1' = 'object_number=32, tag_tid=1, entry_tid=35980' 2026-02-18T08:38:46.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=32, tag_tid=1, entry_tid=35980' '!=' 'object_number=32, tag_tid=1, entry_tid=35980' 2026-02-18T08:38:46.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:38:46.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1.6 2026-02-18T08:38:47.761 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_20 2026-02-18T08:38:47.762 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:38:47.762 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:38:47.762 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_20 2026-02-18T08:38:47.762 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:38:47.762 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_20 ']' 2026-02-18T08:38:47.762 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_20' 2026-02-18T08:38:47.762 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_20 2026-02-18T08:38:47.762 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:38:47.762 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:38:47.763 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:38:47.763 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:38:47.763 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:38:47.763 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:38:47.763 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:38:47.766 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:38:47.766 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:38:47.766 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:38:47.766 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_20 2026-02-18T08:38:47.766 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:38:47.766 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:38:47.766 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:38:47.766 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:38:47.766 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:38:47.767 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:38:47.767 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:38:47.767 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:38:47.767 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:38:47.767 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:38:47.767 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:38:47.767 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:38:47.767 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:38:47.767 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:38:47.767 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:38:47.767 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:38:47.767 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:38:47.768 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:38:47.768 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:38:47.768 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:38:47.768 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:38:47.768 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:38:47.768 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:38:47.768 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:38:47.768 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:38:47.768 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:38:47.768 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:38:47.768 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:38:47.768 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:38:47.768 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:38:47.776 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:38:47.777 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:38:47.777 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:38:47.777 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_20 2026-02-18T08:38:47.828 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_20'] 2026-02-18T08:38:47.828 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:38:47.828 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:38:47.831 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:38:47.831 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_20 2026-02-18T08:38:47.831 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:38:47.831 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:38:47.831 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:38:47.831 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:38:47.832 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:38:47.832 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:38:47.832 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:38:47.832 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:38:47.832 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:38:47.832 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:38:47.832 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:38:47.832 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:38:47.832 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:38:47.832 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:38:47.832 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:38:47.832 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:38:47.832 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:38:47.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:38:47.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:38:47.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:38:47.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:38:47.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:38:47.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:38:47.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:38:47.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:38:47.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:38:47.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:38:47.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:38:47.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:38:47.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:38:47.842 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:38:47.842 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:38:47.842 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:38:47.843 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_20 2026-02-18T08:38:47.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_20'] 2026-02-18T08:38:47.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:38:47.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:38:47.905 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:38:47.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_20 2026-02-18T08:38:47.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:38:47.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:38:47.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:38:47.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:38:47.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:38:47.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:38:47.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:38:47.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:38:47.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:38:47.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:38:47.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:38:47.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:38:47.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:38:47.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:38:47.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:38:47.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:38:47.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:38:47.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:38:47.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:38:47.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:38:47.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:38:47.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:38:47.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:38:47.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:38:47.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:38:47.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:38:47.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:38:47.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:38:47.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:38:47.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:38:47.915 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:38:47.916 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:38:47.916 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:38:47.916 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_20 2026-02-18T08:38:47.987 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:38:47.987 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_20 2026-02-18T08:38:47.987 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:38:47.988 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:38:47.988 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_20 2026-02-18T08:38:47.988 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_20 '' 2026-02-18T08:38:47.988 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:38:47.988 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:38:47.988 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_20 2026-02-18T08:38:47.988 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:38:47.988 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_20.status 2026-02-18T08:38:47.989 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_20.status 2026-02-18T08:38:47.989 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:38:47.989 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_20.status 2026-02-18T08:38:47.990 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_20 2026-02-18T08:38:47.990 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_20.status 2026-02-18T08:38:48.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 8 2026-02-18T08:38:48.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 8 2026-02-18T08:38:48.018 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:38:48.018 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=33, tag_tid=2, entry_tid=1], [object_number=32, tag_tid=2, entry_tid=0], [object_number=35, tag_tid=1, entry_tid=35979], [object_number=34, tag_tid=1, entry_tid=35978]]], state=connected] 2026-02-18T08:38:48.018 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=33, tag_tid=2, entry_tid=1], [object_number=32, tag_tid=2, entry_tid=0], [object_number=35, tag_tid=1, entry_tid=35979], [object_number=34, tag_tid=1, entry_tid=35978]]], state=connected] 2026-02-18T08:38:48.020 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_20.status 2026-02-18T08:38:48.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=33, tag_tid=2, entry_tid=1' 2026-02-18T08:38:48.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_20 2026-02-18T08:38:48.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:38:48.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:38:48.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_20 2026-02-18T08:38:48.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_20 '..*' 2026-02-18T08:38:48.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:38:48.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:38:48.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_20 2026-02-18T08:38:48.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:38:48.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_20.status 2026-02-18T08:38:48.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_20.status 2026-02-18T08:38:48.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:38:48.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_20.status 2026-02-18T08:38:48.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_20 2026-02-18T08:38:48.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_20.status 2026-02-18T08:38:48.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 8 2026-02-18T08:38:48.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 8 2026-02-18T08:38:48.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:38:48.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=33, tag_tid=2, entry_tid=1], [object_number=32, tag_tid=2, entry_tid=0], [object_number=35, tag_tid=1, entry_tid=35979], [object_number=34, tag_tid=1, entry_tid=35978]]], state=connected] 2026-02-18T08:38:48.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=33, tag_tid=2, entry_tid=1], [object_number=32, tag_tid=2, entry_tid=0], [object_number=35, tag_tid=1, entry_tid=35979], [object_number=34, tag_tid=1, entry_tid=35978]]], state=connected] 2026-02-18T08:38:48.055 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_20.status 2026-02-18T08:38:48.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=33, tag_tid=2, entry_tid=1' 2026-02-18T08:38:48.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=33, tag_tid=2, entry_tid=1' -a 'object_number=33, tag_tid=2, entry_tid=1' = 'object_number=33, tag_tid=2, entry_tid=1' 2026-02-18T08:38:48.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:38:48.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_snap_present cluster1 mirror image_20 snap 2026-02-18T08:38:48.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:38:48.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:38:48.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_20 2026-02-18T08:38:48.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap 2026-02-18T08:38:48.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-18T08:38:48.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-18T08:38:48.058 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1 2026-02-18T08:38:49.058 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster1 info mirror/image_20@snap 2026-02-18T08:38:49.086 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:rbd image 'image_20': 2026-02-18T08:38:49.086 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: size 128 MiB in 32 objects 2026-02-18T08:38:49.086 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: order 22 (4 MiB objects) 2026-02-18T08:38:49.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: snapshot_count: 1 2026-02-18T08:38:49.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: id: 11f9c07acdd1 2026-02-18T08:38:49.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: block_name_prefix: rbd_data.11f9c07acdd1 2026-02-18T08:38:49.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: format: 2 2026-02-18T08:38:49.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: features: layering, exclusive-lock, journaling 2026-02-18T08:38:49.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: op_features: 2026-02-18T08:38:49.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: flags: 2026-02-18T08:38:49.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: create_timestamp: Wed Feb 18 08:29:13 2026 2026-02-18T08:38:49.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: access_timestamp: Wed Feb 18 08:29:13 2026 2026-02-18T08:38:49.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: modify_timestamp: Wed Feb 18 08:30:14 2026 2026-02-18T08:38:49.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: protected: False 2026-02-18T08:38:49.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: journal: 11f9c07acdd1 2026-02-18T08:38:49.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring state: enabled 2026-02-18T08:38:49.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring mode: journal 2026-02-18T08:38:49.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring global id: f8f99cb0-70d4-4f77-a1ee-2a8409e2fbc6 2026-02-18T08:38:49.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring primary: false 2026-02-18T08:38:49.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:38:49.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ compare_image_snaps mirror image_20 snap 2026-02-18T08:38:49.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:38:49.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_20 2026-02-18T08:38:49.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap 2026-02-18T08:38:49.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local ret=0 2026-02-18T08:38:49.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local rmt_export=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_20.export 2026-02-18T08:38:49.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loc_export=/tmp/tmp.vybh1m9yJF/cluster1-mirror-image_20.export 2026-02-18T08:38:49.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_20.export /tmp/tmp.vybh1m9yJF/cluster1-mirror-image_20.export 2026-02-18T08:38:49.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 -p mirror export image_20@snap /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_20.export 2026-02-18T08:38:49.231 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-18T08:38:49.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster1 -p mirror export image_20@snap /tmp/tmp.vybh1m9yJF/cluster1-mirror-image_20.export 2026-02-18T08:38:49.474 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-18T08:38:49.485 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmp /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_20.export /tmp/tmp.vybh1m9yJF/cluster1-mirror-image_20.export 2026-02-18T08:38:49.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_20.export /tmp/tmp.vybh1m9yJF/cluster1-mirror-image_20.export 2026-02-18T08:38:49.573 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:38:49.574 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T08:38:49.574 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_21 2026-02-18T08:38:49.574 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_snap cluster2 mirror image_21 snap 2026-02-18T08:38:49.574 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:38:49.574 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:38:49.574 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_21 2026-02-18T08:38:49.574 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap 2026-02-18T08:38:49.574 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 -p mirror snap create image_21@snap --debug-rbd=20 --debug-journaler=20 2026-02-18T08:38:50.924 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_replay_started cluster1 mirror image_21 2026-02-18T08:38:50.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:38:50.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:38:50.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_21 2026-02-18T08:38:50.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_replay_state cluster1 mirror image_21 started 2026-02-18T08:38:50.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:38:50.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:38:50.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_21 2026-02-18T08:38:50.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local state=started 2026-02-18T08:38:50.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-18T08:38:50.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:38:50.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1 2026-02-18T08:38:51.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test_image_replay_state cluster1 mirror image_21 started 2026-02-18T08:38:51.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:38:51.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:38:51.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_21 2026-02-18T08:38:51.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local test_state=started 2026-02-18T08:38:51.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local status_result 2026-02-18T08:38:51.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_state=stopped 2026-02-18T08:38:51.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_21 2026-02-18T08:38:51.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -i state 2026-02-18T08:38:51.928 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_instance=cluster1 2026-02-18T08:38:51.928 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-18T08:38:51.928 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:38:51.928 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance=cluster1 2026-02-18T08:38:51.928 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local loop_instance 2026-02-18T08:38:51.928 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:38:51.928 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sleep 0 2026-02-18T08:38:51.928 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:38:51.931 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ seq 0 3 2026-02-18T08:38:51.931 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:38:51.931 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_21 2026-02-18T08:38:51.931 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:0 2026-02-18T08:38:51.932 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-18T08:38:51.932 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:38:51.932 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:38:51.932 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:38:51.932 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-18T08:38:51.932 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:38:51.932 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:38:51.932 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:38:51.932 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:38:51.932 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-18T08:38:51.932 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:38:51.933 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:38:51.933 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:38:51.933 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:38:51.933 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:0 2026-02-18T08:38:51.933 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-18T08:38:51.933 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-18T08:38:51.933 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:38:51.933 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:0 2026-02-18T08:38:51.933 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T08:38:51.933 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-18T08:38:51.933 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:38:51.933 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:38:51.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-18T08:38:51.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-18T08:38:51.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-18T08:38:51.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=0 2026-02-18T08:38:51.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=0 2026-02-18T08:38:51.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:38:51.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:38:51.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:38:51.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:38:51.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_21 2026-02-18T08:38:51.999 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ return 0 2026-02-18T08:38:52.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ status_result=' "state": "Replaying"' 2026-02-18T08:38:52.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ echo ' "state": "Replaying"' 2026-02-18T08:38:52.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ grep -i Replaying 2026-02-18T08:38:52.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: "state": "Replaying" 2026-02-18T08:38:52.003 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ current_state=started 2026-02-18T08:38:52.003 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test started = started 2026-02-18T08:38:52.003 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:38:52.003 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror image_21 2026-02-18T08:38:52.003 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_cluster=cluster1 2026-02-18T08:38:52.003 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:38:52.003 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_pool=mirror 2026-02-18T08:38:52.003 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local remote_pool=mirror 2026-02-18T08:38:52.004 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_21 2026-02-18T08:38:52.004 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' journal = journal ']' 2026-02-18T08:38:52.004 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror mirror image_21 2026-02-18T08:38:52.004 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_cluster=cluster1 2026-02-18T08:38:52.004 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:38:52.004 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_pool=mirror 2026-02-18T08:38:52.004 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local remote_pool=mirror 2026-02-18T08:38:52.004 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_21 2026-02-18T08:38:52.004 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-02-18T08:38:52.004 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-02-18T08:38:52.004 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:38:52.004 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:38:52.005 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:38:52.204 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_21 2026-02-18T08:38:52.204 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:38:52.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:38:52.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_21 2026-02-18T08:38:52.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:38:52.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_21 ']' 2026-02-18T08:38:52.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_21' 2026-02-18T08:38:52.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_21 2026-02-18T08:38:52.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:38:52.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:38:52.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:38:52.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:38:52.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:38:52.206 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:38:52.206 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:38:52.206 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:38:52.206 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:38:52.208 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:38:52.209 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_21 2026-02-18T08:38:52.209 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:38:52.209 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:38:52.209 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:38:52.209 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:38:52.209 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:38:52.209 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:38:52.209 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:38:52.209 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:38:52.209 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:38:52.209 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:38:52.210 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:38:52.210 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:38:52.210 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:38:52.210 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:38:52.210 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:38:52.210 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:38:52.210 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:38:52.210 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:38:52.210 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:38:52.210 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:38:52.211 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:38:52.211 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:38:52.211 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:38:52.211 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:38:52.211 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:38:52.211 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:38:52.211 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:38:52.212 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:38:52.212 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:38:52.212 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:38:52.219 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:38:52.219 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:38:52.220 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:38:52.220 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_21 2026-02-18T08:38:52.286 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:38:52.286 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_21 2026-02-18T08:38:52.287 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:38:52.287 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:38:52.287 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_21 2026-02-18T08:38:52.287 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_21 '' 2026-02-18T08:38:52.287 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:38:52.287 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:38:52.287 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_21 2026-02-18T08:38:52.287 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:38:52.287 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_21.status 2026-02-18T08:38:52.287 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_21.status 2026-02-18T08:38:52.288 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:38:52.288 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_21.status 2026-02-18T08:38:52.289 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_21 2026-02-18T08:38:52.289 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_21.status 2026-02-18T08:38:52.325 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 4 2026-02-18T08:38:52.325 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 4 2026-02-18T08:38:52.325 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:38:52.325 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=2, entry_tid=1], [object_number=16, tag_tid=2, entry_tid=0], [object_number=19, tag_tid=1, entry_tid=17879], [object_number=18, tag_tid=1, entry_tid=17878]]], state=connected] 2026-02-18T08:38:52.325 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=17, tag_tid=1, entry_tid=17881], [object_number=16, tag_tid=1, entry_tid=17880], [object_number=19, tag_tid=1, entry_tid=17879], [object_number=18, tag_tid=1, entry_tid=17878]]], state=connected] 2026-02-18T08:38:52.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_21.status 2026-02-18T08:38:52.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=17, tag_tid=2, entry_tid=1' 2026-02-18T08:38:52.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_21 2026-02-18T08:38:52.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:38:52.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:38:52.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_21 2026-02-18T08:38:52.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_21 '..*' 2026-02-18T08:38:52.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:38:52.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:38:52.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_21 2026-02-18T08:38:52.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:38:52.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_21.status 2026-02-18T08:38:52.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_21.status 2026-02-18T08:38:52.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:38:52.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_21.status 2026-02-18T08:38:52.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_21.status 2026-02-18T08:38:52.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_21 2026-02-18T08:38:52.360 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 4 2026-02-18T08:38:52.360 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 4 2026-02-18T08:38:52.360 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:38:52.360 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=2, entry_tid=1], [object_number=16, tag_tid=2, entry_tid=0], [object_number=19, tag_tid=1, entry_tid=17879], [object_number=18, tag_tid=1, entry_tid=17878]]], state=connected] 2026-02-18T08:38:52.360 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=17, tag_tid=1, entry_tid=17881], [object_number=16, tag_tid=1, entry_tid=17880], [object_number=19, tag_tid=1, entry_tid=17879], [object_number=18, tag_tid=1, entry_tid=17878]]], state=connected] 2026-02-18T08:38:52.362 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_21.status 2026-02-18T08:38:52.364 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=17, tag_tid=1, entry_tid=17881' 2026-02-18T08:38:52.364 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=17, tag_tid=2, entry_tid=1' -a 'object_number=17, tag_tid=2, entry_tid=1' = 'object_number=17, tag_tid=1, entry_tid=17881' 2026-02-18T08:38:52.364 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=17, tag_tid=1, entry_tid=17881' '!=' '' 2026-02-18T08:38:52.364 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:38:52.364 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=17, tag_tid=1, entry_tid=17881' = '' 2026-02-18T08:38:52.364 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=17, tag_tid=1, entry_tid=17881' 2026-02-18T08:38:52.365 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=17, tag_tid=2, entry_tid=1' 2026-02-18T08:38:52.365 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:38:52.365 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:38:52.366 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=2 2026-02-18T08:38:52.366 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=17, tag_tid=1, entry_tid=17881' 2026-02-18T08:38:52.367 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:38:52.367 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:38:52.368 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=1 2026-02-18T08:38:52.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=17, tag_tid=2, entry_tid=1' 2026-02-18T08:38:52.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:38:52.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:38:52.370 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:38:52.371 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=17, tag_tid=1, entry_tid=17881' 2026-02-18T08:38:52.371 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:38:52.371 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:38:52.372 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=17881 2026-02-18T08:38:52.372 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 2 = 1 -a 1 -le 17881 2026-02-18T08:38:52.372 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:38:52.372 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:38:52.372 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:38:52.574 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_21 2026-02-18T08:38:52.575 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:38:52.575 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:38:52.575 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_21 2026-02-18T08:38:52.575 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:38:52.575 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_21 ']' 2026-02-18T08:38:52.575 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_21' 2026-02-18T08:38:52.575 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_21 2026-02-18T08:38:52.576 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:38:52.576 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:38:52.576 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:38:52.576 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:38:52.576 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:38:52.576 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:38:52.576 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:38:52.576 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:38:52.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:38:52.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:38:52.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_21 2026-02-18T08:38:52.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:38:52.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:38:52.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:38:52.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:38:52.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:38:52.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:38:52.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:38:52.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:38:52.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:38:52.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:38:52.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:38:52.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:38:52.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:38:52.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:38:52.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:38:52.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:38:52.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:38:52.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:38:52.582 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:38:52.582 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:38:52.582 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:38:52.582 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:38:52.582 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:38:52.582 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:38:52.582 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:38:52.582 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:38:52.582 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:38:52.582 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:38:52.582 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:38:52.582 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:38:52.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:38:52.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:38:52.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:38:52.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_21 2026-02-18T08:38:52.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:38:52.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_21 2026-02-18T08:38:52.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:38:52.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:38:52.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_21 2026-02-18T08:38:52.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_21 '' 2026-02-18T08:38:52.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:38:52.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:38:52.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_21 2026-02-18T08:38:52.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:38:52.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_21.status 2026-02-18T08:38:52.658 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_21.status 2026-02-18T08:38:52.658 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:38:52.658 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_21.status 2026-02-18T08:38:52.659 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_21 2026-02-18T08:38:52.659 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_21.status 2026-02-18T08:38:52.683 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 4 2026-02-18T08:38:52.683 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 4 2026-02-18T08:38:52.684 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:38:52.684 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=2, entry_tid=1], [object_number=16, tag_tid=2, entry_tid=0], [object_number=19, tag_tid=1, entry_tid=17879], [object_number=18, tag_tid=1, entry_tid=17878]]], state=connected] 2026-02-18T08:38:52.684 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=17, tag_tid=1, entry_tid=17881], [object_number=16, tag_tid=1, entry_tid=17880], [object_number=19, tag_tid=1, entry_tid=17879], [object_number=18, tag_tid=1, entry_tid=17878]]], state=connected] 2026-02-18T08:38:52.686 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_21.status 2026-02-18T08:38:52.687 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=17, tag_tid=2, entry_tid=1' 2026-02-18T08:38:52.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_21 2026-02-18T08:38:52.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:38:52.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:38:52.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_21 2026-02-18T08:38:52.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_21 '..*' 2026-02-18T08:38:52.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:38:52.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:38:52.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_21 2026-02-18T08:38:52.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:38:52.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_21.status 2026-02-18T08:38:52.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_21.status 2026-02-18T08:38:52.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:38:52.691 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_21.status 2026-02-18T08:38:52.691 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_21 2026-02-18T08:38:52.691 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_21.status 2026-02-18T08:38:52.724 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 4 2026-02-18T08:38:52.725 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 4 2026-02-18T08:38:52.725 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:38:52.725 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=2, entry_tid=1], [object_number=16, tag_tid=2, entry_tid=0], [object_number=19, tag_tid=1, entry_tid=17879], [object_number=18, tag_tid=1, entry_tid=17878]]], state=connected] 2026-02-18T08:38:52.725 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=17, tag_tid=1, entry_tid=17881], [object_number=16, tag_tid=1, entry_tid=17880], [object_number=19, tag_tid=1, entry_tid=17879], [object_number=18, tag_tid=1, entry_tid=17878]]], state=connected] 2026-02-18T08:38:52.728 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_21.status 2026-02-18T08:38:52.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=17, tag_tid=1, entry_tid=17881' 2026-02-18T08:38:52.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=17, tag_tid=2, entry_tid=1' -a 'object_number=17, tag_tid=2, entry_tid=1' = 'object_number=17, tag_tid=1, entry_tid=17881' 2026-02-18T08:38:52.731 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=17, tag_tid=1, entry_tid=17881' '!=' 'object_number=17, tag_tid=1, entry_tid=17881' 2026-02-18T08:38:52.731 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:38:52.731 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.4 2026-02-18T08:38:53.132 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_21 2026-02-18T08:38:53.132 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:38:53.132 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:38:53.132 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_21 2026-02-18T08:38:53.132 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:38:53.132 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_21 ']' 2026-02-18T08:38:53.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_21' 2026-02-18T08:38:53.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_21 2026-02-18T08:38:53.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:38:53.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:38:53.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:38:53.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:38:53.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:38:53.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:38:53.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:38:53.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:38:53.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:38:53.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:38:53.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_21 2026-02-18T08:38:53.136 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:38:53.136 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:38:53.136 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:38:53.136 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:38:53.136 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:38:53.136 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:38:53.136 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:38:53.136 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:38:53.136 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:38:53.136 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:38:53.136 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:38:53.136 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:38:53.136 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:38:53.136 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:38:53.136 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:38:53.136 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:38:53.137 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:38:53.137 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:38:53.137 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:38:53.137 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:38:53.137 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:38:53.137 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:38:53.137 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:38:53.137 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:38:53.137 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:38:53.137 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:38:53.137 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:38:53.137 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:38:53.137 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:38:53.137 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:38:53.146 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:38:53.147 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:38:53.147 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:38:53.147 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_21 2026-02-18T08:38:53.203 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:38:53.204 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_21 2026-02-18T08:38:53.204 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:38:53.204 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:38:53.204 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_21 2026-02-18T08:38:53.204 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_21 '' 2026-02-18T08:38:53.204 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:38:53.204 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:38:53.204 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_21 2026-02-18T08:38:53.204 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:38:53.204 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_21.status 2026-02-18T08:38:53.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_21.status 2026-02-18T08:38:53.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:38:53.206 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_21.status 2026-02-18T08:38:53.206 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_21 2026-02-18T08:38:53.206 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_21.status 2026-02-18T08:38:53.244 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 4 2026-02-18T08:38:53.244 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 4 2026-02-18T08:38:53.244 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:38:53.244 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=2, entry_tid=1], [object_number=16, tag_tid=2, entry_tid=0], [object_number=19, tag_tid=1, entry_tid=17879], [object_number=18, tag_tid=1, entry_tid=17878]]], state=connected] 2026-02-18T08:38:53.244 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=17, tag_tid=1, entry_tid=17881], [object_number=16, tag_tid=1, entry_tid=17880], [object_number=19, tag_tid=1, entry_tid=17879], [object_number=18, tag_tid=1, entry_tid=17878]]], state=connected] 2026-02-18T08:38:53.247 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_21.status 2026-02-18T08:38:53.249 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=17, tag_tid=2, entry_tid=1' 2026-02-18T08:38:53.250 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_21 2026-02-18T08:38:53.250 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:38:53.250 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:38:53.250 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_21 2026-02-18T08:38:53.250 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_21 '..*' 2026-02-18T08:38:53.250 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:38:53.250 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:38:53.250 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_21 2026-02-18T08:38:53.250 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:38:53.250 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_21.status 2026-02-18T08:38:53.250 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_21.status 2026-02-18T08:38:53.250 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:38:53.252 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_21.status 2026-02-18T08:38:53.252 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_21 2026-02-18T08:38:53.252 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_21.status 2026-02-18T08:38:53.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 4 2026-02-18T08:38:53.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 4 2026-02-18T08:38:53.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:38:53.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=2, entry_tid=1], [object_number=16, tag_tid=2, entry_tid=0], [object_number=19, tag_tid=1, entry_tid=17879], [object_number=18, tag_tid=1, entry_tid=17878]]], state=connected] 2026-02-18T08:38:53.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=17, tag_tid=1, entry_tid=17881], [object_number=16, tag_tid=1, entry_tid=17880], [object_number=19, tag_tid=1, entry_tid=17879], [object_number=18, tag_tid=1, entry_tid=17878]]], state=connected] 2026-02-18T08:38:53.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_21.status 2026-02-18T08:38:53.500 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=17, tag_tid=1, entry_tid=17881' 2026-02-18T08:38:53.500 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=17, tag_tid=2, entry_tid=1' -a 'object_number=17, tag_tid=2, entry_tid=1' = 'object_number=17, tag_tid=1, entry_tid=17881' 2026-02-18T08:38:53.500 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=17, tag_tid=1, entry_tid=17881' '!=' 'object_number=17, tag_tid=1, entry_tid=17881' 2026-02-18T08:38:53.500 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:38:53.500 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.8 2026-02-18T08:38:54.301 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_21 2026-02-18T08:38:54.301 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:38:54.302 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:38:54.302 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_21 2026-02-18T08:38:54.302 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:38:54.302 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_21 ']' 2026-02-18T08:38:54.302 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_21' 2026-02-18T08:38:54.302 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_21 2026-02-18T08:38:54.302 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:38:54.302 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:38:54.302 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:38:54.302 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:38:54.302 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:38:54.302 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:38:54.302 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:38:54.304 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:38:54.304 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:38:54.304 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:38:54.304 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_21 2026-02-18T08:38:54.304 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:38:54.304 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:38:54.304 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:38:54.304 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:38:54.304 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:38:54.304 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:38:54.304 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:38:54.304 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:38:54.304 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:38:54.304 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:38:54.304 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:38:54.304 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:38:54.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:38:54.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:38:54.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:38:54.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:38:54.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:38:54.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:38:54.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:38:54.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:38:54.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:38:54.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:38:54.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:38:54.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:38:54.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:38:54.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:38:54.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:38:54.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:38:54.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:38:54.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:38:54.316 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:38:54.317 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:38:54.317 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:38:54.317 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_21 2026-02-18T08:38:54.382 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:38:54.383 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_21 2026-02-18T08:38:54.383 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:38:54.383 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:38:54.383 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_21 2026-02-18T08:38:54.383 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_21 '' 2026-02-18T08:38:54.383 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:38:54.383 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:38:54.383 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_21 2026-02-18T08:38:54.383 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:38:54.384 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_21.status 2026-02-18T08:38:54.384 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_21.status 2026-02-18T08:38:54.384 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:38:54.385 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_21.status 2026-02-18T08:38:54.385 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_21 2026-02-18T08:38:54.385 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_21.status 2026-02-18T08:38:54.413 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 4 2026-02-18T08:38:54.413 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 4 2026-02-18T08:38:54.413 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:38:54.413 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=2, entry_tid=1], [object_number=16, tag_tid=2, entry_tid=0], [object_number=19, tag_tid=1, entry_tid=17879], [object_number=18, tag_tid=1, entry_tid=17878]]], state=connected] 2026-02-18T08:38:54.413 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=17, tag_tid=1, entry_tid=17881], [object_number=16, tag_tid=1, entry_tid=17880], [object_number=19, tag_tid=1, entry_tid=17879], [object_number=18, tag_tid=1, entry_tid=17878]]], state=connected] 2026-02-18T08:38:54.415 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_21.status 2026-02-18T08:38:54.416 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=17, tag_tid=2, entry_tid=1' 2026-02-18T08:38:54.417 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_21 2026-02-18T08:38:54.417 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:38:54.417 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:38:54.417 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_21 2026-02-18T08:38:54.417 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_21 '..*' 2026-02-18T08:38:54.417 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:38:54.417 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:38:54.417 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_21 2026-02-18T08:38:54.417 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:38:54.418 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_21.status 2026-02-18T08:38:54.418 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_21.status 2026-02-18T08:38:54.418 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:38:54.419 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_21.status 2026-02-18T08:38:54.420 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_21 2026-02-18T08:38:54.420 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_21.status 2026-02-18T08:38:54.455 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 4 2026-02-18T08:38:54.455 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 4 2026-02-18T08:38:54.455 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:38:54.455 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=2, entry_tid=1], [object_number=16, tag_tid=2, entry_tid=0], [object_number=19, tag_tid=1, entry_tid=17879], [object_number=18, tag_tid=1, entry_tid=17878]]], state=connected] 2026-02-18T08:38:54.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=17, tag_tid=1, entry_tid=17881], [object_number=16, tag_tid=1, entry_tid=17880], [object_number=19, tag_tid=1, entry_tid=17879], [object_number=18, tag_tid=1, entry_tid=17878]]], state=connected] 2026-02-18T08:38:54.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_21.status 2026-02-18T08:38:54.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=17, tag_tid=1, entry_tid=17881' 2026-02-18T08:38:54.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=17, tag_tid=2, entry_tid=1' -a 'object_number=17, tag_tid=2, entry_tid=1' = 'object_number=17, tag_tid=1, entry_tid=17881' 2026-02-18T08:38:54.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=17, tag_tid=1, entry_tid=17881' '!=' 'object_number=17, tag_tid=1, entry_tid=17881' 2026-02-18T08:38:54.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:38:54.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1.6 2026-02-18T08:38:56.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_21 2026-02-18T08:38:56.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:38:56.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:38:56.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_21 2026-02-18T08:38:56.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:38:56.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_21 ']' 2026-02-18T08:38:56.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_21' 2026-02-18T08:38:56.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_21 2026-02-18T08:38:56.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:38:56.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:38:56.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:38:56.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:38:56.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:38:56.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:38:56.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:38:56.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:38:56.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:38:56.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:38:56.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_21 2026-02-18T08:38:56.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:38:56.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:38:56.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:38:56.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:38:56.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:38:56.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:38:56.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:38:56.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:38:56.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:38:56.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:38:56.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:38:56.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:38:56.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:38:56.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:38:56.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:38:56.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:38:56.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:38:56.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:38:56.067 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:38:56.067 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:38:56.067 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:38:56.067 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:38:56.067 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:38:56.067 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:38:56.067 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:38:56.068 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:38:56.068 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:38:56.068 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:38:56.068 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:38:56.068 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:38:56.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:38:56.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:38:56.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:38:56.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_21 2026-02-18T08:38:56.138 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:38:56.139 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_21 2026-02-18T08:38:56.139 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:38:56.139 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:38:56.139 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_21 2026-02-18T08:38:56.139 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_21 '' 2026-02-18T08:38:56.139 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:38:56.139 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:38:56.139 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_21 2026-02-18T08:38:56.139 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:38:56.139 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_21.status 2026-02-18T08:38:56.140 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_21.status 2026-02-18T08:38:56.140 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:38:56.140 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_21.status 2026-02-18T08:38:56.141 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_21 2026-02-18T08:38:56.141 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_21.status 2026-02-18T08:38:56.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 4 2026-02-18T08:38:56.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 4 2026-02-18T08:38:56.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:38:56.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=2, entry_tid=1], [object_number=16, tag_tid=2, entry_tid=0], [object_number=19, tag_tid=1, entry_tid=17879], [object_number=18, tag_tid=1, entry_tid=17878]]], state=connected] 2026-02-18T08:38:56.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=17, tag_tid=1, entry_tid=17881], [object_number=16, tag_tid=1, entry_tid=17880], [object_number=19, tag_tid=1, entry_tid=17879], [object_number=18, tag_tid=1, entry_tid=17878]]], state=connected] 2026-02-18T08:38:56.171 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_21.status 2026-02-18T08:38:56.173 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=17, tag_tid=2, entry_tid=1' 2026-02-18T08:38:56.174 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_21 2026-02-18T08:38:56.174 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:38:56.174 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:38:56.174 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_21 2026-02-18T08:38:56.174 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_21 '..*' 2026-02-18T08:38:56.174 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:38:56.174 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:38:56.174 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_21 2026-02-18T08:38:56.174 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:38:56.174 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_21.status 2026-02-18T08:38:56.174 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_21.status 2026-02-18T08:38:56.174 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:38:56.176 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_21.status 2026-02-18T08:38:56.176 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_21 2026-02-18T08:38:56.176 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_21.status 2026-02-18T08:38:56.211 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 4 2026-02-18T08:38:56.211 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 4 2026-02-18T08:38:56.211 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:38:56.211 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=2, entry_tid=1], [object_number=16, tag_tid=2, entry_tid=0], [object_number=19, tag_tid=1, entry_tid=17879], [object_number=18, tag_tid=1, entry_tid=17878]]], state=connected] 2026-02-18T08:38:56.211 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=17, tag_tid=1, entry_tid=17881], [object_number=16, tag_tid=1, entry_tid=17880], [object_number=19, tag_tid=1, entry_tid=17879], [object_number=18, tag_tid=1, entry_tid=17878]]], state=connected] 2026-02-18T08:38:56.215 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_21.status 2026-02-18T08:38:56.217 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=17, tag_tid=1, entry_tid=17881' 2026-02-18T08:38:56.217 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=17, tag_tid=2, entry_tid=1' -a 'object_number=17, tag_tid=2, entry_tid=1' = 'object_number=17, tag_tid=1, entry_tid=17881' 2026-02-18T08:38:56.217 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=17, tag_tid=1, entry_tid=17881' '!=' 'object_number=17, tag_tid=1, entry_tid=17881' 2026-02-18T08:38:56.217 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:38:56.217 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 2 2026-02-18T08:38:58.219 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_21 2026-02-18T08:38:58.219 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:38:58.219 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:38:58.219 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_21 2026-02-18T08:38:58.219 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:38:58.219 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_21 ']' 2026-02-18T08:38:58.219 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_21' 2026-02-18T08:38:58.219 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_21 2026-02-18T08:38:58.219 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:38:58.219 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:38:58.219 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:38:58.219 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:38:58.219 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:38:58.220 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:38:58.220 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:38:58.220 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:38:58.220 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:38:58.222 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:38:58.223 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_21 2026-02-18T08:38:58.223 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:38:58.223 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:38:58.223 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:38:58.223 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:38:58.223 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:38:58.223 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:38:58.223 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:38:58.223 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:38:58.223 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:38:58.223 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:38:58.223 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:38:58.223 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:38:58.223 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:38:58.223 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:38:58.223 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:38:58.224 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:38:58.224 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:38:58.224 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:38:58.224 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:38:58.224 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:38:58.224 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:38:58.224 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:38:58.224 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:38:58.224 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:38:58.224 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:38:58.224 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:38:58.224 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:38:58.224 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:38:58.224 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:38:58.224 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:38:58.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:38:58.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:38:58.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:38:58.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_21 2026-02-18T08:38:58.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:38:58.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_21 2026-02-18T08:38:58.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:38:58.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:38:58.304 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_21 2026-02-18T08:38:58.304 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_21 '' 2026-02-18T08:38:58.304 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:38:58.304 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:38:58.304 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_21 2026-02-18T08:38:58.304 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:38:58.304 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_21.status 2026-02-18T08:38:58.304 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_21.status 2026-02-18T08:38:58.304 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:38:58.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_21.status 2026-02-18T08:38:58.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_21 2026-02-18T08:38:58.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_21.status 2026-02-18T08:38:58.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 4 2026-02-18T08:38:58.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 4 2026-02-18T08:38:58.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:38:58.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=2, entry_tid=1], [object_number=16, tag_tid=2, entry_tid=0], [object_number=19, tag_tid=1, entry_tid=17879], [object_number=18, tag_tid=1, entry_tid=17878]]], state=connected] 2026-02-18T08:38:58.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=17, tag_tid=2, entry_tid=1], [object_number=16, tag_tid=2, entry_tid=0], [object_number=19, tag_tid=1, entry_tid=17879], [object_number=18, tag_tid=1, entry_tid=17878]]], state=connected] 2026-02-18T08:38:58.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_21.status 2026-02-18T08:38:58.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=17, tag_tid=2, entry_tid=1' 2026-02-18T08:38:58.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_21 2026-02-18T08:38:58.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:38:58.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:38:58.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_21 2026-02-18T08:38:58.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_21 '..*' 2026-02-18T08:38:58.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:38:58.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:38:58.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_21 2026-02-18T08:38:58.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:38:58.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_21.status 2026-02-18T08:38:58.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_21.status 2026-02-18T08:38:58.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:38:58.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_21.status 2026-02-18T08:38:58.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_21 2026-02-18T08:38:58.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_21.status 2026-02-18T08:38:58.374 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 4 2026-02-18T08:38:58.374 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 4 2026-02-18T08:38:58.374 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:38:58.374 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=2, entry_tid=1], [object_number=16, tag_tid=2, entry_tid=0], [object_number=19, tag_tid=1, entry_tid=17879], [object_number=18, tag_tid=1, entry_tid=17878]]], state=connected] 2026-02-18T08:38:58.374 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=17, tag_tid=2, entry_tid=1], [object_number=16, tag_tid=2, entry_tid=0], [object_number=19, tag_tid=1, entry_tid=17879], [object_number=18, tag_tid=1, entry_tid=17878]]], state=connected] 2026-02-18T08:38:58.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_21.status 2026-02-18T08:38:58.380 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=17, tag_tid=2, entry_tid=1' 2026-02-18T08:38:58.380 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=17, tag_tid=2, entry_tid=1' -a 'object_number=17, tag_tid=2, entry_tid=1' = 'object_number=17, tag_tid=2, entry_tid=1' 2026-02-18T08:38:58.380 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:38:58.380 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_snap_present cluster1 mirror image_21 snap 2026-02-18T08:38:58.380 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:38:58.380 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:38:58.380 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_21 2026-02-18T08:38:58.380 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap 2026-02-18T08:38:58.380 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-18T08:38:58.380 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-18T08:38:58.380 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1 2026-02-18T08:38:59.381 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster1 info mirror/image_21@snap 2026-02-18T08:38:59.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:rbd image 'image_21': 2026-02-18T08:38:59.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: size 128 MiB in 32 objects 2026-02-18T08:38:59.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: order 22 (4 MiB objects) 2026-02-18T08:38:59.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: snapshot_count: 1 2026-02-18T08:38:59.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: id: 10ac72d206d4 2026-02-18T08:38:59.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: block_name_prefix: rbd_data.10ac72d206d4 2026-02-18T08:38:59.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: format: 2 2026-02-18T08:38:59.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: features: layering, exclusive-lock, journaling 2026-02-18T08:38:59.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: op_features: 2026-02-18T08:38:59.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: flags: 2026-02-18T08:38:59.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: create_timestamp: Wed Feb 18 08:29:24 2026 2026-02-18T08:38:59.412 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: access_timestamp: Wed Feb 18 08:29:24 2026 2026-02-18T08:38:59.412 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: modify_timestamp: Wed Feb 18 08:30:28 2026 2026-02-18T08:38:59.412 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: protected: False 2026-02-18T08:38:59.412 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: journal: 10ac72d206d4 2026-02-18T08:38:59.412 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring state: enabled 2026-02-18T08:38:59.412 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring mode: journal 2026-02-18T08:38:59.412 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring global id: 87d5b386-10fa-4a77-a70b-1c819b6b9282 2026-02-18T08:38:59.412 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring primary: false 2026-02-18T08:38:59.414 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:38:59.414 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ compare_image_snaps mirror image_21 snap 2026-02-18T08:38:59.414 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:38:59.414 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_21 2026-02-18T08:38:59.414 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap 2026-02-18T08:38:59.414 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local ret=0 2026-02-18T08:38:59.414 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local rmt_export=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_21.export 2026-02-18T08:38:59.414 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loc_export=/tmp/tmp.vybh1m9yJF/cluster1-mirror-image_21.export 2026-02-18T08:38:59.414 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_21.export /tmp/tmp.vybh1m9yJF/cluster1-mirror-image_21.export 2026-02-18T08:38:59.415 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 -p mirror export image_21@snap /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_21.export 2026-02-18T08:38:59.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-18T08:38:59.558 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster1 -p mirror export image_21@snap /tmp/tmp.vybh1m9yJF/cluster1-mirror-image_21.export 2026-02-18T08:38:59.721 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-18T08:38:59.733 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmp /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_21.export /tmp/tmp.vybh1m9yJF/cluster1-mirror-image_21.export 2026-02-18T08:38:59.788 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_21.export /tmp/tmp.vybh1m9yJF/cluster1-mirror-image_21.export 2026-02-18T08:38:59.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:38:59.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T08:38:59.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_22 2026-02-18T08:38:59.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_snap cluster2 mirror image_22 snap 2026-02-18T08:38:59.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:38:59.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:38:59.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_22 2026-02-18T08:38:59.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap 2026-02-18T08:38:59.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 -p mirror snap create image_22@snap --debug-rbd=20 --debug-journaler=20 2026-02-18T08:39:02.944 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_replay_started cluster1 mirror image_22 2026-02-18T08:39:02.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:39:02.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:39:02.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_22 2026-02-18T08:39:02.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_replay_state cluster1 mirror image_22 started 2026-02-18T08:39:02.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:39:02.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:39:02.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_22 2026-02-18T08:39:02.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local state=started 2026-02-18T08:39:02.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-18T08:39:02.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:39:02.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1 2026-02-18T08:39:03.947 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test_image_replay_state cluster1 mirror image_22 started 2026-02-18T08:39:03.947 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:39:03.948 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:39:03.948 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_22 2026-02-18T08:39:03.948 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local test_state=started 2026-02-18T08:39:03.948 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local status_result 2026-02-18T08:39:03.948 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_state=stopped 2026-02-18T08:39:03.948 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_22 2026-02-18T08:39:03.948 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -i state 2026-02-18T08:39:03.948 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_instance=cluster1 2026-02-18T08:39:03.948 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-18T08:39:03.949 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:39:03.949 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance=cluster1 2026-02-18T08:39:03.949 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local loop_instance 2026-02-18T08:39:03.949 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:39:03.949 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sleep 0 2026-02-18T08:39:03.952 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:39:03.952 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ seq 0 3 2026-02-18T08:39:03.952 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:39:03.952 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_22 2026-02-18T08:39:03.953 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:0 2026-02-18T08:39:03.953 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-18T08:39:03.953 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:39:03.953 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:39:03.953 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:39:03.953 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-18T08:39:03.953 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:39:03.953 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:39:03.953 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:39:03.953 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:39:03.953 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-18T08:39:03.954 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:39:03.954 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:39:03.954 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:39:03.954 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:39:03.954 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:0 2026-02-18T08:39:03.954 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-18T08:39:03.954 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-18T08:39:03.954 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:39:03.954 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:0 2026-02-18T08:39:03.954 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T08:39:03.955 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-18T08:39:03.955 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:39:03.955 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:39:03.955 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-18T08:39:03.955 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-18T08:39:03.955 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-18T08:39:03.955 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=0 2026-02-18T08:39:03.955 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=0 2026-02-18T08:39:03.955 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:39:03.961 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:39:03.962 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:39:03.962 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:39:03.962 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_22 2026-02-18T08:39:04.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_22'] 2026-02-18T08:39:04.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status : get status for rbd mirror 2026-02-18T08:39:04.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:39:04.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:39:04.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_22 2026-02-18T08:39:04.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:1 2026-02-18T08:39:04.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-18T08:39:04.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:39:04.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:39:04.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:39:04.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-18T08:39:04.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:39:04.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:39:04.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:39:04.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:39:04.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-18T08:39:04.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:39:04.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:39:04.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:39:04.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:39:04.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:1 2026-02-18T08:39:04.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-18T08:39:04.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-18T08:39:04.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:39:04.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:1 2026-02-18T08:39:04.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T08:39:04.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-18T08:39:04.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:39:04.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:39:04.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-18T08:39:04.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-18T08:39:04.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-18T08:39:04.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=1 2026-02-18T08:39:04.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=1 2026-02-18T08:39:04.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:39:04.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:39:04.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:39:04.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:39:04.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_22 2026-02-18T08:39:04.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ return 0 2026-02-18T08:39:04.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ status_result=' "state": "Replaying"' 2026-02-18T08:39:04.092 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ echo ' "state": "Replaying"' 2026-02-18T08:39:04.092 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ grep -i Replaying 2026-02-18T08:39:04.093 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: "state": "Replaying" 2026-02-18T08:39:04.093 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ current_state=started 2026-02-18T08:39:04.094 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test started = started 2026-02-18T08:39:04.094 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:39:04.094 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror image_22 2026-02-18T08:39:04.094 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_cluster=cluster1 2026-02-18T08:39:04.094 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:39:04.094 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_pool=mirror 2026-02-18T08:39:04.094 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local remote_pool=mirror 2026-02-18T08:39:04.094 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_22 2026-02-18T08:39:04.094 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' journal = journal ']' 2026-02-18T08:39:04.094 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror mirror image_22 2026-02-18T08:39:04.094 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_cluster=cluster1 2026-02-18T08:39:04.094 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:39:04.094 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_pool=mirror 2026-02-18T08:39:04.094 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local remote_pool=mirror 2026-02-18T08:39:04.094 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_22 2026-02-18T08:39:04.094 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-02-18T08:39:04.094 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-02-18T08:39:04.094 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:39:04.094 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:39:04.094 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:39:04.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_22 2026-02-18T08:39:04.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:39:04.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:39:04.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_22 2026-02-18T08:39:04.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:39:04.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_22 ']' 2026-02-18T08:39:04.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_22' 2026-02-18T08:39:04.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_22 2026-02-18T08:39:04.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:39:04.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:39:04.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:39:04.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:39:04.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:39:04.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:39:04.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:39:04.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:39:04.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:39:04.298 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:39:04.298 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_22 2026-02-18T08:39:04.298 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:39:04.298 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:39:04.298 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:39:04.298 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:39:04.298 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:39:04.298 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:39:04.298 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:39:04.298 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:39:04.298 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:39:04.298 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:39:04.298 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:39:04.298 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:39:04.298 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:39:04.299 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:39:04.299 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:39:04.299 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:39:04.299 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:39:04.299 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:39:04.299 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:39:04.299 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:39:04.300 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:39:04.300 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:39:04.300 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:39:04.300 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:39:04.300 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:39:04.300 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:39:04.300 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:39:04.300 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:39:04.300 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:39:04.300 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:39:04.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:39:04.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:39:04.312 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:39:04.312 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_22 2026-02-18T08:39:04.361 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_22'] 2026-02-18T08:39:04.361 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:39:04.361 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:39:04.363 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:39:04.363 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_22 2026-02-18T08:39:04.363 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:39:04.363 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:39:04.363 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:39:04.363 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:39:04.363 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:39:04.364 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:39:04.364 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:39:04.364 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:39:04.364 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:39:04.364 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:39:04.364 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:39:04.364 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:39:04.364 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:39:04.364 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:39:04.364 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:39:04.364 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:39:04.364 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:39:04.364 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:39:04.364 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:39:04.364 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:39:04.364 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:39:04.364 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:39:04.364 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:39:04.364 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:39:04.364 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:39:04.364 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:39:04.364 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:39:04.364 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:39:04.365 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:39:04.365 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:39:04.374 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:39:04.375 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:39:04.375 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:39:04.375 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_22 2026-02-18T08:39:04.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:39:04.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_22 2026-02-18T08:39:04.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:39:04.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:39:04.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_22 2026-02-18T08:39:04.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_22 '' 2026-02-18T08:39:04.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:39:04.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:39:04.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_22 2026-02-18T08:39:04.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:39:04.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_22.status 2026-02-18T08:39:04.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_22.status 2026-02-18T08:39:04.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:39:04.443 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_22.status 2026-02-18T08:39:04.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_22 2026-02-18T08:39:04.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_22.status 2026-02-18T08:39:04.479 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 10 2026-02-18T08:39:04.479 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 10 2026-02-18T08:39:04.479 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:39:04.480 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=41, tag_tid=2, entry_tid=1], [object_number=40, tag_tid=2, entry_tid=0], [object_number=43, tag_tid=1, entry_tid=44267], [object_number=42, tag_tid=1, entry_tid=44266]]], state=connected] 2026-02-18T08:39:04.480 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=41, tag_tid=1, entry_tid=44269], [object_number=40, tag_tid=1, entry_tid=44268], [object_number=43, tag_tid=1, entry_tid=44267], [object_number=42, tag_tid=1, entry_tid=44266]]], state=connected] 2026-02-18T08:39:04.483 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_22.status 2026-02-18T08:39:04.485 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=41, tag_tid=2, entry_tid=1' 2026-02-18T08:39:04.485 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_22 2026-02-18T08:39:04.485 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:39:04.485 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:39:04.485 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_22 2026-02-18T08:39:04.485 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_22 '..*' 2026-02-18T08:39:04.485 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:39:04.485 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:39:04.486 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_22 2026-02-18T08:39:04.486 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:39:04.486 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_22.status 2026-02-18T08:39:04.486 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_22.status 2026-02-18T08:39:04.486 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:39:04.487 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_22.status 2026-02-18T08:39:04.488 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_22 2026-02-18T08:39:04.488 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_22.status 2026-02-18T08:39:04.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 10 2026-02-18T08:39:04.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 10 2026-02-18T08:39:04.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:39:04.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=41, tag_tid=2, entry_tid=1], [object_number=40, tag_tid=2, entry_tid=0], [object_number=43, tag_tid=1, entry_tid=44267], [object_number=42, tag_tid=1, entry_tid=44266]]], state=connected] 2026-02-18T08:39:04.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=41, tag_tid=1, entry_tid=44269], [object_number=40, tag_tid=1, entry_tid=44268], [object_number=43, tag_tid=1, entry_tid=44267], [object_number=42, tag_tid=1, entry_tid=44266]]], state=connected] 2026-02-18T08:39:04.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_22.status 2026-02-18T08:39:04.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=41, tag_tid=1, entry_tid=44269' 2026-02-18T08:39:04.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=41, tag_tid=2, entry_tid=1' -a 'object_number=41, tag_tid=2, entry_tid=1' = 'object_number=41, tag_tid=1, entry_tid=44269' 2026-02-18T08:39:04.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=41, tag_tid=1, entry_tid=44269' '!=' '' 2026-02-18T08:39:04.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:39:04.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=41, tag_tid=1, entry_tid=44269' = '' 2026-02-18T08:39:04.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=41, tag_tid=1, entry_tid=44269' 2026-02-18T08:39:04.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=41, tag_tid=2, entry_tid=1' 2026-02-18T08:39:04.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:39:04.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:39:04.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=2 2026-02-18T08:39:04.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=41, tag_tid=1, entry_tid=44269' 2026-02-18T08:39:04.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:39:04.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:39:04.524 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=1 2026-02-18T08:39:04.524 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=41, tag_tid=2, entry_tid=1' 2026-02-18T08:39:04.524 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:39:04.524 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:39:04.526 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:39:04.526 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=41, tag_tid=1, entry_tid=44269' 2026-02-18T08:39:04.526 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:39:04.527 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:39:04.528 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=44269 2026-02-18T08:39:04.528 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 2 = 1 -a 1 -le 44269 2026-02-18T08:39:04.528 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:39:04.528 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:39:04.528 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:39:04.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_22 2026-02-18T08:39:04.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:39:04.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:39:04.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_22 2026-02-18T08:39:04.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:39:04.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_22 ']' 2026-02-18T08:39:04.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_22' 2026-02-18T08:39:04.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_22 2026-02-18T08:39:04.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:39:04.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:39:04.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:39:04.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:39:04.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:39:04.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:39:04.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:39:04.731 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:39:04.731 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:39:04.732 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:39:04.733 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_22 2026-02-18T08:39:04.733 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:39:04.733 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:39:04.733 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:39:04.733 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:39:04.733 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:39:04.733 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:39:04.733 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:39:04.733 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:39:04.733 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:39:04.733 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:39:04.733 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:39:04.733 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:39:04.733 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:39:04.733 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:39:04.734 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:39:04.734 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:39:04.734 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:39:04.734 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:39:04.734 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:39:04.734 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:39:04.734 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:39:04.734 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:39:04.734 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:39:04.734 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:39:04.734 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:39:04.734 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:39:04.734 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:39:04.734 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:39:04.734 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:39:04.735 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:39:04.746 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:39:04.746 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:39:04.746 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:39:04.746 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_22 2026-02-18T08:39:04.804 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_22'] 2026-02-18T08:39:04.804 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:39:04.804 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:39:04.806 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:39:04.806 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_22 2026-02-18T08:39:04.806 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:39:04.806 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:39:04.806 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:39:04.806 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:39:04.806 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:39:04.806 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:39:04.806 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:39:04.806 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:39:04.806 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:39:04.806 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:39:04.806 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:39:04.806 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:39:04.807 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:39:04.807 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:39:04.807 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:39:04.807 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:39:04.807 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:39:04.807 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:39:04.807 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:39:04.807 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:39:04.807 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:39:04.807 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:39:04.807 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:39:04.807 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:39:04.807 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:39:04.807 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:39:04.807 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:39:04.807 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:39:04.807 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:39:04.807 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:39:04.818 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:39:04.819 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:39:04.819 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:39:04.819 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_22 2026-02-18T08:39:04.884 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:39:04.884 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_22 2026-02-18T08:39:04.884 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:39:04.884 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:39:04.884 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_22 2026-02-18T08:39:04.884 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_22 '' 2026-02-18T08:39:04.884 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:39:04.884 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:39:04.885 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_22 2026-02-18T08:39:04.885 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:39:04.885 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_22.status 2026-02-18T08:39:04.885 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_22.status 2026-02-18T08:39:04.885 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:39:04.886 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_22.status 2026-02-18T08:39:04.887 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_22 2026-02-18T08:39:04.887 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_22.status 2026-02-18T08:39:04.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 10 2026-02-18T08:39:04.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 10 2026-02-18T08:39:04.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:39:04.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=41, tag_tid=2, entry_tid=1], [object_number=40, tag_tid=2, entry_tid=0], [object_number=43, tag_tid=1, entry_tid=44267], [object_number=42, tag_tid=1, entry_tid=44266]]], state=connected] 2026-02-18T08:39:04.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=41, tag_tid=1, entry_tid=44269], [object_number=40, tag_tid=1, entry_tid=44268], [object_number=43, tag_tid=1, entry_tid=44267], [object_number=42, tag_tid=1, entry_tid=44266]]], state=connected] 2026-02-18T08:39:04.928 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_22.status 2026-02-18T08:39:04.930 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=41, tag_tid=2, entry_tid=1' 2026-02-18T08:39:04.931 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_22 2026-02-18T08:39:04.931 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:39:04.931 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:39:04.931 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_22 2026-02-18T08:39:04.931 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_22 '..*' 2026-02-18T08:39:04.931 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:39:04.931 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:39:04.931 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_22 2026-02-18T08:39:04.931 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:39:04.931 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_22.status 2026-02-18T08:39:04.931 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_22.status 2026-02-18T08:39:04.931 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:39:04.933 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_22.status 2026-02-18T08:39:04.933 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_22 2026-02-18T08:39:04.933 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_22.status 2026-02-18T08:39:04.967 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 10 2026-02-18T08:39:04.967 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 10 2026-02-18T08:39:04.967 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:39:04.967 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=41, tag_tid=2, entry_tid=1], [object_number=40, tag_tid=2, entry_tid=0], [object_number=43, tag_tid=1, entry_tid=44267], [object_number=42, tag_tid=1, entry_tid=44266]]], state=connected] 2026-02-18T08:39:04.967 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=41, tag_tid=1, entry_tid=44269], [object_number=40, tag_tid=1, entry_tid=44268], [object_number=43, tag_tid=1, entry_tid=44267], [object_number=42, tag_tid=1, entry_tid=44266]]], state=connected] 2026-02-18T08:39:04.970 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_22.status 2026-02-18T08:39:04.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=41, tag_tid=1, entry_tid=44269' 2026-02-18T08:39:04.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=41, tag_tid=2, entry_tid=1' -a 'object_number=41, tag_tid=2, entry_tid=1' = 'object_number=41, tag_tid=1, entry_tid=44269' 2026-02-18T08:39:04.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=41, tag_tid=1, entry_tid=44269' '!=' 'object_number=41, tag_tid=1, entry_tid=44269' 2026-02-18T08:39:04.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:39:04.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.4 2026-02-18T08:39:05.374 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_22 2026-02-18T08:39:05.374 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:39:05.374 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:39:05.374 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_22 2026-02-18T08:39:05.374 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:39:05.374 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_22 ']' 2026-02-18T08:39:05.375 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_22' 2026-02-18T08:39:05.375 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_22 2026-02-18T08:39:05.375 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:39:05.375 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:39:05.375 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:39:05.375 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:39:05.375 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:39:05.375 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:39:05.375 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:39:05.375 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:39:05.375 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:39:05.377 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:39:05.377 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_22 2026-02-18T08:39:05.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:39:05.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:39:05.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:39:05.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:39:05.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:39:05.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:39:05.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:39:05.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:39:05.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:39:05.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:39:05.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:39:05.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:39:05.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:39:05.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:39:05.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:39:05.379 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:39:05.379 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:39:05.379 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:39:05.379 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:39:05.379 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:39:05.379 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:39:05.379 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:39:05.379 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:39:05.379 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:39:05.379 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:39:05.379 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:39:05.379 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:39:05.379 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:39:05.379 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:39:05.380 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:39:05.390 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:39:05.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:39:05.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:39:05.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_22 2026-02-18T08:39:05.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_22'] 2026-02-18T08:39:05.443 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:39:05.443 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:39:05.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:39:05.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_22 2026-02-18T08:39:05.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:39:05.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:39:05.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:39:05.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:39:05.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:39:05.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:39:05.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:39:05.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:39:05.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:39:05.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:39:05.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:39:05.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:39:05.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:39:05.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:39:05.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:39:05.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:39:05.446 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:39:05.446 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:39:05.446 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:39:05.446 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:39:05.446 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:39:05.446 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:39:05.446 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:39:05.446 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:39:05.446 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:39:05.446 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:39:05.446 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:39:05.446 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:39:05.446 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:39:05.446 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:39:05.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:39:05.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:39:05.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:39:05.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_22 2026-02-18T08:39:05.516 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:39:05.516 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_22 2026-02-18T08:39:05.516 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:39:05.516 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:39:05.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_22 2026-02-18T08:39:05.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_22 '' 2026-02-18T08:39:05.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:39:05.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:39:05.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_22 2026-02-18T08:39:05.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:39:05.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_22.status 2026-02-18T08:39:05.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_22.status 2026-02-18T08:39:05.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:39:05.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_22.status 2026-02-18T08:39:05.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_22 2026-02-18T08:39:05.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_22.status 2026-02-18T08:39:05.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 10 2026-02-18T08:39:05.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 10 2026-02-18T08:39:05.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:39:05.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=41, tag_tid=2, entry_tid=1], [object_number=40, tag_tid=2, entry_tid=0], [object_number=43, tag_tid=1, entry_tid=44267], [object_number=42, tag_tid=1, entry_tid=44266]]], state=connected] 2026-02-18T08:39:05.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=41, tag_tid=1, entry_tid=44269], [object_number=40, tag_tid=1, entry_tid=44268], [object_number=43, tag_tid=1, entry_tid=44267], [object_number=42, tag_tid=1, entry_tid=44266]]], state=connected] 2026-02-18T08:39:05.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_22.status 2026-02-18T08:39:05.552 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=41, tag_tid=2, entry_tid=1' 2026-02-18T08:39:05.553 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_22 2026-02-18T08:39:05.553 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:39:05.553 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:39:05.553 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_22 2026-02-18T08:39:05.553 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_22 '..*' 2026-02-18T08:39:05.553 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:39:05.553 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:39:05.553 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_22 2026-02-18T08:39:05.553 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:39:05.553 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_22.status 2026-02-18T08:39:05.553 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_22.status 2026-02-18T08:39:05.553 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:39:05.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_22.status 2026-02-18T08:39:05.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_22 2026-02-18T08:39:05.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_22.status 2026-02-18T08:39:05.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 10 2026-02-18T08:39:05.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 10 2026-02-18T08:39:05.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:39:05.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=41, tag_tid=2, entry_tid=1], [object_number=40, tag_tid=2, entry_tid=0], [object_number=43, tag_tid=1, entry_tid=44267], [object_number=42, tag_tid=1, entry_tid=44266]]], state=connected] 2026-02-18T08:39:05.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=41, tag_tid=1, entry_tid=44269], [object_number=40, tag_tid=1, entry_tid=44268], [object_number=43, tag_tid=1, entry_tid=44267], [object_number=42, tag_tid=1, entry_tid=44266]]], state=connected] 2026-02-18T08:39:05.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_22.status 2026-02-18T08:39:05.595 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=41, tag_tid=1, entry_tid=44269' 2026-02-18T08:39:05.595 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=41, tag_tid=2, entry_tid=1' -a 'object_number=41, tag_tid=2, entry_tid=1' = 'object_number=41, tag_tid=1, entry_tid=44269' 2026-02-18T08:39:05.595 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=41, tag_tid=1, entry_tid=44269' '!=' 'object_number=41, tag_tid=1, entry_tid=44269' 2026-02-18T08:39:05.595 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:39:05.595 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.8 2026-02-18T08:39:06.397 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_22 2026-02-18T08:39:06.397 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:39:06.397 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:39:06.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_22 2026-02-18T08:39:06.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:39:06.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_22 ']' 2026-02-18T08:39:06.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_22' 2026-02-18T08:39:06.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_22 2026-02-18T08:39:06.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:39:06.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:39:06.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:39:06.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:39:06.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:39:06.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:39:06.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:39:06.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:39:06.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:39:06.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:39:06.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_22 2026-02-18T08:39:06.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:39:06.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:39:06.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:39:06.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:39:06.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:39:06.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:39:06.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:39:06.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:39:06.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:39:06.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:39:06.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:39:06.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:39:06.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:39:06.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:39:06.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:39:06.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:39:06.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:39:06.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:39:06.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:39:06.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:39:06.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:39:06.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:39:06.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:39:06.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:39:06.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:39:06.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:39:06.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:39:06.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:39:06.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:39:06.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:39:06.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:39:06.412 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:39:06.412 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:39:06.412 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_22 2026-02-18T08:39:06.472 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_22'] 2026-02-18T08:39:06.472 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:39:06.473 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:39:06.475 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:39:06.476 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_22 2026-02-18T08:39:06.476 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:39:06.476 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:39:06.476 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:39:06.476 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:39:06.476 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:39:06.476 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:39:06.476 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:39:06.476 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:39:06.476 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:39:06.476 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:39:06.476 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:39:06.476 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:39:06.476 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:39:06.476 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:39:06.476 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:39:06.476 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:39:06.476 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:39:06.477 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:39:06.477 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:39:06.477 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:39:06.477 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:39:06.477 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:39:06.477 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:39:06.477 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:39:06.477 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:39:06.477 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:39:06.477 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:39:06.477 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:39:06.477 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:39:06.477 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:39:06.487 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:39:06.487 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:39:06.487 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:39:06.487 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_22 2026-02-18T08:39:06.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:39:06.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_22 2026-02-18T08:39:06.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:39:06.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:39:06.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_22 2026-02-18T08:39:06.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_22 '' 2026-02-18T08:39:06.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:39:06.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:39:06.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_22 2026-02-18T08:39:06.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:39:06.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_22.status 2026-02-18T08:39:06.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_22.status 2026-02-18T08:39:06.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:39:06.557 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_22.status 2026-02-18T08:39:06.557 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_22 2026-02-18T08:39:06.558 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_22.status 2026-02-18T08:39:06.586 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 10 2026-02-18T08:39:06.586 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 10 2026-02-18T08:39:06.586 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:39:06.586 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=41, tag_tid=2, entry_tid=1], [object_number=40, tag_tid=2, entry_tid=0], [object_number=43, tag_tid=1, entry_tid=44267], [object_number=42, tag_tid=1, entry_tid=44266]]], state=connected] 2026-02-18T08:39:06.586 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=41, tag_tid=1, entry_tid=44269], [object_number=40, tag_tid=1, entry_tid=44268], [object_number=43, tag_tid=1, entry_tid=44267], [object_number=42, tag_tid=1, entry_tid=44266]]], state=connected] 2026-02-18T08:39:06.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_22.status 2026-02-18T08:39:06.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=41, tag_tid=2, entry_tid=1' 2026-02-18T08:39:06.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_22 2026-02-18T08:39:06.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:39:06.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:39:06.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_22 2026-02-18T08:39:06.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_22 '..*' 2026-02-18T08:39:06.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:39:06.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:39:06.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_22 2026-02-18T08:39:06.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:39:06.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_22.status 2026-02-18T08:39:06.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_22.status 2026-02-18T08:39:06.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:39:06.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_22.status 2026-02-18T08:39:06.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_22 2026-02-18T08:39:06.594 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_22.status 2026-02-18T08:39:06.620 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 10 2026-02-18T08:39:06.620 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 10 2026-02-18T08:39:06.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:39:06.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=41, tag_tid=2, entry_tid=1], [object_number=40, tag_tid=2, entry_tid=0], [object_number=43, tag_tid=1, entry_tid=44267], [object_number=42, tag_tid=1, entry_tid=44266]]], state=connected] 2026-02-18T08:39:06.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=41, tag_tid=1, entry_tid=44269], [object_number=40, tag_tid=1, entry_tid=44268], [object_number=43, tag_tid=1, entry_tid=44267], [object_number=42, tag_tid=1, entry_tid=44266]]], state=connected] 2026-02-18T08:39:06.623 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_22.status 2026-02-18T08:39:06.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=41, tag_tid=1, entry_tid=44269' 2026-02-18T08:39:06.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=41, tag_tid=2, entry_tid=1' -a 'object_number=41, tag_tid=2, entry_tid=1' = 'object_number=41, tag_tid=1, entry_tid=44269' 2026-02-18T08:39:06.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=41, tag_tid=1, entry_tid=44269' '!=' 'object_number=41, tag_tid=1, entry_tid=44269' 2026-02-18T08:39:06.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:39:06.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1.6 2026-02-18T08:39:08.227 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_22 2026-02-18T08:39:08.227 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:39:08.227 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:39:08.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_22 2026-02-18T08:39:08.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:39:08.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_22 ']' 2026-02-18T08:39:08.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_22' 2026-02-18T08:39:08.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_22 2026-02-18T08:39:08.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:39:08.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:39:08.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:39:08.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:39:08.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:39:08.229 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:39:08.229 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:39:08.229 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:39:08.231 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:39:08.231 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:39:08.231 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_22 2026-02-18T08:39:08.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:39:08.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:39:08.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:39:08.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:39:08.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:39:08.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:39:08.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:39:08.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:39:08.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:39:08.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:39:08.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:39:08.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:39:08.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:39:08.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:39:08.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:39:08.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:39:08.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:39:08.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:39:08.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:39:08.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:39:08.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:39:08.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:39:08.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:39:08.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:39:08.234 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:39:08.234 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:39:08.234 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:39:08.234 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:39:08.234 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:39:08.234 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:39:08.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:39:08.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:39:08.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:39:08.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_22 2026-02-18T08:39:08.301 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_22'] 2026-02-18T08:39:08.301 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:39:08.301 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:39:08.304 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:39:08.304 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_22 2026-02-18T08:39:08.304 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:39:08.304 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:39:08.304 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:39:08.304 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:39:08.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:39:08.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:39:08.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:39:08.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:39:08.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:39:08.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:39:08.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:39:08.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:39:08.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:39:08.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:39:08.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:39:08.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:39:08.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:39:08.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:39:08.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:39:08.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:39:08.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:39:08.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:39:08.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:39:08.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:39:08.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:39:08.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:39:08.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:39:08.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:39:08.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:39:08.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:39:08.315 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:39:08.316 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:39:08.316 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:39:08.316 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_22 2026-02-18T08:39:08.385 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:39:08.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_22 2026-02-18T08:39:08.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:39:08.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:39:08.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_22 2026-02-18T08:39:08.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_22 '' 2026-02-18T08:39:08.387 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:39:08.387 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:39:08.387 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_22 2026-02-18T08:39:08.387 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:39:08.387 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_22.status 2026-02-18T08:39:08.387 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_22.status 2026-02-18T08:39:08.387 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:39:08.388 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_22.status 2026-02-18T08:39:08.388 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_22 2026-02-18T08:39:08.388 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_22.status 2026-02-18T08:39:08.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 10 2026-02-18T08:39:08.425 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 10 2026-02-18T08:39:08.425 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:39:08.425 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=41, tag_tid=2, entry_tid=1], [object_number=40, tag_tid=2, entry_tid=0], [object_number=43, tag_tid=1, entry_tid=44267], [object_number=42, tag_tid=1, entry_tid=44266]]], state=connected] 2026-02-18T08:39:08.425 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=41, tag_tid=2, entry_tid=1], [object_number=40, tag_tid=2, entry_tid=0], [object_number=43, tag_tid=1, entry_tid=44267], [object_number=42, tag_tid=1, entry_tid=44266]]], state=connected] 2026-02-18T08:39:08.428 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_22.status 2026-02-18T08:39:08.429 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=41, tag_tid=2, entry_tid=1' 2026-02-18T08:39:08.430 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_22 2026-02-18T08:39:08.431 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:39:08.431 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:39:08.431 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_22 2026-02-18T08:39:08.431 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_22 '..*' 2026-02-18T08:39:08.431 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:39:08.431 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:39:08.431 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_22 2026-02-18T08:39:08.431 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:39:08.431 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_22.status 2026-02-18T08:39:08.432 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_22.status 2026-02-18T08:39:08.432 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:39:08.432 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_22.status 2026-02-18T08:39:08.433 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_22 2026-02-18T08:39:08.433 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_22.status 2026-02-18T08:39:08.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 10 2026-02-18T08:39:08.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 10 2026-02-18T08:39:08.462 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:39:08.462 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=41, tag_tid=2, entry_tid=1], [object_number=40, tag_tid=2, entry_tid=0], [object_number=43, tag_tid=1, entry_tid=44267], [object_number=42, tag_tid=1, entry_tid=44266]]], state=connected] 2026-02-18T08:39:08.462 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=41, tag_tid=2, entry_tid=1], [object_number=40, tag_tid=2, entry_tid=0], [object_number=43, tag_tid=1, entry_tid=44267], [object_number=42, tag_tid=1, entry_tid=44266]]], state=connected] 2026-02-18T08:39:08.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_22.status 2026-02-18T08:39:08.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=41, tag_tid=2, entry_tid=1' 2026-02-18T08:39:08.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=41, tag_tid=2, entry_tid=1' -a 'object_number=41, tag_tid=2, entry_tid=1' = 'object_number=41, tag_tid=2, entry_tid=1' 2026-02-18T08:39:08.467 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:39:08.467 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_snap_present cluster1 mirror image_22 snap 2026-02-18T08:39:08.467 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:39:08.467 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:39:08.467 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_22 2026-02-18T08:39:08.467 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap 2026-02-18T08:39:08.467 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-18T08:39:08.467 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-18T08:39:08.467 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1 2026-02-18T08:39:09.467 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster1 info mirror/image_22@snap 2026-02-18T08:39:09.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:rbd image 'image_22': 2026-02-18T08:39:09.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: size 128 MiB in 32 objects 2026-02-18T08:39:09.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: order 22 (4 MiB objects) 2026-02-18T08:39:09.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: snapshot_count: 1 2026-02-18T08:39:09.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: id: 10ed32fc2f01 2026-02-18T08:39:09.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: block_name_prefix: rbd_data.10ed32fc2f01 2026-02-18T08:39:09.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: format: 2 2026-02-18T08:39:09.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: features: layering, exclusive-lock, journaling 2026-02-18T08:39:09.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: op_features: 2026-02-18T08:39:09.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: flags: 2026-02-18T08:39:09.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: create_timestamp: Wed Feb 18 08:29:29 2026 2026-02-18T08:39:09.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: access_timestamp: Wed Feb 18 08:29:29 2026 2026-02-18T08:39:09.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: modify_timestamp: Wed Feb 18 08:30:30 2026 2026-02-18T08:39:09.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: protected: False 2026-02-18T08:39:09.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: journal: 10ed32fc2f01 2026-02-18T08:39:09.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring state: enabled 2026-02-18T08:39:09.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring mode: journal 2026-02-18T08:39:09.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring global id: e88c55c0-e283-428a-b844-459cfe64b04f 2026-02-18T08:39:09.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring primary: false 2026-02-18T08:39:09.500 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:39:09.500 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ compare_image_snaps mirror image_22 snap 2026-02-18T08:39:09.501 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:39:09.501 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_22 2026-02-18T08:39:09.501 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap 2026-02-18T08:39:09.501 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local ret=0 2026-02-18T08:39:09.501 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local rmt_export=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_22.export 2026-02-18T08:39:09.501 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loc_export=/tmp/tmp.vybh1m9yJF/cluster1-mirror-image_22.export 2026-02-18T08:39:09.501 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_22.export /tmp/tmp.vybh1m9yJF/cluster1-mirror-image_22.export 2026-02-18T08:39:09.501 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 -p mirror export image_22@snap /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_22.export 2026-02-18T08:39:09.653 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-18T08:39:09.663 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster1 -p mirror export image_22@snap /tmp/tmp.vybh1m9yJF/cluster1-mirror-image_22.export 2026-02-18T08:39:09.855 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-18T08:39:09.864 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmp /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_22.export /tmp/tmp.vybh1m9yJF/cluster1-mirror-image_22.export 2026-02-18T08:39:09.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_22.export /tmp/tmp.vybh1m9yJF/cluster1-mirror-image_22.export 2026-02-18T08:39:09.956 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:39:09.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T08:39:09.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_23 2026-02-18T08:39:09.974 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_snap cluster2 mirror image_23 snap 2026-02-18T08:39:09.974 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:39:09.974 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:39:09.974 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_23 2026-02-18T08:39:09.974 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap 2026-02-18T08:39:09.974 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 -p mirror snap create image_23@snap --debug-rbd=20 --debug-journaler=20 2026-02-18T08:39:13.955 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_replay_started cluster1 mirror image_23 2026-02-18T08:39:13.955 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:39:13.955 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:39:13.955 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_23 2026-02-18T08:39:13.955 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_replay_state cluster1 mirror image_23 started 2026-02-18T08:39:13.956 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:39:13.956 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:39:13.956 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_23 2026-02-18T08:39:13.956 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local state=started 2026-02-18T08:39:13.956 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-18T08:39:13.956 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:39:13.956 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1 2026-02-18T08:39:14.956 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test_image_replay_state cluster1 mirror image_23 started 2026-02-18T08:39:14.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:39:14.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:39:14.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_23 2026-02-18T08:39:14.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local test_state=started 2026-02-18T08:39:14.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local status_result 2026-02-18T08:39:14.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_state=stopped 2026-02-18T08:39:14.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_23 2026-02-18T08:39:14.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -i state 2026-02-18T08:39:14.958 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_instance=cluster1 2026-02-18T08:39:14.958 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-18T08:39:14.958 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:39:14.958 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance=cluster1 2026-02-18T08:39:14.958 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local loop_instance 2026-02-18T08:39:14.958 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:39:14.958 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sleep 0 2026-02-18T08:39:14.961 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:39:14.962 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ seq 0 3 2026-02-18T08:39:14.962 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:39:14.962 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_23 2026-02-18T08:39:14.962 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:0 2026-02-18T08:39:14.962 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-18T08:39:14.962 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:39:14.962 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:39:14.962 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:39:14.962 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-18T08:39:14.962 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:39:14.963 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:39:14.963 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:39:14.963 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:39:14.963 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-18T08:39:14.963 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:39:14.963 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:39:14.963 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:39:14.963 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:39:14.963 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:0 2026-02-18T08:39:14.963 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-18T08:39:14.963 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-18T08:39:14.963 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:39:14.964 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:0 2026-02-18T08:39:14.964 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T08:39:14.964 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-18T08:39:14.964 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:39:14.964 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:39:14.964 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-18T08:39:14.964 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-18T08:39:14.964 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-18T08:39:14.964 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=0 2026-02-18T08:39:14.964 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=0 2026-02-18T08:39:14.964 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:39:14.970 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:39:14.971 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:39:14.971 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:39:14.971 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_23 2026-02-18T08:39:15.035 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ return 0 2026-02-18T08:39:15.036 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ status_result=' "state": "Replaying"' 2026-02-18T08:39:15.036 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ echo ' "state": "Replaying"' 2026-02-18T08:39:15.037 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ grep -i Replaying 2026-02-18T08:39:15.038 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: "state": "Replaying" 2026-02-18T08:39:15.039 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ current_state=started 2026-02-18T08:39:15.039 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test started = started 2026-02-18T08:39:15.039 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:39:15.040 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror image_23 2026-02-18T08:39:15.040 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_cluster=cluster1 2026-02-18T08:39:15.040 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:39:15.040 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_pool=mirror 2026-02-18T08:39:15.040 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local remote_pool=mirror 2026-02-18T08:39:15.040 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_23 2026-02-18T08:39:15.040 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' journal = journal ']' 2026-02-18T08:39:15.040 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror mirror image_23 2026-02-18T08:39:15.040 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_cluster=cluster1 2026-02-18T08:39:15.040 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:39:15.040 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_pool=mirror 2026-02-18T08:39:15.041 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local remote_pool=mirror 2026-02-18T08:39:15.041 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_23 2026-02-18T08:39:15.041 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-02-18T08:39:15.041 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-02-18T08:39:15.041 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:39:15.041 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:39:15.041 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:39:15.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_23 2026-02-18T08:39:15.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:39:15.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:39:15.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_23 2026-02-18T08:39:15.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:39:15.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_23 ']' 2026-02-18T08:39:15.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_23' 2026-02-18T08:39:15.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_23 2026-02-18T08:39:15.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:39:15.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:39:15.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:39:15.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:39:15.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:39:15.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:39:15.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:39:15.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:39:15.245 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:39:15.245 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:39:15.245 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_23 2026-02-18T08:39:15.245 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:39:15.245 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:39:15.245 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:39:15.245 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:39:15.245 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:39:15.245 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:39:15.246 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:39:15.246 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:39:15.246 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:39:15.246 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:39:15.246 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:39:15.246 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:39:15.246 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:39:15.246 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:39:15.246 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:39:15.246 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:39:15.246 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:39:15.246 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:39:15.247 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:39:15.247 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:39:15.247 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:39:15.247 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:39:15.247 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:39:15.247 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:39:15.247 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:39:15.247 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:39:15.248 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:39:15.248 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:39:15.248 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:39:15.248 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:39:15.255 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:39:15.255 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:39:15.255 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:39:15.255 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_23 2026-02-18T08:39:15.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:39:15.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_23 2026-02-18T08:39:15.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:39:15.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:39:15.312 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_23 2026-02-18T08:39:15.312 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_23 '' 2026-02-18T08:39:15.312 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:39:15.312 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:39:15.312 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_23 2026-02-18T08:39:15.312 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:39:15.312 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_23.status 2026-02-18T08:39:15.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_23.status 2026-02-18T08:39:15.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:39:15.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_23.status 2026-02-18T08:39:15.314 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_23 2026-02-18T08:39:15.314 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_23.status 2026-02-18T08:39:15.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 11 2026-02-18T08:39:15.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 11 2026-02-18T08:39:15.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:39:15.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=2, entry_tid=1], [object_number=44, tag_tid=2, entry_tid=0], [object_number=47, tag_tid=1, entry_tid=46583], [object_number=46, tag_tid=1, entry_tid=46582]]], state=connected] 2026-02-18T08:39:15.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=47, tag_tid=1, entry_tid=46583], [object_number=46, tag_tid=1, entry_tid=46582], [object_number=45, tag_tid=1, entry_tid=46581], [object_number=44, tag_tid=1, entry_tid=46580]]], state=connected] 2026-02-18T08:39:15.344 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_23.status 2026-02-18T08:39:15.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=45, tag_tid=2, entry_tid=1' 2026-02-18T08:39:15.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_23 2026-02-18T08:39:15.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:39:15.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:39:15.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_23 2026-02-18T08:39:15.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_23 '..*' 2026-02-18T08:39:15.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:39:15.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:39:15.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_23 2026-02-18T08:39:15.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:39:15.348 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_23.status 2026-02-18T08:39:15.348 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_23.status 2026-02-18T08:39:15.348 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:39:15.348 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_23.status 2026-02-18T08:39:15.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_23 2026-02-18T08:39:15.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_23.status 2026-02-18T08:39:15.383 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 11 2026-02-18T08:39:15.383 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 11 2026-02-18T08:39:15.384 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:39:15.384 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=2, entry_tid=1], [object_number=44, tag_tid=2, entry_tid=0], [object_number=47, tag_tid=1, entry_tid=46583], [object_number=46, tag_tid=1, entry_tid=46582]]], state=connected] 2026-02-18T08:39:15.384 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=47, tag_tid=1, entry_tid=46583], [object_number=46, tag_tid=1, entry_tid=46582], [object_number=45, tag_tid=1, entry_tid=46581], [object_number=44, tag_tid=1, entry_tid=46580]]], state=connected] 2026-02-18T08:39:15.387 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_23.status 2026-02-18T08:39:15.388 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=47, tag_tid=1, entry_tid=46583' 2026-02-18T08:39:15.388 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=45, tag_tid=2, entry_tid=1' -a 'object_number=45, tag_tid=2, entry_tid=1' = 'object_number=47, tag_tid=1, entry_tid=46583' 2026-02-18T08:39:15.388 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=47, tag_tid=1, entry_tid=46583' '!=' '' 2026-02-18T08:39:15.389 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:39:15.389 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=47, tag_tid=1, entry_tid=46583' = '' 2026-02-18T08:39:15.389 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=47, tag_tid=1, entry_tid=46583' 2026-02-18T08:39:15.389 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=45, tag_tid=2, entry_tid=1' 2026-02-18T08:39:15.390 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:39:15.390 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:39:15.390 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=2 2026-02-18T08:39:15.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=47, tag_tid=1, entry_tid=46583' 2026-02-18T08:39:15.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:39:15.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:39:15.393 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=1 2026-02-18T08:39:15.394 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=45, tag_tid=2, entry_tid=1' 2026-02-18T08:39:15.394 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:39:15.394 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:39:15.395 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:39:15.396 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=47, tag_tid=1, entry_tid=46583' 2026-02-18T08:39:15.396 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:39:15.396 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:39:15.397 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=46583 2026-02-18T08:39:15.397 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 2 = 1 -a 1 -le 46583 2026-02-18T08:39:15.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:39:15.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:39:15.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:39:15.599 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_23 2026-02-18T08:39:15.599 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:39:15.600 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:39:15.600 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_23 2026-02-18T08:39:15.600 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:39:15.600 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_23 ']' 2026-02-18T08:39:15.600 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_23' 2026-02-18T08:39:15.600 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_23 2026-02-18T08:39:15.600 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:39:15.600 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:39:15.600 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:39:15.600 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:39:15.601 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:39:15.601 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:39:15.601 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:39:15.601 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:39:15.603 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:39:15.604 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:39:15.604 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_23 2026-02-18T08:39:15.604 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:39:15.604 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:39:15.604 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:39:15.604 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:39:15.604 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:39:15.604 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:39:15.604 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:39:15.604 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:39:15.604 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:39:15.605 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:39:15.605 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:39:15.605 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:39:15.605 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:39:15.605 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:39:15.605 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:39:15.605 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:39:15.605 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:39:15.605 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:39:15.605 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:39:15.605 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:39:15.605 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:39:15.605 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:39:15.606 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:39:15.606 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:39:15.606 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:39:15.606 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:39:15.606 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:39:15.606 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:39:15.606 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:39:15.606 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:39:15.613 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:39:15.613 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:39:15.613 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:39:15.613 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_23 2026-02-18T08:39:15.670 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:39:15.670 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_23 2026-02-18T08:39:15.670 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:39:15.670 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:39:15.670 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_23 2026-02-18T08:39:15.671 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_23 '' 2026-02-18T08:39:15.671 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:39:15.671 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:39:15.671 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_23 2026-02-18T08:39:15.671 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:39:15.671 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_23.status 2026-02-18T08:39:15.672 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_23.status 2026-02-18T08:39:15.672 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:39:15.672 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_23.status 2026-02-18T08:39:15.673 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_23 2026-02-18T08:39:15.673 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_23.status 2026-02-18T08:39:15.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 11 2026-02-18T08:39:15.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 11 2026-02-18T08:39:15.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:39:15.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=2, entry_tid=1], [object_number=44, tag_tid=2, entry_tid=0], [object_number=47, tag_tid=1, entry_tid=46583], [object_number=46, tag_tid=1, entry_tid=46582]]], state=connected] 2026-02-18T08:39:15.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=47, tag_tid=1, entry_tid=46583], [object_number=46, tag_tid=1, entry_tid=46582], [object_number=45, tag_tid=1, entry_tid=46581], [object_number=44, tag_tid=1, entry_tid=46580]]], state=connected] 2026-02-18T08:39:15.702 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_23.status 2026-02-18T08:39:15.703 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=45, tag_tid=2, entry_tid=1' 2026-02-18T08:39:15.705 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_23 2026-02-18T08:39:15.705 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:39:15.705 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:39:15.705 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_23 2026-02-18T08:39:15.705 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_23 '..*' 2026-02-18T08:39:15.705 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:39:15.705 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:39:15.705 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_23 2026-02-18T08:39:15.705 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:39:15.705 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_23.status 2026-02-18T08:39:15.705 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_23.status 2026-02-18T08:39:15.706 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:39:15.706 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_23.status 2026-02-18T08:39:15.706 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_23 2026-02-18T08:39:15.706 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_23.status 2026-02-18T08:39:15.733 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 11 2026-02-18T08:39:15.733 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 11 2026-02-18T08:39:15.733 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:39:15.733 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=2, entry_tid=1], [object_number=44, tag_tid=2, entry_tid=0], [object_number=47, tag_tid=1, entry_tid=46583], [object_number=46, tag_tid=1, entry_tid=46582]]], state=connected] 2026-02-18T08:39:15.733 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=47, tag_tid=1, entry_tid=46583], [object_number=46, tag_tid=1, entry_tid=46582], [object_number=45, tag_tid=1, entry_tid=46581], [object_number=44, tag_tid=1, entry_tid=46580]]], state=connected] 2026-02-18T08:39:15.735 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_23.status 2026-02-18T08:39:15.737 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=47, tag_tid=1, entry_tid=46583' 2026-02-18T08:39:15.737 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=45, tag_tid=2, entry_tid=1' -a 'object_number=45, tag_tid=2, entry_tid=1' = 'object_number=47, tag_tid=1, entry_tid=46583' 2026-02-18T08:39:15.737 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=47, tag_tid=1, entry_tid=46583' '!=' 'object_number=47, tag_tid=1, entry_tid=46583' 2026-02-18T08:39:15.737 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:39:15.737 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.4 2026-02-18T08:39:16.139 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_23 2026-02-18T08:39:16.140 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:39:16.140 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:39:16.140 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_23 2026-02-18T08:39:16.140 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:39:16.140 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_23 ']' 2026-02-18T08:39:16.140 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_23' 2026-02-18T08:39:16.140 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_23 2026-02-18T08:39:16.140 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:39:16.140 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:39:16.141 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:39:16.141 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:39:16.141 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:39:16.141 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:39:16.141 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:39:16.141 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:39:16.144 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:39:16.144 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:39:16.144 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_23 2026-02-18T08:39:16.144 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:39:16.144 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:39:16.144 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:39:16.144 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:39:16.144 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:39:16.144 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:39:16.144 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:39:16.145 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:39:16.145 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:39:16.145 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:39:16.145 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:39:16.145 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:39:16.145 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:39:16.145 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:39:16.145 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:39:16.145 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:39:16.145 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:39:16.145 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:39:16.145 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:39:16.146 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:39:16.146 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:39:16.146 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:39:16.146 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:39:16.146 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:39:16.146 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:39:16.146 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:39:16.146 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:39:16.146 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:39:16.146 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:39:16.146 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:39:16.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:39:16.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:39:16.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:39:16.155 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_23 2026-02-18T08:39:16.221 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:39:16.222 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_23 2026-02-18T08:39:16.222 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:39:16.222 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:39:16.222 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_23 2026-02-18T08:39:16.222 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_23 '' 2026-02-18T08:39:16.222 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:39:16.222 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:39:16.222 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_23 2026-02-18T08:39:16.223 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:39:16.223 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_23.status 2026-02-18T08:39:16.223 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_23.status 2026-02-18T08:39:16.223 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:39:16.223 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_23.status 2026-02-18T08:39:16.224 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_23 2026-02-18T08:39:16.224 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_23.status 2026-02-18T08:39:16.261 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 11 2026-02-18T08:39:16.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 11 2026-02-18T08:39:16.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:39:16.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=2, entry_tid=1], [object_number=44, tag_tid=2, entry_tid=0], [object_number=47, tag_tid=1, entry_tid=46583], [object_number=46, tag_tid=1, entry_tid=46582]]], state=connected] 2026-02-18T08:39:16.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=47, tag_tid=1, entry_tid=46583], [object_number=46, tag_tid=1, entry_tid=46582], [object_number=45, tag_tid=1, entry_tid=46581], [object_number=44, tag_tid=1, entry_tid=46580]]], state=connected] 2026-02-18T08:39:16.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_23.status 2026-02-18T08:39:16.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=45, tag_tid=2, entry_tid=1' 2026-02-18T08:39:16.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_23 2026-02-18T08:39:16.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:39:16.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:39:16.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_23 2026-02-18T08:39:16.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_23 '..*' 2026-02-18T08:39:16.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:39:16.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:39:16.269 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_23 2026-02-18T08:39:16.269 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:39:16.269 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_23.status 2026-02-18T08:39:16.269 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_23.status 2026-02-18T08:39:16.269 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:39:16.270 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_23.status 2026-02-18T08:39:16.270 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_23 2026-02-18T08:39:16.270 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_23.status 2026-02-18T08:39:16.297 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 11 2026-02-18T08:39:16.297 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 11 2026-02-18T08:39:16.297 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:39:16.297 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=2, entry_tid=1], [object_number=44, tag_tid=2, entry_tid=0], [object_number=47, tag_tid=1, entry_tid=46583], [object_number=46, tag_tid=1, entry_tid=46582]]], state=connected] 2026-02-18T08:39:16.297 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=47, tag_tid=1, entry_tid=46583], [object_number=46, tag_tid=1, entry_tid=46582], [object_number=45, tag_tid=1, entry_tid=46581], [object_number=44, tag_tid=1, entry_tid=46580]]], state=connected] 2026-02-18T08:39:16.299 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_23.status 2026-02-18T08:39:16.301 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=47, tag_tid=1, entry_tid=46583' 2026-02-18T08:39:16.301 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=45, tag_tid=2, entry_tid=1' -a 'object_number=45, tag_tid=2, entry_tid=1' = 'object_number=47, tag_tid=1, entry_tid=46583' 2026-02-18T08:39:16.301 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=47, tag_tid=1, entry_tid=46583' '!=' 'object_number=47, tag_tid=1, entry_tid=46583' 2026-02-18T08:39:16.301 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:39:16.301 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.8 2026-02-18T08:39:17.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_23 2026-02-18T08:39:17.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:39:17.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:39:17.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_23 2026-02-18T08:39:17.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:39:17.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_23 ']' 2026-02-18T08:39:17.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_23' 2026-02-18T08:39:17.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_23 2026-02-18T08:39:17.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:39:17.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:39:17.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:39:17.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:39:17.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:39:17.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:39:17.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:39:17.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:39:17.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:39:17.106 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:39:17.106 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_23 2026-02-18T08:39:17.106 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:39:17.106 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:39:17.106 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:39:17.106 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:39:17.106 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:39:17.106 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:39:17.106 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:39:17.106 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:39:17.106 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:39:17.106 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:39:17.106 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:39:17.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:39:17.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:39:17.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:39:17.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:39:17.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:39:17.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:39:17.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:39:17.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:39:17.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:39:17.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:39:17.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:39:17.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:39:17.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:39:17.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:39:17.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:39:17.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:39:17.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:39:17.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:39:17.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:39:17.118 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:39:17.119 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:39:17.119 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:39:17.119 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_23 2026-02-18T08:39:17.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:39:17.183 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_23 2026-02-18T08:39:17.183 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:39:17.183 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:39:17.183 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_23 2026-02-18T08:39:17.183 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_23 '' 2026-02-18T08:39:17.183 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:39:17.183 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:39:17.183 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_23 2026-02-18T08:39:17.183 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:39:17.183 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_23.status 2026-02-18T08:39:17.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_23.status 2026-02-18T08:39:17.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:39:17.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_23.status 2026-02-18T08:39:17.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_23 2026-02-18T08:39:17.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_23.status 2026-02-18T08:39:17.213 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 11 2026-02-18T08:39:17.213 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 11 2026-02-18T08:39:17.213 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:39:17.213 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=2, entry_tid=1], [object_number=44, tag_tid=2, entry_tid=0], [object_number=47, tag_tid=1, entry_tid=46583], [object_number=46, tag_tid=1, entry_tid=46582]]], state=connected] 2026-02-18T08:39:17.213 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=47, tag_tid=1, entry_tid=46583], [object_number=46, tag_tid=1, entry_tid=46582], [object_number=45, tag_tid=1, entry_tid=46581], [object_number=44, tag_tid=1, entry_tid=46580]]], state=connected] 2026-02-18T08:39:17.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_23.status 2026-02-18T08:39:17.218 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=45, tag_tid=2, entry_tid=1' 2026-02-18T08:39:17.218 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_23 2026-02-18T08:39:17.218 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:39:17.218 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:39:17.218 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_23 2026-02-18T08:39:17.219 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_23 '..*' 2026-02-18T08:39:17.219 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:39:17.219 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:39:17.219 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_23 2026-02-18T08:39:17.219 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:39:17.219 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_23.status 2026-02-18T08:39:17.219 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_23.status 2026-02-18T08:39:17.219 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:39:17.221 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_23.status 2026-02-18T08:39:17.221 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_23 2026-02-18T08:39:17.221 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_23.status 2026-02-18T08:39:17.249 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 11 2026-02-18T08:39:17.249 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 11 2026-02-18T08:39:17.249 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:39:17.249 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=2, entry_tid=1], [object_number=44, tag_tid=2, entry_tid=0], [object_number=47, tag_tid=1, entry_tid=46583], [object_number=46, tag_tid=1, entry_tid=46582]]], state=connected] 2026-02-18T08:39:17.249 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=47, tag_tid=1, entry_tid=46583], [object_number=46, tag_tid=1, entry_tid=46582], [object_number=45, tag_tid=1, entry_tid=46581], [object_number=44, tag_tid=1, entry_tid=46580]]], state=connected] 2026-02-18T08:39:17.251 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_23.status 2026-02-18T08:39:17.253 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=47, tag_tid=1, entry_tid=46583' 2026-02-18T08:39:17.253 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=45, tag_tid=2, entry_tid=1' -a 'object_number=45, tag_tid=2, entry_tid=1' = 'object_number=47, tag_tid=1, entry_tid=46583' 2026-02-18T08:39:17.253 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=47, tag_tid=1, entry_tid=46583' '!=' 'object_number=47, tag_tid=1, entry_tid=46583' 2026-02-18T08:39:17.253 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:39:17.253 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1.6 2026-02-18T08:39:18.855 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_23 2026-02-18T08:39:18.855 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:39:18.856 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:39:18.856 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_23 2026-02-18T08:39:18.856 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:39:18.856 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_23 ']' 2026-02-18T08:39:18.856 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_23' 2026-02-18T08:39:18.856 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_23 2026-02-18T08:39:18.856 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:39:18.856 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:39:18.856 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:39:18.856 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:39:18.856 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:39:18.856 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:39:18.857 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:39:18.859 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:39:18.859 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:39:18.860 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:39:18.860 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_23 2026-02-18T08:39:18.860 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:39:18.860 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:39:18.860 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:39:18.860 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:39:18.860 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:39:18.860 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:39:18.860 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:39:18.860 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:39:18.860 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:39:18.861 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:39:18.861 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:39:18.861 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:39:18.861 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:39:18.861 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:39:18.861 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:39:18.861 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:39:18.861 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:39:18.861 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:39:18.861 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:39:18.861 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:39:18.861 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:39:18.862 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:39:18.862 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:39:18.862 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:39:18.862 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:39:18.862 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:39:18.862 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:39:18.862 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:39:18.862 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:39:18.862 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:39:18.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:39:18.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:39:18.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:39:18.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_23 2026-02-18T08:39:18.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:39:18.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_23 2026-02-18T08:39:18.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:39:18.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:39:18.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_23 2026-02-18T08:39:18.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_23 '' 2026-02-18T08:39:18.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:39:18.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:39:18.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_23 2026-02-18T08:39:18.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:39:18.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_23.status 2026-02-18T08:39:18.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_23.status 2026-02-18T08:39:18.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:39:18.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_23.status 2026-02-18T08:39:18.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_23 2026-02-18T08:39:18.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_23.status 2026-02-18T08:39:18.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 11 2026-02-18T08:39:18.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 11 2026-02-18T08:39:18.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:39:18.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=2, entry_tid=1], [object_number=44, tag_tid=2, entry_tid=0], [object_number=47, tag_tid=1, entry_tid=46583], [object_number=46, tag_tid=1, entry_tid=46582]]], state=connected] 2026-02-18T08:39:18.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=45, tag_tid=2, entry_tid=1], [object_number=44, tag_tid=2, entry_tid=0], [object_number=47, tag_tid=1, entry_tid=46583], [object_number=46, tag_tid=1, entry_tid=46582]]], state=connected] 2026-02-18T08:39:18.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_23.status 2026-02-18T08:39:18.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=45, tag_tid=2, entry_tid=1' 2026-02-18T08:39:18.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_23 2026-02-18T08:39:18.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:39:18.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:39:18.985 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_23 2026-02-18T08:39:18.985 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_23 '..*' 2026-02-18T08:39:18.985 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:39:18.985 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:39:18.985 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_23 2026-02-18T08:39:18.985 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:39:18.985 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_23.status 2026-02-18T08:39:18.985 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_23.status 2026-02-18T08:39:18.986 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:39:18.986 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_23.status 2026-02-18T08:39:18.986 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_23 2026-02-18T08:39:18.986 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_23.status 2026-02-18T08:39:19.020 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 11 2026-02-18T08:39:19.020 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 11 2026-02-18T08:39:19.020 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:39:19.020 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=2, entry_tid=1], [object_number=44, tag_tid=2, entry_tid=0], [object_number=47, tag_tid=1, entry_tid=46583], [object_number=46, tag_tid=1, entry_tid=46582]]], state=connected] 2026-02-18T08:39:19.020 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=45, tag_tid=2, entry_tid=1], [object_number=44, tag_tid=2, entry_tid=0], [object_number=47, tag_tid=1, entry_tid=46583], [object_number=46, tag_tid=1, entry_tid=46582]]], state=connected] 2026-02-18T08:39:19.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_23.status 2026-02-18T08:39:19.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=45, tag_tid=2, entry_tid=1' 2026-02-18T08:39:19.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=45, tag_tid=2, entry_tid=1' -a 'object_number=45, tag_tid=2, entry_tid=1' = 'object_number=45, tag_tid=2, entry_tid=1' 2026-02-18T08:39:19.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:39:19.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_snap_present cluster1 mirror image_23 snap 2026-02-18T08:39:19.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:39:19.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:39:19.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_23 2026-02-18T08:39:19.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap 2026-02-18T08:39:19.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-18T08:39:19.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-18T08:39:19.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1 2026-02-18T08:39:20.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster1 info mirror/image_23@snap 2026-02-18T08:39:20.055 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:rbd image 'image_23': 2026-02-18T08:39:20.055 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: size 128 MiB in 32 objects 2026-02-18T08:39:20.055 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: order 22 (4 MiB objects) 2026-02-18T08:39:20.055 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: snapshot_count: 1 2026-02-18T08:39:20.055 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: id: 11045aaf9858 2026-02-18T08:39:20.056 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: block_name_prefix: rbd_data.11045aaf9858 2026-02-18T08:39:20.056 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: format: 2 2026-02-18T08:39:20.056 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: features: layering, exclusive-lock, journaling 2026-02-18T08:39:20.056 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: op_features: 2026-02-18T08:39:20.056 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: flags: 2026-02-18T08:39:20.056 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: create_timestamp: Wed Feb 18 08:29:42 2026 2026-02-18T08:39:20.056 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: access_timestamp: Wed Feb 18 08:29:42 2026 2026-02-18T08:39:20.056 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: modify_timestamp: Wed Feb 18 08:30:43 2026 2026-02-18T08:39:20.056 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: protected: False 2026-02-18T08:39:20.056 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: journal: 11045aaf9858 2026-02-18T08:39:20.056 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring state: enabled 2026-02-18T08:39:20.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring mode: journal 2026-02-18T08:39:20.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring global id: 08e1a2de-bb51-4517-a95d-f13dd565c402 2026-02-18T08:39:20.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring primary: false 2026-02-18T08:39:20.059 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:39:20.059 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ compare_image_snaps mirror image_23 snap 2026-02-18T08:39:20.059 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:39:20.059 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_23 2026-02-18T08:39:20.059 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap 2026-02-18T08:39:20.059 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local ret=0 2026-02-18T08:39:20.059 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local rmt_export=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_23.export 2026-02-18T08:39:20.059 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loc_export=/tmp/tmp.vybh1m9yJF/cluster1-mirror-image_23.export 2026-02-18T08:39:20.059 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_23.export /tmp/tmp.vybh1m9yJF/cluster1-mirror-image_23.export 2026-02-18T08:39:20.060 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 -p mirror export image_23@snap /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_23.export 2026-02-18T08:39:20.200 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-18T08:39:20.210 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster1 -p mirror export image_23@snap /tmp/tmp.vybh1m9yJF/cluster1-mirror-image_23.export 2026-02-18T08:39:20.408 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-18T08:39:20.419 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmp /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_23.export /tmp/tmp.vybh1m9yJF/cluster1-mirror-image_23.export 2026-02-18T08:39:20.474 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_23.export /tmp/tmp.vybh1m9yJF/cluster1-mirror-image_23.export 2026-02-18T08:39:20.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:39:20.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T08:39:20.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_24 2026-02-18T08:39:20.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_snap cluster2 mirror image_24 snap 2026-02-18T08:39:20.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:39:20.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:39:20.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_24 2026-02-18T08:39:20.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap 2026-02-18T08:39:20.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 -p mirror snap create image_24@snap --debug-rbd=20 --debug-journaler=20 2026-02-18T08:39:22.965 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_replay_started cluster1 mirror image_24 2026-02-18T08:39:22.965 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:39:22.965 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:39:22.966 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_24 2026-02-18T08:39:22.966 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_replay_state cluster1 mirror image_24 started 2026-02-18T08:39:22.966 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:39:22.966 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:39:22.966 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_24 2026-02-18T08:39:22.966 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local state=started 2026-02-18T08:39:22.966 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-18T08:39:22.966 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:39:22.966 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1 2026-02-18T08:39:23.967 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test_image_replay_state cluster1 mirror image_24 started 2026-02-18T08:39:23.967 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:39:23.967 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:39:23.967 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_24 2026-02-18T08:39:23.968 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local test_state=started 2026-02-18T08:39:23.968 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local status_result 2026-02-18T08:39:23.968 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_state=stopped 2026-02-18T08:39:23.968 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_24 2026-02-18T08:39:23.968 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -i state 2026-02-18T08:39:23.968 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_instance=cluster1 2026-02-18T08:39:23.968 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-18T08:39:23.968 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:39:23.968 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance=cluster1 2026-02-18T08:39:23.968 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local loop_instance 2026-02-18T08:39:23.968 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:39:23.969 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sleep 0 2026-02-18T08:39:23.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:39:23.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ seq 0 3 2026-02-18T08:39:23.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:39:23.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_24 2026-02-18T08:39:23.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:0 2026-02-18T08:39:23.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-18T08:39:23.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:39:23.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:39:23.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:39:23.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-18T08:39:23.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:39:23.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:39:23.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:39:23.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:39:23.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-18T08:39:23.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:39:23.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:39:23.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:39:23.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:39:23.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:0 2026-02-18T08:39:23.974 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-18T08:39:23.974 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-18T08:39:23.974 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:39:23.974 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:0 2026-02-18T08:39:23.974 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T08:39:23.974 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-18T08:39:23.974 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:39:23.974 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:39:23.974 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-18T08:39:23.974 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-18T08:39:23.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-18T08:39:23.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=0 2026-02-18T08:39:23.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=0 2026-02-18T08:39:23.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:39:23.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:39:23.982 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:39:23.982 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:39:23.982 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_24 2026-02-18T08:39:24.040 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_24'] 2026-02-18T08:39:24.041 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status : get status for rbd mirror 2026-02-18T08:39:24.041 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:39:24.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:39:24.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_24 2026-02-18T08:39:24.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:1 2026-02-18T08:39:24.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-18T08:39:24.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:39:24.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:39:24.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:39:24.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-18T08:39:24.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:39:24.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:39:24.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:39:24.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:39:24.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-18T08:39:24.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:39:24.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:39:24.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:39:24.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:39:24.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:1 2026-02-18T08:39:24.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-18T08:39:24.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-18T08:39:24.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:39:24.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:1 2026-02-18T08:39:24.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T08:39:24.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-18T08:39:24.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:39:24.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:39:24.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-18T08:39:24.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-18T08:39:24.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-18T08:39:24.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=1 2026-02-18T08:39:24.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=1 2026-02-18T08:39:24.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:39:24.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:39:24.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:39:24.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:39:24.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_24 2026-02-18T08:39:24.113 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ return 0 2026-02-18T08:39:24.114 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ status_result=' "state": "Replaying"' 2026-02-18T08:39:24.114 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ echo ' "state": "Replaying"' 2026-02-18T08:39:24.114 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ grep -i Replaying 2026-02-18T08:39:24.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: "state": "Replaying" 2026-02-18T08:39:24.118 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ current_state=started 2026-02-18T08:39:24.118 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test started = started 2026-02-18T08:39:24.118 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:39:24.118 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror image_24 2026-02-18T08:39:24.118 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_cluster=cluster1 2026-02-18T08:39:24.118 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:39:24.118 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_pool=mirror 2026-02-18T08:39:24.118 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local remote_pool=mirror 2026-02-18T08:39:24.118 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_24 2026-02-18T08:39:24.119 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' journal = journal ']' 2026-02-18T08:39:24.119 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror mirror image_24 2026-02-18T08:39:24.119 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_cluster=cluster1 2026-02-18T08:39:24.119 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:39:24.119 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_pool=mirror 2026-02-18T08:39:24.119 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local remote_pool=mirror 2026-02-18T08:39:24.119 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_24 2026-02-18T08:39:24.119 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-02-18T08:39:24.119 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-02-18T08:39:24.119 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:39:24.119 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:39:24.119 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:39:24.318 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_24 2026-02-18T08:39:24.318 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:39:24.318 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:39:24.318 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_24 2026-02-18T08:39:24.318 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:39:24.319 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_24 ']' 2026-02-18T08:39:24.319 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_24' 2026-02-18T08:39:24.319 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_24 2026-02-18T08:39:24.319 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:39:24.319 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:39:24.319 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:39:24.319 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:39:24.319 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:39:24.319 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:39:24.319 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:39:24.323 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:39:24.323 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:39:24.323 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:39:24.323 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_24 2026-02-18T08:39:24.323 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:39:24.323 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:39:24.324 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:39:24.324 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:39:24.324 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:39:24.324 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:39:24.324 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:39:24.324 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:39:24.324 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:39:24.324 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:39:24.324 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:39:24.324 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:39:24.325 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:39:24.325 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:39:24.325 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:39:24.325 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:39:24.325 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:39:24.325 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:39:24.325 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:39:24.325 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:39:24.325 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:39:24.326 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:39:24.326 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:39:24.326 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:39:24.326 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:39:24.326 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:39:24.326 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:39:24.326 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:39:24.326 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:39:24.326 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:39:24.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:39:24.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:39:24.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:39:24.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_24 2026-02-18T08:39:24.387 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_24'] 2026-02-18T08:39:24.387 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:39:24.387 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:39:24.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:39:24.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_24 2026-02-18T08:39:24.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:39:24.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:39:24.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:39:24.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:39:24.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:39:24.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:39:24.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:39:24.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:39:24.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:39:24.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:39:24.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:39:24.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:39:24.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:39:24.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:39:24.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:39:24.393 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:39:24.393 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:39:24.393 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:39:24.393 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:39:24.393 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:39:24.393 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:39:24.393 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:39:24.393 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:39:24.393 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:39:24.393 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:39:24.393 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:39:24.394 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:39:24.394 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:39:24.394 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:39:24.394 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:39:24.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:39:24.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:39:24.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:39:24.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_24 2026-02-18T08:39:24.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:39:24.463 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_24 2026-02-18T08:39:24.463 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:39:24.463 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:39:24.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_24 2026-02-18T08:39:24.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_24 '' 2026-02-18T08:39:24.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:39:24.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:39:24.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_24 2026-02-18T08:39:24.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:39:24.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_24.status 2026-02-18T08:39:24.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_24.status 2026-02-18T08:39:24.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:39:24.465 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_24.status 2026-02-18T08:39:24.465 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_24 2026-02-18T08:39:24.465 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_24.status 2026-02-18T08:39:24.492 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 8 2026-02-18T08:39:24.492 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 8 2026-02-18T08:39:24.492 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:39:24.493 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=33, tag_tid=2, entry_tid=1], [object_number=32, tag_tid=2, entry_tid=0], [object_number=35, tag_tid=1, entry_tid=35939], [object_number=34, tag_tid=1, entry_tid=35938]]], state=connected] 2026-02-18T08:39:24.493 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=33, tag_tid=1, entry_tid=35941], [object_number=32, tag_tid=1, entry_tid=35940], [object_number=35, tag_tid=1, entry_tid=35939], [object_number=34, tag_tid=1, entry_tid=35938]]], state=connected] 2026-02-18T08:39:24.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_24.status 2026-02-18T08:39:24.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=33, tag_tid=2, entry_tid=1' 2026-02-18T08:39:24.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_24 2026-02-18T08:39:24.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:39:24.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:39:24.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_24 2026-02-18T08:39:24.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_24 '..*' 2026-02-18T08:39:24.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:39:24.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:39:24.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_24 2026-02-18T08:39:24.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:39:24.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_24.status 2026-02-18T08:39:24.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_24.status 2026-02-18T08:39:24.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:39:24.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_24.status 2026-02-18T08:39:24.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_24 2026-02-18T08:39:24.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_24.status 2026-02-18T08:39:24.535 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 8 2026-02-18T08:39:24.536 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 8 2026-02-18T08:39:24.536 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:39:24.536 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=33, tag_tid=2, entry_tid=1], [object_number=32, tag_tid=2, entry_tid=0], [object_number=35, tag_tid=1, entry_tid=35939], [object_number=34, tag_tid=1, entry_tid=35938]]], state=connected] 2026-02-18T08:39:24.536 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=33, tag_tid=1, entry_tid=35941], [object_number=32, tag_tid=1, entry_tid=35940], [object_number=35, tag_tid=1, entry_tid=35939], [object_number=34, tag_tid=1, entry_tid=35938]]], state=connected] 2026-02-18T08:39:24.539 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_24.status 2026-02-18T08:39:24.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=33, tag_tid=1, entry_tid=35941' 2026-02-18T08:39:24.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=33, tag_tid=2, entry_tid=1' -a 'object_number=33, tag_tid=2, entry_tid=1' = 'object_number=33, tag_tid=1, entry_tid=35941' 2026-02-18T08:39:24.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=33, tag_tid=1, entry_tid=35941' '!=' '' 2026-02-18T08:39:24.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:39:24.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=33, tag_tid=1, entry_tid=35941' = '' 2026-02-18T08:39:24.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=33, tag_tid=1, entry_tid=35941' 2026-02-18T08:39:24.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=33, tag_tid=2, entry_tid=1' 2026-02-18T08:39:24.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:39:24.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:39:24.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=2 2026-02-18T08:39:24.545 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=33, tag_tid=1, entry_tid=35941' 2026-02-18T08:39:24.545 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:39:24.545 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:39:24.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=1 2026-02-18T08:39:24.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=33, tag_tid=2, entry_tid=1' 2026-02-18T08:39:24.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:39:24.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:39:24.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:39:24.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=33, tag_tid=1, entry_tid=35941' 2026-02-18T08:39:24.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:39:24.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:39:24.551 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=35941 2026-02-18T08:39:24.551 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 2 = 1 -a 1 -le 35941 2026-02-18T08:39:24.551 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:39:24.551 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:39:24.551 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:39:24.753 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_24 2026-02-18T08:39:24.753 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:39:24.753 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:39:24.753 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_24 2026-02-18T08:39:24.753 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:39:24.754 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_24 ']' 2026-02-18T08:39:24.754 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_24' 2026-02-18T08:39:24.754 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_24 2026-02-18T08:39:24.754 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:39:24.754 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:39:24.754 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:39:24.754 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:39:24.754 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:39:24.754 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:39:24.755 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:39:24.755 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:39:24.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:39:24.758 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:39:24.758 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_24 2026-02-18T08:39:24.758 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:39:24.758 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:39:24.758 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:39:24.758 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:39:24.758 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:39:24.758 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:39:24.759 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:39:24.759 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:39:24.759 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:39:24.759 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:39:24.759 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:39:24.759 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:39:24.759 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:39:24.760 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:39:24.760 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:39:24.760 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:39:24.760 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:39:24.760 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:39:24.760 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:39:24.760 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:39:24.760 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:39:24.760 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:39:24.760 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:39:24.760 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:39:24.760 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:39:24.760 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:39:24.761 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:39:24.761 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:39:24.761 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:39:24.761 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:39:24.768 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:39:24.768 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:39:24.768 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:39:24.768 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_24 2026-02-18T08:39:24.826 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_24'] 2026-02-18T08:39:24.826 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:39:24.827 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:39:24.829 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:39:24.829 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_24 2026-02-18T08:39:24.830 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:39:24.830 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:39:24.830 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:39:24.830 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:39:24.830 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:39:24.830 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:39:24.830 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:39:24.830 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:39:24.830 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:39:24.831 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:39:24.831 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:39:24.831 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:39:24.831 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:39:24.831 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:39:24.831 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:39:24.831 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:39:24.831 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:39:24.831 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:39:24.832 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:39:24.832 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:39:24.832 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:39:24.832 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:39:24.832 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:39:24.832 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:39:24.832 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:39:24.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:39:24.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:39:24.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:39:24.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:39:24.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:39:24.839 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:39:24.839 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:39:24.840 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:39:24.840 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_24 2026-02-18T08:39:24.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:39:24.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_24 2026-02-18T08:39:24.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:39:24.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:39:24.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_24 2026-02-18T08:39:24.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_24 '' 2026-02-18T08:39:24.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:39:24.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:39:24.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_24 2026-02-18T08:39:24.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:39:24.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_24.status 2026-02-18T08:39:24.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_24.status 2026-02-18T08:39:24.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:39:24.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_24.status 2026-02-18T08:39:24.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_24 2026-02-18T08:39:24.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_24.status 2026-02-18T08:39:24.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 8 2026-02-18T08:39:24.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 8 2026-02-18T08:39:24.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:39:24.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=33, tag_tid=2, entry_tid=1], [object_number=32, tag_tid=2, entry_tid=0], [object_number=35, tag_tid=1, entry_tid=35939], [object_number=34, tag_tid=1, entry_tid=35938]]], state=connected] 2026-02-18T08:39:24.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=33, tag_tid=1, entry_tid=35941], [object_number=32, tag_tid=1, entry_tid=35940], [object_number=35, tag_tid=1, entry_tid=35939], [object_number=34, tag_tid=1, entry_tid=35938]]], state=connected] 2026-02-18T08:39:24.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_24.status 2026-02-18T08:39:24.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=33, tag_tid=2, entry_tid=1' 2026-02-18T08:39:24.944 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_24 2026-02-18T08:39:24.944 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:39:24.944 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:39:24.944 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_24 2026-02-18T08:39:24.944 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_24 '..*' 2026-02-18T08:39:24.944 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:39:24.944 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:39:24.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_24 2026-02-18T08:39:24.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:39:24.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_24.status 2026-02-18T08:39:24.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_24.status 2026-02-18T08:39:24.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:39:24.946 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_24.status 2026-02-18T08:39:24.946 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_24 2026-02-18T08:39:24.946 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_24.status 2026-02-18T08:39:24.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 8 2026-02-18T08:39:24.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 8 2026-02-18T08:39:24.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:39:24.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=33, tag_tid=2, entry_tid=1], [object_number=32, tag_tid=2, entry_tid=0], [object_number=35, tag_tid=1, entry_tid=35939], [object_number=34, tag_tid=1, entry_tid=35938]]], state=connected] 2026-02-18T08:39:24.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=33, tag_tid=1, entry_tid=35941], [object_number=32, tag_tid=1, entry_tid=35940], [object_number=35, tag_tid=1, entry_tid=35939], [object_number=34, tag_tid=1, entry_tid=35938]]], state=connected] 2026-02-18T08:39:24.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_24.status 2026-02-18T08:39:24.985 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=33, tag_tid=1, entry_tid=35941' 2026-02-18T08:39:24.985 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=33, tag_tid=2, entry_tid=1' -a 'object_number=33, tag_tid=2, entry_tid=1' = 'object_number=33, tag_tid=1, entry_tid=35941' 2026-02-18T08:39:24.985 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=33, tag_tid=1, entry_tid=35941' '!=' 'object_number=33, tag_tid=1, entry_tid=35941' 2026-02-18T08:39:24.985 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:39:24.985 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.4 2026-02-18T08:39:25.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_24 2026-02-18T08:39:25.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:39:25.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:39:25.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_24 2026-02-18T08:39:25.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:39:25.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_24 ']' 2026-02-18T08:39:25.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_24' 2026-02-18T08:39:25.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_24 2026-02-18T08:39:25.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:39:25.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:39:25.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:39:25.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:39:25.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:39:25.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:39:25.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:39:25.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:39:25.387 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:39:25.388 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:39:25.388 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_24 2026-02-18T08:39:25.388 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:39:25.388 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:39:25.388 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:39:25.388 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:39:25.388 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:39:25.388 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:39:25.388 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:39:25.388 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:39:25.388 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:39:25.388 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:39:25.388 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:39:25.389 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:39:25.389 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:39:25.389 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:39:25.389 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:39:25.390 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:39:25.390 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:39:25.390 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:39:25.390 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:39:25.390 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:39:25.390 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:39:25.390 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:39:25.390 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:39:25.390 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:39:25.390 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:39:25.390 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:39:25.390 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:39:25.390 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:39:25.390 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:39:25.390 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:39:25.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:39:25.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:39:25.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:39:25.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_24 2026-02-18T08:39:25.450 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_24'] 2026-02-18T08:39:25.450 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:39:25.450 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:39:25.453 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:39:25.453 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_24 2026-02-18T08:39:25.453 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:39:25.454 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:39:25.454 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:39:25.454 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:39:25.454 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:39:25.454 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:39:25.454 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:39:25.454 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:39:25.454 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:39:25.454 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:39:25.455 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:39:25.455 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:39:25.455 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:39:25.455 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:39:25.455 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:39:25.455 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:39:25.455 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:39:25.455 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:39:25.455 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:39:25.455 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:39:25.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:39:25.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:39:25.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:39:25.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:39:25.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:39:25.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:39:25.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:39:25.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:39:25.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:39:25.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:39:25.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:39:25.465 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:39:25.465 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:39:25.465 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_24 2026-02-18T08:39:25.532 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:39:25.532 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_24 2026-02-18T08:39:25.532 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:39:25.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:39:25.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_24 2026-02-18T08:39:25.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_24 '' 2026-02-18T08:39:25.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:39:25.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:39:25.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_24 2026-02-18T08:39:25.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:39:25.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_24.status 2026-02-18T08:39:25.534 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_24.status 2026-02-18T08:39:25.534 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:39:25.534 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_24.status 2026-02-18T08:39:25.535 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_24 2026-02-18T08:39:25.535 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_24.status 2026-02-18T08:39:25.562 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 8 2026-02-18T08:39:25.562 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 8 2026-02-18T08:39:25.562 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:39:25.562 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=33, tag_tid=2, entry_tid=1], [object_number=32, tag_tid=2, entry_tid=0], [object_number=35, tag_tid=1, entry_tid=35939], [object_number=34, tag_tid=1, entry_tid=35938]]], state=connected] 2026-02-18T08:39:25.562 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=33, tag_tid=1, entry_tid=35941], [object_number=32, tag_tid=1, entry_tid=35940], [object_number=35, tag_tid=1, entry_tid=35939], [object_number=34, tag_tid=1, entry_tid=35938]]], state=connected] 2026-02-18T08:39:25.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_24.status 2026-02-18T08:39:25.566 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=33, tag_tid=2, entry_tid=1' 2026-02-18T08:39:25.567 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_24 2026-02-18T08:39:25.567 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:39:25.567 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:39:25.567 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_24 2026-02-18T08:39:25.567 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_24 '..*' 2026-02-18T08:39:25.567 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:39:25.568 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:39:25.568 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_24 2026-02-18T08:39:25.568 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:39:25.568 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_24.status 2026-02-18T08:39:25.568 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_24.status 2026-02-18T08:39:25.569 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:39:25.569 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_24.status 2026-02-18T08:39:25.569 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_24 2026-02-18T08:39:25.570 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_24.status 2026-02-18T08:39:25.596 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 8 2026-02-18T08:39:25.596 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 8 2026-02-18T08:39:25.596 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:39:25.596 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=33, tag_tid=2, entry_tid=1], [object_number=32, tag_tid=2, entry_tid=0], [object_number=35, tag_tid=1, entry_tid=35939], [object_number=34, tag_tid=1, entry_tid=35938]]], state=connected] 2026-02-18T08:39:25.596 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=33, tag_tid=1, entry_tid=35941], [object_number=32, tag_tid=1, entry_tid=35940], [object_number=35, tag_tid=1, entry_tid=35939], [object_number=34, tag_tid=1, entry_tid=35938]]], state=connected] 2026-02-18T08:39:25.598 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_24.status 2026-02-18T08:39:25.600 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=33, tag_tid=1, entry_tid=35941' 2026-02-18T08:39:25.600 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=33, tag_tid=2, entry_tid=1' -a 'object_number=33, tag_tid=2, entry_tid=1' = 'object_number=33, tag_tid=1, entry_tid=35941' 2026-02-18T08:39:25.601 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=33, tag_tid=1, entry_tid=35941' '!=' 'object_number=33, tag_tid=1, entry_tid=35941' 2026-02-18T08:39:25.601 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:39:25.601 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.8 2026-02-18T08:39:26.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_24 2026-02-18T08:39:26.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:39:26.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:39:26.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_24 2026-02-18T08:39:26.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:39:26.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_24 ']' 2026-02-18T08:39:26.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_24' 2026-02-18T08:39:26.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_24 2026-02-18T08:39:26.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:39:26.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:39:26.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:39:26.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:39:26.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:39:26.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:39:26.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:39:26.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:39:26.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:39:26.403 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:39:26.403 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_24 2026-02-18T08:39:26.403 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:39:26.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:39:26.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:39:26.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:39:26.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:39:26.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:39:26.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:39:26.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:39:26.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:39:26.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:39:26.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:39:26.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:39:26.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:39:26.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:39:26.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:39:26.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:39:26.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:39:26.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:39:26.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:39:26.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:39:26.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:39:26.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:39:26.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:39:26.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:39:26.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:39:26.405 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:39:26.405 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:39:26.405 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:39:26.405 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:39:26.405 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:39:26.415 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:39:26.416 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:39:26.416 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:39:26.416 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_24 2026-02-18T08:39:26.477 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_24'] 2026-02-18T08:39:26.477 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:39:26.477 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:39:26.478 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:39:26.478 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_24 2026-02-18T08:39:26.478 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:39:26.478 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:39:26.478 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:39:26.478 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:39:26.478 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:39:26.478 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:39:26.478 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:39:26.478 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:39:26.478 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:39:26.478 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:39:26.478 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:39:26.479 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:39:26.479 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:39:26.479 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:39:26.479 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:39:26.479 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:39:26.479 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:39:26.479 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:39:26.479 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:39:26.479 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:39:26.479 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:39:26.479 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:39:26.479 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:39:26.479 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:39:26.479 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:39:26.479 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:39:26.479 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:39:26.479 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:39:26.479 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:39:26.479 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:39:26.490 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:39:26.490 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:39:26.490 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:39:26.490 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_24 2026-02-18T08:39:26.558 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:39:26.558 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_24 2026-02-18T08:39:26.558 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:39:26.558 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:39:26.558 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_24 2026-02-18T08:39:26.558 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_24 '' 2026-02-18T08:39:26.559 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:39:26.559 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:39:26.559 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_24 2026-02-18T08:39:26.559 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:39:26.559 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_24.status 2026-02-18T08:39:26.559 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_24.status 2026-02-18T08:39:26.559 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:39:26.561 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_24.status 2026-02-18T08:39:26.561 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_24 2026-02-18T08:39:26.561 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_24.status 2026-02-18T08:39:26.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 8 2026-02-18T08:39:26.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 8 2026-02-18T08:39:26.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:39:26.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=33, tag_tid=2, entry_tid=1], [object_number=32, tag_tid=2, entry_tid=0], [object_number=35, tag_tid=1, entry_tid=35939], [object_number=34, tag_tid=1, entry_tid=35938]]], state=connected] 2026-02-18T08:39:26.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=33, tag_tid=1, entry_tid=35941], [object_number=32, tag_tid=1, entry_tid=35940], [object_number=35, tag_tid=1, entry_tid=35939], [object_number=34, tag_tid=1, entry_tid=35938]]], state=connected] 2026-02-18T08:39:26.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_24.status 2026-02-18T08:39:26.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=33, tag_tid=2, entry_tid=1' 2026-02-18T08:39:26.594 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_24 2026-02-18T08:39:26.594 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:39:26.594 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:39:26.594 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_24 2026-02-18T08:39:26.594 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_24 '..*' 2026-02-18T08:39:26.594 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:39:26.594 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:39:26.594 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_24 2026-02-18T08:39:26.594 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:39:26.594 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_24.status 2026-02-18T08:39:26.594 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_24.status 2026-02-18T08:39:26.595 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:39:26.596 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_24.status 2026-02-18T08:39:26.596 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_24.status 2026-02-18T08:39:26.597 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_24 2026-02-18T08:39:26.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 8 2026-02-18T08:39:26.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 8 2026-02-18T08:39:26.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:39:26.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=33, tag_tid=2, entry_tid=1], [object_number=32, tag_tid=2, entry_tid=0], [object_number=35, tag_tid=1, entry_tid=35939], [object_number=34, tag_tid=1, entry_tid=35938]]], state=connected] 2026-02-18T08:39:26.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=33, tag_tid=1, entry_tid=35941], [object_number=32, tag_tid=1, entry_tid=35940], [object_number=35, tag_tid=1, entry_tid=35939], [object_number=34, tag_tid=1, entry_tid=35938]]], state=connected] 2026-02-18T08:39:26.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_24.status 2026-02-18T08:39:26.628 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=33, tag_tid=1, entry_tid=35941' 2026-02-18T08:39:26.628 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=33, tag_tid=2, entry_tid=1' -a 'object_number=33, tag_tid=2, entry_tid=1' = 'object_number=33, tag_tid=1, entry_tid=35941' 2026-02-18T08:39:26.628 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=33, tag_tid=1, entry_tid=35941' '!=' 'object_number=33, tag_tid=1, entry_tid=35941' 2026-02-18T08:39:26.628 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:39:26.628 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1.6 2026-02-18T08:39:28.231 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_24 2026-02-18T08:39:28.231 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:39:28.231 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:39:28.231 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_24 2026-02-18T08:39:28.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:39:28.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_24 ']' 2026-02-18T08:39:28.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_24' 2026-02-18T08:39:28.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_24 2026-02-18T08:39:28.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:39:28.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:39:28.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:39:28.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:39:28.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:39:28.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:39:28.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:39:28.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:39:28.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:39:28.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:39:28.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_24 2026-02-18T08:39:28.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:39:28.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:39:28.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:39:28.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:39:28.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:39:28.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:39:28.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:39:28.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:39:28.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:39:28.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:39:28.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:39:28.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:39:28.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:39:28.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:39:28.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:39:28.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:39:28.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:39:28.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:39:28.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:39:28.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:39:28.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:39:28.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:39:28.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:39:28.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:39:28.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:39:28.238 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:39:28.238 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:39:28.238 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:39:28.238 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:39:28.238 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:39:28.246 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:39:28.247 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:39:28.247 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:39:28.247 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_24 2026-02-18T08:39:28.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_24'] 2026-02-18T08:39:28.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:39:28.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:39:28.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:39:28.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_24 2026-02-18T08:39:28.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:39:28.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:39:28.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:39:28.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:39:28.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:39:28.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:39:28.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:39:28.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:39:28.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:39:28.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:39:28.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:39:28.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:39:28.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:39:28.310 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:39:28.310 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:39:28.310 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:39:28.310 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:39:28.310 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:39:28.310 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:39:28.310 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:39:28.310 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:39:28.310 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:39:28.310 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:39:28.310 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:39:28.310 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:39:28.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:39:28.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:39:28.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:39:28.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:39:28.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:39:28.320 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:39:28.320 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:39:28.320 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:39:28.320 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_24 2026-02-18T08:39:28.385 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:39:28.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_24 2026-02-18T08:39:28.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:39:28.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:39:28.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_24 2026-02-18T08:39:28.387 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_24 '' 2026-02-18T08:39:28.387 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:39:28.387 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:39:28.387 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_24 2026-02-18T08:39:28.387 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:39:28.387 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_24.status 2026-02-18T08:39:28.387 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_24.status 2026-02-18T08:39:28.387 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:39:28.388 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_24.status 2026-02-18T08:39:28.388 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_24 2026-02-18T08:39:28.388 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_24.status 2026-02-18T08:39:28.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 8 2026-02-18T08:39:28.425 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 8 2026-02-18T08:39:28.425 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:39:28.425 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=33, tag_tid=2, entry_tid=1], [object_number=32, tag_tid=2, entry_tid=0], [object_number=35, tag_tid=1, entry_tid=35939], [object_number=34, tag_tid=1, entry_tid=35938]]], state=connected] 2026-02-18T08:39:28.425 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=33, tag_tid=1, entry_tid=35941], [object_number=32, tag_tid=1, entry_tid=35940], [object_number=35, tag_tid=1, entry_tid=35939], [object_number=34, tag_tid=1, entry_tid=35938]]], state=connected] 2026-02-18T08:39:28.428 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_24.status 2026-02-18T08:39:28.430 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=33, tag_tid=2, entry_tid=1' 2026-02-18T08:39:28.431 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_24 2026-02-18T08:39:28.431 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:39:28.431 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:39:28.431 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_24 2026-02-18T08:39:28.431 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_24 '..*' 2026-02-18T08:39:28.431 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:39:28.432 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:39:28.432 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_24 2026-02-18T08:39:28.432 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:39:28.432 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_24.status 2026-02-18T08:39:28.432 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_24.status 2026-02-18T08:39:28.432 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:39:28.432 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_24.status 2026-02-18T08:39:28.433 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_24 2026-02-18T08:39:28.433 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_24.status 2026-02-18T08:39:28.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 8 2026-02-18T08:39:28.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 8 2026-02-18T08:39:28.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:39:28.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=33, tag_tid=2, entry_tid=1], [object_number=32, tag_tid=2, entry_tid=0], [object_number=35, tag_tid=1, entry_tid=35939], [object_number=34, tag_tid=1, entry_tid=35938]]], state=connected] 2026-02-18T08:39:28.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=33, tag_tid=1, entry_tid=35941], [object_number=32, tag_tid=1, entry_tid=35940], [object_number=35, tag_tid=1, entry_tid=35939], [object_number=34, tag_tid=1, entry_tid=35938]]], state=connected] 2026-02-18T08:39:28.462 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_24.status 2026-02-18T08:39:28.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=33, tag_tid=1, entry_tid=35941' 2026-02-18T08:39:28.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=33, tag_tid=2, entry_tid=1' -a 'object_number=33, tag_tid=2, entry_tid=1' = 'object_number=33, tag_tid=1, entry_tid=35941' 2026-02-18T08:39:28.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=33, tag_tid=1, entry_tid=35941' '!=' 'object_number=33, tag_tid=1, entry_tid=35941' 2026-02-18T08:39:28.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:39:28.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 2 2026-02-18T08:39:30.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_24 2026-02-18T08:39:30.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:39:30.467 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:39:30.467 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_24 2026-02-18T08:39:30.467 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:39:30.467 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_24 ']' 2026-02-18T08:39:30.467 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_24' 2026-02-18T08:39:30.467 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_24 2026-02-18T08:39:30.467 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:39:30.467 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:39:30.467 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:39:30.467 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:39:30.467 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:39:30.468 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:39:30.468 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:39:30.468 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:39:30.468 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:39:30.470 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:39:30.470 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_24 2026-02-18T08:39:30.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:39:30.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:39:30.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:39:30.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:39:30.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:39:30.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:39:30.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:39:30.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:39:30.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:39:30.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:39:30.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:39:30.472 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:39:30.472 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:39:30.472 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:39:30.472 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:39:30.472 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:39:30.472 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:39:30.472 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:39:30.472 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:39:30.472 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:39:30.472 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:39:30.472 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:39:30.472 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:39:30.472 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:39:30.473 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:39:30.473 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:39:30.473 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:39:30.473 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:39:30.473 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:39:30.473 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:39:30.479 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:39:30.479 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:39:30.480 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:39:30.480 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_24 2026-02-18T08:39:30.539 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_24'] 2026-02-18T08:39:30.539 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:39:30.539 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:39:30.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:39:30.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_24 2026-02-18T08:39:30.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:39:30.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:39:30.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:39:30.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:39:30.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:39:30.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:39:30.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:39:30.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:39:30.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:39:30.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:39:30.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:39:30.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:39:30.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:39:30.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:39:30.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:39:30.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:39:30.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:39:30.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:39:30.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:39:30.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:39:30.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:39:30.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:39:30.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:39:30.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:39:30.545 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:39:30.545 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:39:30.545 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:39:30.545 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:39:30.545 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:39:30.545 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:39:30.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:39:30.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:39:30.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:39:30.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_24 2026-02-18T08:39:30.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:39:30.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_24 2026-02-18T08:39:30.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:39:30.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:39:30.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_24 2026-02-18T08:39:30.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_24 '' 2026-02-18T08:39:30.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:39:30.628 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:39:30.628 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_24 2026-02-18T08:39:30.628 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:39:30.628 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_24.status 2026-02-18T08:39:30.628 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_24.status 2026-02-18T08:39:30.628 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:39:30.629 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_24.status 2026-02-18T08:39:30.629 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_24.status 2026-02-18T08:39:30.629 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_24 2026-02-18T08:39:30.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 8 2026-02-18T08:39:30.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 8 2026-02-18T08:39:30.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:39:30.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=33, tag_tid=2, entry_tid=1], [object_number=32, tag_tid=2, entry_tid=0], [object_number=35, tag_tid=1, entry_tid=35939], [object_number=34, tag_tid=1, entry_tid=35938]]], state=connected] 2026-02-18T08:39:30.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=33, tag_tid=2, entry_tid=1], [object_number=32, tag_tid=2, entry_tid=0], [object_number=35, tag_tid=1, entry_tid=35939], [object_number=34, tag_tid=1, entry_tid=35938]]], state=connected] 2026-02-18T08:39:30.660 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_24.status 2026-02-18T08:39:30.661 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=33, tag_tid=2, entry_tid=1' 2026-02-18T08:39:30.662 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_24 2026-02-18T08:39:30.662 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:39:30.662 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:39:30.662 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_24 2026-02-18T08:39:30.663 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_24 '..*' 2026-02-18T08:39:30.663 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:39:30.663 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:39:30.663 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_24 2026-02-18T08:39:30.663 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:39:30.663 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_24.status 2026-02-18T08:39:30.663 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_24.status 2026-02-18T08:39:30.663 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:39:30.664 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_24.status 2026-02-18T08:39:30.664 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_24 2026-02-18T08:39:30.664 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_24.status 2026-02-18T08:39:30.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 8 2026-02-18T08:39:30.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 8 2026-02-18T08:39:30.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:39:30.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=33, tag_tid=2, entry_tid=1], [object_number=32, tag_tid=2, entry_tid=0], [object_number=35, tag_tid=1, entry_tid=35939], [object_number=34, tag_tid=1, entry_tid=35938]]], state=connected] 2026-02-18T08:39:30.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=33, tag_tid=2, entry_tid=1], [object_number=32, tag_tid=2, entry_tid=0], [object_number=35, tag_tid=1, entry_tid=35939], [object_number=34, tag_tid=1, entry_tid=35938]]], state=connected] 2026-02-18T08:39:30.702 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_24.status 2026-02-18T08:39:30.705 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=33, tag_tid=2, entry_tid=1' 2026-02-18T08:39:30.705 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=33, tag_tid=2, entry_tid=1' -a 'object_number=33, tag_tid=2, entry_tid=1' = 'object_number=33, tag_tid=2, entry_tid=1' 2026-02-18T08:39:30.705 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:39:30.705 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_snap_present cluster1 mirror image_24 snap 2026-02-18T08:39:30.705 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:39:30.705 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:39:30.705 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_24 2026-02-18T08:39:30.705 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap 2026-02-18T08:39:30.705 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-18T08:39:30.705 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-18T08:39:30.706 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1 2026-02-18T08:39:31.705 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster1 info mirror/image_24@snap 2026-02-18T08:39:31.735 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:rbd image 'image_24': 2026-02-18T08:39:31.735 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: size 128 MiB in 32 objects 2026-02-18T08:39:31.735 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: order 22 (4 MiB objects) 2026-02-18T08:39:31.735 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: snapshot_count: 1 2026-02-18T08:39:31.735 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: id: 11f9bd0d720 2026-02-18T08:39:31.735 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: block_name_prefix: rbd_data.11f9bd0d720 2026-02-18T08:39:31.735 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: format: 2 2026-02-18T08:39:31.735 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: features: layering, exclusive-lock, journaling 2026-02-18T08:39:31.735 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: op_features: 2026-02-18T08:39:31.735 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: flags: 2026-02-18T08:39:31.735 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: create_timestamp: Wed Feb 18 08:29:55 2026 2026-02-18T08:39:31.735 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: access_timestamp: Wed Feb 18 08:29:55 2026 2026-02-18T08:39:31.735 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: modify_timestamp: Wed Feb 18 08:30:56 2026 2026-02-18T08:39:31.735 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: protected: False 2026-02-18T08:39:31.736 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: journal: 11f9bd0d720 2026-02-18T08:39:31.736 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring state: enabled 2026-02-18T08:39:31.736 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring mode: journal 2026-02-18T08:39:31.736 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring global id: 9ae9d7b2-1f5f-4926-a75e-4b507e6fa8b9 2026-02-18T08:39:31.736 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring primary: false 2026-02-18T08:39:31.738 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:39:31.738 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ compare_image_snaps mirror image_24 snap 2026-02-18T08:39:31.738 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:39:31.738 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_24 2026-02-18T08:39:31.738 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap 2026-02-18T08:39:31.738 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local ret=0 2026-02-18T08:39:31.738 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local rmt_export=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_24.export 2026-02-18T08:39:31.738 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loc_export=/tmp/tmp.vybh1m9yJF/cluster1-mirror-image_24.export 2026-02-18T08:39:31.738 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_24.export /tmp/tmp.vybh1m9yJF/cluster1-mirror-image_24.export 2026-02-18T08:39:31.739 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 -p mirror export image_24@snap /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_24.export 2026-02-18T08:39:31.877 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-18T08:39:31.886 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster1 -p mirror export image_24@snap /tmp/tmp.vybh1m9yJF/cluster1-mirror-image_24.export 2026-02-18T08:39:32.068 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-18T08:39:32.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmp /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_24.export /tmp/tmp.vybh1m9yJF/cluster1-mirror-image_24.export 2026-02-18T08:39:32.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_24.export /tmp/tmp.vybh1m9yJF/cluster1-mirror-image_24.export 2026-02-18T08:39:32.177 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:39:32.177 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T08:39:32.177 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_25 2026-02-18T08:39:32.178 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_snap cluster2 mirror image_25 snap 2026-02-18T08:39:32.178 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:39:32.178 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:39:32.178 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_25 2026-02-18T08:39:32.178 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap 2026-02-18T08:39:32.178 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 -p mirror snap create image_25@snap --debug-rbd=20 --debug-journaler=20 2026-02-18T08:39:35.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_replay_started cluster1 mirror image_25 2026-02-18T08:39:35.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:39:35.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:39:35.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_25 2026-02-18T08:39:35.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_replay_state cluster1 mirror image_25 started 2026-02-18T08:39:35.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:39:35.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:39:35.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_25 2026-02-18T08:39:35.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local state=started 2026-02-18T08:39:35.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-18T08:39:35.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:39:35.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1 2026-02-18T08:39:36.974 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test_image_replay_state cluster1 mirror image_25 started 2026-02-18T08:39:36.974 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:39:36.974 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:39:36.974 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_25 2026-02-18T08:39:36.974 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local test_state=started 2026-02-18T08:39:36.974 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local status_result 2026-02-18T08:39:36.974 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_state=stopped 2026-02-18T08:39:36.974 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_25 2026-02-18T08:39:36.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -i state 2026-02-18T08:39:36.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_instance=cluster1 2026-02-18T08:39:36.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-18T08:39:36.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:39:36.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance=cluster1 2026-02-18T08:39:36.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local loop_instance 2026-02-18T08:39:36.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:39:36.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sleep 0 2026-02-18T08:39:36.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:39:36.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ seq 0 3 2026-02-18T08:39:36.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:39:36.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_25 2026-02-18T08:39:36.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:0 2026-02-18T08:39:36.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-18T08:39:36.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:39:36.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:39:36.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:39:36.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-18T08:39:36.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:39:36.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:39:36.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:39:36.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:39:36.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-18T08:39:36.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:39:36.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:39:36.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:39:36.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:39:36.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:0 2026-02-18T08:39:36.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-18T08:39:36.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-18T08:39:36.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:39:36.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:0 2026-02-18T08:39:36.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T08:39:36.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-18T08:39:36.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:39:36.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:39:36.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-18T08:39:36.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-18T08:39:36.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-18T08:39:36.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=0 2026-02-18T08:39:36.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=0 2026-02-18T08:39:36.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:39:36.989 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:39:36.990 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:39:36.990 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:39:36.990 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_25 2026-02-18T08:39:37.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_25'] 2026-02-18T08:39:37.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status : get status for rbd mirror 2026-02-18T08:39:37.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:39:37.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:39:37.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_25 2026-02-18T08:39:37.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:1 2026-02-18T08:39:37.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-18T08:39:37.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:39:37.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:39:37.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:39:37.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-18T08:39:37.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:39:37.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:39:37.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:39:37.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:39:37.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-18T08:39:37.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:39:37.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:39:37.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:39:37.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:39:37.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:1 2026-02-18T08:39:37.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-18T08:39:37.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-18T08:39:37.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:39:37.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:1 2026-02-18T08:39:37.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T08:39:37.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-18T08:39:37.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:39:37.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:39:37.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-18T08:39:37.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-18T08:39:37.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-18T08:39:37.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=1 2026-02-18T08:39:37.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=1 2026-02-18T08:39:37.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:39:37.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:39:37.062 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:39:37.062 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:39:37.062 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_25 2026-02-18T08:39:37.122 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ return 0 2026-02-18T08:39:37.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ status_result=' "state": "Replaying"' 2026-02-18T08:39:37.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ echo ' "state": "Replaying"' 2026-02-18T08:39:37.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ grep -i Replaying 2026-02-18T08:39:37.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: "state": "Replaying" 2026-02-18T08:39:37.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ current_state=started 2026-02-18T08:39:37.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test started = started 2026-02-18T08:39:37.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:39:37.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror image_25 2026-02-18T08:39:37.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_cluster=cluster1 2026-02-18T08:39:37.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:39:37.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_pool=mirror 2026-02-18T08:39:37.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local remote_pool=mirror 2026-02-18T08:39:37.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_25 2026-02-18T08:39:37.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' journal = journal ']' 2026-02-18T08:39:37.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror mirror image_25 2026-02-18T08:39:37.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_cluster=cluster1 2026-02-18T08:39:37.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:39:37.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_pool=mirror 2026-02-18T08:39:37.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local remote_pool=mirror 2026-02-18T08:39:37.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_25 2026-02-18T08:39:37.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-02-18T08:39:37.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-02-18T08:39:37.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:39:37.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:39:37.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:39:37.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_25 2026-02-18T08:39:37.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:39:37.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:39:37.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_25 2026-02-18T08:39:37.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:39:37.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_25 ']' 2026-02-18T08:39:37.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_25' 2026-02-18T08:39:37.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_25 2026-02-18T08:39:37.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:39:37.329 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:39:37.329 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:39:37.329 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:39:37.329 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:39:37.329 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:39:37.329 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:39:37.329 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:39:37.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:39:37.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:39:37.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_25 2026-02-18T08:39:37.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:39:37.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:39:37.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:39:37.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:39:37.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:39:37.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:39:37.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:39:37.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:39:37.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:39:37.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:39:37.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:39:37.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:39:37.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:39:37.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:39:37.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:39:37.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:39:37.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:39:37.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:39:37.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:39:37.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:39:37.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:39:37.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:39:37.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:39:37.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:39:37.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:39:37.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:39:37.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:39:37.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:39:37.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:39:37.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:39:37.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:39:37.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:39:37.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:39:37.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_25 2026-02-18T08:39:37.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_25'] 2026-02-18T08:39:37.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:39:37.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:39:37.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:39:37.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_25 2026-02-18T08:39:37.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:39:37.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:39:37.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:39:37.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:39:37.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:39:37.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:39:37.403 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:39:37.403 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:39:37.403 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:39:37.403 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:39:37.403 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:39:37.403 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:39:37.403 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:39:37.403 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:39:37.403 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:39:37.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:39:37.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:39:37.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:39:37.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:39:37.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:39:37.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:39:37.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:39:37.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:39:37.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:39:37.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:39:37.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:39:37.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:39:37.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:39:37.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:39:37.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:39:37.414 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:39:37.414 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:39:37.415 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:39:37.415 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_25 2026-02-18T08:39:37.473 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:39:37.473 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_25 2026-02-18T08:39:37.473 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:39:37.473 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:39:37.473 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_25 2026-02-18T08:39:37.473 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_25 '' 2026-02-18T08:39:37.473 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:39:37.473 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:39:37.473 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_25 2026-02-18T08:39:37.473 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:39:37.474 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_25.status 2026-02-18T08:39:37.474 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_25.status 2026-02-18T08:39:37.474 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:39:37.476 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_25.status 2026-02-18T08:39:37.476 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_25 2026-02-18T08:39:37.476 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_25.status 2026-02-18T08:39:37.503 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 6 2026-02-18T08:39:37.504 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 6 2026-02-18T08:39:37.504 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:39:37.504 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=25, tag_tid=2, entry_tid=1], [object_number=24, tag_tid=2, entry_tid=0], [object_number=27, tag_tid=1, entry_tid=26199], [object_number=26, tag_tid=1, entry_tid=26198]]], state=connected] 2026-02-18T08:39:37.504 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=25, tag_tid=1, entry_tid=26201], [object_number=24, tag_tid=1, entry_tid=26200], [object_number=27, tag_tid=1, entry_tid=26199], [object_number=26, tag_tid=1, entry_tid=26198]]], state=connected] 2026-02-18T08:39:37.506 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_25.status 2026-02-18T08:39:37.507 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=25, tag_tid=2, entry_tid=1' 2026-02-18T08:39:37.508 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_25 2026-02-18T08:39:37.508 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:39:37.508 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:39:37.508 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_25 2026-02-18T08:39:37.509 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_25 '..*' 2026-02-18T08:39:37.509 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:39:37.509 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:39:37.509 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_25 2026-02-18T08:39:37.509 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:39:37.509 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_25.status 2026-02-18T08:39:37.510 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_25.status 2026-02-18T08:39:37.510 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:39:37.510 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_25.status 2026-02-18T08:39:37.510 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_25 2026-02-18T08:39:37.511 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_25.status 2026-02-18T08:39:37.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 6 2026-02-18T08:39:37.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 6 2026-02-18T08:39:37.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:39:37.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=25, tag_tid=2, entry_tid=1], [object_number=24, tag_tid=2, entry_tid=0], [object_number=27, tag_tid=1, entry_tid=26199], [object_number=26, tag_tid=1, entry_tid=26198]]], state=connected] 2026-02-18T08:39:37.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=25, tag_tid=1, entry_tid=26201], [object_number=24, tag_tid=1, entry_tid=26200], [object_number=27, tag_tid=1, entry_tid=26199], [object_number=26, tag_tid=1, entry_tid=26198]]], state=connected] 2026-02-18T08:39:37.540 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_25.status 2026-02-18T08:39:37.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=25, tag_tid=1, entry_tid=26201' 2026-02-18T08:39:37.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=25, tag_tid=2, entry_tid=1' -a 'object_number=25, tag_tid=2, entry_tid=1' = 'object_number=25, tag_tid=1, entry_tid=26201' 2026-02-18T08:39:37.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=25, tag_tid=1, entry_tid=26201' '!=' '' 2026-02-18T08:39:37.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:39:37.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=25, tag_tid=1, entry_tid=26201' = '' 2026-02-18T08:39:37.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=25, tag_tid=1, entry_tid=26201' 2026-02-18T08:39:37.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=25, tag_tid=2, entry_tid=1' 2026-02-18T08:39:37.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:39:37.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:39:37.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=2 2026-02-18T08:39:37.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=25, tag_tid=1, entry_tid=26201' 2026-02-18T08:39:37.545 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:39:37.545 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:39:37.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=1 2026-02-18T08:39:37.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=25, tag_tid=2, entry_tid=1' 2026-02-18T08:39:37.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:39:37.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:39:37.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:39:37.549 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.0 with skip (0.40702748681162715) > weight (0.25) 2026-02-18T08:39:37.549 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.1 with skip (0.9859806335910718) > weight (0.25) 2026-02-18T08:39:37.549 INFO:tasks.rbd_mirror_thrash:kill cluster2.client.mirror.2 2026-02-18T08:39:37.549 INFO:tasks.rbd_mirror.cluster2.client.mirror.2:Sent signal 15 2026-02-18T08:39:37.549 INFO:tasks.rbd_mirror_thrash:waiting for 2.971635337043541 secs before reviving daemons 2026-02-18T08:39:37.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=25, tag_tid=1, entry_tid=26201' 2026-02-18T08:39:37.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:39:37.551 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:39:37.551 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial163.stderr:2026-02-18T08:39:37.541+0000 7f79b62b3640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.2 (PID: 15028) UID: 1000 2026-02-18T08:39:37.552 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=26201 2026-02-18T08:39:37.553 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 2 = 1 -a 1 -le 26201 2026-02-18T08:39:37.553 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:39:37.553 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:39:37.553 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:39:37.753 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_25 2026-02-18T08:39:37.753 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:39:37.753 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:39:37.753 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_25 2026-02-18T08:39:37.754 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:39:37.754 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_25 ']' 2026-02-18T08:39:37.754 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_25' 2026-02-18T08:39:37.754 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_25 2026-02-18T08:39:37.754 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:39:37.754 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:39:37.754 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:39:37.754 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:39:37.754 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:39:37.754 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:39:37.754 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:39:37.754 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:39:37.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:39:37.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:39:37.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_25 2026-02-18T08:39:37.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:39:37.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:39:37.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:39:37.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:39:37.758 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:39:37.758 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:39:37.758 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:39:37.758 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:39:37.758 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:39:37.758 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:39:37.758 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:39:37.758 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:39:37.758 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:39:37.758 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:39:37.758 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:39:37.758 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:39:37.759 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:39:37.759 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:39:37.759 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:39:37.759 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:39:37.759 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:39:37.759 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:39:37.759 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:39:37.759 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:39:37.759 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:39:37.759 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:39:37.759 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:39:37.759 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:39:37.760 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:39:37.761 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:39:37.767 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:39:37.768 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:39:37.768 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:39:37.768 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_25 2026-02-18T08:39:37.826 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_25'] 2026-02-18T08:39:37.827 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:39:37.827 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:39:37.828 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:39:37.828 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_25 2026-02-18T08:39:37.828 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:39:37.828 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:39:37.828 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:39:37.828 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:39:37.828 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:39:37.828 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:39:37.828 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:39:37.829 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:39:37.829 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:39:37.829 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:39:37.829 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:39:37.829 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:39:37.829 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:39:37.829 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:39:37.829 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:39:37.829 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:39:37.829 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:39:37.829 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:39:37.829 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:39:37.829 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:39:37.829 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:39:37.829 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:39:37.829 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:39:37.829 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:39:37.829 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:39:37.829 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:39:37.829 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:39:37.829 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:39:37.829 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:39:37.829 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:39:37.839 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:39:37.840 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:39:37.840 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:39:37.840 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_25 2026-02-18T08:39:37.897 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:39:37.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_25 2026-02-18T08:39:37.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:39:37.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:39:37.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_25 2026-02-18T08:39:37.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_25 '' 2026-02-18T08:39:37.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:39:37.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:39:37.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_25 2026-02-18T08:39:37.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:39:37.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_25.status 2026-02-18T08:39:37.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_25.status 2026-02-18T08:39:37.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:39:37.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_25.status 2026-02-18T08:39:37.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_25 2026-02-18T08:39:37.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_25.status 2026-02-18T08:39:37.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 6 2026-02-18T08:39:37.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 6 2026-02-18T08:39:37.928 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:39:37.928 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=25, tag_tid=2, entry_tid=1], [object_number=24, tag_tid=2, entry_tid=0], [object_number=27, tag_tid=1, entry_tid=26199], [object_number=26, tag_tid=1, entry_tid=26198]]], state=connected] 2026-02-18T08:39:37.928 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=25, tag_tid=1, entry_tid=26201], [object_number=24, tag_tid=1, entry_tid=26200], [object_number=27, tag_tid=1, entry_tid=26199], [object_number=26, tag_tid=1, entry_tid=26198]]], state=connected] 2026-02-18T08:39:37.930 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_25.status 2026-02-18T08:39:37.931 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=25, tag_tid=2, entry_tid=1' 2026-02-18T08:39:37.933 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_25 2026-02-18T08:39:37.933 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:39:37.933 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:39:37.933 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_25 2026-02-18T08:39:37.933 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_25 '..*' 2026-02-18T08:39:37.933 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:39:37.933 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:39:37.933 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_25 2026-02-18T08:39:37.933 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:39:37.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_25.status 2026-02-18T08:39:37.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_25.status 2026-02-18T08:39:37.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:39:37.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_25.status 2026-02-18T08:39:37.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_25 2026-02-18T08:39:37.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_25.status 2026-02-18T08:39:37.970 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 6 2026-02-18T08:39:37.970 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 6 2026-02-18T08:39:37.970 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:39:37.970 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=25, tag_tid=2, entry_tid=1], [object_number=24, tag_tid=2, entry_tid=0], [object_number=27, tag_tid=1, entry_tid=26199], [object_number=26, tag_tid=1, entry_tid=26198]]], state=connected] 2026-02-18T08:39:37.971 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=25, tag_tid=1, entry_tid=26201], [object_number=24, tag_tid=1, entry_tid=26200], [object_number=27, tag_tid=1, entry_tid=26199], [object_number=26, tag_tid=1, entry_tid=26198]]], state=connected] 2026-02-18T08:39:37.974 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_25.status 2026-02-18T08:39:37.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=25, tag_tid=1, entry_tid=26201' 2026-02-18T08:39:37.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=25, tag_tid=2, entry_tid=1' -a 'object_number=25, tag_tid=2, entry_tid=1' = 'object_number=25, tag_tid=1, entry_tid=26201' 2026-02-18T08:39:37.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=25, tag_tid=1, entry_tid=26201' '!=' 'object_number=25, tag_tid=1, entry_tid=26201' 2026-02-18T08:39:37.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:39:37.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.4 2026-02-18T08:39:38.377 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_25 2026-02-18T08:39:38.377 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:39:38.377 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:39:38.377 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_25 2026-02-18T08:39:38.377 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:39:38.377 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_25 ']' 2026-02-18T08:39:38.377 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_25' 2026-02-18T08:39:38.377 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_25 2026-02-18T08:39:38.377 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:39:38.377 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:39:38.377 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:39:38.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:39:38.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:39:38.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:39:38.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:39:38.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:39:38.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:39:38.379 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:39:38.379 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_25 2026-02-18T08:39:38.379 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:39:38.379 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:39:38.379 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:39:38.380 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:39:38.380 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:39:38.380 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:39:38.381 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:39:38.381 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:39:38.381 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:39:38.381 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:39:38.381 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:39:38.381 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:39:38.381 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:39:38.381 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:39:38.381 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:39:38.381 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:39:38.381 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:39:38.381 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:39:38.381 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:39:38.381 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:39:38.381 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:39:38.381 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:39:38.381 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:39:38.381 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:39:38.381 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:39:38.381 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:39:38.381 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:39:38.381 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:39:38.381 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:39:38.382 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:39:38.393 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:39:38.394 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:39:38.394 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:39:38.394 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_25 2026-02-18T08:39:38.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_25'] 2026-02-18T08:39:38.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:39:38.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:39:38.458 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:39:38.458 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_25 2026-02-18T08:39:38.458 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:39:38.458 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:39:38.458 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:39:38.458 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:39:38.458 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:39:38.458 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:39:38.458 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:39:38.458 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:39:38.458 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:39:38.458 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:39:38.458 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:39:38.458 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:39:38.458 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:39:38.458 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:39:38.458 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:39:38.458 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:39:38.458 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:39:38.458 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:39:38.458 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:39:38.458 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:39:38.458 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:39:38.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:39:38.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:39:38.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:39:38.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:39:38.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:39:38.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:39:38.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:39:38.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:39:38.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:39:38.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:39:38.470 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:39:38.470 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:39:38.470 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_25 2026-02-18T08:39:38.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:39:38.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_25 2026-02-18T08:39:38.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:39:38.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:39:38.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_25 2026-02-18T08:39:38.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_25 '' 2026-02-18T08:39:38.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:39:38.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:39:38.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_25 2026-02-18T08:39:38.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:39:38.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_25.status 2026-02-18T08:39:38.539 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_25.status 2026-02-18T08:39:38.539 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:39:38.540 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_25.status 2026-02-18T08:39:38.540 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_25 2026-02-18T08:39:38.540 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_25.status 2026-02-18T08:39:38.568 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 6 2026-02-18T08:39:38.568 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 6 2026-02-18T08:39:38.568 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:39:38.568 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=25, tag_tid=2, entry_tid=1], [object_number=24, tag_tid=2, entry_tid=0], [object_number=27, tag_tid=1, entry_tid=26199], [object_number=26, tag_tid=1, entry_tid=26198]]], state=connected] 2026-02-18T08:39:38.568 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=25, tag_tid=1, entry_tid=26201], [object_number=24, tag_tid=1, entry_tid=26200], [object_number=27, tag_tid=1, entry_tid=26199], [object_number=26, tag_tid=1, entry_tid=26198]]], state=connected] 2026-02-18T08:39:38.570 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_25.status 2026-02-18T08:39:38.572 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=25, tag_tid=2, entry_tid=1' 2026-02-18T08:39:38.573 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_25 2026-02-18T08:39:38.573 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:39:38.573 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:39:38.573 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_25 2026-02-18T08:39:38.573 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_25 '..*' 2026-02-18T08:39:38.573 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:39:38.573 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:39:38.573 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_25 2026-02-18T08:39:38.573 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:39:38.573 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_25.status 2026-02-18T08:39:38.574 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_25.status 2026-02-18T08:39:38.574 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:39:38.575 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_25.status 2026-02-18T08:39:38.575 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_25 2026-02-18T08:39:38.576 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_25.status 2026-02-18T08:39:38.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 6 2026-02-18T08:39:38.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 6 2026-02-18T08:39:38.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:39:38.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=25, tag_tid=2, entry_tid=1], [object_number=24, tag_tid=2, entry_tid=0], [object_number=27, tag_tid=1, entry_tid=26199], [object_number=26, tag_tid=1, entry_tid=26198]]], state=connected] 2026-02-18T08:39:38.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=25, tag_tid=1, entry_tid=26201], [object_number=24, tag_tid=1, entry_tid=26200], [object_number=27, tag_tid=1, entry_tid=26199], [object_number=26, tag_tid=1, entry_tid=26198]]], state=connected] 2026-02-18T08:39:38.620 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_25.status 2026-02-18T08:39:38.622 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=25, tag_tid=1, entry_tid=26201' 2026-02-18T08:39:38.622 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=25, tag_tid=2, entry_tid=1' -a 'object_number=25, tag_tid=2, entry_tid=1' = 'object_number=25, tag_tid=1, entry_tid=26201' 2026-02-18T08:39:38.622 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=25, tag_tid=1, entry_tid=26201' '!=' 'object_number=25, tag_tid=1, entry_tid=26201' 2026-02-18T08:39:38.622 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:39:38.622 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.8 2026-02-18T08:39:39.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_25 2026-02-18T08:39:39.425 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:39:39.425 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:39:39.425 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_25 2026-02-18T08:39:39.425 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:39:39.425 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_25 ']' 2026-02-18T08:39:39.425 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_25' 2026-02-18T08:39:39.425 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_25 2026-02-18T08:39:39.425 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:39:39.425 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:39:39.425 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:39:39.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:39:39.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:39:39.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:39:39.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:39:39.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:39:39.428 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:39:39.429 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:39:39.429 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_25 2026-02-18T08:39:39.429 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:39:39.429 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:39:39.429 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:39:39.429 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:39:39.429 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:39:39.429 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:39:39.429 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:39:39.429 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:39:39.430 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:39:39.430 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:39:39.430 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:39:39.430 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:39:39.430 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:39:39.430 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:39:39.430 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:39:39.430 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:39:39.430 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:39:39.430 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:39:39.430 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:39:39.430 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:39:39.430 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:39:39.431 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:39:39.431 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:39:39.431 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:39:39.431 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:39:39.431 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:39:39.431 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:39:39.431 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:39:39.431 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:39:39.431 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:39:39.440 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:39:39.440 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:39:39.440 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:39:39.440 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_25 2026-02-18T08:39:39.503 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_25'] 2026-02-18T08:39:39.504 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:39:39.504 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:39:39.506 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:39:39.507 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_25 2026-02-18T08:39:39.507 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:39:39.507 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:39:39.507 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:39:39.507 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:39:39.507 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:39:39.507 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:39:39.507 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:39:39.507 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:39:39.507 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:39:39.507 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:39:39.508 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:39:39.508 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:39:39.508 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:39:39.508 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:39:39.508 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:39:39.508 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:39:39.508 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:39:39.508 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:39:39.508 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:39:39.508 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:39:39.508 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:39:39.508 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:39:39.508 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:39:39.509 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:39:39.509 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:39:39.509 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:39:39.509 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:39:39.509 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:39:39.509 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:39:39.509 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:39:39.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:39:39.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:39:39.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:39:39.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_25 2026-02-18T08:39:39.584 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:39:39.585 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_25 2026-02-18T08:39:39.585 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:39:39.586 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:39:39.586 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_25 2026-02-18T08:39:39.586 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_25 '' 2026-02-18T08:39:39.586 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:39:39.586 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:39:39.586 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_25 2026-02-18T08:39:39.586 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:39:39.586 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_25.status 2026-02-18T08:39:39.586 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_25.status 2026-02-18T08:39:39.586 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:39:39.587 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_25.status 2026-02-18T08:39:39.587 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_25 2026-02-18T08:39:39.587 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_25.status 2026-02-18T08:39:39.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 6 2026-02-18T08:39:39.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 6 2026-02-18T08:39:39.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:39:39.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=25, tag_tid=2, entry_tid=1], [object_number=24, tag_tid=2, entry_tid=0], [object_number=27, tag_tid=1, entry_tid=26199], [object_number=26, tag_tid=1, entry_tid=26198]]], state=connected] 2026-02-18T08:39:39.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=25, tag_tid=1, entry_tid=26201], [object_number=24, tag_tid=1, entry_tid=26200], [object_number=27, tag_tid=1, entry_tid=26199], [object_number=26, tag_tid=1, entry_tid=26198]]], state=connected] 2026-02-18T08:39:39.629 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_25.status 2026-02-18T08:39:39.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=25, tag_tid=2, entry_tid=1' 2026-02-18T08:39:39.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_25 2026-02-18T08:39:39.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:39:39.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:39:39.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_25 2026-02-18T08:39:39.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_25 '..*' 2026-02-18T08:39:39.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:39:39.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:39:39.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_25 2026-02-18T08:39:39.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:39:39.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_25.status 2026-02-18T08:39:39.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_25.status 2026-02-18T08:39:39.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:39:39.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_25.status 2026-02-18T08:39:39.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_25 2026-02-18T08:39:39.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_25.status 2026-02-18T08:39:39.661 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 6 2026-02-18T08:39:39.661 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 6 2026-02-18T08:39:39.661 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:39:39.661 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=25, tag_tid=2, entry_tid=1], [object_number=24, tag_tid=2, entry_tid=0], [object_number=27, tag_tid=1, entry_tid=26199], [object_number=26, tag_tid=1, entry_tid=26198]]], state=connected] 2026-02-18T08:39:39.661 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=25, tag_tid=1, entry_tid=26201], [object_number=24, tag_tid=1, entry_tid=26200], [object_number=27, tag_tid=1, entry_tid=26199], [object_number=26, tag_tid=1, entry_tid=26198]]], state=connected] 2026-02-18T08:39:39.663 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_25.status 2026-02-18T08:39:39.664 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=25, tag_tid=1, entry_tid=26201' 2026-02-18T08:39:39.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=25, tag_tid=2, entry_tid=1' -a 'object_number=25, tag_tid=2, entry_tid=1' = 'object_number=25, tag_tid=1, entry_tid=26201' 2026-02-18T08:39:39.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=25, tag_tid=1, entry_tid=26201' '!=' 'object_number=25, tag_tid=1, entry_tid=26201' 2026-02-18T08:39:39.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:39:39.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1.6 2026-02-18T08:39:40.522 INFO:tasks.rbd_mirror_thrash:waiting for cluster2.client.mirror.2 2026-02-18T08:39:40.522 INFO:teuthology.orchestra.run:waiting for 600 2026-02-18T08:39:40.523 INFO:tasks.rbd_mirror_thrash:reviving cluster2.client.mirror.2 2026-02-18T08:39:40.523 INFO:tasks.rbd_mirror.cluster2.client.mirror.2:Restarting daemon 2026-02-18T08:39:40.523 DEBUG:teuthology.orchestra.run.trial163:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.2 2026-02-18T08:39:40.525 INFO:tasks.rbd_mirror.cluster2.client.mirror.2:Started 2026-02-18T08:39:40.525 INFO:tasks.rbd_mirror_thrash:waiting for 68.15880374174726 secs before thrashing 2026-02-18T08:39:41.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_25 2026-02-18T08:39:41.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:39:41.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:39:41.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_25 2026-02-18T08:39:41.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:39:41.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_25 ']' 2026-02-18T08:39:41.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_25' 2026-02-18T08:39:41.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_25 2026-02-18T08:39:41.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:39:41.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:39:41.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:39:41.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:39:41.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:39:41.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:39:41.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:39:41.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:39:41.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:39:41.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:39:41.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_25 2026-02-18T08:39:41.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:39:41.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:39:41.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:39:41.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:39:41.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:39:41.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:39:41.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:39:41.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:39:41.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:39:41.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:39:41.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:39:41.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:39:41.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:39:41.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:39:41.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:39:41.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:39:41.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:39:41.273 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:39:41.273 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:39:41.273 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:39:41.273 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:39:41.273 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:39:41.273 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:39:41.273 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:39:41.273 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:39:41.273 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:39:41.273 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:39:41.273 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:39:41.273 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:39:41.274 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:39:41.282 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:39:41.282 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:39:41.283 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:39:41.283 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_25 2026-02-18T08:39:41.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_25'] 2026-02-18T08:39:41.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:39:41.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:39:41.344 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:39:41.344 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_25 2026-02-18T08:39:41.344 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:39:41.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:39:41.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:39:41.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:39:41.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:39:41.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:39:41.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:39:41.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:39:41.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:39:41.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:39:41.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:39:41.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:39:41.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:39:41.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:39:41.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:39:41.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:39:41.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:39:41.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:39:41.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:39:41.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:39:41.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:39:41.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:39:41.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:39:41.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:39:41.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:39:41.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:39:41.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:39:41.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:39:41.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:39:41.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:39:41.355 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:39:41.355 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:39:41.355 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:39:41.355 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_25 2026-02-18T08:39:41.421 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:39:41.423 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_25 2026-02-18T08:39:41.423 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:39:41.423 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:39:41.423 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_25 2026-02-18T08:39:41.423 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_25 '' 2026-02-18T08:39:41.423 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:39:41.423 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:39:41.423 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_25 2026-02-18T08:39:41.423 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:39:41.423 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_25.status 2026-02-18T08:39:41.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_25.status 2026-02-18T08:39:41.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:39:41.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_25.status 2026-02-18T08:39:41.425 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_25 2026-02-18T08:39:41.425 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_25.status 2026-02-18T08:39:41.462 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 6 2026-02-18T08:39:41.462 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 6 2026-02-18T08:39:41.462 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:39:41.462 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=25, tag_tid=2, entry_tid=1], [object_number=24, tag_tid=2, entry_tid=0], [object_number=27, tag_tid=1, entry_tid=26199], [object_number=26, tag_tid=1, entry_tid=26198]]], state=connected] 2026-02-18T08:39:41.462 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=25, tag_tid=1, entry_tid=26201], [object_number=24, tag_tid=1, entry_tid=26200], [object_number=27, tag_tid=1, entry_tid=26199], [object_number=26, tag_tid=1, entry_tid=26198]]], state=connected] 2026-02-18T08:39:41.465 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_25.status 2026-02-18T08:39:41.467 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=25, tag_tid=2, entry_tid=1' 2026-02-18T08:39:41.468 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_25 2026-02-18T08:39:41.468 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:39:41.468 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:39:41.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_25 2026-02-18T08:39:41.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_25 '..*' 2026-02-18T08:39:41.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:39:41.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:39:41.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_25 2026-02-18T08:39:41.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:39:41.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_25.status 2026-02-18T08:39:41.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_25.status 2026-02-18T08:39:41.470 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:39:41.470 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_25.status 2026-02-18T08:39:41.470 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_25 2026-02-18T08:39:41.470 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_25.status 2026-02-18T08:39:41.504 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 6 2026-02-18T08:39:41.505 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 6 2026-02-18T08:39:41.505 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:39:41.505 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=25, tag_tid=2, entry_tid=1], [object_number=24, tag_tid=2, entry_tid=0], [object_number=27, tag_tid=1, entry_tid=26199], [object_number=26, tag_tid=1, entry_tid=26198]]], state=connected] 2026-02-18T08:39:41.505 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=25, tag_tid=1, entry_tid=26201], [object_number=24, tag_tid=1, entry_tid=26200], [object_number=27, tag_tid=1, entry_tid=26199], [object_number=26, tag_tid=1, entry_tid=26198]]], state=connected] 2026-02-18T08:39:41.508 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_25.status 2026-02-18T08:39:41.510 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=25, tag_tid=1, entry_tid=26201' 2026-02-18T08:39:41.510 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=25, tag_tid=2, entry_tid=1' -a 'object_number=25, tag_tid=2, entry_tid=1' = 'object_number=25, tag_tid=1, entry_tid=26201' 2026-02-18T08:39:41.510 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=25, tag_tid=1, entry_tid=26201' '!=' 'object_number=25, tag_tid=1, entry_tid=26201' 2026-02-18T08:39:41.511 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:39:41.511 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 2 2026-02-18T08:39:43.513 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_25 2026-02-18T08:39:43.513 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:39:43.513 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:39:43.513 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_25 2026-02-18T08:39:43.513 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:39:43.513 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_25 ']' 2026-02-18T08:39:43.513 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_25' 2026-02-18T08:39:43.513 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_25 2026-02-18T08:39:43.513 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:39:43.514 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:39:43.514 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:39:43.514 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:39:43.514 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:39:43.514 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:39:43.514 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:39:43.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:39:43.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:39:43.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:39:43.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_25 2026-02-18T08:39:43.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:39:43.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:39:43.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:39:43.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:39:43.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:39:43.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:39:43.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:39:43.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:39:43.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:39:43.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:39:43.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:39:43.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:39:43.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:39:43.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:39:43.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:39:43.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:39:43.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:39:43.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:39:43.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:39:43.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:39:43.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:39:43.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:39:43.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:39:43.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:39:43.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:39:43.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:39:43.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:39:43.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:39:43.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:39:43.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:39:43.528 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:39:43.528 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:39:43.528 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:39:43.529 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_25 2026-02-18T08:39:43.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_25'] 2026-02-18T08:39:43.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:39:43.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:39:43.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:39:43.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_25 2026-02-18T08:39:43.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:39:43.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:39:43.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:39:43.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:39:43.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:39:43.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:39:43.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:39:43.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:39:43.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:39:43.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:39:43.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:39:43.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:39:43.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:39:43.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:39:43.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:39:43.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:39:43.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:39:43.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:39:43.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:39:43.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:39:43.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:39:43.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:39:43.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:39:43.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:39:43.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:39:43.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:39:43.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:39:43.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:39:43.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:39:43.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:39:43.602 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:39:43.603 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:39:43.603 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:39:43.603 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_25 2026-02-18T08:39:43.674 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:39:43.675 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_25 2026-02-18T08:39:43.675 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:39:43.675 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:39:43.675 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_25 2026-02-18T08:39:43.675 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_25 '' 2026-02-18T08:39:43.675 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:39:43.675 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:39:43.675 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_25 2026-02-18T08:39:43.676 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:39:43.676 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_25.status 2026-02-18T08:39:43.676 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_25.status 2026-02-18T08:39:43.676 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:39:43.676 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_25.status 2026-02-18T08:39:43.677 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_25 2026-02-18T08:39:43.677 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_25.status 2026-02-18T08:39:43.714 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 6 2026-02-18T08:39:43.714 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 6 2026-02-18T08:39:43.714 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:39:43.714 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=25, tag_tid=2, entry_tid=1], [object_number=24, tag_tid=2, entry_tid=0], [object_number=27, tag_tid=1, entry_tid=26199], [object_number=26, tag_tid=1, entry_tid=26198]]], state=connected] 2026-02-18T08:39:43.714 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=25, tag_tid=2, entry_tid=1], [object_number=24, tag_tid=2, entry_tid=0], [object_number=27, tag_tid=1, entry_tid=26199], [object_number=26, tag_tid=1, entry_tid=26198]]], state=connected] 2026-02-18T08:39:43.717 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_25.status 2026-02-18T08:39:43.719 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=25, tag_tid=2, entry_tid=1' 2026-02-18T08:39:43.720 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_25 2026-02-18T08:39:43.720 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:39:43.720 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:39:43.720 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_25 2026-02-18T08:39:43.720 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_25 '..*' 2026-02-18T08:39:43.720 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:39:43.720 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:39:43.720 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_25 2026-02-18T08:39:43.721 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:39:43.721 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_25.status 2026-02-18T08:39:43.721 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_25.status 2026-02-18T08:39:43.721 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:39:43.721 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_25.status 2026-02-18T08:39:43.722 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_25 2026-02-18T08:39:43.722 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_25.status 2026-02-18T08:39:43.756 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 6 2026-02-18T08:39:43.756 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 6 2026-02-18T08:39:43.756 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:39:43.756 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=25, tag_tid=2, entry_tid=1], [object_number=24, tag_tid=2, entry_tid=0], [object_number=27, tag_tid=1, entry_tid=26199], [object_number=26, tag_tid=1, entry_tid=26198]]], state=connected] 2026-02-18T08:39:43.756 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=25, tag_tid=2, entry_tid=1], [object_number=24, tag_tid=2, entry_tid=0], [object_number=27, tag_tid=1, entry_tid=26199], [object_number=26, tag_tid=1, entry_tid=26198]]], state=connected] 2026-02-18T08:39:43.760 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_25.status 2026-02-18T08:39:43.762 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=25, tag_tid=2, entry_tid=1' 2026-02-18T08:39:43.762 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=25, tag_tid=2, entry_tid=1' -a 'object_number=25, tag_tid=2, entry_tid=1' = 'object_number=25, tag_tid=2, entry_tid=1' 2026-02-18T08:39:43.763 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:39:43.763 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_snap_present cluster1 mirror image_25 snap 2026-02-18T08:39:43.763 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:39:43.763 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:39:43.763 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_25 2026-02-18T08:39:43.763 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap 2026-02-18T08:39:43.763 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-18T08:39:43.763 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-18T08:39:43.763 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1 2026-02-18T08:39:44.763 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster1 info mirror/image_25@snap 2026-02-18T08:39:44.794 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:rbd image 'image_25': 2026-02-18T08:39:44.794 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: size 128 MiB in 32 objects 2026-02-18T08:39:44.794 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: order 22 (4 MiB objects) 2026-02-18T08:39:44.794 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: snapshot_count: 1 2026-02-18T08:39:44.794 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: id: 120d75657bc6 2026-02-18T08:39:44.794 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: block_name_prefix: rbd_data.120d75657bc6 2026-02-18T08:39:44.795 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: format: 2 2026-02-18T08:39:44.795 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: features: layering, exclusive-lock, journaling 2026-02-18T08:39:44.795 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: op_features: 2026-02-18T08:39:44.795 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: flags: 2026-02-18T08:39:44.795 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: create_timestamp: Wed Feb 18 08:30:06 2026 2026-02-18T08:39:44.795 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: access_timestamp: Wed Feb 18 08:30:06 2026 2026-02-18T08:39:44.795 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: modify_timestamp: Wed Feb 18 08:31:07 2026 2026-02-18T08:39:44.795 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: protected: False 2026-02-18T08:39:44.795 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: journal: 120d75657bc6 2026-02-18T08:39:44.795 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring state: enabled 2026-02-18T08:39:44.795 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring mode: journal 2026-02-18T08:39:44.796 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring global id: 0422029f-eed3-4fd5-abba-e559aca1c156 2026-02-18T08:39:44.796 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring primary: false 2026-02-18T08:39:44.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:39:44.798 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ compare_image_snaps mirror image_25 snap 2026-02-18T08:39:44.798 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:39:44.798 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_25 2026-02-18T08:39:44.798 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap 2026-02-18T08:39:44.798 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local ret=0 2026-02-18T08:39:44.798 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local rmt_export=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_25.export 2026-02-18T08:39:44.798 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loc_export=/tmp/tmp.vybh1m9yJF/cluster1-mirror-image_25.export 2026-02-18T08:39:44.798 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_25.export /tmp/tmp.vybh1m9yJF/cluster1-mirror-image_25.export 2026-02-18T08:39:44.798 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 -p mirror export image_25@snap /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_25.export 2026-02-18T08:39:44.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-18T08:39:44.950 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster1 -p mirror export image_25@snap /tmp/tmp.vybh1m9yJF/cluster1-mirror-image_25.export 2026-02-18T08:39:45.118 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-18T08:39:45.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmp /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_25.export /tmp/tmp.vybh1m9yJF/cluster1-mirror-image_25.export 2026-02-18T08:39:45.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_25.export /tmp/tmp.vybh1m9yJF/cluster1-mirror-image_25.export 2026-02-18T08:39:45.230 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:39:45.230 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T08:39:45.230 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_26 2026-02-18T08:39:45.230 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_snap cluster2 mirror image_26 snap 2026-02-18T08:39:45.230 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:39:45.230 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:39:45.230 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_26 2026-02-18T08:39:45.231 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap 2026-02-18T08:39:45.231 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 -p mirror snap create image_26@snap --debug-rbd=20 --debug-journaler=20 2026-02-18T08:39:47.992 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_replay_started cluster1 mirror image_26 2026-02-18T08:39:47.992 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:39:47.993 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:39:47.993 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_26 2026-02-18T08:39:47.993 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_replay_state cluster1 mirror image_26 started 2026-02-18T08:39:47.993 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:39:47.993 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:39:47.993 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_26 2026-02-18T08:39:47.993 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local state=started 2026-02-18T08:39:47.993 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-18T08:39:47.993 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:39:47.993 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1 2026-02-18T08:39:48.124 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.0 with skip (0.5994827876765736) > weight (0.25) 2026-02-18T08:39:48.125 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.1 with skip (0.26539496464661716) > weight (0.25) 2026-02-18T08:39:48.125 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.2 with skip (0.46853713257654694) > weight (0.25) 2026-02-18T08:39:48.125 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.3 with skip (0.9733084084041643) > weight (0.25) 2026-02-18T08:39:48.125 INFO:tasks.rbd_mirror_thrash:waiting for 99.34175013157923 secs before thrashing 2026-02-18T08:39:48.994 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test_image_replay_state cluster1 mirror image_26 started 2026-02-18T08:39:48.994 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:39:48.994 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:39:48.994 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_26 2026-02-18T08:39:48.994 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local test_state=started 2026-02-18T08:39:48.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local status_result 2026-02-18T08:39:48.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_state=stopped 2026-02-18T08:39:48.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_26 2026-02-18T08:39:48.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -i state 2026-02-18T08:39:48.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_instance=cluster1 2026-02-18T08:39:48.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-18T08:39:48.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:39:48.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance=cluster1 2026-02-18T08:39:48.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local loop_instance 2026-02-18T08:39:48.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:39:48.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sleep 0 2026-02-18T08:39:48.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:39:48.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ seq 0 3 2026-02-18T08:39:48.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:39:48.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_26 2026-02-18T08:39:48.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:0 2026-02-18T08:39:48.999 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-18T08:39:48.999 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:39:48.999 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:39:48.999 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:39:48.999 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-18T08:39:48.999 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:39:48.999 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:39:48.999 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:39:48.999 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:39:48.999 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-18T08:39:48.999 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:39:48.999 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:39:49.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:39:49.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:39:49.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:0 2026-02-18T08:39:49.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-18T08:39:49.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-18T08:39:49.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:39:49.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:0 2026-02-18T08:39:49.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T08:39:49.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-18T08:39:49.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:39:49.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:39:49.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-18T08:39:49.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-18T08:39:49.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-18T08:39:49.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=0 2026-02-18T08:39:49.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=0 2026-02-18T08:39:49.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:39:49.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:39:49.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:39:49.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:39:49.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_26 2026-02-18T08:39:49.068 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ return 0 2026-02-18T08:39:49.068 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ status_result=' "state": "Replaying"' 2026-02-18T08:39:49.069 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ echo ' "state": "Replaying"' 2026-02-18T08:39:49.069 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ grep -i Replaying 2026-02-18T08:39:49.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: "state": "Replaying" 2026-02-18T08:39:49.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ current_state=started 2026-02-18T08:39:49.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test started = started 2026-02-18T08:39:49.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:39:49.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror image_26 2026-02-18T08:39:49.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_cluster=cluster1 2026-02-18T08:39:49.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:39:49.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_pool=mirror 2026-02-18T08:39:49.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local remote_pool=mirror 2026-02-18T08:39:49.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_26 2026-02-18T08:39:49.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' journal = journal ']' 2026-02-18T08:39:49.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror mirror image_26 2026-02-18T08:39:49.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_cluster=cluster1 2026-02-18T08:39:49.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:39:49.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_pool=mirror 2026-02-18T08:39:49.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local remote_pool=mirror 2026-02-18T08:39:49.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_26 2026-02-18T08:39:49.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-02-18T08:39:49.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-02-18T08:39:49.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:39:49.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:39:49.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:39:49.273 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_26 2026-02-18T08:39:49.273 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:39:49.273 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:39:49.273 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_26 2026-02-18T08:39:49.273 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:39:49.273 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_26 ']' 2026-02-18T08:39:49.273 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_26' 2026-02-18T08:39:49.273 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_26 2026-02-18T08:39:49.273 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:39:49.274 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:39:49.274 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:39:49.274 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:39:49.274 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:39:49.274 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:39:49.274 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:39:49.274 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:39:49.277 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:39:49.277 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:39:49.277 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_26 2026-02-18T08:39:49.277 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:39:49.277 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:39:49.278 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:39:49.278 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:39:49.278 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:39:49.278 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:39:49.278 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:39:49.278 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:39:49.278 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:39:49.278 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:39:49.278 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:39:49.278 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:39:49.278 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:39:49.278 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:39:49.279 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:39:49.279 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:39:49.279 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:39:49.279 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:39:49.279 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:39:49.279 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:39:49.279 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:39:49.279 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:39:49.279 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:39:49.279 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:39:49.279 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:39:49.280 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:39:49.280 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:39:49.280 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:39:49.280 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:39:49.280 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:39:49.288 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:39:49.288 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:39:49.288 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:39:49.289 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_26 2026-02-18T08:39:49.353 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:39:49.354 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_26 2026-02-18T08:39:49.354 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:39:49.354 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:39:49.354 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_26 2026-02-18T08:39:49.355 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_26 '' 2026-02-18T08:39:49.355 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:39:49.355 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:39:49.355 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_26 2026-02-18T08:39:49.355 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:39:49.355 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_26.status 2026-02-18T08:39:49.355 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_26.status 2026-02-18T08:39:49.355 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:39:49.356 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_26.status 2026-02-18T08:39:49.356 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_26 2026-02-18T08:39:49.356 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_26.status 2026-02-18T08:39:49.393 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 5 2026-02-18T08:39:49.393 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 5 2026-02-18T08:39:49.393 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:39:49.393 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=21, tag_tid=2, entry_tid=1], [object_number=20, tag_tid=2, entry_tid=0], [object_number=23, tag_tid=1, entry_tid=24815], [object_number=22, tag_tid=1, entry_tid=24814]]], state=connected] 2026-02-18T08:39:49.393 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=21, tag_tid=1, entry_tid=24817], [object_number=20, tag_tid=1, entry_tid=24816], [object_number=23, tag_tid=1, entry_tid=24815], [object_number=22, tag_tid=1, entry_tid=24814]]], state=connected] 2026-02-18T08:39:49.397 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_26.status 2026-02-18T08:39:49.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=21, tag_tid=2, entry_tid=1' 2026-02-18T08:39:49.399 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_26 2026-02-18T08:39:49.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:39:49.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:39:49.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_26 2026-02-18T08:39:49.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_26 '..*' 2026-02-18T08:39:49.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:39:49.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:39:49.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_26 2026-02-18T08:39:49.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:39:49.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_26.status 2026-02-18T08:39:49.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_26.status 2026-02-18T08:39:49.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:39:49.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_26.status 2026-02-18T08:39:49.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_26 2026-02-18T08:39:49.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_26.status 2026-02-18T08:39:49.428 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 5 2026-02-18T08:39:49.429 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 5 2026-02-18T08:39:49.429 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:39:49.429 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=21, tag_tid=2, entry_tid=1], [object_number=20, tag_tid=2, entry_tid=0], [object_number=23, tag_tid=1, entry_tid=24815], [object_number=22, tag_tid=1, entry_tid=24814]]], state=connected] 2026-02-18T08:39:49.429 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=21, tag_tid=1, entry_tid=24817], [object_number=20, tag_tid=1, entry_tid=24816], [object_number=23, tag_tid=1, entry_tid=24815], [object_number=22, tag_tid=1, entry_tid=24814]]], state=connected] 2026-02-18T08:39:49.431 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_26.status 2026-02-18T08:39:49.433 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=21, tag_tid=1, entry_tid=24817' 2026-02-18T08:39:49.433 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=21, tag_tid=2, entry_tid=1' -a 'object_number=21, tag_tid=2, entry_tid=1' = 'object_number=21, tag_tid=1, entry_tid=24817' 2026-02-18T08:39:49.433 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=21, tag_tid=1, entry_tid=24817' '!=' '' 2026-02-18T08:39:49.433 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:39:49.433 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=21, tag_tid=1, entry_tid=24817' = '' 2026-02-18T08:39:49.433 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=21, tag_tid=1, entry_tid=24817' 2026-02-18T08:39:49.434 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=21, tag_tid=2, entry_tid=1' 2026-02-18T08:39:49.434 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:39:49.434 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:39:49.435 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=2 2026-02-18T08:39:49.436 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=21, tag_tid=1, entry_tid=24817' 2026-02-18T08:39:49.436 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:39:49.436 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:39:49.437 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=1 2026-02-18T08:39:49.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=21, tag_tid=2, entry_tid=1' 2026-02-18T08:39:49.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:39:49.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:39:49.440 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:39:49.440 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=21, tag_tid=1, entry_tid=24817' 2026-02-18T08:39:49.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:39:49.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:39:49.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=24817 2026-02-18T08:39:49.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 2 = 1 -a 1 -le 24817 2026-02-18T08:39:49.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:39:49.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:39:49.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:39:49.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_26 2026-02-18T08:39:49.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:39:49.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:39:49.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_26 2026-02-18T08:39:49.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:39:49.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_26 ']' 2026-02-18T08:39:49.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_26' 2026-02-18T08:39:49.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_26 2026-02-18T08:39:49.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:39:49.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:39:49.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:39:49.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:39:49.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:39:49.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:39:49.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:39:49.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:39:49.648 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:39:49.648 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:39:49.648 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_26 2026-02-18T08:39:49.648 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:39:49.648 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:39:49.648 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:39:49.648 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:39:49.648 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:39:49.648 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:39:49.649 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:39:49.649 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:39:49.649 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:39:49.649 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:39:49.649 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:39:49.649 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:39:49.649 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:39:49.649 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:39:49.649 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:39:49.649 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:39:49.649 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:39:49.649 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:39:49.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:39:49.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:39:49.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:39:49.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:39:49.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:39:49.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:39:49.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:39:49.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:39:49.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:39:49.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:39:49.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:39:49.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:39:49.658 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:39:49.658 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:39:49.658 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:39:49.659 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_26 2026-02-18T08:39:49.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:39:49.724 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_26 2026-02-18T08:39:49.724 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:39:49.724 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:39:49.724 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_26 2026-02-18T08:39:49.724 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_26 '' 2026-02-18T08:39:49.724 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:39:49.725 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:39:49.725 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_26 2026-02-18T08:39:49.725 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:39:49.725 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_26.status 2026-02-18T08:39:49.725 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_26.status 2026-02-18T08:39:49.725 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:39:49.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_26.status 2026-02-18T08:39:49.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_26 2026-02-18T08:39:49.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_26.status 2026-02-18T08:39:49.753 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 5 2026-02-18T08:39:49.754 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 5 2026-02-18T08:39:49.754 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:39:49.754 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=21, tag_tid=2, entry_tid=1], [object_number=20, tag_tid=2, entry_tid=0], [object_number=23, tag_tid=1, entry_tid=24815], [object_number=22, tag_tid=1, entry_tid=24814]]], state=connected] 2026-02-18T08:39:49.754 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=21, tag_tid=1, entry_tid=24817], [object_number=20, tag_tid=1, entry_tid=24816], [object_number=23, tag_tid=1, entry_tid=24815], [object_number=22, tag_tid=1, entry_tid=24814]]], state=connected] 2026-02-18T08:39:49.756 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_26.status 2026-02-18T08:39:49.758 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=21, tag_tid=2, entry_tid=1' 2026-02-18T08:39:49.759 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_26 2026-02-18T08:39:49.759 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:39:49.759 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:39:49.760 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_26 2026-02-18T08:39:49.760 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_26 '..*' 2026-02-18T08:39:49.760 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:39:49.760 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:39:49.760 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_26 2026-02-18T08:39:49.760 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:39:49.760 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_26.status 2026-02-18T08:39:49.760 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_26.status 2026-02-18T08:39:49.760 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:39:49.761 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_26.status 2026-02-18T08:39:49.761 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_26 2026-02-18T08:39:49.761 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_26.status 2026-02-18T08:39:49.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 5 2026-02-18T08:39:49.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 5 2026-02-18T08:39:49.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:39:49.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=21, tag_tid=2, entry_tid=1], [object_number=20, tag_tid=2, entry_tid=0], [object_number=23, tag_tid=1, entry_tid=24815], [object_number=22, tag_tid=1, entry_tid=24814]]], state=connected] 2026-02-18T08:39:49.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=21, tag_tid=1, entry_tid=24817], [object_number=20, tag_tid=1, entry_tid=24816], [object_number=23, tag_tid=1, entry_tid=24815], [object_number=22, tag_tid=1, entry_tid=24814]]], state=connected] 2026-02-18T08:39:50.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_26.status 2026-02-18T08:39:50.003 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=21, tag_tid=1, entry_tid=24817' 2026-02-18T08:39:50.003 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=21, tag_tid=2, entry_tid=1' -a 'object_number=21, tag_tid=2, entry_tid=1' = 'object_number=21, tag_tid=1, entry_tid=24817' 2026-02-18T08:39:50.003 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=21, tag_tid=1, entry_tid=24817' '!=' 'object_number=21, tag_tid=1, entry_tid=24817' 2026-02-18T08:39:50.003 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:39:50.003 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.4 2026-02-18T08:39:50.405 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_26 2026-02-18T08:39:50.405 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:39:50.406 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:39:50.406 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_26 2026-02-18T08:39:50.406 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:39:50.406 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_26 ']' 2026-02-18T08:39:50.406 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_26' 2026-02-18T08:39:50.406 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_26 2026-02-18T08:39:50.406 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:39:50.406 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:39:50.406 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:39:50.406 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:39:50.406 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:39:50.407 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:39:50.407 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:39:50.407 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:39:50.409 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:39:50.409 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:39:50.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_26 2026-02-18T08:39:50.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:39:50.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:39:50.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:39:50.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:39:50.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:39:50.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:39:50.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:39:50.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:39:50.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:39:50.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:39:50.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:39:50.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:39:50.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:39:50.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:39:50.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:39:50.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:39:50.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:39:50.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:39:50.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:39:50.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:39:50.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:39:50.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:39:50.412 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:39:50.412 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:39:50.412 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:39:50.412 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:39:50.412 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:39:50.412 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:39:50.412 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:39:50.412 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:39:50.425 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:39:50.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:39:50.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:39:50.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_26 2026-02-18T08:39:50.486 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:39:50.488 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_26 2026-02-18T08:39:50.488 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:39:50.488 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:39:50.488 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_26 2026-02-18T08:39:50.488 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_26 '' 2026-02-18T08:39:50.488 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:39:50.488 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:39:50.488 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_26 2026-02-18T08:39:50.488 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:39:50.488 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_26.status 2026-02-18T08:39:50.489 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_26.status 2026-02-18T08:39:50.489 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:39:50.489 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_26.status 2026-02-18T08:39:50.490 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_26 2026-02-18T08:39:50.490 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_26.status 2026-02-18T08:39:50.516 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 5 2026-02-18T08:39:50.516 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 5 2026-02-18T08:39:50.516 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:39:50.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=21, tag_tid=2, entry_tid=1], [object_number=20, tag_tid=2, entry_tid=0], [object_number=23, tag_tid=1, entry_tid=24815], [object_number=22, tag_tid=1, entry_tid=24814]]], state=connected] 2026-02-18T08:39:50.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=21, tag_tid=1, entry_tid=24817], [object_number=20, tag_tid=1, entry_tid=24816], [object_number=23, tag_tid=1, entry_tid=24815], [object_number=22, tag_tid=1, entry_tid=24814]]], state=connected] 2026-02-18T08:39:50.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_26.status 2026-02-18T08:39:50.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=21, tag_tid=2, entry_tid=1' 2026-02-18T08:39:50.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_26 2026-02-18T08:39:50.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:39:50.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:39:50.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_26 2026-02-18T08:39:50.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_26 '..*' 2026-02-18T08:39:50.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:39:50.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:39:50.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_26 2026-02-18T08:39:50.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:39:50.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_26.status 2026-02-18T08:39:50.523 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_26.status 2026-02-18T08:39:50.523 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:39:50.524 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_26.status 2026-02-18T08:39:50.524 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_26 2026-02-18T08:39:50.524 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_26.status 2026-02-18T08:39:50.560 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 5 2026-02-18T08:39:50.560 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 5 2026-02-18T08:39:50.560 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:39:50.560 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=21, tag_tid=2, entry_tid=1], [object_number=20, tag_tid=2, entry_tid=0], [object_number=23, tag_tid=1, entry_tid=24815], [object_number=22, tag_tid=1, entry_tid=24814]]], state=connected] 2026-02-18T08:39:50.560 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=21, tag_tid=1, entry_tid=24817], [object_number=20, tag_tid=1, entry_tid=24816], [object_number=23, tag_tid=1, entry_tid=24815], [object_number=22, tag_tid=1, entry_tid=24814]]], state=connected] 2026-02-18T08:39:50.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_26.status 2026-02-18T08:39:50.565 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=21, tag_tid=1, entry_tid=24817' 2026-02-18T08:39:50.565 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=21, tag_tid=2, entry_tid=1' -a 'object_number=21, tag_tid=2, entry_tid=1' = 'object_number=21, tag_tid=1, entry_tid=24817' 2026-02-18T08:39:50.565 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=21, tag_tid=1, entry_tid=24817' '!=' 'object_number=21, tag_tid=1, entry_tid=24817' 2026-02-18T08:39:50.565 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:39:50.565 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.8 2026-02-18T08:39:51.367 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_26 2026-02-18T08:39:51.367 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:39:51.367 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:39:51.367 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_26 2026-02-18T08:39:51.367 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:39:51.368 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_26 ']' 2026-02-18T08:39:51.368 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_26' 2026-02-18T08:39:51.368 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_26 2026-02-18T08:39:51.368 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:39:51.368 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:39:51.368 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:39:51.368 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:39:51.368 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:39:51.368 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:39:51.368 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:39:51.371 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:39:51.371 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:39:51.371 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:39:51.371 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_26 2026-02-18T08:39:51.371 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:39:51.371 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:39:51.372 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:39:51.372 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:39:51.372 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:39:51.372 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:39:51.372 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:39:51.372 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:39:51.372 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:39:51.372 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:39:51.372 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:39:51.372 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:39:51.372 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:39:51.372 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:39:51.373 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:39:51.373 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:39:51.373 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:39:51.373 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:39:51.373 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:39:51.373 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:39:51.373 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:39:51.373 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:39:51.373 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:39:51.373 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:39:51.373 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:39:51.373 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:39:51.374 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:39:51.374 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:39:51.374 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:39:51.374 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:39:51.382 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:39:51.382 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:39:51.382 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:39:51.383 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_26 2026-02-18T08:39:51.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:39:51.449 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_26 2026-02-18T08:39:51.449 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:39:51.449 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:39:51.449 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_26 2026-02-18T08:39:51.449 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_26 '' 2026-02-18T08:39:51.449 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:39:51.449 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:39:51.449 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_26 2026-02-18T08:39:51.449 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:39:51.450 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_26.status 2026-02-18T08:39:51.450 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_26.status 2026-02-18T08:39:51.450 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:39:51.450 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_26.status 2026-02-18T08:39:51.450 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_26 2026-02-18T08:39:51.451 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_26.status 2026-02-18T08:39:51.487 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 5 2026-02-18T08:39:51.487 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 5 2026-02-18T08:39:51.487 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:39:51.487 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=21, tag_tid=2, entry_tid=1], [object_number=20, tag_tid=2, entry_tid=0], [object_number=23, tag_tid=1, entry_tid=24815], [object_number=22, tag_tid=1, entry_tid=24814]]], state=connected] 2026-02-18T08:39:51.487 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=21, tag_tid=1, entry_tid=24817], [object_number=20, tag_tid=1, entry_tid=24816], [object_number=23, tag_tid=1, entry_tid=24815], [object_number=22, tag_tid=1, entry_tid=24814]]], state=connected] 2026-02-18T08:39:51.491 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_26.status 2026-02-18T08:39:51.492 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=21, tag_tid=2, entry_tid=1' 2026-02-18T08:39:51.493 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_26 2026-02-18T08:39:51.493 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:39:51.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:39:51.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_26 2026-02-18T08:39:51.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_26 '..*' 2026-02-18T08:39:51.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:39:51.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:39:51.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_26 2026-02-18T08:39:51.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:39:51.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_26.status 2026-02-18T08:39:51.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_26.status 2026-02-18T08:39:51.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:39:51.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_26.status 2026-02-18T08:39:51.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_26 2026-02-18T08:39:51.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_26.status 2026-02-18T08:39:51.531 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 5 2026-02-18T08:39:51.531 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 5 2026-02-18T08:39:51.531 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:39:51.531 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=21, tag_tid=2, entry_tid=1], [object_number=20, tag_tid=2, entry_tid=0], [object_number=23, tag_tid=1, entry_tid=24815], [object_number=22, tag_tid=1, entry_tid=24814]]], state=connected] 2026-02-18T08:39:51.531 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=21, tag_tid=1, entry_tid=24817], [object_number=20, tag_tid=1, entry_tid=24816], [object_number=23, tag_tid=1, entry_tid=24815], [object_number=22, tag_tid=1, entry_tid=24814]]], state=connected] 2026-02-18T08:39:51.534 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_26.status 2026-02-18T08:39:51.536 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=21, tag_tid=1, entry_tid=24817' 2026-02-18T08:39:51.536 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=21, tag_tid=2, entry_tid=1' -a 'object_number=21, tag_tid=2, entry_tid=1' = 'object_number=21, tag_tid=1, entry_tid=24817' 2026-02-18T08:39:51.536 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=21, tag_tid=1, entry_tid=24817' '!=' 'object_number=21, tag_tid=1, entry_tid=24817' 2026-02-18T08:39:51.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:39:51.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1.6 2026-02-18T08:39:53.138 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_26 2026-02-18T08:39:53.139 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:39:53.139 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:39:53.139 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_26 2026-02-18T08:39:53.139 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:39:53.139 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_26 ']' 2026-02-18T08:39:53.139 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_26' 2026-02-18T08:39:53.139 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_26 2026-02-18T08:39:53.139 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:39:53.139 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:39:53.140 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:39:53.140 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:39:53.140 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:39:53.140 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:39:53.140 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:39:53.140 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:39:53.142 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:39:53.143 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:39:53.143 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_26 2026-02-18T08:39:53.143 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:39:53.143 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:39:53.143 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:39:53.143 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:39:53.143 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:39:53.143 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:39:53.143 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:39:53.143 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:39:53.143 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:39:53.144 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:39:53.144 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:39:53.144 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:39:53.144 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:39:53.144 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:39:53.144 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:39:53.144 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:39:53.144 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:39:53.144 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:39:53.144 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:39:53.144 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:39:53.144 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:39:53.145 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:39:53.145 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:39:53.145 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:39:53.145 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:39:53.145 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:39:53.145 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:39:53.145 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:39:53.145 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:39:53.145 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:39:53.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:39:53.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:39:53.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:39:53.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_26 2026-02-18T08:39:53.210 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:39:53.210 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_26 2026-02-18T08:39:53.210 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:39:53.210 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:39:53.211 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_26 2026-02-18T08:39:53.211 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_26 '' 2026-02-18T08:39:53.211 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:39:53.211 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:39:53.211 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_26 2026-02-18T08:39:53.211 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:39:53.211 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_26.status 2026-02-18T08:39:53.212 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_26.status 2026-02-18T08:39:53.212 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:39:53.212 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_26.status 2026-02-18T08:39:53.213 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_26 2026-02-18T08:39:53.213 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_26.status 2026-02-18T08:39:53.249 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 5 2026-02-18T08:39:53.249 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 5 2026-02-18T08:39:53.249 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:39:53.250 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=21, tag_tid=2, entry_tid=1], [object_number=20, tag_tid=2, entry_tid=0], [object_number=23, tag_tid=1, entry_tid=24815], [object_number=22, tag_tid=1, entry_tid=24814]]], state=connected] 2026-02-18T08:39:53.250 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=21, tag_tid=1, entry_tid=24817], [object_number=20, tag_tid=1, entry_tid=24816], [object_number=23, tag_tid=1, entry_tid=24815], [object_number=22, tag_tid=1, entry_tid=24814]]], state=connected] 2026-02-18T08:39:53.253 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_26.status 2026-02-18T08:39:53.255 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=21, tag_tid=2, entry_tid=1' 2026-02-18T08:39:53.256 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_26 2026-02-18T08:39:53.256 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:39:53.256 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:39:53.256 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_26 2026-02-18T08:39:53.256 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_26 '..*' 2026-02-18T08:39:53.256 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:39:53.256 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:39:53.257 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_26 2026-02-18T08:39:53.257 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:39:53.257 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_26.status 2026-02-18T08:39:53.257 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_26.status 2026-02-18T08:39:53.257 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:39:53.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_26.status 2026-02-18T08:39:53.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_26 2026-02-18T08:39:53.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_26.status 2026-02-18T08:39:53.283 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 5 2026-02-18T08:39:53.284 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 5 2026-02-18T08:39:53.284 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:39:53.284 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=21, tag_tid=2, entry_tid=1], [object_number=20, tag_tid=2, entry_tid=0], [object_number=23, tag_tid=1, entry_tid=24815], [object_number=22, tag_tid=1, entry_tid=24814]]], state=connected] 2026-02-18T08:39:53.284 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=21, tag_tid=1, entry_tid=24817], [object_number=20, tag_tid=1, entry_tid=24816], [object_number=23, tag_tid=1, entry_tid=24815], [object_number=22, tag_tid=1, entry_tid=24814]]], state=connected] 2026-02-18T08:39:53.286 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_26.status 2026-02-18T08:39:53.288 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=21, tag_tid=1, entry_tid=24817' 2026-02-18T08:39:53.288 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=21, tag_tid=2, entry_tid=1' -a 'object_number=21, tag_tid=2, entry_tid=1' = 'object_number=21, tag_tid=1, entry_tid=24817' 2026-02-18T08:39:53.288 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=21, tag_tid=1, entry_tid=24817' '!=' 'object_number=21, tag_tid=1, entry_tid=24817' 2026-02-18T08:39:53.288 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:39:53.288 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 2 2026-02-18T08:39:55.290 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_26 2026-02-18T08:39:55.290 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:39:55.290 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:39:55.290 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_26 2026-02-18T08:39:55.290 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:39:55.290 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_26 ']' 2026-02-18T08:39:55.291 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_26' 2026-02-18T08:39:55.291 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_26 2026-02-18T08:39:55.291 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:39:55.291 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:39:55.291 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:39:55.291 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:39:55.291 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:39:55.291 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:39:55.291 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:39:55.294 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:39:55.294 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:39:55.294 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:39:55.294 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_26 2026-02-18T08:39:55.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:39:55.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:39:55.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:39:55.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:39:55.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:39:55.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:39:55.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:39:55.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:39:55.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:39:55.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:39:55.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:39:55.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:39:55.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:39:55.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:39:55.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:39:55.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:39:55.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:39:55.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:39:55.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:39:55.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:39:55.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:39:55.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:39:55.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:39:55.297 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:39:55.297 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:39:55.297 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:39:55.297 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:39:55.297 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:39:55.297 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:39:55.297 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:39:55.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:39:55.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:39:55.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:39:55.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_26 2026-02-18T08:39:55.363 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:39:55.364 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_26 2026-02-18T08:39:55.364 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:39:55.364 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:39:55.364 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_26 2026-02-18T08:39:55.364 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_26 '' 2026-02-18T08:39:55.364 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:39:55.365 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:39:55.365 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_26 2026-02-18T08:39:55.365 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:39:55.365 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_26.status 2026-02-18T08:39:55.365 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_26.status 2026-02-18T08:39:55.365 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:39:55.365 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_26.status 2026-02-18T08:39:55.366 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_26 2026-02-18T08:39:55.366 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_26.status 2026-02-18T08:39:55.393 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 5 2026-02-18T08:39:55.393 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 5 2026-02-18T08:39:55.393 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:39:55.393 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=21, tag_tid=2, entry_tid=1], [object_number=20, tag_tid=2, entry_tid=0], [object_number=23, tag_tid=1, entry_tid=24815], [object_number=22, tag_tid=1, entry_tid=24814]]], state=connected] 2026-02-18T08:39:55.393 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=21, tag_tid=1, entry_tid=24817], [object_number=20, tag_tid=1, entry_tid=24816], [object_number=23, tag_tid=1, entry_tid=24815], [object_number=22, tag_tid=1, entry_tid=24814]]], state=connected] 2026-02-18T08:39:55.395 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_26.status 2026-02-18T08:39:55.397 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=21, tag_tid=2, entry_tid=1' 2026-02-18T08:39:55.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_26 2026-02-18T08:39:55.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:39:55.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:39:55.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_26 2026-02-18T08:39:55.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_26 '..*' 2026-02-18T08:39:55.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:39:55.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:39:55.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_26 2026-02-18T08:39:55.399 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:39:55.399 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_26.status 2026-02-18T08:39:55.399 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_26.status 2026-02-18T08:39:55.399 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:39:55.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_26.status 2026-02-18T08:39:55.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_26 2026-02-18T08:39:55.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_26.status 2026-02-18T08:39:55.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 5 2026-02-18T08:39:55.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 5 2026-02-18T08:39:55.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:39:55.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=21, tag_tid=2, entry_tid=1], [object_number=20, tag_tid=2, entry_tid=0], [object_number=23, tag_tid=1, entry_tid=24815], [object_number=22, tag_tid=1, entry_tid=24814]]], state=connected] 2026-02-18T08:39:55.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=21, tag_tid=1, entry_tid=24817], [object_number=20, tag_tid=1, entry_tid=24816], [object_number=23, tag_tid=1, entry_tid=24815], [object_number=22, tag_tid=1, entry_tid=24814]]], state=connected] 2026-02-18T08:39:55.636 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_26.status 2026-02-18T08:39:55.639 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=21, tag_tid=1, entry_tid=24817' 2026-02-18T08:39:55.639 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=21, tag_tid=2, entry_tid=1' -a 'object_number=21, tag_tid=2, entry_tid=1' = 'object_number=21, tag_tid=1, entry_tid=24817' 2026-02-18T08:39:55.639 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=21, tag_tid=1, entry_tid=24817' '!=' 'object_number=21, tag_tid=1, entry_tid=24817' 2026-02-18T08:39:55.639 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:39:55.639 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 2 2026-02-18T08:39:57.641 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_26 2026-02-18T08:39:57.641 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:39:57.641 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:39:57.641 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_26 2026-02-18T08:39:57.641 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:39:57.642 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_26 ']' 2026-02-18T08:39:57.642 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_26' 2026-02-18T08:39:57.642 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_26 2026-02-18T08:39:57.642 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:39:57.642 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:39:57.642 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:39:57.642 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:39:57.642 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:39:57.642 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:39:57.642 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:39:57.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:39:57.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:39:57.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:39:57.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_26 2026-02-18T08:39:57.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:39:57.646 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:39:57.646 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:39:57.646 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:39:57.646 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:39:57.646 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:39:57.646 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:39:57.646 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:39:57.646 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:39:57.646 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:39:57.646 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:39:57.646 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:39:57.646 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:39:57.647 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:39:57.647 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:39:57.647 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:39:57.647 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:39:57.647 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:39:57.647 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:39:57.647 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:39:57.647 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:39:57.647 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:39:57.647 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:39:57.647 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:39:57.647 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:39:57.648 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:39:57.648 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:39:57.648 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:39:57.648 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:39:57.648 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:39:57.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:39:57.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:39:57.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:39:57.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_26 2026-02-18T08:39:57.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:39:57.717 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_26 2026-02-18T08:39:57.717 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:39:57.717 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:39:57.717 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_26 2026-02-18T08:39:57.717 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_26 '' 2026-02-18T08:39:57.718 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:39:57.718 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:39:57.718 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_26 2026-02-18T08:39:57.718 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:39:57.718 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_26.status 2026-02-18T08:39:57.718 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_26.status 2026-02-18T08:39:57.718 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:39:57.719 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_26.status 2026-02-18T08:39:57.719 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_26 2026-02-18T08:39:57.719 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_26.status 2026-02-18T08:39:57.756 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 5 2026-02-18T08:39:57.756 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 5 2026-02-18T08:39:57.756 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:39:57.756 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=21, tag_tid=2, entry_tid=1], [object_number=20, tag_tid=2, entry_tid=0], [object_number=23, tag_tid=1, entry_tid=24815], [object_number=22, tag_tid=1, entry_tid=24814]]], state=connected] 2026-02-18T08:39:57.756 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=21, tag_tid=2, entry_tid=1], [object_number=20, tag_tid=2, entry_tid=0], [object_number=23, tag_tid=1, entry_tid=24815], [object_number=22, tag_tid=1, entry_tid=24814]]], state=connected] 2026-02-18T08:39:57.759 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_26.status 2026-02-18T08:39:57.761 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=21, tag_tid=2, entry_tid=1' 2026-02-18T08:39:57.762 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_26 2026-02-18T08:39:57.762 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:39:57.762 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:39:57.763 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_26 2026-02-18T08:39:57.763 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_26 '..*' 2026-02-18T08:39:57.763 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:39:57.763 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:39:57.763 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_26 2026-02-18T08:39:57.763 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:39:57.763 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_26.status 2026-02-18T08:39:57.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_26.status 2026-02-18T08:39:57.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:39:57.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_26.status 2026-02-18T08:39:57.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_26 2026-02-18T08:39:57.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_26.status 2026-02-18T08:39:57.801 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 5 2026-02-18T08:39:57.801 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 5 2026-02-18T08:39:57.801 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:39:57.801 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=21, tag_tid=2, entry_tid=1], [object_number=20, tag_tid=2, entry_tid=0], [object_number=23, tag_tid=1, entry_tid=24815], [object_number=22, tag_tid=1, entry_tid=24814]]], state=connected] 2026-02-18T08:39:57.801 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=21, tag_tid=2, entry_tid=1], [object_number=20, tag_tid=2, entry_tid=0], [object_number=23, tag_tid=1, entry_tid=24815], [object_number=22, tag_tid=1, entry_tid=24814]]], state=connected] 2026-02-18T08:39:57.804 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_26.status 2026-02-18T08:39:57.805 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=21, tag_tid=2, entry_tid=1' 2026-02-18T08:39:57.805 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=21, tag_tid=2, entry_tid=1' -a 'object_number=21, tag_tid=2, entry_tid=1' = 'object_number=21, tag_tid=2, entry_tid=1' 2026-02-18T08:39:57.805 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:39:57.805 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_snap_present cluster1 mirror image_26 snap 2026-02-18T08:39:57.806 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:39:57.806 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:39:57.806 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_26 2026-02-18T08:39:57.806 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap 2026-02-18T08:39:57.806 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-18T08:39:57.806 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-18T08:39:57.806 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1 2026-02-18T08:39:58.806 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster1 info mirror/image_26@snap 2026-02-18T08:39:58.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:rbd image 'image_26': 2026-02-18T08:39:58.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: size 128 MiB in 32 objects 2026-02-18T08:39:58.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: order 22 (4 MiB objects) 2026-02-18T08:39:58.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: snapshot_count: 1 2026-02-18T08:39:58.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: id: 120db61d5d0f 2026-02-18T08:39:58.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: block_name_prefix: rbd_data.120db61d5d0f 2026-02-18T08:39:58.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: format: 2 2026-02-18T08:39:58.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: features: layering, exclusive-lock, journaling 2026-02-18T08:39:58.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: op_features: 2026-02-18T08:39:58.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: flags: 2026-02-18T08:39:58.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: create_timestamp: Wed Feb 18 08:30:14 2026 2026-02-18T08:39:58.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: access_timestamp: Wed Feb 18 08:30:14 2026 2026-02-18T08:39:58.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: modify_timestamp: Wed Feb 18 08:31:15 2026 2026-02-18T08:39:58.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: protected: False 2026-02-18T08:39:58.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: journal: 120db61d5d0f 2026-02-18T08:39:58.835 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring state: enabled 2026-02-18T08:39:58.835 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring mode: journal 2026-02-18T08:39:58.835 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring global id: 90c48a58-c075-4efa-a6a6-c10971a7608e 2026-02-18T08:39:58.835 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring primary: false 2026-02-18T08:39:58.836 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:39:58.836 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ compare_image_snaps mirror image_26 snap 2026-02-18T08:39:58.836 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:39:58.837 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_26 2026-02-18T08:39:58.837 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap 2026-02-18T08:39:58.837 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local ret=0 2026-02-18T08:39:58.837 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local rmt_export=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_26.export 2026-02-18T08:39:58.837 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loc_export=/tmp/tmp.vybh1m9yJF/cluster1-mirror-image_26.export 2026-02-18T08:39:58.837 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_26.export /tmp/tmp.vybh1m9yJF/cluster1-mirror-image_26.export 2026-02-18T08:39:58.837 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 -p mirror export image_26@snap /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_26.export 2026-02-18T08:39:58.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-18T08:39:58.985 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster1 -p mirror export image_26@snap /tmp/tmp.vybh1m9yJF/cluster1-mirror-image_26.export 2026-02-18T08:39:59.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-18T08:39:59.201 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmp /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_26.export /tmp/tmp.vybh1m9yJF/cluster1-mirror-image_26.export 2026-02-18T08:39:59.255 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_26.export /tmp/tmp.vybh1m9yJF/cluster1-mirror-image_26.export 2026-02-18T08:39:59.301 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:39:59.301 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T08:39:59.301 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_27 2026-02-18T08:39:59.302 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_snap cluster2 mirror image_27 snap 2026-02-18T08:39:59.302 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:39:59.302 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:39:59.302 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_27 2026-02-18T08:39:59.302 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap 2026-02-18T08:39:59.302 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 -p mirror snap create image_27@snap --debug-rbd=20 --debug-journaler=20 2026-02-18T08:40:03.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_replay_started cluster1 mirror image_27 2026-02-18T08:40:03.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:40:03.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:40:03.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_27 2026-02-18T08:40:03.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_replay_state cluster1 mirror image_27 started 2026-02-18T08:40:03.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:40:03.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:40:03.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_27 2026-02-18T08:40:03.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local state=started 2026-02-18T08:40:03.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-18T08:40:03.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:40:03.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1 2026-02-18T08:40:04.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test_image_replay_state cluster1 mirror image_27 started 2026-02-18T08:40:04.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:40:04.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:40:04.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_27 2026-02-18T08:40:04.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local test_state=started 2026-02-18T08:40:04.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local status_result 2026-02-18T08:40:04.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_state=stopped 2026-02-18T08:40:04.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_27 2026-02-18T08:40:04.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -i state 2026-02-18T08:40:04.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_instance=cluster1 2026-02-18T08:40:04.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-18T08:40:04.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:40:04.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance=cluster1 2026-02-18T08:40:04.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local loop_instance 2026-02-18T08:40:04.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:40:04.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sleep 0 2026-02-18T08:40:04.019 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:40:04.020 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ seq 0 3 2026-02-18T08:40:04.020 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:40:04.020 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_27 2026-02-18T08:40:04.020 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:0 2026-02-18T08:40:04.020 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-18T08:40:04.020 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:40:04.020 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:40:04.020 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:40:04.020 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-18T08:40:04.020 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:40:04.020 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:40:04.021 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:40:04.021 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:40:04.021 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-18T08:40:04.021 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:40:04.021 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:40:04.021 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:40:04.021 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:40:04.021 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:0 2026-02-18T08:40:04.021 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-18T08:40:04.021 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-18T08:40:04.021 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:40:04.021 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:0 2026-02-18T08:40:04.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T08:40:04.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-18T08:40:04.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:40:04.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:40:04.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-18T08:40:04.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-18T08:40:04.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-18T08:40:04.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=0 2026-02-18T08:40:04.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=0 2026-02-18T08:40:04.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:40:04.029 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:40:04.029 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:40:04.029 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:40:04.029 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_27 2026-02-18T08:40:04.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_27'] 2026-02-18T08:40:04.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status : get status for rbd mirror 2026-02-18T08:40:04.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:40:04.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:40:04.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_27 2026-02-18T08:40:04.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:1 2026-02-18T08:40:04.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-18T08:40:04.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:40:04.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:40:04.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:40:04.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-18T08:40:04.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:40:04.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:40:04.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:40:04.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:40:04.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-18T08:40:04.086 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:40:04.086 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:40:04.086 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:40:04.086 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:40:04.086 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:1 2026-02-18T08:40:04.086 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-18T08:40:04.086 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-18T08:40:04.086 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:40:04.086 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:1 2026-02-18T08:40:04.086 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T08:40:04.086 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-18T08:40:04.086 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:40:04.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:40:04.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-18T08:40:04.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-18T08:40:04.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-18T08:40:04.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=1 2026-02-18T08:40:04.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=1 2026-02-18T08:40:04.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:40:04.095 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:40:04.096 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:40:04.096 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:40:04.096 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_27 2026-02-18T08:40:04.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_27'] 2026-02-18T08:40:04.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status : get status for rbd mirror 2026-02-18T08:40:04.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:40:04.163 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:40:04.163 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_27 2026-02-18T08:40:04.163 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:2 2026-02-18T08:40:04.163 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-18T08:40:04.164 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:40:04.164 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:40:04.164 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:40:04.164 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-18T08:40:04.164 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:40:04.164 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:40:04.164 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:40:04.164 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:40:04.164 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-18T08:40:04.164 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:40:04.164 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:40:04.164 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:40:04.165 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:40:04.165 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:2 2026-02-18T08:40:04.165 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-18T08:40:04.165 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-18T08:40:04.165 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:40:04.165 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:2 2026-02-18T08:40:04.165 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T08:40:04.165 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-18T08:40:04.165 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:40:04.165 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:40:04.165 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-18T08:40:04.165 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-18T08:40:04.165 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-18T08:40:04.166 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=2 2026-02-18T08:40:04.166 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=2 2026-02-18T08:40:04.166 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:40:04.174 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:40:04.175 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:40:04.175 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:40:04.175 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_27 2026-02-18T08:40:04.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_27'] 2026-02-18T08:40:04.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status : get status for rbd mirror 2026-02-18T08:40:04.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:40:04.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:40:04.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/image_27 2026-02-18T08:40:04.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:3 2026-02-18T08:40:04.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-18T08:40:04.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:40:04.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:40:04.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:40:04.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-18T08:40:04.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:40:04.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:40:04.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:40:04.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:40:04.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-18T08:40:04.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:40:04.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:40:04.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:40:04.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:40:04.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:3 2026-02-18T08:40:04.244 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-18T08:40:04.244 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-18T08:40:04.244 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:40:04.244 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:3 2026-02-18T08:40:04.244 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T08:40:04.244 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-18T08:40:04.244 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:40:04.244 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:40:04.244 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-18T08:40:04.244 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-18T08:40:04.244 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-18T08:40:04.244 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=3 2026-02-18T08:40:04.244 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=3 2026-02-18T08:40:04.245 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:40:04.253 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:40:04.254 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:40:04.254 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:40:04.254 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/image_27 2026-02-18T08:40:04.312 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ return 0 2026-02-18T08:40:04.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ status_result=' "state": "Replaying"' 2026-02-18T08:40:04.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ echo ' "state": "Replaying"' 2026-02-18T08:40:04.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ grep -i Replaying 2026-02-18T08:40:04.314 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: "state": "Replaying" 2026-02-18T08:40:04.316 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ current_state=started 2026-02-18T08:40:04.316 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test started = started 2026-02-18T08:40:04.316 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:40:04.316 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror image_27 2026-02-18T08:40:04.316 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_cluster=cluster1 2026-02-18T08:40:04.316 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:40:04.316 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_pool=mirror 2026-02-18T08:40:04.317 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local remote_pool=mirror 2026-02-18T08:40:04.317 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_27 2026-02-18T08:40:04.317 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' journal = journal ']' 2026-02-18T08:40:04.317 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror mirror image_27 2026-02-18T08:40:04.317 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_cluster=cluster1 2026-02-18T08:40:04.317 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:40:04.317 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_pool=mirror 2026-02-18T08:40:04.317 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local remote_pool=mirror 2026-02-18T08:40:04.317 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_27 2026-02-18T08:40:04.317 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-02-18T08:40:04.317 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-02-18T08:40:04.317 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:40:04.318 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:40:04.318 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:40:04.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_27 2026-02-18T08:40:04.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:40:04.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:40:04.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_27 2026-02-18T08:40:04.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:40:04.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_27 ']' 2026-02-18T08:40:04.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_27' 2026-02-18T08:40:04.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_27 2026-02-18T08:40:04.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:40:04.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:40:04.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:40:04.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:40:04.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:40:04.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:40:04.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:40:04.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:40:04.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:40:04.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:40:04.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_27 2026-02-18T08:40:04.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:40:04.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:40:04.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:40:04.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:40:04.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:40:04.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:40:04.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:40:04.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:40:04.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:40:04.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:40:04.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:40:04.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:40:04.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:40:04.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:40:04.523 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:40:04.523 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:40:04.523 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:40:04.523 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:40:04.523 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:40:04.523 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:40:04.523 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:40:04.523 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:40:04.523 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:40:04.523 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:40:04.523 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:40:04.523 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:40:04.524 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:40:04.524 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:40:04.524 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:40:04.524 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:40:04.530 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:40:04.531 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:40:04.531 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:40:04.531 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_27 2026-02-18T08:40:04.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_27'] 2026-02-18T08:40:04.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:40:04.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:40:04.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:40:04.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_27 2026-02-18T08:40:04.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:40:04.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:40:04.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:40:04.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:40:04.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:40:04.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:40:04.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:40:04.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:40:04.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:40:04.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:40:04.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:40:04.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:40:04.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:40:04.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:40:04.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:40:04.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:40:04.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:40:04.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:40:04.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:40:04.594 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:40:04.594 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:40:04.594 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:40:04.594 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:40:04.594 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:40:04.594 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:40:04.594 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:40:04.594 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:40:04.594 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:40:04.594 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:40:04.594 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:40:04.602 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:40:04.602 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:40:04.602 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:40:04.603 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_27 2026-02-18T08:40:04.663 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_27'] 2026-02-18T08:40:04.663 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:40:04.663 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:40:04.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:40:04.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_27 2026-02-18T08:40:04.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:40:04.667 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:40:04.667 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:40:04.667 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:40:04.667 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:40:04.667 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:40:04.667 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:40:04.667 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:40:04.667 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:40:04.667 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:40:04.667 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:40:04.667 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:40:04.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:40:04.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:40:04.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:40:04.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:40:04.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:40:04.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:40:04.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:40:04.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:40:04.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:40:04.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:40:04.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:40:04.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:40:04.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:40:04.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:40:04.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:40:04.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:40:04.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:40:04.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:40:04.677 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:40:04.678 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:40:04.678 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:40:04.678 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_27 2026-02-18T08:40:04.729 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_27'] 2026-02-18T08:40:04.729 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:40:04.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:40:04.732 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:40:04.732 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/image_27 2026-02-18T08:40:04.732 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:40:04.732 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:40:04.732 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:40:04.732 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:40:04.733 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:40:04.733 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:40:04.733 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:40:04.733 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:40:04.733 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:40:04.733 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:40:04.733 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:40:04.733 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:40:04.733 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:40:04.733 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:40:04.733 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:40:04.733 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:40:04.734 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:40:04.734 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:40:04.734 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:40:04.734 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:40:04.734 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:40:04.734 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:40:04.734 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:40:04.734 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:40:04.734 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:40:04.734 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:40:04.734 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:40:04.734 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:40:04.735 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:40:04.735 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:40:04.742 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:40:04.743 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:40:04.743 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:40:04.743 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/image_27 2026-02-18T08:40:04.809 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:40:04.810 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_27 2026-02-18T08:40:04.810 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:40:04.810 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:40:04.810 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_27 2026-02-18T08:40:04.810 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_27 '' 2026-02-18T08:40:04.810 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:40:04.810 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:40:04.811 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_27 2026-02-18T08:40:04.811 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:40:04.811 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_27.status 2026-02-18T08:40:04.811 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_27.status 2026-02-18T08:40:04.811 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:40:04.811 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_27.status 2026-02-18T08:40:04.812 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_27 2026-02-18T08:40:04.812 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_27.status 2026-02-18T08:40:04.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 10 2026-02-18T08:40:04.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 10 2026-02-18T08:40:04.850 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:40:04.850 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=41, tag_tid=2, entry_tid=1], [object_number=40, tag_tid=2, entry_tid=0], [object_number=43, tag_tid=1, entry_tid=44535], [object_number=42, tag_tid=1, entry_tid=44534]]], state=connected] 2026-02-18T08:40:04.850 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=40, tag_tid=1, entry_tid=44536], [object_number=43, tag_tid=1, entry_tid=44535], [object_number=42, tag_tid=1, entry_tid=44534], [object_number=41, tag_tid=1, entry_tid=44533]]], state=connected] 2026-02-18T08:40:04.853 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_27.status 2026-02-18T08:40:04.854 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=41, tag_tid=2, entry_tid=1' 2026-02-18T08:40:04.855 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_27 2026-02-18T08:40:04.855 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:40:04.855 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:40:04.856 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_27 2026-02-18T08:40:04.856 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_27 '..*' 2026-02-18T08:40:04.856 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:40:04.856 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:40:04.856 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_27 2026-02-18T08:40:04.856 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:40:04.856 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_27.status 2026-02-18T08:40:04.857 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_27.status 2026-02-18T08:40:04.857 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:40:04.857 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_27.status 2026-02-18T08:40:04.858 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_27 2026-02-18T08:40:04.858 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_27.status 2026-02-18T08:40:04.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 10 2026-02-18T08:40:04.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 10 2026-02-18T08:40:04.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:40:04.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=41, tag_tid=2, entry_tid=1], [object_number=40, tag_tid=2, entry_tid=0], [object_number=43, tag_tid=1, entry_tid=44535], [object_number=42, tag_tid=1, entry_tid=44534]]], state=connected] 2026-02-18T08:40:04.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=40, tag_tid=1, entry_tid=44536], [object_number=43, tag_tid=1, entry_tid=44535], [object_number=42, tag_tid=1, entry_tid=44534], [object_number=41, tag_tid=1, entry_tid=44533]]], state=connected] 2026-02-18T08:40:04.895 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_27.status 2026-02-18T08:40:04.897 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=40, tag_tid=1, entry_tid=44536' 2026-02-18T08:40:04.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=41, tag_tid=2, entry_tid=1' -a 'object_number=41, tag_tid=2, entry_tid=1' = 'object_number=40, tag_tid=1, entry_tid=44536' 2026-02-18T08:40:04.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=40, tag_tid=1, entry_tid=44536' '!=' '' 2026-02-18T08:40:04.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:40:04.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=40, tag_tid=1, entry_tid=44536' = '' 2026-02-18T08:40:04.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=40, tag_tid=1, entry_tid=44536' 2026-02-18T08:40:04.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=41, tag_tid=2, entry_tid=1' 2026-02-18T08:40:04.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:40:04.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:40:04.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=2 2026-02-18T08:40:04.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=40, tag_tid=1, entry_tid=44536' 2026-02-18T08:40:04.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:40:04.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:40:04.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=1 2026-02-18T08:40:04.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=41, tag_tid=2, entry_tid=1' 2026-02-18T08:40:04.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:40:04.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:40:04.904 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:40:04.905 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=40, tag_tid=1, entry_tid=44536' 2026-02-18T08:40:04.905 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:40:04.905 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:40:04.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=44536 2026-02-18T08:40:04.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 2 = 1 -a 1 -le 44536 2026-02-18T08:40:04.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:40:04.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:40:04.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:40:05.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_27 2026-02-18T08:40:05.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:40:05.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:40:05.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_27 2026-02-18T08:40:05.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:40:05.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_27 ']' 2026-02-18T08:40:05.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_27' 2026-02-18T08:40:05.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_27 2026-02-18T08:40:05.110 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:40:05.110 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:40:05.110 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:40:05.110 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:40:05.110 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:40:05.110 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:40:05.110 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:40:05.113 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:40:05.113 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:40:05.113 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:40:05.113 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_27 2026-02-18T08:40:05.113 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:40:05.113 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:40:05.113 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:40:05.113 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:40:05.114 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:40:05.114 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:40:05.114 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:40:05.114 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:40:05.114 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:40:05.114 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:40:05.114 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:40:05.114 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:40:05.114 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:40:05.114 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:40:05.114 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:40:05.114 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:40:05.114 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:40:05.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:40:05.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:40:05.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:40:05.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:40:05.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:40:05.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:40:05.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:40:05.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:40:05.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:40:05.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:40:05.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:40:05.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:40:05.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:40:05.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:40:05.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:40:05.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:40:05.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_27 2026-02-18T08:40:05.176 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_27'] 2026-02-18T08:40:05.176 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:40:05.176 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:40:05.179 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:40:05.179 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_27 2026-02-18T08:40:05.179 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:40:05.179 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:40:05.179 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:40:05.179 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:40:05.179 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:40:05.179 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:40:05.179 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:40:05.180 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:40:05.180 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:40:05.180 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:40:05.180 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:40:05.180 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:40:05.180 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:40:05.180 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:40:05.180 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:40:05.180 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:40:05.180 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:40:05.180 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:40:05.180 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:40:05.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:40:05.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:40:05.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:40:05.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:40:05.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:40:05.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:40:05.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:40:05.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:40:05.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:40:05.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:40:05.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:40:05.190 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:40:05.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:40:05.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:40:05.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_27 2026-02-18T08:40:05.251 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_27'] 2026-02-18T08:40:05.251 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:40:05.251 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:40:05.254 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:40:05.254 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_27 2026-02-18T08:40:05.254 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:40:05.254 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:40:05.254 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:40:05.254 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:40:05.255 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:40:05.255 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:40:05.255 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:40:05.255 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:40:05.255 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:40:05.255 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:40:05.255 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:40:05.255 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:40:05.255 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:40:05.255 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:40:05.255 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:40:05.255 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:40:05.256 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:40:05.256 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:40:05.256 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:40:05.256 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:40:05.256 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:40:05.256 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:40:05.256 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:40:05.256 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:40:05.256 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:40:05.256 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:40:05.256 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:40:05.256 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:40:05.257 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:40:05.257 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:40:05.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:40:05.266 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:40:05.266 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:40:05.266 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_27 2026-02-18T08:40:05.326 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_27'] 2026-02-18T08:40:05.327 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:40:05.327 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:40:05.329 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:40:05.329 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/image_27 2026-02-18T08:40:05.329 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:40:05.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:40:05.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:40:05.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:40:05.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:40:05.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:40:05.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:40:05.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:40:05.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:40:05.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:40:05.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:40:05.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:40:05.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:40:05.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:40:05.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:40:05.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:40:05.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:40:05.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:40:05.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:40:05.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:40:05.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:40:05.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:40:05.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:40:05.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:40:05.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:40:05.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:40:05.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:40:05.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:40:05.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:40:05.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:40:05.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:40:05.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:40:05.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:40:05.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/image_27 2026-02-18T08:40:05.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:40:05.406 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_27 2026-02-18T08:40:05.406 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:40:05.406 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:40:05.406 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_27 2026-02-18T08:40:05.406 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_27 '' 2026-02-18T08:40:05.406 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:40:05.406 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:40:05.406 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_27 2026-02-18T08:40:05.406 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:40:05.406 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_27.status 2026-02-18T08:40:05.406 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_27.status 2026-02-18T08:40:05.407 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:40:05.407 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_27.status 2026-02-18T08:40:05.407 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_27 2026-02-18T08:40:05.408 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_27.status 2026-02-18T08:40:05.443 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 10 2026-02-18T08:40:05.443 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 10 2026-02-18T08:40:05.443 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:40:05.443 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=41, tag_tid=2, entry_tid=1], [object_number=40, tag_tid=2, entry_tid=0], [object_number=43, tag_tid=1, entry_tid=44535], [object_number=42, tag_tid=1, entry_tid=44534]]], state=connected] 2026-02-18T08:40:05.443 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=40, tag_tid=1, entry_tid=44536], [object_number=43, tag_tid=1, entry_tid=44535], [object_number=42, tag_tid=1, entry_tid=44534], [object_number=41, tag_tid=1, entry_tid=44533]]], state=connected] 2026-02-18T08:40:05.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_27.status 2026-02-18T08:40:05.448 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=41, tag_tid=2, entry_tid=1' 2026-02-18T08:40:05.449 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_27 2026-02-18T08:40:05.449 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:40:05.449 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:40:05.449 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_27 2026-02-18T08:40:05.450 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_27 '..*' 2026-02-18T08:40:05.450 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:40:05.450 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:40:05.450 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_27 2026-02-18T08:40:05.450 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:40:05.450 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_27.status 2026-02-18T08:40:05.450 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_27.status 2026-02-18T08:40:05.451 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:40:05.451 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_27.status 2026-02-18T08:40:05.451 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_27 2026-02-18T08:40:05.452 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_27.status 2026-02-18T08:40:05.478 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 10 2026-02-18T08:40:05.478 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 10 2026-02-18T08:40:05.478 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:40:05.478 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=41, tag_tid=2, entry_tid=1], [object_number=40, tag_tid=2, entry_tid=0], [object_number=43, tag_tid=1, entry_tid=44535], [object_number=42, tag_tid=1, entry_tid=44534]]], state=connected] 2026-02-18T08:40:05.478 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=40, tag_tid=1, entry_tid=44536], [object_number=43, tag_tid=1, entry_tid=44535], [object_number=42, tag_tid=1, entry_tid=44534], [object_number=41, tag_tid=1, entry_tid=44533]]], state=connected] 2026-02-18T08:40:05.480 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_27.status 2026-02-18T08:40:05.482 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=40, tag_tid=1, entry_tid=44536' 2026-02-18T08:40:05.482 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=41, tag_tid=2, entry_tid=1' -a 'object_number=41, tag_tid=2, entry_tid=1' = 'object_number=40, tag_tid=1, entry_tid=44536' 2026-02-18T08:40:05.483 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=40, tag_tid=1, entry_tid=44536' '!=' 'object_number=40, tag_tid=1, entry_tid=44536' 2026-02-18T08:40:05.483 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:40:05.483 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.4 2026-02-18T08:40:05.884 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_27 2026-02-18T08:40:05.885 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:40:05.885 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:40:05.885 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_27 2026-02-18T08:40:05.885 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:40:05.885 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_27 ']' 2026-02-18T08:40:05.885 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_27' 2026-02-18T08:40:05.885 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_27 2026-02-18T08:40:05.885 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:40:05.886 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:40:05.886 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:40:05.886 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:40:05.886 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:40:05.886 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:40:05.886 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:40:05.886 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:40:05.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:40:05.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:40:05.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_27 2026-02-18T08:40:05.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:40:05.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:40:05.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:40:05.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:40:05.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:40:05.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:40:05.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:40:05.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:40:05.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:40:05.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:40:05.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:40:05.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:40:05.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:40:05.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:40:05.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:40:05.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:40:05.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:40:05.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:40:05.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:40:05.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:40:05.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:40:05.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:40:05.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:40:05.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:40:05.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:40:05.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:40:05.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:40:05.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:40:05.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:40:05.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:40:05.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:40:05.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:40:05.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:40:05.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_27 2026-02-18T08:40:05.958 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_27'] 2026-02-18T08:40:05.958 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:40:05.958 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:40:05.961 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:40:05.961 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_27 2026-02-18T08:40:05.961 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:40:05.961 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:40:05.961 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:40:05.961 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:40:05.961 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:40:05.961 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:40:05.961 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:40:05.961 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:40:05.962 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:40:05.962 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:40:05.962 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:40:05.962 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:40:05.962 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:40:05.962 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:40:05.962 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:40:05.962 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:40:05.962 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:40:05.962 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:40:05.962 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:40:05.962 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:40:05.963 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:40:05.963 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:40:05.963 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:40:05.963 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:40:05.963 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:40:05.963 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:40:05.963 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:40:05.963 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:40:05.963 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:40:05.963 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:40:05.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:40:05.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:40:05.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:40:05.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_27 2026-02-18T08:40:06.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_27'] 2026-02-18T08:40:06.027 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:40:06.027 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:40:06.029 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:40:06.029 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_27 2026-02-18T08:40:06.029 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:40:06.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:40:06.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:40:06.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:40:06.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:40:06.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:40:06.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:40:06.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:40:06.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:40:06.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:40:06.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:40:06.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:40:06.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:40:06.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:40:06.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:40:06.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:40:06.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:40:06.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:40:06.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:40:06.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:40:06.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:40:06.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:40:06.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:40:06.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:40:06.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:40:06.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:40:06.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:40:06.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:40:06.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:40:06.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:40:06.040 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:40:06.040 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:40:06.040 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:40:06.040 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_27 2026-02-18T08:40:06.100 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_27'] 2026-02-18T08:40:06.100 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:40:06.100 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:40:06.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:40:06.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/image_27 2026-02-18T08:40:06.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:40:06.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:40:06.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:40:06.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:40:06.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:40:06.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:40:06.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:40:06.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:40:06.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:40:06.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:40:06.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:40:06.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:40:06.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:40:06.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:40:06.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:40:06.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:40:06.105 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:40:06.105 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:40:06.105 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:40:06.105 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:40:06.105 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:40:06.105 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:40:06.105 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:40:06.105 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:40:06.105 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:40:06.105 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:40:06.105 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:40:06.105 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:40:06.106 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:40:06.106 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:40:06.114 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:40:06.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:40:06.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:40:06.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/image_27 2026-02-18T08:40:06.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:40:06.188 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_27 2026-02-18T08:40:06.188 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:40:06.188 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:40:06.188 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_27 2026-02-18T08:40:06.188 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_27 '' 2026-02-18T08:40:06.188 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:40:06.188 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:40:06.189 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_27 2026-02-18T08:40:06.189 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:40:06.189 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_27.status 2026-02-18T08:40:06.189 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_27.status 2026-02-18T08:40:06.189 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:40:06.189 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_27.status 2026-02-18T08:40:06.190 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_27 2026-02-18T08:40:06.190 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_27.status 2026-02-18T08:40:06.223 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 10 2026-02-18T08:40:06.223 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 10 2026-02-18T08:40:06.223 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:40:06.223 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=41, tag_tid=2, entry_tid=1], [object_number=40, tag_tid=2, entry_tid=0], [object_number=43, tag_tid=1, entry_tid=44535], [object_number=42, tag_tid=1, entry_tid=44534]]], state=connected] 2026-02-18T08:40:06.223 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=41, tag_tid=2, entry_tid=1], [object_number=40, tag_tid=2, entry_tid=0], [object_number=43, tag_tid=1, entry_tid=44535], [object_number=42, tag_tid=1, entry_tid=44534]]], state=connected] 2026-02-18T08:40:06.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_27.status 2026-02-18T08:40:06.227 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=41, tag_tid=2, entry_tid=1' 2026-02-18T08:40:06.229 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_27 2026-02-18T08:40:06.229 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:40:06.229 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:40:06.229 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_27 2026-02-18T08:40:06.229 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_27 '..*' 2026-02-18T08:40:06.229 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:40:06.229 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:40:06.229 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_27 2026-02-18T08:40:06.230 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:40:06.230 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_27.status 2026-02-18T08:40:06.230 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_27.status 2026-02-18T08:40:06.230 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:40:06.231 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_27.status 2026-02-18T08:40:06.231 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_27 2026-02-18T08:40:06.231 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_27.status 2026-02-18T08:40:06.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 10 2026-02-18T08:40:06.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 10 2026-02-18T08:40:06.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:40:06.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=41, tag_tid=2, entry_tid=1], [object_number=40, tag_tid=2, entry_tid=0], [object_number=43, tag_tid=1, entry_tid=44535], [object_number=42, tag_tid=1, entry_tid=44534]]], state=connected] 2026-02-18T08:40:06.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=41, tag_tid=2, entry_tid=1], [object_number=40, tag_tid=2, entry_tid=0], [object_number=43, tag_tid=1, entry_tid=44535], [object_number=42, tag_tid=1, entry_tid=44534]]], state=connected] 2026-02-18T08:40:06.270 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_27.status 2026-02-18T08:40:06.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=41, tag_tid=2, entry_tid=1' 2026-02-18T08:40:06.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=41, tag_tid=2, entry_tid=1' -a 'object_number=41, tag_tid=2, entry_tid=1' = 'object_number=41, tag_tid=2, entry_tid=1' 2026-02-18T08:40:06.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:40:06.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_snap_present cluster1 mirror image_27 snap 2026-02-18T08:40:06.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:40:06.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:40:06.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_27 2026-02-18T08:40:06.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap 2026-02-18T08:40:06.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-18T08:40:06.273 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-18T08:40:06.273 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1 2026-02-18T08:40:07.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster1 info mirror/image_27@snap 2026-02-18T08:40:07.302 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:rbd image 'image_27': 2026-02-18T08:40:07.302 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: size 128 MiB in 32 objects 2026-02-18T08:40:07.302 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: order 22 (4 MiB objects) 2026-02-18T08:40:07.302 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: snapshot_count: 1 2026-02-18T08:40:07.302 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: id: 120dbd2ea7bf 2026-02-18T08:40:07.302 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: block_name_prefix: rbd_data.120dbd2ea7bf 2026-02-18T08:40:07.302 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: format: 2 2026-02-18T08:40:07.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: features: layering, exclusive-lock, journaling 2026-02-18T08:40:07.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: op_features: 2026-02-18T08:40:07.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: flags: 2026-02-18T08:40:07.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: create_timestamp: Wed Feb 18 08:30:22 2026 2026-02-18T08:40:07.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: access_timestamp: Wed Feb 18 08:30:22 2026 2026-02-18T08:40:07.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: modify_timestamp: Wed Feb 18 08:31:23 2026 2026-02-18T08:40:07.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: protected: False 2026-02-18T08:40:07.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: journal: 120dbd2ea7bf 2026-02-18T08:40:07.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring state: enabled 2026-02-18T08:40:07.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring mode: journal 2026-02-18T08:40:07.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring global id: a59efa5e-e154-4db1-82aa-0677e43533a1 2026-02-18T08:40:07.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring primary: false 2026-02-18T08:40:07.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:40:07.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ compare_image_snaps mirror image_27 snap 2026-02-18T08:40:07.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:40:07.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_27 2026-02-18T08:40:07.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap 2026-02-18T08:40:07.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local ret=0 2026-02-18T08:40:07.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local rmt_export=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_27.export 2026-02-18T08:40:07.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loc_export=/tmp/tmp.vybh1m9yJF/cluster1-mirror-image_27.export 2026-02-18T08:40:07.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_27.export /tmp/tmp.vybh1m9yJF/cluster1-mirror-image_27.export 2026-02-18T08:40:07.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 -p mirror export image_27@snap /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_27.export 2026-02-18T08:40:07.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-18T08:40:07.451 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster1 -p mirror export image_27@snap /tmp/tmp.vybh1m9yJF/cluster1-mirror-image_27.export 2026-02-18T08:40:07.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-18T08:40:07.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmp /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_27.export /tmp/tmp.vybh1m9yJF/cluster1-mirror-image_27.export 2026-02-18T08:40:07.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_27.export /tmp/tmp.vybh1m9yJF/cluster1-mirror-image_27.export 2026-02-18T08:40:07.769 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:40:07.769 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T08:40:07.770 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_28 2026-02-18T08:40:07.770 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_snap cluster2 mirror image_28 snap 2026-02-18T08:40:07.770 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:40:07.770 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:40:07.770 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_28 2026-02-18T08:40:07.770 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap 2026-02-18T08:40:07.770 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 -p mirror snap create image_28@snap --debug-rbd=20 --debug-journaler=20 2026-02-18T08:40:08.149 INFO:tasks.ceph.cluster2.osd.0.trial163.stderr:2026-02-18T08:40:08.140+0000 7f2129686640 -1 reset not still connected to 0x561765c096c0 2026-02-18T08:40:12.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_replay_started cluster1 mirror image_28 2026-02-18T08:40:12.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:40:12.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:40:12.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_28 2026-02-18T08:40:12.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_replay_state cluster1 mirror image_28 started 2026-02-18T08:40:12.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:40:12.027 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:40:12.027 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_28 2026-02-18T08:40:12.027 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local state=started 2026-02-18T08:40:12.027 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-18T08:40:12.027 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:40:12.027 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1 2026-02-18T08:40:13.028 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test_image_replay_state cluster1 mirror image_28 started 2026-02-18T08:40:13.028 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:40:13.028 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:40:13.028 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_28 2026-02-18T08:40:13.028 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local test_state=started 2026-02-18T08:40:13.028 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local status_result 2026-02-18T08:40:13.028 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_state=stopped 2026-02-18T08:40:13.028 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_28 2026-02-18T08:40:13.029 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -i state 2026-02-18T08:40:13.029 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_instance=cluster1 2026-02-18T08:40:13.029 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-18T08:40:13.029 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:40:13.029 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance=cluster1 2026-02-18T08:40:13.029 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local loop_instance 2026-02-18T08:40:13.029 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:40:13.029 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sleep 0 2026-02-18T08:40:13.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:40:13.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ seq 0 3 2026-02-18T08:40:13.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:40:13.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_28 2026-02-18T08:40:13.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:0 2026-02-18T08:40:13.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-18T08:40:13.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:40:13.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:40:13.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:40:13.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-18T08:40:13.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:40:13.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:40:13.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:40:13.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:40:13.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-18T08:40:13.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:40:13.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:40:13.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:40:13.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:40:13.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:0 2026-02-18T08:40:13.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-18T08:40:13.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-18T08:40:13.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:40:13.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:0 2026-02-18T08:40:13.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T08:40:13.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-18T08:40:13.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:40:13.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:40:13.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-18T08:40:13.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-18T08:40:13.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-18T08:40:13.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=0 2026-02-18T08:40:13.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=0 2026-02-18T08:40:13.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:40:13.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:40:13.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:40:13.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:40:13.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_28 2026-02-18T08:40:13.095 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_28'] 2026-02-18T08:40:13.095 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status : get status for rbd mirror 2026-02-18T08:40:13.095 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:40:13.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:40:13.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_28 2026-02-18T08:40:13.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:1 2026-02-18T08:40:13.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-18T08:40:13.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:40:13.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:40:13.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:40:13.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-18T08:40:13.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:40:13.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:40:13.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:40:13.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:40:13.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-18T08:40:13.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:40:13.100 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:40:13.100 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:40:13.100 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:40:13.100 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:1 2026-02-18T08:40:13.100 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-18T08:40:13.100 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-18T08:40:13.100 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:40:13.100 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:1 2026-02-18T08:40:13.100 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T08:40:13.100 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-18T08:40:13.100 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:40:13.100 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:40:13.100 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-18T08:40:13.101 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-18T08:40:13.101 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-18T08:40:13.101 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=1 2026-02-18T08:40:13.101 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=1 2026-02-18T08:40:13.101 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:40:13.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:40:13.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:40:13.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:40:13.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_28 2026-02-18T08:40:13.173 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_28'] 2026-02-18T08:40:13.173 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status : get status for rbd mirror 2026-02-18T08:40:13.173 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:40:13.176 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:40:13.176 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_28 2026-02-18T08:40:13.176 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:2 2026-02-18T08:40:13.176 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-18T08:40:13.176 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:40:13.176 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:40:13.176 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:40:13.177 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-18T08:40:13.177 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:40:13.177 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:40:13.177 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:40:13.177 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:40:13.177 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-18T08:40:13.177 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:40:13.177 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:40:13.177 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:40:13.177 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:40:13.177 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:2 2026-02-18T08:40:13.177 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-18T08:40:13.178 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-18T08:40:13.178 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:40:13.178 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:2 2026-02-18T08:40:13.178 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T08:40:13.178 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-18T08:40:13.178 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:40:13.178 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:40:13.178 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-18T08:40:13.178 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-18T08:40:13.178 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-18T08:40:13.178 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=2 2026-02-18T08:40:13.178 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=2 2026-02-18T08:40:13.178 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:40:13.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:40:13.187 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:40:13.187 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:40:13.187 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_28 2026-02-18T08:40:13.249 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_28'] 2026-02-18T08:40:13.250 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status : get status for rbd mirror 2026-02-18T08:40:13.250 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:40:13.252 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:40:13.252 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/image_28 2026-02-18T08:40:13.252 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:3 2026-02-18T08:40:13.253 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-18T08:40:13.253 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:40:13.253 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:40:13.253 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:40:13.253 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-18T08:40:13.253 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:40:13.253 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:40:13.253 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:40:13.253 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:40:13.253 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-18T08:40:13.253 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:40:13.253 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:40:13.254 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:40:13.254 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:40:13.254 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:3 2026-02-18T08:40:13.254 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-18T08:40:13.254 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-18T08:40:13.254 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:40:13.254 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:3 2026-02-18T08:40:13.254 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T08:40:13.254 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-18T08:40:13.254 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:40:13.254 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:40:13.254 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-18T08:40:13.254 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-18T08:40:13.255 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-18T08:40:13.255 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=3 2026-02-18T08:40:13.255 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=3 2026-02-18T08:40:13.255 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:40:13.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:40:13.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:40:13.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:40:13.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/image_28 2026-02-18T08:40:13.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ return 0 2026-02-18T08:40:13.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ status_result=' "state": "Replaying"' 2026-02-18T08:40:13.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ echo ' "state": "Replaying"' 2026-02-18T08:40:13.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ grep -i Replaying 2026-02-18T08:40:13.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: "state": "Replaying" 2026-02-18T08:40:13.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ current_state=started 2026-02-18T08:40:13.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test started = started 2026-02-18T08:40:13.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:40:13.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror image_28 2026-02-18T08:40:13.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_cluster=cluster1 2026-02-18T08:40:13.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:40:13.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_pool=mirror 2026-02-18T08:40:13.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local remote_pool=mirror 2026-02-18T08:40:13.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_28 2026-02-18T08:40:13.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' journal = journal ']' 2026-02-18T08:40:13.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror mirror image_28 2026-02-18T08:40:13.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_cluster=cluster1 2026-02-18T08:40:13.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:40:13.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_pool=mirror 2026-02-18T08:40:13.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local remote_pool=mirror 2026-02-18T08:40:13.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_28 2026-02-18T08:40:13.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-02-18T08:40:13.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-02-18T08:40:13.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:40:13.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:40:13.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:40:13.535 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_28 2026-02-18T08:40:13.535 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:40:13.535 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:40:13.535 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_28 2026-02-18T08:40:13.535 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:40:13.535 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_28 ']' 2026-02-18T08:40:13.535 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_28' 2026-02-18T08:40:13.535 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_28 2026-02-18T08:40:13.535 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:40:13.536 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:40:13.536 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:40:13.536 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:40:13.536 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:40:13.536 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:40:13.536 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:40:13.536 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:40:13.539 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:40:13.539 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:40:13.539 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_28 2026-02-18T08:40:13.539 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:40:13.539 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:40:13.539 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:40:13.539 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:40:13.539 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:40:13.539 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:40:13.540 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:40:13.540 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:40:13.540 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:40:13.540 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:40:13.540 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:40:13.540 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:40:13.540 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:40:13.540 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:40:13.540 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:40:13.540 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:40:13.540 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:40:13.540 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:40:13.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:40:13.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:40:13.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:40:13.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:40:13.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:40:13.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:40:13.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:40:13.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:40:13.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:40:13.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:40:13.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:40:13.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:40:13.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:40:13.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:40:13.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:40:13.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_28 2026-02-18T08:40:13.599 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_28'] 2026-02-18T08:40:13.599 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:40:13.599 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:40:13.602 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:40:13.602 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_28 2026-02-18T08:40:13.602 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:40:13.602 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:40:13.603 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:40:13.603 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:40:13.603 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:40:13.603 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:40:13.603 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:40:13.603 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:40:13.603 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:40:13.603 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:40:13.603 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:40:13.603 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:40:13.603 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:40:13.603 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:40:13.604 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:40:13.604 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:40:13.604 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:40:13.604 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:40:13.604 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:40:13.604 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:40:13.604 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:40:13.604 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:40:13.604 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:40:13.604 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:40:13.604 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:40:13.604 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:40:13.604 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:40:13.605 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:40:13.605 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:40:13.605 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:40:13.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:40:13.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:40:13.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:40:13.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_28 2026-02-18T08:40:13.675 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_28'] 2026-02-18T08:40:13.675 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:40:13.675 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:40:13.678 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:40:13.678 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_28 2026-02-18T08:40:13.678 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:40:13.678 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:40:13.678 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:40:13.678 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:40:13.678 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:40:13.678 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:40:13.678 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:40:13.678 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:40:13.679 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:40:13.679 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:40:13.679 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:40:13.679 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:40:13.679 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:40:13.679 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:40:13.679 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:40:13.679 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:40:13.679 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:40:13.679 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:40:13.679 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:40:13.679 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:40:13.679 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:40:13.680 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:40:13.680 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:40:13.680 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:40:13.680 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:40:13.680 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:40:13.680 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:40:13.680 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:40:13.680 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:40:13.680 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:40:13.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:40:13.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:40:13.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:40:13.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_28 2026-02-18T08:40:13.742 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_28'] 2026-02-18T08:40:13.742 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:40:13.742 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:40:13.745 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:40:13.745 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/image_28 2026-02-18T08:40:13.745 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:40:13.745 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:40:13.745 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:40:13.745 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:40:13.745 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:40:13.745 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:40:13.745 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:40:13.745 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:40:13.746 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:40:13.746 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:40:13.746 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:40:13.746 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:40:13.746 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:40:13.746 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:40:13.746 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:40:13.746 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:40:13.746 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:40:13.746 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:40:13.746 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:40:13.746 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:40:13.747 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:40:13.747 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:40:13.747 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:40:13.747 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:40:13.747 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:40:13.747 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:40:13.747 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:40:13.747 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:40:13.747 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:40:13.747 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:40:13.755 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:40:13.755 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:40:13.755 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:40:13.755 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/image_28 2026-02-18T08:40:13.813 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:40:13.814 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_28 2026-02-18T08:40:13.814 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:40:13.814 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:40:13.814 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_28 2026-02-18T08:40:13.814 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_28 '' 2026-02-18T08:40:13.814 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:40:13.814 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:40:13.815 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_28 2026-02-18T08:40:13.815 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:40:13.815 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_28.status 2026-02-18T08:40:13.815 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_28.status 2026-02-18T08:40:13.815 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:40:13.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_28.status 2026-02-18T08:40:13.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_28 2026-02-18T08:40:13.817 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_28.status 2026-02-18T08:40:13.853 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 7 2026-02-18T08:40:13.853 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 7 2026-02-18T08:40:13.853 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:40:13.853 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=29, tag_tid=2, entry_tid=1], [object_number=28, tag_tid=2, entry_tid=0], [object_number=31, tag_tid=1, entry_tid=30863], [object_number=30, tag_tid=1, entry_tid=30862]]], state=connected] 2026-02-18T08:40:13.853 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=29, tag_tid=1, entry_tid=30865], [object_number=28, tag_tid=1, entry_tid=30864], [object_number=31, tag_tid=1, entry_tid=30863], [object_number=30, tag_tid=1, entry_tid=30862]]], state=connected] 2026-02-18T08:40:13.857 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_28.status 2026-02-18T08:40:13.858 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=29, tag_tid=2, entry_tid=1' 2026-02-18T08:40:13.859 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_28 2026-02-18T08:40:13.859 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:40:13.859 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:40:13.860 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_28 2026-02-18T08:40:13.860 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_28 '..*' 2026-02-18T08:40:13.860 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:40:13.860 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:40:13.860 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_28 2026-02-18T08:40:13.860 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:40:13.860 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_28.status 2026-02-18T08:40:13.861 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_28.status 2026-02-18T08:40:13.861 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:40:13.861 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_28.status 2026-02-18T08:40:13.862 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_28 2026-02-18T08:40:13.862 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_28.status 2026-02-18T08:40:13.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 7 2026-02-18T08:40:13.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 7 2026-02-18T08:40:13.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:40:13.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=29, tag_tid=2, entry_tid=1], [object_number=28, tag_tid=2, entry_tid=0], [object_number=31, tag_tid=1, entry_tid=30863], [object_number=30, tag_tid=1, entry_tid=30862]]], state=connected] 2026-02-18T08:40:13.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=29, tag_tid=1, entry_tid=30865], [object_number=28, tag_tid=1, entry_tid=30864], [object_number=31, tag_tid=1, entry_tid=30863], [object_number=30, tag_tid=1, entry_tid=30862]]], state=connected] 2026-02-18T08:40:13.896 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_28.status 2026-02-18T08:40:13.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=29, tag_tid=1, entry_tid=30865' 2026-02-18T08:40:13.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=29, tag_tid=2, entry_tid=1' -a 'object_number=29, tag_tid=2, entry_tid=1' = 'object_number=29, tag_tid=1, entry_tid=30865' 2026-02-18T08:40:13.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=29, tag_tid=1, entry_tid=30865' '!=' '' 2026-02-18T08:40:13.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:40:13.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=29, tag_tid=1, entry_tid=30865' = '' 2026-02-18T08:40:13.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=29, tag_tid=1, entry_tid=30865' 2026-02-18T08:40:13.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=29, tag_tid=2, entry_tid=1' 2026-02-18T08:40:13.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:40:13.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:40:13.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=2 2026-02-18T08:40:13.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=29, tag_tid=1, entry_tid=30865' 2026-02-18T08:40:13.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:40:13.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:40:13.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=1 2026-02-18T08:40:13.904 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=29, tag_tid=2, entry_tid=1' 2026-02-18T08:40:13.904 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:40:13.904 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:40:13.905 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:40:13.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=29, tag_tid=1, entry_tid=30865' 2026-02-18T08:40:13.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:40:13.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:40:13.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=30865 2026-02-18T08:40:13.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 2 = 1 -a 1 -le 30865 2026-02-18T08:40:13.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:40:13.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:40:13.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:40:14.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_28 2026-02-18T08:40:14.110 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:40:14.110 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:40:14.110 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_28 2026-02-18T08:40:14.110 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:40:14.110 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_28 ']' 2026-02-18T08:40:14.110 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_28' 2026-02-18T08:40:14.111 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_28 2026-02-18T08:40:14.111 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:40:14.111 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:40:14.111 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:40:14.111 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:40:14.111 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:40:14.111 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:40:14.111 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:40:14.111 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:40:14.111 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:40:14.114 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:40:14.114 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_28 2026-02-18T08:40:14.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:40:14.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:40:14.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:40:14.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:40:14.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:40:14.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:40:14.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:40:14.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:40:14.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:40:14.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:40:14.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:40:14.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:40:14.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:40:14.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:40:14.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:40:14.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:40:14.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:40:14.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:40:14.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:40:14.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:40:14.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:40:14.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:40:14.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:40:14.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:40:14.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:40:14.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:40:14.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:40:14.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:40:14.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:40:14.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:40:14.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:40:14.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:40:14.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:40:14.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_28 2026-02-18T08:40:14.180 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_28'] 2026-02-18T08:40:14.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:40:14.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:40:14.183 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:40:14.183 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_28 2026-02-18T08:40:14.183 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:40:14.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:40:14.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:40:14.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:40:14.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:40:14.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:40:14.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:40:14.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:40:14.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:40:14.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:40:14.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:40:14.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:40:14.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:40:14.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:40:14.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:40:14.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:40:14.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:40:14.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:40:14.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:40:14.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:40:14.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:40:14.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:40:14.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:40:14.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:40:14.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:40:14.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:40:14.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:40:14.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:40:14.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:40:14.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:40:14.194 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:40:14.195 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:40:14.195 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:40:14.195 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_28 2026-02-18T08:40:14.254 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_28'] 2026-02-18T08:40:14.254 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:40:14.254 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:40:14.257 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:40:14.257 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_28 2026-02-18T08:40:14.257 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:40:14.257 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:40:14.257 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:40:14.257 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:40:14.257 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:40:14.257 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:40:14.257 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:40:14.257 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:40:14.257 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:40:14.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:40:14.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:40:14.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:40:14.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:40:14.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:40:14.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:40:14.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:40:14.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:40:14.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:40:14.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:40:14.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:40:14.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:40:14.259 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:40:14.259 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:40:14.259 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:40:14.259 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:40:14.259 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:40:14.259 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:40:14.259 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:40:14.259 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:40:14.259 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:40:14.266 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:40:14.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:40:14.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:40:14.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_28 2026-02-18T08:40:14.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_28'] 2026-02-18T08:40:14.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:40:14.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:40:14.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:40:14.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/image_28 2026-02-18T08:40:14.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:40:14.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:40:14.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:40:14.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:40:14.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:40:14.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:40:14.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:40:14.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:40:14.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:40:14.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:40:14.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:40:14.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:40:14.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:40:14.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:40:14.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:40:14.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:40:14.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:40:14.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:40:14.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:40:14.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:40:14.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:40:14.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:40:14.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:40:14.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:40:14.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:40:14.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:40:14.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:40:14.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:40:14.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:40:14.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:40:14.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:40:14.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:40:14.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:40:14.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/image_28 2026-02-18T08:40:14.406 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:40:14.407 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_28 2026-02-18T08:40:14.408 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:40:14.408 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:40:14.408 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_28 2026-02-18T08:40:14.408 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_28 '' 2026-02-18T08:40:14.408 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:40:14.408 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:40:14.408 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_28 2026-02-18T08:40:14.408 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:40:14.408 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_28.status 2026-02-18T08:40:14.408 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_28.status 2026-02-18T08:40:14.409 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:40:14.409 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_28.status 2026-02-18T08:40:14.409 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_28 2026-02-18T08:40:14.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_28.status 2026-02-18T08:40:14.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 7 2026-02-18T08:40:14.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 7 2026-02-18T08:40:14.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:40:14.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=29, tag_tid=2, entry_tid=1], [object_number=28, tag_tid=2, entry_tid=0], [object_number=31, tag_tid=1, entry_tid=30863], [object_number=30, tag_tid=1, entry_tid=30862]]], state=connected] 2026-02-18T08:40:14.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=29, tag_tid=1, entry_tid=30865], [object_number=28, tag_tid=1, entry_tid=30864], [object_number=31, tag_tid=1, entry_tid=30863], [object_number=30, tag_tid=1, entry_tid=30862]]], state=connected] 2026-02-18T08:40:14.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_28.status 2026-02-18T08:40:14.443 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=29, tag_tid=2, entry_tid=1' 2026-02-18T08:40:14.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_28 2026-02-18T08:40:14.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:40:14.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:40:14.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_28 2026-02-18T08:40:14.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_28 '..*' 2026-02-18T08:40:14.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:40:14.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:40:14.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_28 2026-02-18T08:40:14.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:40:14.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_28.status 2026-02-18T08:40:14.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_28.status 2026-02-18T08:40:14.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:40:14.446 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_28.status 2026-02-18T08:40:14.446 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_28 2026-02-18T08:40:14.446 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_28.status 2026-02-18T08:40:14.480 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 7 2026-02-18T08:40:14.480 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 7 2026-02-18T08:40:14.480 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:40:14.480 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=29, tag_tid=2, entry_tid=1], [object_number=28, tag_tid=2, entry_tid=0], [object_number=31, tag_tid=1, entry_tid=30863], [object_number=30, tag_tid=1, entry_tid=30862]]], state=connected] 2026-02-18T08:40:14.480 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=29, tag_tid=1, entry_tid=30865], [object_number=28, tag_tid=1, entry_tid=30864], [object_number=31, tag_tid=1, entry_tid=30863], [object_number=30, tag_tid=1, entry_tid=30862]]], state=connected] 2026-02-18T08:40:14.483 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_28.status 2026-02-18T08:40:14.485 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=29, tag_tid=1, entry_tid=30865' 2026-02-18T08:40:14.485 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=29, tag_tid=2, entry_tid=1' -a 'object_number=29, tag_tid=2, entry_tid=1' = 'object_number=29, tag_tid=1, entry_tid=30865' 2026-02-18T08:40:14.485 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=29, tag_tid=1, entry_tid=30865' '!=' 'object_number=29, tag_tid=1, entry_tid=30865' 2026-02-18T08:40:14.485 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:40:14.485 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.4 2026-02-18T08:40:14.887 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_28 2026-02-18T08:40:14.887 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:40:14.887 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:40:14.887 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_28 2026-02-18T08:40:14.887 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:40:14.887 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_28 ']' 2026-02-18T08:40:14.887 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_28' 2026-02-18T08:40:14.887 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_28 2026-02-18T08:40:14.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:40:14.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:40:14.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:40:14.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:40:14.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:40:14.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:40:14.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:40:14.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:40:14.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:40:14.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:40:14.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_28 2026-02-18T08:40:14.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:40:14.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:40:14.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:40:14.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:40:14.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:40:14.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:40:14.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:40:14.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:40:14.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:40:14.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:40:14.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:40:14.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:40:14.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:40:14.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:40:14.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:40:14.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:40:14.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:40:14.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:40:14.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:40:14.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:40:14.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:40:14.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:40:14.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:40:14.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:40:14.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:40:14.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:40:14.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:40:14.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:40:14.894 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:40:14.894 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:40:14.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:40:14.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:40:14.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:40:14.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_28 2026-02-18T08:40:14.962 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_28'] 2026-02-18T08:40:14.962 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:40:14.962 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:40:14.965 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:40:14.965 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_28 2026-02-18T08:40:14.965 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:40:14.965 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:40:14.965 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:40:14.965 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:40:14.965 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:40:14.965 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:40:14.966 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:40:14.966 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:40:14.966 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:40:14.966 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:40:14.966 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:40:14.966 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:40:14.966 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:40:14.966 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:40:14.966 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:40:14.966 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:40:14.966 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:40:14.966 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:40:14.967 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:40:14.967 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:40:14.967 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:40:14.967 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:40:14.967 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:40:14.967 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:40:14.967 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:40:14.967 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:40:14.967 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:40:14.967 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:40:14.967 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:40:14.967 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:40:14.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:40:14.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:40:14.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:40:14.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_28 2026-02-18T08:40:15.038 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_28'] 2026-02-18T08:40:15.039 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:40:15.039 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:40:15.041 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:40:15.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_28 2026-02-18T08:40:15.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:40:15.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:40:15.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:40:15.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:40:15.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:40:15.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:40:15.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:40:15.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:40:15.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:40:15.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:40:15.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:40:15.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:40:15.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:40:15.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:40:15.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:40:15.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:40:15.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:40:15.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:40:15.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:40:15.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:40:15.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:40:15.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:40:15.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:40:15.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:40:15.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:40:15.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:40:15.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:40:15.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:40:15.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:40:15.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:40:15.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:40:15.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:40:15.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:40:15.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_28 2026-02-18T08:40:15.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_28'] 2026-02-18T08:40:15.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:40:15.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:40:15.111 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:40:15.111 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/image_28 2026-02-18T08:40:15.111 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:40:15.111 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:40:15.111 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:40:15.111 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:40:15.111 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:40:15.111 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:40:15.112 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:40:15.112 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:40:15.112 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:40:15.112 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:40:15.112 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:40:15.112 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:40:15.112 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:40:15.112 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:40:15.112 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:40:15.112 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:40:15.112 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:40:15.112 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:40:15.112 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:40:15.113 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:40:15.113 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:40:15.113 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:40:15.113 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:40:15.113 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:40:15.113 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:40:15.113 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:40:15.113 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:40:15.113 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:40:15.113 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:40:15.113 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:40:15.121 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:40:15.122 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:40:15.122 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:40:15.122 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/image_28 2026-02-18T08:40:15.189 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:40:15.190 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_28 2026-02-18T08:40:15.190 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:40:15.190 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:40:15.190 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_28 2026-02-18T08:40:15.190 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_28 '' 2026-02-18T08:40:15.190 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:40:15.190 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:40:15.190 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_28 2026-02-18T08:40:15.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:40:15.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_28.status 2026-02-18T08:40:15.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_28.status 2026-02-18T08:40:15.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:40:15.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_28.status 2026-02-18T08:40:15.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_28 2026-02-18T08:40:15.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_28.status 2026-02-18T08:40:15.229 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 7 2026-02-18T08:40:15.229 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 7 2026-02-18T08:40:15.229 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:40:15.229 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=29, tag_tid=2, entry_tid=1], [object_number=28, tag_tid=2, entry_tid=0], [object_number=31, tag_tid=1, entry_tid=30863], [object_number=30, tag_tid=1, entry_tid=30862]]], state=connected] 2026-02-18T08:40:15.229 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=29, tag_tid=1, entry_tid=30865], [object_number=28, tag_tid=1, entry_tid=30864], [object_number=31, tag_tid=1, entry_tid=30863], [object_number=30, tag_tid=1, entry_tid=30862]]], state=connected] 2026-02-18T08:40:15.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_28.status 2026-02-18T08:40:15.234 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=29, tag_tid=2, entry_tid=1' 2026-02-18T08:40:15.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_28 2026-02-18T08:40:15.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:40:15.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:40:15.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_28 2026-02-18T08:40:15.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_28 '..*' 2026-02-18T08:40:15.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:40:15.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:40:15.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_28 2026-02-18T08:40:15.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:40:15.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_28.status 2026-02-18T08:40:15.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_28.status 2026-02-18T08:40:15.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:40:15.238 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_28.status 2026-02-18T08:40:15.238 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_28 2026-02-18T08:40:15.238 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_28.status 2026-02-18T08:40:15.266 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 7 2026-02-18T08:40:15.266 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 7 2026-02-18T08:40:15.266 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:40:15.266 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=29, tag_tid=2, entry_tid=1], [object_number=28, tag_tid=2, entry_tid=0], [object_number=31, tag_tid=1, entry_tid=30863], [object_number=30, tag_tid=1, entry_tid=30862]]], state=connected] 2026-02-18T08:40:15.266 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=29, tag_tid=1, entry_tid=30865], [object_number=28, tag_tid=1, entry_tid=30864], [object_number=31, tag_tid=1, entry_tid=30863], [object_number=30, tag_tid=1, entry_tid=30862]]], state=connected] 2026-02-18T08:40:15.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_28.status 2026-02-18T08:40:15.270 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=29, tag_tid=1, entry_tid=30865' 2026-02-18T08:40:15.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=29, tag_tid=2, entry_tid=1' -a 'object_number=29, tag_tid=2, entry_tid=1' = 'object_number=29, tag_tid=1, entry_tid=30865' 2026-02-18T08:40:15.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=29, tag_tid=1, entry_tid=30865' '!=' 'object_number=29, tag_tid=1, entry_tid=30865' 2026-02-18T08:40:15.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:40:15.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.8 2026-02-18T08:40:16.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_28 2026-02-18T08:40:16.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:40:16.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:40:16.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_28 2026-02-18T08:40:16.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:40:16.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_28 ']' 2026-02-18T08:40:16.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_28' 2026-02-18T08:40:16.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_28 2026-02-18T08:40:16.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:40:16.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:40:16.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:40:16.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:40:16.074 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:40:16.074 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:40:16.074 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:40:16.076 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:40:16.076 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:40:16.076 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:40:16.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_28 2026-02-18T08:40:16.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:40:16.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:40:16.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:40:16.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:40:16.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:40:16.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:40:16.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:40:16.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:40:16.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:40:16.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:40:16.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:40:16.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:40:16.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:40:16.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:40:16.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:40:16.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:40:16.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:40:16.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:40:16.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:40:16.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:40:16.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:40:16.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:40:16.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:40:16.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:40:16.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:40:16.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:40:16.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:40:16.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:40:16.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:40:16.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:40:16.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:40:16.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:40:16.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:40:16.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_28 2026-02-18T08:40:16.147 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_28'] 2026-02-18T08:40:16.147 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:40:16.147 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:40:16.150 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:40:16.150 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_28 2026-02-18T08:40:16.150 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:40:16.150 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:40:16.150 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:40:16.150 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:40:16.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:40:16.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:40:16.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:40:16.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:40:16.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:40:16.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:40:16.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:40:16.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:40:16.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:40:16.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:40:16.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:40:16.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:40:16.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:40:16.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:40:16.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:40:16.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:40:16.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:40:16.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:40:16.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:40:16.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:40:16.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:40:16.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:40:16.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:40:16.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:40:16.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:40:16.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:40:16.162 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:40:16.163 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:40:16.163 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:40:16.163 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_28 2026-02-18T08:40:16.214 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_28'] 2026-02-18T08:40:16.214 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:40:16.214 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:40:16.217 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:40:16.217 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_28 2026-02-18T08:40:16.217 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:40:16.217 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:40:16.217 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:40:16.217 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:40:16.217 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:40:16.217 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:40:16.218 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:40:16.218 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:40:16.218 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:40:16.218 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:40:16.218 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:40:16.218 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:40:16.218 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:40:16.218 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:40:16.218 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:40:16.218 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:40:16.218 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:40:16.219 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:40:16.219 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:40:16.219 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:40:16.219 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:40:16.219 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:40:16.219 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:40:16.219 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:40:16.219 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:40:16.219 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:40:16.219 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:40:16.219 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:40:16.219 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:40:16.219 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:40:16.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:40:16.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:40:16.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:40:16.229 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_28 2026-02-18T08:40:16.279 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_28'] 2026-02-18T08:40:16.279 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:40:16.279 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:40:16.282 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:40:16.282 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/image_28 2026-02-18T08:40:16.282 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:40:16.282 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:40:16.283 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:40:16.283 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:40:16.283 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:40:16.283 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:40:16.283 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:40:16.283 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:40:16.283 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:40:16.283 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:40:16.283 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:40:16.283 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:40:16.284 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:40:16.284 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:40:16.284 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:40:16.284 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:40:16.284 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:40:16.284 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:40:16.284 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:40:16.284 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:40:16.284 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:40:16.284 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:40:16.284 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:40:16.284 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:40:16.285 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:40:16.285 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:40:16.285 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:40:16.285 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:40:16.285 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:40:16.285 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:40:16.293 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:40:16.294 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:40:16.294 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:40:16.294 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/image_28 2026-02-18T08:40:16.360 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:40:16.360 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_28 2026-02-18T08:40:16.360 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:40:16.360 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:40:16.361 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_28 2026-02-18T08:40:16.361 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_28 '' 2026-02-18T08:40:16.361 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:40:16.361 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:40:16.361 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_28 2026-02-18T08:40:16.361 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:40:16.361 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_28.status 2026-02-18T08:40:16.362 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_28.status 2026-02-18T08:40:16.362 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:40:16.362 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_28.status 2026-02-18T08:40:16.362 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_28 2026-02-18T08:40:16.362 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_28.status 2026-02-18T08:40:16.397 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 7 2026-02-18T08:40:16.397 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 7 2026-02-18T08:40:16.397 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:40:16.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=29, tag_tid=2, entry_tid=1], [object_number=28, tag_tid=2, entry_tid=0], [object_number=31, tag_tid=1, entry_tid=30863], [object_number=30, tag_tid=1, entry_tid=30862]]], state=connected] 2026-02-18T08:40:16.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=29, tag_tid=1, entry_tid=30865], [object_number=28, tag_tid=1, entry_tid=30864], [object_number=31, tag_tid=1, entry_tid=30863], [object_number=30, tag_tid=1, entry_tid=30862]]], state=connected] 2026-02-18T08:40:16.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_28.status 2026-02-18T08:40:16.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=29, tag_tid=2, entry_tid=1' 2026-02-18T08:40:16.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_28 2026-02-18T08:40:16.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:40:16.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:40:16.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_28 2026-02-18T08:40:16.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_28 '..*' 2026-02-18T08:40:16.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:40:16.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:40:16.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_28 2026-02-18T08:40:16.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:40:16.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_28.status 2026-02-18T08:40:16.405 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_28.status 2026-02-18T08:40:16.405 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:40:16.405 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_28.status 2026-02-18T08:40:16.406 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_28 2026-02-18T08:40:16.406 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_28.status 2026-02-18T08:40:16.432 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 7 2026-02-18T08:40:16.432 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 7 2026-02-18T08:40:16.433 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:40:16.433 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=29, tag_tid=2, entry_tid=1], [object_number=28, tag_tid=2, entry_tid=0], [object_number=31, tag_tid=1, entry_tid=30863], [object_number=30, tag_tid=1, entry_tid=30862]]], state=connected] 2026-02-18T08:40:16.433 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=29, tag_tid=1, entry_tid=30865], [object_number=28, tag_tid=1, entry_tid=30864], [object_number=31, tag_tid=1, entry_tid=30863], [object_number=30, tag_tid=1, entry_tid=30862]]], state=connected] 2026-02-18T08:40:16.435 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_28.status 2026-02-18T08:40:16.436 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=29, tag_tid=1, entry_tid=30865' 2026-02-18T08:40:16.436 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=29, tag_tid=2, entry_tid=1' -a 'object_number=29, tag_tid=2, entry_tid=1' = 'object_number=29, tag_tid=1, entry_tid=30865' 2026-02-18T08:40:16.437 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=29, tag_tid=1, entry_tid=30865' '!=' 'object_number=29, tag_tid=1, entry_tid=30865' 2026-02-18T08:40:16.437 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:40:16.437 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1.6 2026-02-18T08:40:18.038 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_28 2026-02-18T08:40:18.038 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:40:18.038 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:40:18.039 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_28 2026-02-18T08:40:18.039 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:40:18.039 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_28 ']' 2026-02-18T08:40:18.039 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_28' 2026-02-18T08:40:18.039 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_28 2026-02-18T08:40:18.039 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:40:18.039 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:40:18.039 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:40:18.039 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:40:18.039 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:40:18.039 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:40:18.040 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:40:18.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:40:18.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:40:18.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:40:18.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_28 2026-02-18T08:40:18.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:40:18.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:40:18.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:40:18.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:40:18.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:40:18.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:40:18.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:40:18.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:40:18.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:40:18.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:40:18.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:40:18.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:40:18.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:40:18.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:40:18.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:40:18.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:40:18.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:40:18.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:40:18.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:40:18.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:40:18.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:40:18.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:40:18.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:40:18.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:40:18.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:40:18.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:40:18.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:40:18.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:40:18.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:40:18.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:40:18.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:40:18.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:40:18.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:40:18.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_28 2026-02-18T08:40:18.113 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_28'] 2026-02-18T08:40:18.113 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:40:18.114 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:40:18.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:40:18.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_28 2026-02-18T08:40:18.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:40:18.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:40:18.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:40:18.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:40:18.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:40:18.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:40:18.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:40:18.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:40:18.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:40:18.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:40:18.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:40:18.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:40:18.118 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:40:18.118 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:40:18.118 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:40:18.118 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:40:18.118 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:40:18.118 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:40:18.118 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:40:18.118 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:40:18.118 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:40:18.118 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:40:18.118 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:40:18.118 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:40:18.119 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:40:18.119 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:40:18.119 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:40:18.119 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:40:18.119 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:40:18.119 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:40:18.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:40:18.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:40:18.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:40:18.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_28 2026-02-18T08:40:18.189 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_28'] 2026-02-18T08:40:18.189 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:40:18.189 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:40:18.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:40:18.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_28 2026-02-18T08:40:18.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:40:18.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:40:18.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:40:18.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:40:18.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:40:18.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:40:18.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:40:18.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:40:18.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:40:18.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:40:18.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:40:18.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:40:18.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:40:18.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:40:18.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:40:18.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:40:18.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:40:18.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:40:18.194 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:40:18.194 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:40:18.194 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:40:18.194 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:40:18.194 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:40:18.194 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:40:18.194 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:40:18.194 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:40:18.194 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:40:18.194 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:40:18.194 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:40:18.194 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:40:18.202 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:40:18.203 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:40:18.203 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:40:18.203 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_28 2026-02-18T08:40:18.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_28'] 2026-02-18T08:40:18.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:40:18.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:40:18.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:40:18.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/image_28 2026-02-18T08:40:18.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:40:18.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:40:18.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:40:18.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:40:18.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:40:18.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:40:18.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:40:18.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:40:18.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:40:18.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:40:18.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:40:18.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:40:18.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:40:18.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:40:18.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:40:18.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:40:18.269 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:40:18.269 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:40:18.269 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:40:18.269 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:40:18.269 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:40:18.269 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:40:18.269 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:40:18.269 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:40:18.269 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:40:18.269 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:40:18.269 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:40:18.269 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:40:18.270 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:40:18.270 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:40:18.278 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:40:18.278 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:40:18.278 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:40:18.279 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/image_28 2026-02-18T08:40:18.344 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:40:18.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_28 2026-02-18T08:40:18.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:40:18.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:40:18.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_28 2026-02-18T08:40:18.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_28 '' 2026-02-18T08:40:18.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:40:18.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:40:18.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_28 2026-02-18T08:40:18.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:40:18.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_28.status 2026-02-18T08:40:18.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_28.status 2026-02-18T08:40:18.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:40:18.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_28.status 2026-02-18T08:40:18.348 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_28 2026-02-18T08:40:18.348 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_28.status 2026-02-18T08:40:18.385 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 7 2026-02-18T08:40:18.385 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 7 2026-02-18T08:40:18.385 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:40:18.385 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=29, tag_tid=2, entry_tid=1], [object_number=28, tag_tid=2, entry_tid=0], [object_number=31, tag_tid=1, entry_tid=30863], [object_number=30, tag_tid=1, entry_tid=30862]]], state=connected] 2026-02-18T08:40:18.385 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=29, tag_tid=1, entry_tid=30865], [object_number=28, tag_tid=1, entry_tid=30864], [object_number=31, tag_tid=1, entry_tid=30863], [object_number=30, tag_tid=1, entry_tid=30862]]], state=connected] 2026-02-18T08:40:18.388 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_28.status 2026-02-18T08:40:18.389 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=29, tag_tid=2, entry_tid=1' 2026-02-18T08:40:18.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_28 2026-02-18T08:40:18.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:40:18.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:40:18.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_28 2026-02-18T08:40:18.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_28 '..*' 2026-02-18T08:40:18.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:40:18.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:40:18.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_28 2026-02-18T08:40:18.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:40:18.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_28.status 2026-02-18T08:40:18.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_28.status 2026-02-18T08:40:18.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:40:18.393 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_28.status 2026-02-18T08:40:18.393 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_28.status 2026-02-18T08:40:18.393 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_28 2026-02-18T08:40:18.421 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 7 2026-02-18T08:40:18.421 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 7 2026-02-18T08:40:18.421 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:40:18.421 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=29, tag_tid=2, entry_tid=1], [object_number=28, tag_tid=2, entry_tid=0], [object_number=31, tag_tid=1, entry_tid=30863], [object_number=30, tag_tid=1, entry_tid=30862]]], state=connected] 2026-02-18T08:40:18.421 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=29, tag_tid=1, entry_tid=30865], [object_number=28, tag_tid=1, entry_tid=30864], [object_number=31, tag_tid=1, entry_tid=30863], [object_number=30, tag_tid=1, entry_tid=30862]]], state=connected] 2026-02-18T08:40:18.423 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_28.status 2026-02-18T08:40:18.425 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=29, tag_tid=1, entry_tid=30865' 2026-02-18T08:40:18.425 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=29, tag_tid=2, entry_tid=1' -a 'object_number=29, tag_tid=2, entry_tid=1' = 'object_number=29, tag_tid=1, entry_tid=30865' 2026-02-18T08:40:18.425 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=29, tag_tid=1, entry_tid=30865' '!=' 'object_number=29, tag_tid=1, entry_tid=30865' 2026-02-18T08:40:18.425 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:40:18.425 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 2 2026-02-18T08:40:20.427 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_28 2026-02-18T08:40:20.428 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:40:20.428 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:40:20.428 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_28 2026-02-18T08:40:20.428 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:40:20.428 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_28 ']' 2026-02-18T08:40:20.428 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_28' 2026-02-18T08:40:20.428 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_28 2026-02-18T08:40:20.428 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:40:20.429 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:40:20.429 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:40:20.429 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:40:20.429 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:40:20.429 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:40:20.429 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:40:20.429 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:40:20.429 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:40:20.431 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:40:20.432 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_28 2026-02-18T08:40:20.432 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:40:20.432 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:40:20.432 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:40:20.432 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:40:20.432 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:40:20.432 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:40:20.432 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:40:20.432 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:40:20.432 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:40:20.433 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:40:20.433 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:40:20.433 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:40:20.433 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:40:20.433 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:40:20.433 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:40:20.433 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:40:20.433 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:40:20.433 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:40:20.433 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:40:20.433 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:40:20.433 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:40:20.434 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:40:20.434 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:40:20.434 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:40:20.434 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:40:20.434 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:40:20.434 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:40:20.434 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:40:20.434 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:40:20.434 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:40:20.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:40:20.443 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:40:20.443 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:40:20.443 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_28 2026-02-18T08:40:20.504 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_28'] 2026-02-18T08:40:20.505 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:40:20.505 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:40:20.507 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:40:20.508 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_28 2026-02-18T08:40:20.508 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:40:20.508 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:40:20.508 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:40:20.508 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:40:20.508 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:40:20.508 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:40:20.508 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:40:20.508 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:40:20.508 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:40:20.508 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:40:20.509 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:40:20.509 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:40:20.509 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:40:20.509 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:40:20.509 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:40:20.509 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:40:20.509 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:40:20.509 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:40:20.509 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:40:20.509 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:40:20.509 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:40:20.509 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:40:20.509 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:40:20.510 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:40:20.510 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:40:20.510 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:40:20.510 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:40:20.510 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:40:20.510 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:40:20.510 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:40:20.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:40:20.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:40:20.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:40:20.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_28 2026-02-18T08:40:20.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_28'] 2026-02-18T08:40:20.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:40:20.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:40:20.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:40:20.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_28 2026-02-18T08:40:20.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:40:20.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:40:20.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:40:20.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:40:20.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:40:20.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:40:20.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:40:20.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:40:20.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:40:20.584 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:40:20.584 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:40:20.584 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:40:20.584 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:40:20.584 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:40:20.584 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:40:20.584 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:40:20.584 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:40:20.584 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:40:20.584 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:40:20.584 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:40:20.584 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:40:20.585 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:40:20.585 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:40:20.585 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:40:20.585 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:40:20.585 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:40:20.585 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:40:20.585 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:40:20.585 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:40:20.585 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:40:20.594 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:40:20.594 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:40:20.594 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:40:20.594 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_28 2026-02-18T08:40:20.647 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_28'] 2026-02-18T08:40:20.647 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:40:20.647 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:40:20.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:40:20.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/image_28 2026-02-18T08:40:20.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:40:20.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:40:20.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:40:20.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:40:20.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:40:20.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:40:20.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:40:20.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:40:20.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:40:20.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:40:20.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:40:20.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:40:20.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:40:20.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:40:20.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:40:20.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:40:20.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:40:20.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:40:20.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:40:20.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:40:20.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:40:20.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:40:20.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:40:20.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:40:20.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:40:20.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:40:20.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:40:20.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:40:20.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:40:20.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:40:20.661 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:40:20.661 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:40:20.662 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:40:20.662 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/image_28 2026-02-18T08:40:20.722 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:40:20.722 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_28 2026-02-18T08:40:20.722 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:40:20.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:40:20.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_28 2026-02-18T08:40:20.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_28 '' 2026-02-18T08:40:20.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:40:20.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:40:20.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_28 2026-02-18T08:40:20.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:40:20.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_28.status 2026-02-18T08:40:20.724 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_28.status 2026-02-18T08:40:20.724 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:40:20.724 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_28.status 2026-02-18T08:40:20.725 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_28 2026-02-18T08:40:20.725 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_28.status 2026-02-18T08:40:20.754 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 7 2026-02-18T08:40:20.754 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 7 2026-02-18T08:40:20.754 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:40:20.754 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=29, tag_tid=2, entry_tid=1], [object_number=28, tag_tid=2, entry_tid=0], [object_number=31, tag_tid=1, entry_tid=30863], [object_number=30, tag_tid=1, entry_tid=30862]]], state=connected] 2026-02-18T08:40:20.754 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=29, tag_tid=2, entry_tid=1], [object_number=28, tag_tid=2, entry_tid=0], [object_number=31, tag_tid=1, entry_tid=30863], [object_number=30, tag_tid=1, entry_tid=30862]]], state=connected] 2026-02-18T08:40:20.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_28.status 2026-02-18T08:40:20.758 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=29, tag_tid=2, entry_tid=1' 2026-02-18T08:40:20.759 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_28 2026-02-18T08:40:20.760 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:40:20.760 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:40:20.760 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_28 2026-02-18T08:40:20.760 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_28 '..*' 2026-02-18T08:40:20.760 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:40:20.760 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:40:20.760 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_28 2026-02-18T08:40:20.760 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:40:20.760 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_28.status 2026-02-18T08:40:20.761 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_28.status 2026-02-18T08:40:20.761 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:40:20.761 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_28.status 2026-02-18T08:40:20.762 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_28 2026-02-18T08:40:20.762 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_28.status 2026-02-18T08:40:20.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 7 2026-02-18T08:40:20.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 7 2026-02-18T08:40:20.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:40:20.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=29, tag_tid=2, entry_tid=1], [object_number=28, tag_tid=2, entry_tid=0], [object_number=31, tag_tid=1, entry_tid=30863], [object_number=30, tag_tid=1, entry_tid=30862]]], state=connected] 2026-02-18T08:40:20.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=29, tag_tid=2, entry_tid=1], [object_number=28, tag_tid=2, entry_tid=0], [object_number=31, tag_tid=1, entry_tid=30863], [object_number=30, tag_tid=1, entry_tid=30862]]], state=connected] 2026-02-18T08:40:20.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_28.status 2026-02-18T08:40:20.793 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=29, tag_tid=2, entry_tid=1' 2026-02-18T08:40:20.794 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=29, tag_tid=2, entry_tid=1' -a 'object_number=29, tag_tid=2, entry_tid=1' = 'object_number=29, tag_tid=2, entry_tid=1' 2026-02-18T08:40:20.794 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:40:20.794 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_snap_present cluster1 mirror image_28 snap 2026-02-18T08:40:20.794 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:40:20.794 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:40:20.794 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_28 2026-02-18T08:40:20.794 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap 2026-02-18T08:40:20.794 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-18T08:40:20.794 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-18T08:40:20.794 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1 2026-02-18T08:40:21.795 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster1 info mirror/image_28@snap 2026-02-18T08:40:21.825 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:rbd image 'image_28': 2026-02-18T08:40:21.825 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: size 128 MiB in 32 objects 2026-02-18T08:40:21.825 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: order 22 (4 MiB objects) 2026-02-18T08:40:21.825 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: snapshot_count: 1 2026-02-18T08:40:21.825 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: id: 120d449cad1a 2026-02-18T08:40:21.825 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: block_name_prefix: rbd_data.120d449cad1a 2026-02-18T08:40:21.826 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: format: 2 2026-02-18T08:40:21.826 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: features: layering, exclusive-lock, journaling 2026-02-18T08:40:21.826 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: op_features: 2026-02-18T08:40:21.826 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: flags: 2026-02-18T08:40:21.826 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: create_timestamp: Wed Feb 18 08:30:35 2026 2026-02-18T08:40:21.826 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: access_timestamp: Wed Feb 18 08:30:35 2026 2026-02-18T08:40:21.826 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: modify_timestamp: Wed Feb 18 08:31:36 2026 2026-02-18T08:40:21.826 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: protected: False 2026-02-18T08:40:21.826 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: journal: 120d449cad1a 2026-02-18T08:40:21.826 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring state: enabled 2026-02-18T08:40:21.826 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring mode: journal 2026-02-18T08:40:21.826 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring global id: a185b94b-f441-49f1-b59a-2359936b5cba 2026-02-18T08:40:21.827 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring primary: false 2026-02-18T08:40:21.828 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:40:21.828 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ compare_image_snaps mirror image_28 snap 2026-02-18T08:40:21.828 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:40:21.829 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_28 2026-02-18T08:40:21.829 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap 2026-02-18T08:40:21.829 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local ret=0 2026-02-18T08:40:21.829 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local rmt_export=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_28.export 2026-02-18T08:40:21.829 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loc_export=/tmp/tmp.vybh1m9yJF/cluster1-mirror-image_28.export 2026-02-18T08:40:21.829 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_28.export /tmp/tmp.vybh1m9yJF/cluster1-mirror-image_28.export 2026-02-18T08:40:21.829 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 -p mirror export image_28@snap /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_28.export 2026-02-18T08:40:21.967 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-18T08:40:21.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster1 -p mirror export image_28@snap /tmp/tmp.vybh1m9yJF/cluster1-mirror-image_28.export 2026-02-18T08:40:22.173 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-18T08:40:22.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmp /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_28.export /tmp/tmp.vybh1m9yJF/cluster1-mirror-image_28.export 2026-02-18T08:40:22.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_28.export /tmp/tmp.vybh1m9yJF/cluster1-mirror-image_28.export 2026-02-18T08:40:22.281 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:40:22.281 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T08:40:22.281 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_29 2026-02-18T08:40:22.281 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_snap cluster2 mirror image_29 snap 2026-02-18T08:40:22.281 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:40:22.281 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:40:22.281 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_29 2026-02-18T08:40:22.281 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap 2026-02-18T08:40:22.281 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 -p mirror snap create image_29@snap --debug-rbd=20 --debug-journaler=20 2026-02-18T08:40:26.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_replay_started cluster1 mirror image_29 2026-02-18T08:40:26.035 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:40:26.035 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:40:26.035 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_29 2026-02-18T08:40:26.035 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_replay_state cluster1 mirror image_29 started 2026-02-18T08:40:26.035 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:40:26.035 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:40:26.035 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_29 2026-02-18T08:40:26.035 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local state=started 2026-02-18T08:40:26.035 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-18T08:40:26.036 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:40:26.036 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1 2026-02-18T08:40:27.035 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test_image_replay_state cluster1 mirror image_29 started 2026-02-18T08:40:27.036 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:40:27.036 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:40:27.036 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_29 2026-02-18T08:40:27.036 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local test_state=started 2026-02-18T08:40:27.036 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local status_result 2026-02-18T08:40:27.036 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_state=stopped 2026-02-18T08:40:27.038 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_29 2026-02-18T08:40:27.038 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -i state 2026-02-18T08:40:27.038 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_instance=cluster1 2026-02-18T08:40:27.038 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-18T08:40:27.038 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:40:27.038 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance=cluster1 2026-02-18T08:40:27.038 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local loop_instance 2026-02-18T08:40:27.038 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:40:27.038 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sleep 0 2026-02-18T08:40:27.038 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:40:27.039 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ seq 0 3 2026-02-18T08:40:27.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:40:27.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_29 2026-02-18T08:40:27.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:0 2026-02-18T08:40:27.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-18T08:40:27.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:40:27.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:40:27.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:40:27.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-18T08:40:27.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:40:27.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:40:27.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:40:27.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:40:27.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-18T08:40:27.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:40:27.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:40:27.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:40:27.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:40:27.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:0 2026-02-18T08:40:27.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-18T08:40:27.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-18T08:40:27.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:40:27.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:0 2026-02-18T08:40:27.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T08:40:27.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-18T08:40:27.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:40:27.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:40:27.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-18T08:40:27.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-18T08:40:27.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-18T08:40:27.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=0 2026-02-18T08:40:27.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=0 2026-02-18T08:40:27.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:40:27.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:40:27.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:40:27.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:40:27.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_29 2026-02-18T08:40:27.112 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_29'] 2026-02-18T08:40:27.112 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status : get status for rbd mirror 2026-02-18T08:40:27.112 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:40:27.114 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:40:27.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_29 2026-02-18T08:40:27.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:1 2026-02-18T08:40:27.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-18T08:40:27.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:40:27.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:40:27.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:40:27.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-18T08:40:27.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:40:27.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:40:27.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:40:27.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:40:27.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-18T08:40:27.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:40:27.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:40:27.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:40:27.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:40:27.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:1 2026-02-18T08:40:27.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-18T08:40:27.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-18T08:40:27.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:40:27.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:1 2026-02-18T08:40:27.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T08:40:27.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-18T08:40:27.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:40:27.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:40:27.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-18T08:40:27.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-18T08:40:27.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-18T08:40:27.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=1 2026-02-18T08:40:27.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=1 2026-02-18T08:40:27.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:40:27.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:40:27.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:40:27.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:40:27.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_29 2026-02-18T08:40:27.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_29'] 2026-02-18T08:40:27.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status : get status for rbd mirror 2026-02-18T08:40:27.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:40:27.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:40:27.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_29 2026-02-18T08:40:27.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:2 2026-02-18T08:40:27.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-18T08:40:27.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:40:27.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:40:27.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:40:27.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-18T08:40:27.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:40:27.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:40:27.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:40:27.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:40:27.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-18T08:40:27.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:40:27.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:40:27.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:40:27.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:40:27.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:2 2026-02-18T08:40:27.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-18T08:40:27.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-18T08:40:27.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:40:27.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:2 2026-02-18T08:40:27.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T08:40:27.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-18T08:40:27.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:40:27.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:40:27.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-18T08:40:27.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-18T08:40:27.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-18T08:40:27.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=2 2026-02-18T08:40:27.187 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=2 2026-02-18T08:40:27.187 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:40:27.195 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:40:27.195 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:40:27.195 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:40:27.196 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_29 2026-02-18T08:40:27.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ return 0 2026-02-18T08:40:27.259 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ status_result=' "state": "Replaying"' 2026-02-18T08:40:27.259 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ echo ' "state": "Replaying"' 2026-02-18T08:40:27.259 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ grep -i Replaying 2026-02-18T08:40:27.260 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: "state": "Replaying" 2026-02-18T08:40:27.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ current_state=started 2026-02-18T08:40:27.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test started = started 2026-02-18T08:40:27.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:40:27.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror image_29 2026-02-18T08:40:27.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_cluster=cluster1 2026-02-18T08:40:27.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:40:27.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_pool=mirror 2026-02-18T08:40:27.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local remote_pool=mirror 2026-02-18T08:40:27.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_29 2026-02-18T08:40:27.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' journal = journal ']' 2026-02-18T08:40:27.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror mirror image_29 2026-02-18T08:40:27.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_cluster=cluster1 2026-02-18T08:40:27.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:40:27.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_pool=mirror 2026-02-18T08:40:27.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local remote_pool=mirror 2026-02-18T08:40:27.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_29 2026-02-18T08:40:27.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-02-18T08:40:27.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-02-18T08:40:27.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:40:27.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:40:27.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:40:27.463 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_29 2026-02-18T08:40:27.463 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:40:27.463 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:40:27.463 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_29 2026-02-18T08:40:27.463 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:40:27.463 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_29 ']' 2026-02-18T08:40:27.463 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_29' 2026-02-18T08:40:27.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_29 2026-02-18T08:40:27.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:40:27.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:40:27.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:40:27.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:40:27.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:40:27.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:40:27.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:40:27.467 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:40:27.467 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:40:27.467 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:40:27.467 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_29 2026-02-18T08:40:27.467 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:40:27.467 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:40:27.467 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:40:27.467 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:40:27.467 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:40:27.467 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:40:27.467 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:40:27.468 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:40:27.468 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:40:27.468 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:40:27.468 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:40:27.468 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:40:27.468 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:40:27.468 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:40:27.468 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:40:27.468 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:40:27.468 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:40:27.468 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:40:27.468 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:40:27.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:40:27.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:40:27.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:40:27.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:40:27.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:40:27.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:40:27.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:40:27.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:40:27.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:40:27.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:40:27.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:40:27.478 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:40:27.479 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:40:27.479 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:40:27.479 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_29 2026-02-18T08:40:27.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_29'] 2026-02-18T08:40:27.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:40:27.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:40:27.540 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:40:27.540 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_29 2026-02-18T08:40:27.540 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:40:27.540 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:40:27.540 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:40:27.540 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:40:27.540 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:40:27.540 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:40:27.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:40:27.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:40:27.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:40:27.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:40:27.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:40:27.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:40:27.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:40:27.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:40:27.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:40:27.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:40:27.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:40:27.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:40:27.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:40:27.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:40:27.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:40:27.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:40:27.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:40:27.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:40:27.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:40:27.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:40:27.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:40:27.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:40:27.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:40:27.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:40:27.551 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:40:27.552 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:40:27.552 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:40:27.552 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_29 2026-02-18T08:40:27.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_29'] 2026-02-18T08:40:27.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:40:27.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:40:27.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:40:27.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_29 2026-02-18T08:40:27.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:40:27.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:40:27.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:40:27.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:40:27.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:40:27.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:40:27.618 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:40:27.618 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:40:27.618 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:40:27.618 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:40:27.618 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:40:27.618 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:40:27.618 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:40:27.618 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:40:27.618 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:40:27.618 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:40:27.618 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:40:27.618 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:40:27.619 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:40:27.619 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:40:27.619 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:40:27.619 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:40:27.619 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:40:27.619 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:40:27.619 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:40:27.619 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:40:27.619 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:40:27.619 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:40:27.619 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:40:27.619 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:40:27.628 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:40:27.629 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:40:27.629 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:40:27.629 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_29 2026-02-18T08:40:27.687 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:40:27.688 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_29 2026-02-18T08:40:27.688 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:40:27.688 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:40:27.688 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_29 2026-02-18T08:40:27.688 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_29 '' 2026-02-18T08:40:27.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:40:27.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:40:27.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_29 2026-02-18T08:40:27.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:40:27.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_29.status 2026-02-18T08:40:27.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_29.status 2026-02-18T08:40:27.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:40:27.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_29.status 2026-02-18T08:40:27.691 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_29 2026-02-18T08:40:27.691 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_29.status 2026-02-18T08:40:27.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 6 2026-02-18T08:40:27.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 6 2026-02-18T08:40:27.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:40:27.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=25, tag_tid=2, entry_tid=1], [object_number=24, tag_tid=2, entry_tid=0], [object_number=27, tag_tid=1, entry_tid=27279], [object_number=26, tag_tid=1, entry_tid=27278]]], state=connected] 2026-02-18T08:40:27.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=25, tag_tid=1, entry_tid=27281], [object_number=24, tag_tid=1, entry_tid=27280], [object_number=27, tag_tid=1, entry_tid=27279], [object_number=26, tag_tid=1, entry_tid=27278]]], state=connected] 2026-02-18T08:40:27.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_29.status 2026-02-18T08:40:27.731 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=25, tag_tid=2, entry_tid=1' 2026-02-18T08:40:27.732 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_29 2026-02-18T08:40:27.732 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:40:27.732 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:40:27.732 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_29 2026-02-18T08:40:27.732 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_29 '..*' 2026-02-18T08:40:27.733 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:40:27.733 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:40:27.733 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_29 2026-02-18T08:40:27.733 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:40:27.733 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_29.status 2026-02-18T08:40:27.733 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_29.status 2026-02-18T08:40:27.733 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:40:27.734 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_29.status 2026-02-18T08:40:27.734 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_29 2026-02-18T08:40:27.735 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_29.status 2026-02-18T08:40:27.768 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 6 2026-02-18T08:40:27.769 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 6 2026-02-18T08:40:27.769 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:40:27.769 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=25, tag_tid=2, entry_tid=1], [object_number=24, tag_tid=2, entry_tid=0], [object_number=27, tag_tid=1, entry_tid=27279], [object_number=26, tag_tid=1, entry_tid=27278]]], state=connected] 2026-02-18T08:40:27.769 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=25, tag_tid=1, entry_tid=27281], [object_number=24, tag_tid=1, entry_tid=27280], [object_number=27, tag_tid=1, entry_tid=27279], [object_number=26, tag_tid=1, entry_tid=27278]]], state=connected] 2026-02-18T08:40:27.772 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_29.status 2026-02-18T08:40:27.774 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=25, tag_tid=1, entry_tid=27281' 2026-02-18T08:40:27.774 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=25, tag_tid=2, entry_tid=1' -a 'object_number=25, tag_tid=2, entry_tid=1' = 'object_number=25, tag_tid=1, entry_tid=27281' 2026-02-18T08:40:27.774 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=25, tag_tid=1, entry_tid=27281' '!=' '' 2026-02-18T08:40:27.774 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:40:27.774 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=25, tag_tid=1, entry_tid=27281' = '' 2026-02-18T08:40:27.774 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=25, tag_tid=1, entry_tid=27281' 2026-02-18T08:40:27.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=25, tag_tid=2, entry_tid=1' 2026-02-18T08:40:27.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:40:27.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:40:27.776 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=2 2026-02-18T08:40:27.777 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=25, tag_tid=1, entry_tid=27281' 2026-02-18T08:40:27.777 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:40:27.777 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:40:27.778 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=1 2026-02-18T08:40:27.779 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=25, tag_tid=2, entry_tid=1' 2026-02-18T08:40:27.779 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:40:27.779 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:40:27.780 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:40:27.781 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=25, tag_tid=1, entry_tid=27281' 2026-02-18T08:40:27.781 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:40:27.781 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:40:27.783 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=27281 2026-02-18T08:40:27.783 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 2 = 1 -a 1 -le 27281 2026-02-18T08:40:27.783 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:40:27.783 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:40:27.783 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:40:27.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_29 2026-02-18T08:40:27.985 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:40:27.985 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:40:27.985 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_29 2026-02-18T08:40:27.985 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:40:27.985 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_29 ']' 2026-02-18T08:40:27.985 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_29' 2026-02-18T08:40:27.985 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_29 2026-02-18T08:40:27.985 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:40:27.985 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:40:27.985 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:40:27.985 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:40:27.985 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:40:27.986 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:40:27.986 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:40:27.988 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:40:27.988 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:40:27.988 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:40:27.988 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_29 2026-02-18T08:40:27.989 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:40:27.989 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:40:27.989 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:40:27.989 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:40:27.989 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:40:27.989 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:40:27.989 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:40:27.989 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:40:27.989 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:40:27.989 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:40:27.989 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:40:27.989 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:40:27.990 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:40:27.990 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:40:27.990 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:40:27.990 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:40:27.990 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:40:27.990 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:40:27.990 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:40:27.990 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:40:27.990 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:40:27.990 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:40:27.990 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:40:27.990 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:40:27.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:40:27.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:40:27.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:40:27.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:40:27.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:40:27.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:40:27.999 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:40:27.999 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:40:28.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:40:28.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_29 2026-02-18T08:40:28.058 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_29'] 2026-02-18T08:40:28.059 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:40:28.059 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:40:28.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:40:28.062 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_29 2026-02-18T08:40:28.062 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:40:28.062 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:40:28.062 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:40:28.062 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:40:28.062 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:40:28.062 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:40:28.062 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:40:28.062 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:40:28.062 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:40:28.062 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:40:28.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:40:28.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:40:28.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:40:28.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:40:28.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:40:28.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:40:28.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:40:28.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:40:28.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:40:28.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:40:28.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:40:28.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:40:28.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:40:28.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:40:28.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:40:28.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:40:28.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:40:28.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:40:28.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:40:28.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:40:28.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:40:28.074 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:40:28.074 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:40:28.074 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_29 2026-02-18T08:40:28.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_29'] 2026-02-18T08:40:28.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:40:28.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:40:28.131 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:40:28.131 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_29 2026-02-18T08:40:28.131 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:40:28.131 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:40:28.131 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:40:28.131 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:40:28.131 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:40:28.131 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:40:28.131 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:40:28.132 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:40:28.132 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:40:28.132 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:40:28.132 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:40:28.132 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:40:28.132 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:40:28.132 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:40:28.132 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:40:28.132 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:40:28.132 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:40:28.132 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:40:28.132 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:40:28.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:40:28.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:40:28.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:40:28.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:40:28.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:40:28.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:40:28.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:40:28.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:40:28.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:40:28.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:40:28.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:40:28.142 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:40:28.143 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:40:28.143 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:40:28.143 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_29 2026-02-18T08:40:28.201 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:40:28.202 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_29 2026-02-18T08:40:28.203 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:40:28.203 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:40:28.203 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_29 2026-02-18T08:40:28.203 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_29 '' 2026-02-18T08:40:28.203 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:40:28.203 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:40:28.203 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_29 2026-02-18T08:40:28.203 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:40:28.203 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_29.status 2026-02-18T08:40:28.204 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_29.status 2026-02-18T08:40:28.204 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:40:28.204 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_29.status 2026-02-18T08:40:28.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_29 2026-02-18T08:40:28.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_29.status 2026-02-18T08:40:28.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 6 2026-02-18T08:40:28.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 6 2026-02-18T08:40:28.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:40:28.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=25, tag_tid=2, entry_tid=1], [object_number=24, tag_tid=2, entry_tid=0], [object_number=27, tag_tid=1, entry_tid=27279], [object_number=26, tag_tid=1, entry_tid=27278]]], state=connected] 2026-02-18T08:40:28.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=25, tag_tid=1, entry_tid=27281], [object_number=24, tag_tid=1, entry_tid=27280], [object_number=27, tag_tid=1, entry_tid=27279], [object_number=26, tag_tid=1, entry_tid=27278]]], state=connected] 2026-02-18T08:40:28.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_29.status 2026-02-18T08:40:28.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=25, tag_tid=2, entry_tid=1' 2026-02-18T08:40:28.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_29 2026-02-18T08:40:28.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:40:28.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:40:28.238 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_29 2026-02-18T08:40:28.238 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_29 '..*' 2026-02-18T08:40:28.238 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:40:28.238 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:40:28.238 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_29 2026-02-18T08:40:28.238 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:40:28.238 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_29.status 2026-02-18T08:40:28.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_29.status 2026-02-18T08:40:28.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:40:28.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_29.status 2026-02-18T08:40:28.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_29 2026-02-18T08:40:28.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_29.status 2026-02-18T08:40:28.266 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 6 2026-02-18T08:40:28.266 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 6 2026-02-18T08:40:28.266 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:40:28.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=25, tag_tid=2, entry_tid=1], [object_number=24, tag_tid=2, entry_tid=0], [object_number=27, tag_tid=1, entry_tid=27279], [object_number=26, tag_tid=1, entry_tid=27278]]], state=connected] 2026-02-18T08:40:28.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=25, tag_tid=1, entry_tid=27281], [object_number=24, tag_tid=1, entry_tid=27280], [object_number=27, tag_tid=1, entry_tid=27279], [object_number=26, tag_tid=1, entry_tid=27278]]], state=connected] 2026-02-18T08:40:28.269 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_29.status 2026-02-18T08:40:28.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=25, tag_tid=1, entry_tid=27281' 2026-02-18T08:40:28.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=25, tag_tid=2, entry_tid=1' -a 'object_number=25, tag_tid=2, entry_tid=1' = 'object_number=25, tag_tid=1, entry_tid=27281' 2026-02-18T08:40:28.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=25, tag_tid=1, entry_tid=27281' '!=' 'object_number=25, tag_tid=1, entry_tid=27281' 2026-02-18T08:40:28.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:40:28.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.4 2026-02-18T08:40:28.673 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_29 2026-02-18T08:40:28.673 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:40:28.673 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:40:28.673 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_29 2026-02-18T08:40:28.673 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:40:28.673 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_29 ']' 2026-02-18T08:40:28.673 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_29' 2026-02-18T08:40:28.674 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_29 2026-02-18T08:40:28.674 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:40:28.674 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:40:28.674 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:40:28.674 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:40:28.674 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:40:28.674 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:40:28.674 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:40:28.677 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:40:28.677 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:40:28.677 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:40:28.677 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_29 2026-02-18T08:40:28.677 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:40:28.677 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:40:28.677 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:40:28.677 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:40:28.677 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:40:28.678 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:40:28.678 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:40:28.678 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:40:28.678 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:40:28.678 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:40:28.678 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:40:28.678 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:40:28.678 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:40:28.678 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:40:28.678 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:40:28.678 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:40:28.678 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:40:28.678 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:40:28.679 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:40:28.679 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:40:28.679 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:40:28.679 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:40:28.679 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:40:28.679 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:40:28.679 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:40:28.679 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:40:28.679 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:40:28.679 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:40:28.679 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:40:28.679 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:40:28.688 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:40:28.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:40:28.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:40:28.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_29 2026-02-18T08:40:28.739 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_29'] 2026-02-18T08:40:28.740 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:40:28.740 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:40:28.742 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:40:28.742 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_29 2026-02-18T08:40:28.743 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:40:28.743 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:40:28.743 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:40:28.743 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:40:28.743 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:40:28.743 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:40:28.743 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:40:28.743 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:40:28.743 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:40:28.743 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:40:28.743 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:40:28.743 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:40:28.744 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:40:28.744 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:40:28.744 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:40:28.744 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:40:28.744 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:40:28.744 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:40:28.744 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:40:28.744 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:40:28.744 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:40:28.744 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:40:28.744 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:40:28.744 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:40:28.745 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:40:28.745 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:40:28.745 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:40:28.745 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:40:28.745 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:40:28.745 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:40:28.754 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:40:28.755 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:40:28.755 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:40:28.755 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_29 2026-02-18T08:40:28.809 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_29'] 2026-02-18T08:40:28.810 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:40:28.810 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:40:28.812 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:40:28.813 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_29 2026-02-18T08:40:28.813 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:40:28.813 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:40:28.813 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:40:28.813 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:40:28.813 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:40:28.813 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:40:28.813 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:40:28.813 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:40:28.813 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:40:28.814 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:40:28.814 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:40:28.814 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:40:28.814 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:40:28.814 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:40:28.814 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:40:28.814 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:40:28.814 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:40:28.814 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:40:28.814 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:40:28.814 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:40:28.814 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:40:28.815 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:40:28.815 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:40:28.815 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:40:28.815 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:40:28.815 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:40:28.815 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:40:28.815 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:40:28.815 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:40:28.815 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:40:28.823 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:40:28.823 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:40:28.823 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:40:28.823 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_29 2026-02-18T08:40:28.882 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:40:28.883 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_29 2026-02-18T08:40:28.883 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:40:28.883 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:40:28.883 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_29 2026-02-18T08:40:28.883 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_29 '' 2026-02-18T08:40:28.883 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:40:28.883 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:40:28.883 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_29 2026-02-18T08:40:28.883 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:40:28.884 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_29.status 2026-02-18T08:40:28.884 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_29.status 2026-02-18T08:40:28.884 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:40:28.884 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_29.status 2026-02-18T08:40:28.885 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_29 2026-02-18T08:40:28.885 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_29.status 2026-02-18T08:40:28.912 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 6 2026-02-18T08:40:28.912 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 6 2026-02-18T08:40:28.912 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:40:28.912 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=25, tag_tid=2, entry_tid=1], [object_number=24, tag_tid=2, entry_tid=0], [object_number=27, tag_tid=1, entry_tid=27279], [object_number=26, tag_tid=1, entry_tid=27278]]], state=connected] 2026-02-18T08:40:28.913 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=25, tag_tid=1, entry_tid=27281], [object_number=24, tag_tid=1, entry_tid=27280], [object_number=27, tag_tid=1, entry_tid=27279], [object_number=26, tag_tid=1, entry_tid=27278]]], state=connected] 2026-02-18T08:40:28.915 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_29.status 2026-02-18T08:40:28.916 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=25, tag_tid=2, entry_tid=1' 2026-02-18T08:40:28.917 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_29 2026-02-18T08:40:28.917 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:40:28.918 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:40:28.918 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_29 2026-02-18T08:40:28.918 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_29 '..*' 2026-02-18T08:40:28.918 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:40:28.918 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:40:28.918 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_29 2026-02-18T08:40:28.918 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:40:28.918 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_29.status 2026-02-18T08:40:28.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_29.status 2026-02-18T08:40:28.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:40:28.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_29.status 2026-02-18T08:40:28.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_29 2026-02-18T08:40:28.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_29.status 2026-02-18T08:40:28.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 6 2026-02-18T08:40:28.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 6 2026-02-18T08:40:28.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:40:28.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=25, tag_tid=2, entry_tid=1], [object_number=24, tag_tid=2, entry_tid=0], [object_number=27, tag_tid=1, entry_tid=27279], [object_number=26, tag_tid=1, entry_tid=27278]]], state=connected] 2026-02-18T08:40:28.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=25, tag_tid=1, entry_tid=27281], [object_number=24, tag_tid=1, entry_tid=27280], [object_number=27, tag_tid=1, entry_tid=27279], [object_number=26, tag_tid=1, entry_tid=27278]]], state=connected] 2026-02-18T08:40:28.947 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_29.status 2026-02-18T08:40:28.949 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=25, tag_tid=1, entry_tid=27281' 2026-02-18T08:40:28.949 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=25, tag_tid=2, entry_tid=1' -a 'object_number=25, tag_tid=2, entry_tid=1' = 'object_number=25, tag_tid=1, entry_tid=27281' 2026-02-18T08:40:28.949 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=25, tag_tid=1, entry_tid=27281' '!=' 'object_number=25, tag_tid=1, entry_tid=27281' 2026-02-18T08:40:28.950 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:40:28.950 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.8 2026-02-18T08:40:29.751 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_29 2026-02-18T08:40:29.752 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:40:29.752 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:40:29.752 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_29 2026-02-18T08:40:29.752 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:40:29.752 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_29 ']' 2026-02-18T08:40:29.752 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_29' 2026-02-18T08:40:29.752 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_29 2026-02-18T08:40:29.752 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:40:29.752 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:40:29.753 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:40:29.753 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:40:29.753 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:40:29.753 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:40:29.753 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:40:29.753 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:40:29.756 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:40:29.756 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:40:29.756 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_29 2026-02-18T08:40:29.756 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:40:29.756 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:40:29.756 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:40:29.756 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:40:29.756 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:40:29.756 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:40:29.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:40:29.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:40:29.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:40:29.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:40:29.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:40:29.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:40:29.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:40:29.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:40:29.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:40:29.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:40:29.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:40:29.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:40:29.758 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:40:29.758 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:40:29.758 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:40:29.758 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:40:29.758 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:40:29.758 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:40:29.758 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:40:29.758 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:40:29.758 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:40:29.758 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:40:29.758 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:40:29.758 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:40:29.767 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:40:29.767 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:40:29.768 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:40:29.768 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_29 2026-02-18T08:40:29.818 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_29'] 2026-02-18T08:40:29.818 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:40:29.818 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:40:29.821 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:40:29.821 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_29 2026-02-18T08:40:29.821 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:40:29.821 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:40:29.821 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:40:29.822 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:40:29.822 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:40:29.822 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:40:29.822 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:40:29.822 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:40:29.822 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:40:29.822 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:40:29.822 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:40:29.822 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:40:29.822 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:40:29.822 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:40:29.822 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:40:29.822 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:40:29.823 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:40:29.823 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:40:29.823 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:40:29.823 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:40:29.823 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:40:29.823 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:40:29.823 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:40:29.823 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:40:29.823 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:40:29.823 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:40:29.823 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:40:29.823 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:40:29.824 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:40:29.824 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:40:29.831 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:40:29.832 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:40:29.832 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:40:29.832 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_29 2026-02-18T08:40:29.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_29'] 2026-02-18T08:40:29.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:40:29.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:40:29.894 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:40:29.894 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_29 2026-02-18T08:40:29.894 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:40:29.894 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:40:29.895 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:40:29.895 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:40:29.895 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:40:29.895 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:40:29.895 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:40:29.895 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:40:29.895 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:40:29.895 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:40:29.895 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:40:29.895 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:40:29.895 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:40:29.895 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:40:29.896 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:40:29.896 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:40:29.896 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:40:29.896 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:40:29.896 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:40:29.896 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:40:29.896 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:40:29.896 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:40:29.896 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:40:29.896 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:40:29.896 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:40:29.896 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:40:29.896 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:40:29.897 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:40:29.897 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:40:29.897 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:40:29.905 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:40:29.905 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:40:29.905 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:40:29.905 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_29 2026-02-18T08:40:29.965 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:40:29.966 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_29 2026-02-18T08:40:29.966 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:40:29.966 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:40:29.966 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_29 2026-02-18T08:40:29.966 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_29 '' 2026-02-18T08:40:29.966 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:40:29.966 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:40:29.966 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_29 2026-02-18T08:40:29.967 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:40:29.967 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_29.status 2026-02-18T08:40:29.967 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_29.status 2026-02-18T08:40:29.967 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:40:29.967 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_29.status 2026-02-18T08:40:29.968 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_29 2026-02-18T08:40:29.968 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_29.status 2026-02-18T08:40:30.005 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 6 2026-02-18T08:40:30.006 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 6 2026-02-18T08:40:30.006 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:40:30.006 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=25, tag_tid=2, entry_tid=1], [object_number=24, tag_tid=2, entry_tid=0], [object_number=27, tag_tid=1, entry_tid=27279], [object_number=26, tag_tid=1, entry_tid=27278]]], state=connected] 2026-02-18T08:40:30.006 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=25, tag_tid=1, entry_tid=27281], [object_number=24, tag_tid=1, entry_tid=27280], [object_number=27, tag_tid=1, entry_tid=27279], [object_number=26, tag_tid=1, entry_tid=27278]]], state=connected] 2026-02-18T08:40:30.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_29.status 2026-02-18T08:40:30.012 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=25, tag_tid=2, entry_tid=1' 2026-02-18T08:40:30.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_29 2026-02-18T08:40:30.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:40:30.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:40:30.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_29 2026-02-18T08:40:30.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_29 '..*' 2026-02-18T08:40:30.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:40:30.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:40:30.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_29 2026-02-18T08:40:30.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:40:30.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_29.status 2026-02-18T08:40:30.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_29.status 2026-02-18T08:40:30.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:40:30.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_29.status 2026-02-18T08:40:30.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_29 2026-02-18T08:40:30.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_29.status 2026-02-18T08:40:30.041 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 6 2026-02-18T08:40:30.041 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 6 2026-02-18T08:40:30.041 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:40:30.041 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=25, tag_tid=2, entry_tid=1], [object_number=24, tag_tid=2, entry_tid=0], [object_number=27, tag_tid=1, entry_tid=27279], [object_number=26, tag_tid=1, entry_tid=27278]]], state=connected] 2026-02-18T08:40:30.041 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=25, tag_tid=1, entry_tid=27281], [object_number=24, tag_tid=1, entry_tid=27280], [object_number=27, tag_tid=1, entry_tid=27279], [object_number=26, tag_tid=1, entry_tid=27278]]], state=connected] 2026-02-18T08:40:30.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_29.status 2026-02-18T08:40:30.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=25, tag_tid=1, entry_tid=27281' 2026-02-18T08:40:30.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=25, tag_tid=2, entry_tid=1' -a 'object_number=25, tag_tid=2, entry_tid=1' = 'object_number=25, tag_tid=1, entry_tid=27281' 2026-02-18T08:40:30.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=25, tag_tid=1, entry_tid=27281' '!=' 'object_number=25, tag_tid=1, entry_tid=27281' 2026-02-18T08:40:30.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:40:30.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1.6 2026-02-18T08:40:31.647 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_29 2026-02-18T08:40:31.648 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:40:31.648 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:40:31.648 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_29 2026-02-18T08:40:31.648 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:40:31.648 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_29 ']' 2026-02-18T08:40:31.648 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_29' 2026-02-18T08:40:31.648 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_29 2026-02-18T08:40:31.648 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:40:31.648 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:40:31.648 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:40:31.649 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:40:31.649 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:40:31.649 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:40:31.649 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:40:31.649 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:40:31.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:40:31.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:40:31.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_29 2026-02-18T08:40:31.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:40:31.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:40:31.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:40:31.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:40:31.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:40:31.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:40:31.653 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:40:31.653 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:40:31.653 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:40:31.653 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:40:31.653 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:40:31.653 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:40:31.653 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:40:31.653 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:40:31.653 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:40:31.653 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:40:31.653 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:40:31.653 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:40:31.654 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:40:31.654 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:40:31.654 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:40:31.654 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:40:31.654 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:40:31.654 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:40:31.654 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:40:31.654 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:40:31.654 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:40:31.654 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:40:31.654 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:40:31.654 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:40:31.663 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:40:31.663 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:40:31.664 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:40:31.664 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_29 2026-02-18T08:40:31.721 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_29'] 2026-02-18T08:40:31.721 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:40:31.721 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:40:31.724 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:40:31.724 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_29 2026-02-18T08:40:31.724 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:40:31.724 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:40:31.724 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:40:31.724 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:40:31.724 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:40:31.724 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:40:31.725 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:40:31.725 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:40:31.725 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:40:31.725 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:40:31.725 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:40:31.725 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:40:31.725 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:40:31.725 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:40:31.725 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:40:31.725 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:40:31.725 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:40:31.725 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:40:31.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:40:31.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:40:31.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:40:31.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:40:31.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:40:31.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:40:31.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:40:31.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:40:31.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:40:31.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:40:31.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:40:31.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:40:31.734 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:40:31.735 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:40:31.735 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:40:31.735 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_29 2026-02-18T08:40:31.794 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_29'] 2026-02-18T08:40:31.795 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:40:31.795 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:40:31.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:40:31.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_29 2026-02-18T08:40:31.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:40:31.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:40:31.798 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:40:31.798 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:40:31.798 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:40:31.798 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:40:31.798 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:40:31.798 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:40:31.798 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:40:31.798 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:40:31.798 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:40:31.798 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:40:31.798 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:40:31.798 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:40:31.798 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:40:31.799 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:40:31.799 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:40:31.799 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:40:31.799 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:40:31.799 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:40:31.799 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:40:31.799 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:40:31.799 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:40:31.799 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:40:31.799 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:40:31.799 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:40:31.799 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:40:31.800 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:40:31.800 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:40:31.800 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:40:31.807 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:40:31.807 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:40:31.808 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:40:31.808 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_29 2026-02-18T08:40:31.877 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:40:31.878 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_29 2026-02-18T08:40:31.879 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:40:31.879 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:40:31.879 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_29 2026-02-18T08:40:31.879 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_29 '' 2026-02-18T08:40:31.879 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:40:31.879 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:40:31.879 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_29 2026-02-18T08:40:31.879 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:40:31.879 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_29.status 2026-02-18T08:40:31.879 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_29.status 2026-02-18T08:40:31.879 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:40:31.880 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_29.status 2026-02-18T08:40:31.880 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_29 2026-02-18T08:40:31.881 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_29.status 2026-02-18T08:40:31.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 6 2026-02-18T08:40:31.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 6 2026-02-18T08:40:31.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:40:31.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=25, tag_tid=2, entry_tid=1], [object_number=24, tag_tid=2, entry_tid=0], [object_number=27, tag_tid=1, entry_tid=27279], [object_number=26, tag_tid=1, entry_tid=27278]]], state=connected] 2026-02-18T08:40:31.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=25, tag_tid=2, entry_tid=1], [object_number=24, tag_tid=2, entry_tid=0], [object_number=27, tag_tid=1, entry_tid=27279], [object_number=26, tag_tid=1, entry_tid=27278]]], state=connected] 2026-02-18T08:40:31.910 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_29.status 2026-02-18T08:40:31.912 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=25, tag_tid=2, entry_tid=1' 2026-02-18T08:40:31.913 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_29 2026-02-18T08:40:31.913 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:40:31.913 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:40:31.914 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_29 2026-02-18T08:40:31.914 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_29 '..*' 2026-02-18T08:40:31.914 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:40:31.914 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:40:31.914 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_29 2026-02-18T08:40:31.914 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:40:31.914 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_29.status 2026-02-18T08:40:31.914 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_29.status 2026-02-18T08:40:31.914 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:40:31.915 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_29.status 2026-02-18T08:40:31.915 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_29 2026-02-18T08:40:31.915 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_29.status 2026-02-18T08:40:32.155 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 6 2026-02-18T08:40:32.155 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 6 2026-02-18T08:40:32.155 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:40:32.155 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=25, tag_tid=2, entry_tid=1], [object_number=24, tag_tid=2, entry_tid=0], [object_number=27, tag_tid=1, entry_tid=27279], [object_number=26, tag_tid=1, entry_tid=27278]]], state=connected] 2026-02-18T08:40:32.155 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=25, tag_tid=2, entry_tid=1], [object_number=24, tag_tid=2, entry_tid=0], [object_number=27, tag_tid=1, entry_tid=27279], [object_number=26, tag_tid=1, entry_tid=27278]]], state=connected] 2026-02-18T08:40:32.158 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_29.status 2026-02-18T08:40:32.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=25, tag_tid=2, entry_tid=1' 2026-02-18T08:40:32.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=25, tag_tid=2, entry_tid=1' -a 'object_number=25, tag_tid=2, entry_tid=1' = 'object_number=25, tag_tid=2, entry_tid=1' 2026-02-18T08:40:32.161 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:40:32.161 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_snap_present cluster1 mirror image_29 snap 2026-02-18T08:40:32.161 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:40:32.161 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:40:32.161 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_29 2026-02-18T08:40:32.161 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap 2026-02-18T08:40:32.161 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-18T08:40:32.161 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-18T08:40:32.161 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1 2026-02-18T08:40:33.161 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster1 info mirror/image_29@snap 2026-02-18T08:40:33.190 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:rbd image 'image_29': 2026-02-18T08:40:33.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: size 128 MiB in 32 objects 2026-02-18T08:40:33.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: order 22 (4 MiB objects) 2026-02-18T08:40:33.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: snapshot_count: 1 2026-02-18T08:40:33.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: id: 10ed82764e04 2026-02-18T08:40:33.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: block_name_prefix: rbd_data.10ed82764e04 2026-02-18T08:40:33.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: format: 2 2026-02-18T08:40:33.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: features: layering, exclusive-lock, journaling 2026-02-18T08:40:33.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: op_features: 2026-02-18T08:40:33.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: flags: 2026-02-18T08:40:33.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: create_timestamp: Wed Feb 18 08:30:45 2026 2026-02-18T08:40:33.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: access_timestamp: Wed Feb 18 08:30:45 2026 2026-02-18T08:40:33.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: modify_timestamp: Wed Feb 18 08:31:46 2026 2026-02-18T08:40:33.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: protected: False 2026-02-18T08:40:33.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: journal: 10ed82764e04 2026-02-18T08:40:33.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring state: enabled 2026-02-18T08:40:33.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring mode: journal 2026-02-18T08:40:33.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring global id: 03402811-042f-4703-8d1d-9ff4cb1b7926 2026-02-18T08:40:33.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring primary: false 2026-02-18T08:40:33.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:40:33.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ compare_image_snaps mirror image_29 snap 2026-02-18T08:40:33.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:40:33.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_29 2026-02-18T08:40:33.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap 2026-02-18T08:40:33.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local ret=0 2026-02-18T08:40:33.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local rmt_export=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_29.export 2026-02-18T08:40:33.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loc_export=/tmp/tmp.vybh1m9yJF/cluster1-mirror-image_29.export 2026-02-18T08:40:33.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_29.export /tmp/tmp.vybh1m9yJF/cluster1-mirror-image_29.export 2026-02-18T08:40:33.194 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 -p mirror export image_29@snap /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_29.export 2026-02-18T08:40:33.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-18T08:40:33.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster1 -p mirror export image_29@snap /tmp/tmp.vybh1m9yJF/cluster1-mirror-image_29.export 2026-02-18T08:40:33.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-18T08:40:33.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmp /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_29.export /tmp/tmp.vybh1m9yJF/cluster1-mirror-image_29.export 2026-02-18T08:40:33.598 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_29.export /tmp/tmp.vybh1m9yJF/cluster1-mirror-image_29.export 2026-02-18T08:40:33.642 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:40:33.642 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T08:40:33.642 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_30 2026-02-18T08:40:33.642 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_snap cluster2 mirror image_30 snap 2026-02-18T08:40:33.643 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:40:33.643 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:40:33.643 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_30 2026-02-18T08:40:33.643 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap 2026-02-18T08:40:33.643 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 -p mirror snap create image_30@snap --debug-rbd=20 --debug-journaler=20 2026-02-18T08:40:36.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_replay_started cluster1 mirror image_30 2026-02-18T08:40:36.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:40:36.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:40:36.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_30 2026-02-18T08:40:36.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_replay_state cluster1 mirror image_30 started 2026-02-18T08:40:36.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:40:36.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:40:36.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_30 2026-02-18T08:40:36.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local state=started 2026-02-18T08:40:36.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-18T08:40:36.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:40:36.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1 2026-02-18T08:40:37.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test_image_replay_state cluster1 mirror image_30 started 2026-02-18T08:40:37.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:40:37.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:40:37.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_30 2026-02-18T08:40:37.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local test_state=started 2026-02-18T08:40:37.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local status_result 2026-02-18T08:40:37.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_state=stopped 2026-02-18T08:40:37.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_30 2026-02-18T08:40:37.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -i state 2026-02-18T08:40:37.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_instance=cluster1 2026-02-18T08:40:37.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-18T08:40:37.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:40:37.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance=cluster1 2026-02-18T08:40:37.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local loop_instance 2026-02-18T08:40:37.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:40:37.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sleep 0 2026-02-18T08:40:37.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:40:37.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ seq 0 3 2026-02-18T08:40:37.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:40:37.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_30 2026-02-18T08:40:37.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:0 2026-02-18T08:40:37.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-18T08:40:37.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:40:37.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:40:37.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:40:37.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-18T08:40:37.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:40:37.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:40:37.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:40:37.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:40:37.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-18T08:40:37.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:40:37.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:40:37.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:40:37.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:40:37.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:0 2026-02-18T08:40:37.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-18T08:40:37.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-18T08:40:37.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:40:37.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:0 2026-02-18T08:40:37.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T08:40:37.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-18T08:40:37.055 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:40:37.055 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:40:37.055 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-18T08:40:37.055 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-18T08:40:37.055 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-18T08:40:37.055 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=0 2026-02-18T08:40:37.055 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=0 2026-02-18T08:40:37.055 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:40:37.062 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:40:37.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:40:37.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:40:37.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_30 2026-02-18T08:40:37.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_30'] 2026-02-18T08:40:37.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status : get status for rbd mirror 2026-02-18T08:40:37.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:40:37.118 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:40:37.118 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_30 2026-02-18T08:40:37.118 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:1 2026-02-18T08:40:37.118 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-18T08:40:37.118 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:40:37.118 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:40:37.118 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:40:37.119 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-18T08:40:37.119 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:40:37.119 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:40:37.119 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:40:37.119 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:40:37.119 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-18T08:40:37.119 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:40:37.119 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:40:37.119 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:40:37.119 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:40:37.119 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:1 2026-02-18T08:40:37.119 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-18T08:40:37.120 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-18T08:40:37.120 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:40:37.120 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:1 2026-02-18T08:40:37.120 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T08:40:37.120 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-18T08:40:37.120 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:40:37.120 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:40:37.120 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-18T08:40:37.120 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-18T08:40:37.120 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-18T08:40:37.120 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=1 2026-02-18T08:40:37.120 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=1 2026-02-18T08:40:37.121 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:40:37.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:40:37.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:40:37.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:40:37.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_30 2026-02-18T08:40:37.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_30'] 2026-02-18T08:40:37.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status : get status for rbd mirror 2026-02-18T08:40:37.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:40:37.194 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:40:37.194 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_30 2026-02-18T08:40:37.195 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:2 2026-02-18T08:40:37.195 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-18T08:40:37.195 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:40:37.195 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:40:37.195 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:40:37.195 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-18T08:40:37.195 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:40:37.195 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:40:37.195 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:40:37.195 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:40:37.196 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-18T08:40:37.196 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:40:37.196 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:40:37.196 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:40:37.196 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:40:37.196 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:2 2026-02-18T08:40:37.196 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-18T08:40:37.196 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-18T08:40:37.196 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:40:37.196 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:2 2026-02-18T08:40:37.197 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T08:40:37.197 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-18T08:40:37.197 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:40:37.197 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:40:37.197 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-18T08:40:37.197 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-18T08:40:37.197 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-18T08:40:37.197 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=2 2026-02-18T08:40:37.197 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=2 2026-02-18T08:40:37.197 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:40:37.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:40:37.206 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:40:37.206 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:40:37.206 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_30 2026-02-18T08:40:37.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ return 0 2026-02-18T08:40:37.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ status_result=' "state": "Replaying"' 2026-02-18T08:40:37.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ echo ' "state": "Replaying"' 2026-02-18T08:40:37.269 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ grep -i Replaying 2026-02-18T08:40:37.269 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: "state": "Replaying" 2026-02-18T08:40:37.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ current_state=started 2026-02-18T08:40:37.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test started = started 2026-02-18T08:40:37.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:40:37.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror image_30 2026-02-18T08:40:37.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_cluster=cluster1 2026-02-18T08:40:37.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:40:37.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_pool=mirror 2026-02-18T08:40:37.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local remote_pool=mirror 2026-02-18T08:40:37.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_30 2026-02-18T08:40:37.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' journal = journal ']' 2026-02-18T08:40:37.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror mirror image_30 2026-02-18T08:40:37.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_cluster=cluster1 2026-02-18T08:40:37.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:40:37.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_pool=mirror 2026-02-18T08:40:37.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local remote_pool=mirror 2026-02-18T08:40:37.273 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_30 2026-02-18T08:40:37.273 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-02-18T08:40:37.273 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-02-18T08:40:37.273 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:40:37.273 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:40:37.273 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:40:37.472 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_30 2026-02-18T08:40:37.472 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:40:37.472 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:40:37.472 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_30 2026-02-18T08:40:37.473 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:40:37.473 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_30 ']' 2026-02-18T08:40:37.473 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_30' 2026-02-18T08:40:37.473 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_30 2026-02-18T08:40:37.473 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:40:37.473 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:40:37.473 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:40:37.473 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:40:37.473 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:40:37.474 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:40:37.474 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:40:37.474 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:40:37.474 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:40:37.477 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:40:37.477 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_30 2026-02-18T08:40:37.477 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:40:37.477 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:40:37.477 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:40:37.477 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:40:37.477 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:40:37.477 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:40:37.478 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:40:37.478 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:40:37.478 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:40:37.478 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:40:37.478 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:40:37.478 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:40:37.478 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:40:37.478 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:40:37.478 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:40:37.479 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:40:37.479 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:40:37.479 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:40:37.479 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:40:37.479 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:40:37.479 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:40:37.479 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:40:37.479 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:40:37.479 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:40:37.479 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:40:37.479 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:40:37.479 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:40:37.480 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:40:37.480 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:40:37.480 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:40:37.486 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:40:37.487 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:40:37.487 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:40:37.487 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_30 2026-02-18T08:40:37.545 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_30'] 2026-02-18T08:40:37.545 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:40:37.545 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:40:37.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:40:37.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_30 2026-02-18T08:40:37.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:40:37.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:40:37.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:40:37.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:40:37.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:40:37.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:40:37.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:40:37.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:40:37.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:40:37.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:40:37.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:40:37.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:40:37.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:40:37.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:40:37.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:40:37.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:40:37.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:40:37.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:40:37.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:40:37.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:40:37.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:40:37.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:40:37.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:40:37.551 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:40:37.551 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:40:37.551 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:40:37.551 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:40:37.551 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:40:37.551 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:40:37.551 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:40:37.559 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:40:37.560 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:40:37.560 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:40:37.560 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_30 2026-02-18T08:40:37.612 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_30'] 2026-02-18T08:40:37.612 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:40:37.612 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:40:37.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:40:37.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_30 2026-02-18T08:40:37.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:40:37.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:40:37.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:40:37.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:40:37.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:40:37.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:40:37.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:40:37.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:40:37.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:40:37.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:40:37.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:40:37.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:40:37.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:40:37.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:40:37.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:40:37.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:40:37.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:40:37.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:40:37.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:40:37.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:40:37.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:40:37.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:40:37.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:40:37.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:40:37.618 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:40:37.618 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:40:37.618 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:40:37.618 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:40:37.618 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:40:37.618 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:40:37.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:40:37.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:40:37.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:40:37.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_30 2026-02-18T08:40:37.693 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:40:37.694 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_30 2026-02-18T08:40:37.694 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:40:37.694 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:40:37.694 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_30 2026-02-18T08:40:37.695 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_30 '' 2026-02-18T08:40:37.695 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:40:37.695 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:40:37.695 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_30 2026-02-18T08:40:37.695 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:40:37.695 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_30.status 2026-02-18T08:40:37.695 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_30.status 2026-02-18T08:40:37.695 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:40:37.696 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_30.status 2026-02-18T08:40:37.696 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_30 2026-02-18T08:40:37.696 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_30.status 2026-02-18T08:40:37.733 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 9 2026-02-18T08:40:37.733 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 9 2026-02-18T08:40:37.733 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:40:37.734 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=37, tag_tid=2, entry_tid=1], [object_number=36, tag_tid=2, entry_tid=0], [object_number=39, tag_tid=1, entry_tid=39135], [object_number=38, tag_tid=1, entry_tid=39134]]], state=connected] 2026-02-18T08:40:37.734 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=39, tag_tid=1, entry_tid=39135], [object_number=38, tag_tid=1, entry_tid=39134], [object_number=37, tag_tid=1, entry_tid=39133], [object_number=36, tag_tid=1, entry_tid=39132]]], state=connected] 2026-02-18T08:40:37.736 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_30.status 2026-02-18T08:40:37.738 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=37, tag_tid=2, entry_tid=1' 2026-02-18T08:40:37.739 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_30 2026-02-18T08:40:37.739 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:40:37.739 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:40:37.740 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_30 2026-02-18T08:40:37.740 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_30 '..*' 2026-02-18T08:40:37.740 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:40:37.740 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:40:37.740 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_30 2026-02-18T08:40:37.740 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:40:37.740 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_30.status 2026-02-18T08:40:37.741 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_30.status 2026-02-18T08:40:37.741 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:40:37.742 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_30.status 2026-02-18T08:40:37.742 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_30 2026-02-18T08:40:37.742 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_30.status 2026-02-18T08:40:37.768 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 9 2026-02-18T08:40:37.768 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 9 2026-02-18T08:40:37.768 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:40:37.768 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=37, tag_tid=2, entry_tid=1], [object_number=36, tag_tid=2, entry_tid=0], [object_number=39, tag_tid=1, entry_tid=39135], [object_number=38, tag_tid=1, entry_tid=39134]]], state=connected] 2026-02-18T08:40:37.768 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=39, tag_tid=1, entry_tid=39135], [object_number=38, tag_tid=1, entry_tid=39134], [object_number=37, tag_tid=1, entry_tid=39133], [object_number=36, tag_tid=1, entry_tid=39132]]], state=connected] 2026-02-18T08:40:37.770 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_30.status 2026-02-18T08:40:37.772 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=39, tag_tid=1, entry_tid=39135' 2026-02-18T08:40:37.773 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=37, tag_tid=2, entry_tid=1' -a 'object_number=37, tag_tid=2, entry_tid=1' = 'object_number=39, tag_tid=1, entry_tid=39135' 2026-02-18T08:40:37.773 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=39, tag_tid=1, entry_tid=39135' '!=' '' 2026-02-18T08:40:37.773 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:40:37.773 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=39, tag_tid=1, entry_tid=39135' = '' 2026-02-18T08:40:37.773 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=39, tag_tid=1, entry_tid=39135' 2026-02-18T08:40:37.774 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=37, tag_tid=2, entry_tid=1' 2026-02-18T08:40:37.774 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:40:37.774 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:40:37.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=2 2026-02-18T08:40:37.776 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=39, tag_tid=1, entry_tid=39135' 2026-02-18T08:40:37.776 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:40:37.776 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:40:37.777 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=1 2026-02-18T08:40:37.778 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=37, tag_tid=2, entry_tid=1' 2026-02-18T08:40:37.778 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:40:37.778 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:40:37.779 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:40:37.780 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=39, tag_tid=1, entry_tid=39135' 2026-02-18T08:40:37.780 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:40:37.780 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:40:37.782 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=39135 2026-02-18T08:40:37.782 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 2 = 1 -a 1 -le 39135 2026-02-18T08:40:37.782 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:40:37.782 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:40:37.782 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:40:37.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_30 2026-02-18T08:40:37.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:40:37.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:40:37.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_30 2026-02-18T08:40:37.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:40:37.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_30 ']' 2026-02-18T08:40:37.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_30' 2026-02-18T08:40:37.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_30 2026-02-18T08:40:37.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:40:37.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:40:37.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:40:37.985 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:40:37.985 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:40:37.985 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:40:37.985 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:40:37.985 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:40:37.987 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:40:37.987 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:40:37.988 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_30 2026-02-18T08:40:37.988 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:40:37.988 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:40:37.988 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:40:37.988 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:40:37.988 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:40:37.988 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:40:37.988 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:40:37.988 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:40:37.988 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:40:37.988 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:40:37.989 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:40:37.989 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:40:37.989 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:40:37.989 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:40:37.989 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:40:37.989 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:40:37.989 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:40:37.989 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:40:37.989 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:40:37.989 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:40:37.990 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:40:37.990 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:40:37.990 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:40:37.990 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:40:37.990 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:40:37.990 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:40:37.990 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:40:37.990 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:40:37.990 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:40:37.990 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:40:37.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:40:37.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:40:37.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:40:37.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_30 2026-02-18T08:40:38.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_30'] 2026-02-18T08:40:38.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:40:38.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:40:38.060 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:40:38.060 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_30 2026-02-18T08:40:38.060 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:40:38.060 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:40:38.060 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:40:38.060 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:40:38.060 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:40:38.060 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:40:38.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:40:38.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:40:38.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:40:38.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:40:38.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:40:38.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:40:38.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:40:38.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:40:38.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:40:38.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:40:38.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:40:38.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:40:38.062 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:40:38.062 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:40:38.062 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:40:38.062 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:40:38.062 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:40:38.062 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:40:38.062 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:40:38.062 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:40:38.062 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:40:38.062 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:40:38.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:40:38.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:40:38.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:40:38.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:40:38.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:40:38.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_30 2026-02-18T08:40:38.132 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_30'] 2026-02-18T08:40:38.132 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:40:38.132 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:40:38.134 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:40:38.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_30 2026-02-18T08:40:38.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:40:38.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:40:38.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:40:38.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:40:38.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:40:38.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:40:38.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:40:38.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:40:38.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:40:38.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:40:38.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:40:38.136 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:40:38.136 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:40:38.136 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:40:38.136 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:40:38.136 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:40:38.136 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:40:38.136 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:40:38.136 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:40:38.136 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:40:38.136 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:40:38.136 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:40:38.136 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:40:38.137 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:40:38.137 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:40:38.137 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:40:38.137 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:40:38.137 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:40:38.137 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:40:38.137 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:40:38.145 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:40:38.146 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:40:38.146 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:40:38.146 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_30 2026-02-18T08:40:38.206 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:40:38.207 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_30 2026-02-18T08:40:38.207 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:40:38.208 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:40:38.208 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_30 2026-02-18T08:40:38.208 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_30 '' 2026-02-18T08:40:38.208 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:40:38.208 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:40:38.208 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_30 2026-02-18T08:40:38.208 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:40:38.208 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_30.status 2026-02-18T08:40:38.208 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_30.status 2026-02-18T08:40:38.208 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:40:38.209 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_30.status 2026-02-18T08:40:38.209 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_30 2026-02-18T08:40:38.209 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_30.status 2026-02-18T08:40:38.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 9 2026-02-18T08:40:38.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 9 2026-02-18T08:40:38.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:40:38.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=37, tag_tid=2, entry_tid=1], [object_number=36, tag_tid=2, entry_tid=0], [object_number=39, tag_tid=1, entry_tid=39135], [object_number=38, tag_tid=1, entry_tid=39134]]], state=connected] 2026-02-18T08:40:38.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=39, tag_tid=1, entry_tid=39135], [object_number=38, tag_tid=1, entry_tid=39134], [object_number=37, tag_tid=1, entry_tid=39133], [object_number=36, tag_tid=1, entry_tid=39132]]], state=connected] 2026-02-18T08:40:38.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_30.status 2026-02-18T08:40:38.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=37, tag_tid=2, entry_tid=1' 2026-02-18T08:40:38.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_30 2026-02-18T08:40:38.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:40:38.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:40:38.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_30 2026-02-18T08:40:38.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_30 '..*' 2026-02-18T08:40:38.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:40:38.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:40:38.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_30 2026-02-18T08:40:38.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:40:38.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_30.status 2026-02-18T08:40:38.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_30.status 2026-02-18T08:40:38.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:40:38.244 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_30.status 2026-02-18T08:40:38.244 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_30 2026-02-18T08:40:38.244 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_30.status 2026-02-18T08:40:38.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 9 2026-02-18T08:40:38.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 9 2026-02-18T08:40:38.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:40:38.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=37, tag_tid=2, entry_tid=1], [object_number=36, tag_tid=2, entry_tid=0], [object_number=39, tag_tid=1, entry_tid=39135], [object_number=38, tag_tid=1, entry_tid=39134]]], state=connected] 2026-02-18T08:40:38.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=39, tag_tid=1, entry_tid=39135], [object_number=38, tag_tid=1, entry_tid=39134], [object_number=37, tag_tid=1, entry_tid=39133], [object_number=36, tag_tid=1, entry_tid=39132]]], state=connected] 2026-02-18T08:40:38.274 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_30.status 2026-02-18T08:40:38.276 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=39, tag_tid=1, entry_tid=39135' 2026-02-18T08:40:38.276 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=37, tag_tid=2, entry_tid=1' -a 'object_number=37, tag_tid=2, entry_tid=1' = 'object_number=39, tag_tid=1, entry_tid=39135' 2026-02-18T08:40:38.276 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=39, tag_tid=1, entry_tid=39135' '!=' 'object_number=39, tag_tid=1, entry_tid=39135' 2026-02-18T08:40:38.277 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:40:38.277 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.4 2026-02-18T08:40:38.678 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_30 2026-02-18T08:40:38.678 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:40:38.679 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:40:38.679 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_30 2026-02-18T08:40:38.679 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:40:38.679 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_30 ']' 2026-02-18T08:40:38.679 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_30' 2026-02-18T08:40:38.679 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_30 2026-02-18T08:40:38.679 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:40:38.679 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:40:38.679 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:40:38.679 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:40:38.679 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:40:38.680 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:40:38.680 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:40:38.680 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:40:38.683 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:40:38.683 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:40:38.683 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_30 2026-02-18T08:40:38.683 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:40:38.683 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:40:38.683 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:40:38.683 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:40:38.683 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:40:38.683 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:40:38.684 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:40:38.684 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:40:38.684 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:40:38.684 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:40:38.684 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:40:38.684 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:40:38.684 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:40:38.684 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:40:38.684 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:40:38.684 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:40:38.684 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:40:38.684 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:40:38.684 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:40:38.685 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:40:38.685 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:40:38.685 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:40:38.685 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:40:38.685 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:40:38.685 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:40:38.685 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:40:38.685 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:40:38.685 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:40:38.685 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:40:38.685 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:40:38.694 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:40:38.694 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:40:38.695 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:40:38.695 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_30 2026-02-18T08:40:38.753 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_30'] 2026-02-18T08:40:38.753 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:40:38.753 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:40:38.756 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:40:38.756 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_30 2026-02-18T08:40:38.756 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:40:38.756 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:40:38.756 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:40:38.756 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:40:38.756 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:40:38.756 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:40:38.756 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:40:38.756 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:40:38.756 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:40:38.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:40:38.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:40:38.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:40:38.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:40:38.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:40:38.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:40:38.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:40:38.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:40:38.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:40:38.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:40:38.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:40:38.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:40:38.758 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:40:38.758 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:40:38.758 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:40:38.758 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:40:38.758 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:40:38.758 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:40:38.758 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:40:38.758 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:40:38.758 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:40:38.766 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:40:38.767 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:40:38.767 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:40:38.767 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_30 2026-02-18T08:40:38.829 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_30'] 2026-02-18T08:40:38.829 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:40:38.829 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:40:38.832 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:40:38.832 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_30 2026-02-18T08:40:38.832 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:40:38.832 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:40:38.832 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:40:38.832 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:40:38.832 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:40:38.832 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:40:38.832 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:40:38.832 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:40:38.832 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:40:38.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:40:38.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:40:38.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:40:38.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:40:38.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:40:38.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:40:38.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:40:38.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:40:38.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:40:38.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:40:38.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:40:38.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:40:38.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:40:38.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:40:38.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:40:38.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:40:38.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:40:38.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:40:38.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:40:38.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:40:38.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:40:38.842 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:40:38.843 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:40:38.843 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:40:38.843 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_30 2026-02-18T08:40:38.910 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:40:38.911 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_30 2026-02-18T08:40:38.911 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:40:38.911 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:40:38.912 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_30 2026-02-18T08:40:38.912 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_30 '' 2026-02-18T08:40:38.912 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:40:38.912 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:40:38.912 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_30 2026-02-18T08:40:38.912 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:40:38.912 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_30.status 2026-02-18T08:40:38.912 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_30.status 2026-02-18T08:40:38.912 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:40:38.913 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_30.status 2026-02-18T08:40:38.913 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_30 2026-02-18T08:40:38.913 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_30.status 2026-02-18T08:40:38.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 9 2026-02-18T08:40:38.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 9 2026-02-18T08:40:38.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:40:38.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=37, tag_tid=2, entry_tid=1], [object_number=36, tag_tid=2, entry_tid=0], [object_number=39, tag_tid=1, entry_tid=39135], [object_number=38, tag_tid=1, entry_tid=39134]]], state=connected] 2026-02-18T08:40:38.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=39, tag_tid=1, entry_tid=39135], [object_number=38, tag_tid=1, entry_tid=39134], [object_number=37, tag_tid=1, entry_tid=39133], [object_number=36, tag_tid=1, entry_tid=39132]]], state=connected] 2026-02-18T08:40:38.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_30.status 2026-02-18T08:40:38.947 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=37, tag_tid=2, entry_tid=1' 2026-02-18T08:40:38.948 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_30 2026-02-18T08:40:38.948 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:40:38.948 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:40:38.948 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_30 2026-02-18T08:40:38.948 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_30 '..*' 2026-02-18T08:40:38.948 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:40:38.949 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:40:38.949 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_30 2026-02-18T08:40:38.949 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:40:38.949 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_30.status 2026-02-18T08:40:38.950 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_30.status 2026-02-18T08:40:38.950 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:40:38.950 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_30.status 2026-02-18T08:40:38.950 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_30 2026-02-18T08:40:38.950 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_30.status 2026-02-18T08:40:38.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 9 2026-02-18T08:40:38.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 9 2026-02-18T08:40:38.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:40:38.985 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=37, tag_tid=2, entry_tid=1], [object_number=36, tag_tid=2, entry_tid=0], [object_number=39, tag_tid=1, entry_tid=39135], [object_number=38, tag_tid=1, entry_tid=39134]]], state=connected] 2026-02-18T08:40:38.985 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=39, tag_tid=1, entry_tid=39135], [object_number=38, tag_tid=1, entry_tid=39134], [object_number=37, tag_tid=1, entry_tid=39133], [object_number=36, tag_tid=1, entry_tid=39132]]], state=connected] 2026-02-18T08:40:38.988 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_30.status 2026-02-18T08:40:38.990 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=39, tag_tid=1, entry_tid=39135' 2026-02-18T08:40:38.990 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=37, tag_tid=2, entry_tid=1' -a 'object_number=37, tag_tid=2, entry_tid=1' = 'object_number=39, tag_tid=1, entry_tid=39135' 2026-02-18T08:40:38.990 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=39, tag_tid=1, entry_tid=39135' '!=' 'object_number=39, tag_tid=1, entry_tid=39135' 2026-02-18T08:40:38.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:40:38.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.8 2026-02-18T08:40:39.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_30 2026-02-18T08:40:39.793 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:40:39.793 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:40:39.793 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_30 2026-02-18T08:40:39.793 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:40:39.793 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_30 ']' 2026-02-18T08:40:39.793 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_30' 2026-02-18T08:40:39.793 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_30 2026-02-18T08:40:39.793 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:40:39.793 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:40:39.794 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:40:39.794 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:40:39.794 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:40:39.794 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:40:39.794 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:40:39.794 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:40:39.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:40:39.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:40:39.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_30 2026-02-18T08:40:39.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:40:39.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:40:39.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:40:39.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:40:39.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:40:39.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:40:39.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:40:39.798 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:40:39.798 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:40:39.798 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:40:39.798 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:40:39.798 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:40:39.798 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:40:39.798 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:40:39.798 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:40:39.798 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:40:39.798 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:40:39.798 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:40:39.798 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:40:39.799 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:40:39.799 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:40:39.799 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:40:39.799 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:40:39.799 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:40:39.799 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:40:39.799 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:40:39.799 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:40:39.799 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:40:39.799 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:40:39.799 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:40:39.807 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:40:39.808 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:40:39.808 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:40:39.808 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_30 2026-02-18T08:40:39.866 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_30'] 2026-02-18T08:40:39.867 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:40:39.867 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:40:39.869 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:40:39.869 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_30 2026-02-18T08:40:39.869 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:40:39.870 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:40:39.870 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:40:39.870 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:40:39.870 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:40:39.870 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:40:39.870 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:40:39.870 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:40:39.870 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:40:39.870 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:40:39.870 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:40:39.870 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:40:39.870 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:40:39.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:40:39.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:40:39.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:40:39.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:40:39.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:40:39.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:40:39.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:40:39.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:40:39.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:40:39.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:40:39.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:40:39.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:40:39.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:40:39.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:40:39.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:40:39.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:40:39.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:40:39.881 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:40:39.882 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:40:39.882 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:40:39.882 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_30 2026-02-18T08:40:39.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_30'] 2026-02-18T08:40:39.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:40:39.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:40:39.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:40:39.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_30 2026-02-18T08:40:39.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:40:39.946 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:40:39.946 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:40:39.946 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:40:39.946 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:40:39.946 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:40:39.946 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:40:39.946 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:40:39.946 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:40:39.946 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:40:39.946 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:40:39.946 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:40:39.947 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:40:39.947 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:40:39.947 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:40:39.947 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:40:39.947 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:40:39.947 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:40:39.947 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:40:39.947 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:40:39.947 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:40:39.947 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:40:39.947 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:40:39.947 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:40:39.947 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:40:39.948 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:40:39.948 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:40:39.948 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:40:39.948 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:40:39.948 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:40:39.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:40:39.958 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:40:39.958 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:40:39.958 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_30 2026-02-18T08:40:40.020 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:40:40.021 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_30 2026-02-18T08:40:40.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:40:40.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:40:40.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_30 2026-02-18T08:40:40.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_30 '' 2026-02-18T08:40:40.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:40:40.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:40:40.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_30 2026-02-18T08:40:40.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:40:40.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_30.status 2026-02-18T08:40:40.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_30.status 2026-02-18T08:40:40.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:40:40.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_30.status 2026-02-18T08:40:40.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_30 2026-02-18T08:40:40.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_30.status 2026-02-18T08:40:40.059 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 9 2026-02-18T08:40:40.059 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 9 2026-02-18T08:40:40.059 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:40:40.059 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=37, tag_tid=2, entry_tid=1], [object_number=36, tag_tid=2, entry_tid=0], [object_number=39, tag_tid=1, entry_tid=39135], [object_number=38, tag_tid=1, entry_tid=39134]]], state=connected] 2026-02-18T08:40:40.059 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=37, tag_tid=2, entry_tid=1], [object_number=36, tag_tid=2, entry_tid=0], [object_number=39, tag_tid=1, entry_tid=39135], [object_number=38, tag_tid=1, entry_tid=39134]]], state=connected] 2026-02-18T08:40:40.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_30.status 2026-02-18T08:40:40.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=37, tag_tid=2, entry_tid=1' 2026-02-18T08:40:40.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_30 2026-02-18T08:40:40.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:40:40.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:40:40.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_30 2026-02-18T08:40:40.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_30 '..*' 2026-02-18T08:40:40.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:40:40.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:40:40.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_30 2026-02-18T08:40:40.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:40:40.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_30.status 2026-02-18T08:40:40.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_30.status 2026-02-18T08:40:40.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:40:40.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_30.status 2026-02-18T08:40:40.067 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_30 2026-02-18T08:40:40.067 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_30.status 2026-02-18T08:40:40.102 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 9 2026-02-18T08:40:40.102 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 9 2026-02-18T08:40:40.102 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:40:40.102 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=37, tag_tid=2, entry_tid=1], [object_number=36, tag_tid=2, entry_tid=0], [object_number=39, tag_tid=1, entry_tid=39135], [object_number=38, tag_tid=1, entry_tid=39134]]], state=connected] 2026-02-18T08:40:40.102 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=37, tag_tid=2, entry_tid=1], [object_number=36, tag_tid=2, entry_tid=0], [object_number=39, tag_tid=1, entry_tid=39135], [object_number=38, tag_tid=1, entry_tid=39134]]], state=connected] 2026-02-18T08:40:40.106 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_30.status 2026-02-18T08:40:40.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=37, tag_tid=2, entry_tid=1' 2026-02-18T08:40:40.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=37, tag_tid=2, entry_tid=1' -a 'object_number=37, tag_tid=2, entry_tid=1' = 'object_number=37, tag_tid=2, entry_tid=1' 2026-02-18T08:40:40.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:40:40.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_snap_present cluster1 mirror image_30 snap 2026-02-18T08:40:40.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:40:40.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:40:40.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_30 2026-02-18T08:40:40.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap 2026-02-18T08:40:40.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-18T08:40:40.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-18T08:40:40.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1 2026-02-18T08:40:41.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster1 info mirror/image_30@snap 2026-02-18T08:40:41.136 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:rbd image 'image_30': 2026-02-18T08:40:41.136 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: size 128 MiB in 32 objects 2026-02-18T08:40:41.136 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: order 22 (4 MiB objects) 2026-02-18T08:40:41.136 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: snapshot_count: 1 2026-02-18T08:40:41.136 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: id: 11048bb7c603 2026-02-18T08:40:41.136 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: block_name_prefix: rbd_data.11048bb7c603 2026-02-18T08:40:41.137 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: format: 2 2026-02-18T08:40:41.137 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: features: layering, exclusive-lock, journaling 2026-02-18T08:40:41.137 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: op_features: 2026-02-18T08:40:41.137 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: flags: 2026-02-18T08:40:41.137 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: create_timestamp: Wed Feb 18 08:30:53 2026 2026-02-18T08:40:41.137 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: access_timestamp: Wed Feb 18 08:30:53 2026 2026-02-18T08:40:41.137 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: modify_timestamp: Wed Feb 18 08:31:54 2026 2026-02-18T08:40:41.137 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: protected: False 2026-02-18T08:40:41.137 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: journal: 11048bb7c603 2026-02-18T08:40:41.137 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring state: enabled 2026-02-18T08:40:41.137 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring mode: journal 2026-02-18T08:40:41.138 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring global id: dea6413c-9d2e-4bcd-b43e-cd1926edecd5 2026-02-18T08:40:41.138 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring primary: false 2026-02-18T08:40:41.139 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:40:41.139 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ compare_image_snaps mirror image_30 snap 2026-02-18T08:40:41.139 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:40:41.140 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_30 2026-02-18T08:40:41.140 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap 2026-02-18T08:40:41.140 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local ret=0 2026-02-18T08:40:41.140 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local rmt_export=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_30.export 2026-02-18T08:40:41.140 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loc_export=/tmp/tmp.vybh1m9yJF/cluster1-mirror-image_30.export 2026-02-18T08:40:41.140 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_30.export /tmp/tmp.vybh1m9yJF/cluster1-mirror-image_30.export 2026-02-18T08:40:41.140 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 -p mirror export image_30@snap /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_30.export 2026-02-18T08:40:41.285 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-18T08:40:41.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster1 -p mirror export image_30@snap /tmp/tmp.vybh1m9yJF/cluster1-mirror-image_30.export 2026-02-18T08:40:41.682 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-18T08:40:41.694 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmp /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_30.export /tmp/tmp.vybh1m9yJF/cluster1-mirror-image_30.export 2026-02-18T08:40:41.749 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_30.export /tmp/tmp.vybh1m9yJF/cluster1-mirror-image_30.export 2026-02-18T08:40:41.794 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:40:41.794 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T08:40:41.794 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_31 2026-02-18T08:40:41.794 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_snap cluster2 mirror image_31 snap 2026-02-18T08:40:41.794 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:40:41.794 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:40:41.794 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_31 2026-02-18T08:40:41.795 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap 2026-02-18T08:40:41.795 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 -p mirror snap create image_31@snap --debug-rbd=20 --debug-journaler=20 2026-02-18T08:40:43.058 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_replay_started cluster1 mirror image_31 2026-02-18T08:40:43.059 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:40:43.059 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:40:43.059 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_31 2026-02-18T08:40:43.059 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_replay_state cluster1 mirror image_31 started 2026-02-18T08:40:43.059 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:40:43.059 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:40:43.059 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_31 2026-02-18T08:40:43.059 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local state=started 2026-02-18T08:40:43.059 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-18T08:40:43.059 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:40:43.059 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1 2026-02-18T08:40:44.060 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test_image_replay_state cluster1 mirror image_31 started 2026-02-18T08:40:44.060 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:40:44.060 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:40:44.060 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_31 2026-02-18T08:40:44.060 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local test_state=started 2026-02-18T08:40:44.060 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local status_result 2026-02-18T08:40:44.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_state=stopped 2026-02-18T08:40:44.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_31 2026-02-18T08:40:44.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -i state 2026-02-18T08:40:44.062 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_instance=cluster1 2026-02-18T08:40:44.062 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-18T08:40:44.062 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:40:44.062 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance=cluster1 2026-02-18T08:40:44.062 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local loop_instance 2026-02-18T08:40:44.062 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:40:44.062 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sleep 0 2026-02-18T08:40:44.062 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:40:44.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ seq 0 3 2026-02-18T08:40:44.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:40:44.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_31 2026-02-18T08:40:44.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:0 2026-02-18T08:40:44.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-18T08:40:44.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:40:44.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:40:44.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:40:44.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-18T08:40:44.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:40:44.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:40:44.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:40:44.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:40:44.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-18T08:40:44.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:40:44.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:40:44.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:40:44.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:40:44.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:0 2026-02-18T08:40:44.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-18T08:40:44.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-18T08:40:44.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:40:44.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:0 2026-02-18T08:40:44.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T08:40:44.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-18T08:40:44.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:40:44.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:40:44.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-18T08:40:44.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-18T08:40:44.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-18T08:40:44.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=0 2026-02-18T08:40:44.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=0 2026-02-18T08:40:44.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:40:44.074 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:40:44.075 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:40:44.075 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:40:44.075 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_31 2026-02-18T08:40:44.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_31'] 2026-02-18T08:40:44.134 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status : get status for rbd mirror 2026-02-18T08:40:44.134 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:40:44.136 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:40:44.137 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_31 2026-02-18T08:40:44.137 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:1 2026-02-18T08:40:44.137 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-18T08:40:44.137 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:40:44.137 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:40:44.137 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:40:44.137 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-18T08:40:44.137 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:40:44.137 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:40:44.137 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:40:44.137 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:40:44.138 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-18T08:40:44.138 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:40:44.138 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:40:44.138 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:40:44.138 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:40:44.138 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:1 2026-02-18T08:40:44.138 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-18T08:40:44.138 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-18T08:40:44.138 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:40:44.138 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:1 2026-02-18T08:40:44.138 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T08:40:44.138 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-18T08:40:44.139 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:40:44.139 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:40:44.139 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-18T08:40:44.139 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-18T08:40:44.139 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-18T08:40:44.139 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=1 2026-02-18T08:40:44.139 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=1 2026-02-18T08:40:44.139 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:40:44.146 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:40:44.147 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:40:44.147 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:40:44.147 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_31 2026-02-18T08:40:44.214 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ return 0 2026-02-18T08:40:44.215 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ status_result=' "state": "Replaying"' 2026-02-18T08:40:44.215 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ echo ' "state": "Replaying"' 2026-02-18T08:40:44.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ grep -i Replaying 2026-02-18T08:40:44.217 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: "state": "Replaying" 2026-02-18T08:40:44.218 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ current_state=started 2026-02-18T08:40:44.218 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test started = started 2026-02-18T08:40:44.218 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:40:44.219 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror image_31 2026-02-18T08:40:44.219 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_cluster=cluster1 2026-02-18T08:40:44.219 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:40:44.219 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_pool=mirror 2026-02-18T08:40:44.219 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local remote_pool=mirror 2026-02-18T08:40:44.219 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_31 2026-02-18T08:40:44.219 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' journal = journal ']' 2026-02-18T08:40:44.219 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror mirror image_31 2026-02-18T08:40:44.219 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_cluster=cluster1 2026-02-18T08:40:44.219 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:40:44.219 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_pool=mirror 2026-02-18T08:40:44.219 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local remote_pool=mirror 2026-02-18T08:40:44.220 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_31 2026-02-18T08:40:44.220 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-02-18T08:40:44.220 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-02-18T08:40:44.220 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:40:44.220 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:40:44.220 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:40:44.419 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_31 2026-02-18T08:40:44.419 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:40:44.420 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:40:44.420 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_31 2026-02-18T08:40:44.420 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:40:44.420 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_31 ']' 2026-02-18T08:40:44.420 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_31' 2026-02-18T08:40:44.420 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_31 2026-02-18T08:40:44.420 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:40:44.420 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:40:44.420 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:40:44.420 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:40:44.420 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:40:44.421 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:40:44.421 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:40:44.423 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:40:44.423 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:40:44.423 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:40:44.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_31 2026-02-18T08:40:44.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:40:44.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:40:44.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:40:44.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:40:44.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:40:44.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:40:44.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:40:44.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:40:44.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:40:44.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:40:44.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:40:44.425 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:40:44.425 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:40:44.425 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:40:44.425 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:40:44.425 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:40:44.425 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:40:44.425 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:40:44.425 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:40:44.425 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:40:44.425 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:40:44.425 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:40:44.425 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:40:44.425 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:40:44.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:40:44.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:40:44.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:40:44.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:40:44.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:40:44.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:40:44.434 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:40:44.434 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:40:44.435 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:40:44.435 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_31 2026-02-18T08:40:44.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_31'] 2026-02-18T08:40:44.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:40:44.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:40:44.500 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:40:44.500 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_31 2026-02-18T08:40:44.500 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:40:44.500 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:40:44.500 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:40:44.500 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:40:44.500 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:40:44.500 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:40:44.501 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:40:44.501 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:40:44.501 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:40:44.501 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:40:44.501 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:40:44.501 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:40:44.501 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:40:44.501 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:40:44.501 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:40:44.501 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:40:44.501 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:40:44.501 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:40:44.501 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:40:44.502 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:40:44.502 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:40:44.502 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:40:44.502 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:40:44.502 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:40:44.502 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:40:44.502 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:40:44.502 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:40:44.502 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:40:44.502 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:40:44.502 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:40:44.511 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:40:44.511 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:40:44.511 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:40:44.512 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_31 2026-02-18T08:40:44.568 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:40:44.570 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_31 2026-02-18T08:40:44.570 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:40:44.570 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:40:44.570 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_31 2026-02-18T08:40:44.570 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_31 '' 2026-02-18T08:40:44.570 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:40:44.570 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:40:44.570 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_31 2026-02-18T08:40:44.570 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:40:44.570 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_31.status 2026-02-18T08:40:44.571 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_31.status 2026-02-18T08:40:44.571 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:40:44.571 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_31.status 2026-02-18T08:40:44.572 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_31 2026-02-18T08:40:44.572 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_31.status 2026-02-18T08:40:44.599 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 8 2026-02-18T08:40:44.599 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 8 2026-02-18T08:40:44.599 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:40:44.599 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=33, tag_tid=2, entry_tid=1], [object_number=32, tag_tid=2, entry_tid=0], [object_number=35, tag_tid=1, entry_tid=34707], [object_number=34, tag_tid=1, entry_tid=34706]]], state=connected] 2026-02-18T08:40:44.599 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=33, tag_tid=1, entry_tid=34709], [object_number=32, tag_tid=1, entry_tid=34708], [object_number=35, tag_tid=1, entry_tid=34707], [object_number=34, tag_tid=1, entry_tid=34706]]], state=connected] 2026-02-18T08:40:44.602 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_31.status 2026-02-18T08:40:44.603 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=33, tag_tid=2, entry_tid=1' 2026-02-18T08:40:44.604 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_31 2026-02-18T08:40:44.604 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:40:44.605 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:40:44.605 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_31 2026-02-18T08:40:44.605 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_31 '..*' 2026-02-18T08:40:44.605 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:40:44.605 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:40:44.605 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_31 2026-02-18T08:40:44.605 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:40:44.605 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_31.status 2026-02-18T08:40:44.605 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_31.status 2026-02-18T08:40:44.606 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:40:44.606 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_31.status 2026-02-18T08:40:44.606 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_31 2026-02-18T08:40:44.607 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_31.status 2026-02-18T08:40:44.642 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 8 2026-02-18T08:40:44.642 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 8 2026-02-18T08:40:44.642 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:40:44.642 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=33, tag_tid=2, entry_tid=1], [object_number=32, tag_tid=2, entry_tid=0], [object_number=35, tag_tid=1, entry_tid=34707], [object_number=34, tag_tid=1, entry_tid=34706]]], state=connected] 2026-02-18T08:40:44.642 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=33, tag_tid=1, entry_tid=34709], [object_number=32, tag_tid=1, entry_tid=34708], [object_number=35, tag_tid=1, entry_tid=34707], [object_number=34, tag_tid=1, entry_tid=34706]]], state=connected] 2026-02-18T08:40:44.646 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_31.status 2026-02-18T08:40:44.647 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=33, tag_tid=1, entry_tid=34709' 2026-02-18T08:40:44.648 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=33, tag_tid=2, entry_tid=1' -a 'object_number=33, tag_tid=2, entry_tid=1' = 'object_number=33, tag_tid=1, entry_tid=34709' 2026-02-18T08:40:44.648 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=33, tag_tid=1, entry_tid=34709' '!=' '' 2026-02-18T08:40:44.648 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:40:44.648 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=33, tag_tid=1, entry_tid=34709' = '' 2026-02-18T08:40:44.648 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=33, tag_tid=1, entry_tid=34709' 2026-02-18T08:40:44.649 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=33, tag_tid=2, entry_tid=1' 2026-02-18T08:40:44.649 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:40:44.649 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:40:44.649 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=2 2026-02-18T08:40:44.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=33, tag_tid=1, entry_tid=34709' 2026-02-18T08:40:44.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:40:44.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:40:44.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=1 2026-02-18T08:40:44.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=33, tag_tid=2, entry_tid=1' 2026-02-18T08:40:44.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:40:44.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:40:44.654 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:40:44.654 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=33, tag_tid=1, entry_tid=34709' 2026-02-18T08:40:44.654 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:40:44.655 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:40:44.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=34709 2026-02-18T08:40:44.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 2 = 1 -a 1 -le 34709 2026-02-18T08:40:44.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:40:44.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:40:44.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:40:44.858 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_31 2026-02-18T08:40:44.858 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:40:44.858 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:40:44.859 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_31 2026-02-18T08:40:44.859 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:40:44.859 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_31 ']' 2026-02-18T08:40:44.859 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_31' 2026-02-18T08:40:44.859 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_31 2026-02-18T08:40:44.859 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:40:44.859 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:40:44.859 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:40:44.859 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:40:44.859 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:40:44.860 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:40:44.860 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:40:44.860 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:40:44.862 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:40:44.862 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:40:44.862 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_31 2026-02-18T08:40:44.863 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:40:44.863 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:40:44.863 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:40:44.863 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:40:44.863 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:40:44.863 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:40:44.863 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:40:44.863 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:40:44.863 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:40:44.863 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:40:44.863 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:40:44.863 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:40:44.864 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:40:44.864 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:40:44.864 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:40:44.864 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:40:44.864 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:40:44.864 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:40:44.864 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:40:44.864 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:40:44.864 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:40:44.864 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:40:44.864 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:40:44.864 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:40:44.864 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:40:44.865 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:40:44.865 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:40:44.865 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:40:44.865 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:40:44.865 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:40:44.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:40:44.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:40:44.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:40:44.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_31 2026-02-18T08:40:44.933 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_31'] 2026-02-18T08:40:44.933 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:40:44.933 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:40:44.936 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:40:44.936 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_31 2026-02-18T08:40:44.936 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:40:44.936 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:40:44.936 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:40:44.936 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:40:44.936 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:40:44.936 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:40:44.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:40:44.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:40:44.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:40:44.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:40:44.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:40:44.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:40:44.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:40:44.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:40:44.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:40:44.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:40:44.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:40:44.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:40:44.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:40:44.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:40:44.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:40:44.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:40:44.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:40:44.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:40:44.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:40:44.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:40:44.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:40:44.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:40:44.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:40:44.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:40:44.946 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:40:44.947 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:40:44.947 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:40:44.947 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_31 2026-02-18T08:40:45.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:40:45.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_31 2026-02-18T08:40:45.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:40:45.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:40:45.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_31 2026-02-18T08:40:45.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_31 '' 2026-02-18T08:40:45.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:40:45.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:40:45.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_31 2026-02-18T08:40:45.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:40:45.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_31.status 2026-02-18T08:40:45.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_31.status 2026-02-18T08:40:45.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:40:45.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_31.status 2026-02-18T08:40:45.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_31.status 2026-02-18T08:40:45.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_31 2026-02-18T08:40:45.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 8 2026-02-18T08:40:45.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 8 2026-02-18T08:40:45.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:40:45.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=33, tag_tid=2, entry_tid=1], [object_number=32, tag_tid=2, entry_tid=0], [object_number=35, tag_tid=1, entry_tid=34707], [object_number=34, tag_tid=1, entry_tid=34706]]], state=connected] 2026-02-18T08:40:45.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=33, tag_tid=1, entry_tid=34709], [object_number=32, tag_tid=1, entry_tid=34708], [object_number=35, tag_tid=1, entry_tid=34707], [object_number=34, tag_tid=1, entry_tid=34706]]], state=connected] 2026-02-18T08:40:45.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_31.status 2026-02-18T08:40:45.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=33, tag_tid=2, entry_tid=1' 2026-02-18T08:40:45.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_31 2026-02-18T08:40:45.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:40:45.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:40:45.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_31 2026-02-18T08:40:45.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_31 '..*' 2026-02-18T08:40:45.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:40:45.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:40:45.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_31 2026-02-18T08:40:45.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:40:45.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_31.status 2026-02-18T08:40:45.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_31.status 2026-02-18T08:40:45.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:40:45.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_31.status 2026-02-18T08:40:45.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_31 2026-02-18T08:40:45.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_31.status 2026-02-18T08:40:45.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 8 2026-02-18T08:40:45.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 8 2026-02-18T08:40:45.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:40:45.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=33, tag_tid=2, entry_tid=1], [object_number=32, tag_tid=2, entry_tid=0], [object_number=35, tag_tid=1, entry_tid=34707], [object_number=34, tag_tid=1, entry_tid=34706]]], state=connected] 2026-02-18T08:40:45.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=33, tag_tid=1, entry_tid=34709], [object_number=32, tag_tid=1, entry_tid=34708], [object_number=35, tag_tid=1, entry_tid=34707], [object_number=34, tag_tid=1, entry_tid=34706]]], state=connected] 2026-02-18T08:40:45.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_31.status 2026-02-18T08:40:45.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=33, tag_tid=1, entry_tid=34709' 2026-02-18T08:40:45.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=33, tag_tid=2, entry_tid=1' -a 'object_number=33, tag_tid=2, entry_tid=1' = 'object_number=33, tag_tid=1, entry_tid=34709' 2026-02-18T08:40:45.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=33, tag_tid=1, entry_tid=34709' '!=' 'object_number=33, tag_tid=1, entry_tid=34709' 2026-02-18T08:40:45.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:40:45.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.4 2026-02-18T08:40:45.484 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_31 2026-02-18T08:40:45.484 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:40:45.484 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:40:45.484 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_31 2026-02-18T08:40:45.485 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:40:45.485 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_31 ']' 2026-02-18T08:40:45.485 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_31' 2026-02-18T08:40:45.485 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_31 2026-02-18T08:40:45.485 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:40:45.485 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:40:45.485 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:40:45.485 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:40:45.485 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:40:45.485 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:40:45.485 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:40:45.488 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:40:45.488 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:40:45.488 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:40:45.488 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_31 2026-02-18T08:40:45.489 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:40:45.489 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:40:45.489 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:40:45.489 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:40:45.489 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:40:45.489 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:40:45.489 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:40:45.489 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:40:45.489 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:40:45.489 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:40:45.489 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:40:45.489 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:40:45.489 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:40:45.490 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:40:45.490 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:40:45.490 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:40:45.490 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:40:45.490 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:40:45.490 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:40:45.490 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:40:45.490 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:40:45.490 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:40:45.490 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:40:45.490 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:40:45.490 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:40:45.490 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:40:45.491 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:40:45.491 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:40:45.491 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:40:45.491 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:40:45.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:40:45.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:40:45.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:40:45.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_31 2026-02-18T08:40:45.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_31'] 2026-02-18T08:40:45.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:40:45.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:40:45.559 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:40:45.559 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_31 2026-02-18T08:40:45.559 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:40:45.559 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:40:45.560 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:40:45.560 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:40:45.560 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:40:45.560 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:40:45.560 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:40:45.560 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:40:45.560 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:40:45.560 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:40:45.560 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:40:45.560 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:40:45.560 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:40:45.560 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:40:45.561 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:40:45.561 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:40:45.561 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:40:45.561 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:40:45.561 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:40:45.561 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:40:45.561 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:40:45.561 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:40:45.561 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:40:45.561 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:40:45.561 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:40:45.561 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:40:45.562 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:40:45.562 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:40:45.562 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:40:45.562 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:40:45.571 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:40:45.571 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:40:45.571 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:40:45.572 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_31 2026-02-18T08:40:45.638 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:40:45.639 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_31 2026-02-18T08:40:45.639 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:40:45.639 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:40:45.639 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_31 2026-02-18T08:40:45.639 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_31 '' 2026-02-18T08:40:45.639 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:40:45.640 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:40:45.640 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_31 2026-02-18T08:40:45.640 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:40:45.640 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_31.status 2026-02-18T08:40:45.640 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_31.status 2026-02-18T08:40:45.640 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:40:45.641 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_31.status 2026-02-18T08:40:45.641 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_31 2026-02-18T08:40:45.642 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_31.status 2026-02-18T08:40:45.667 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 8 2026-02-18T08:40:45.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 8 2026-02-18T08:40:45.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:40:45.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=33, tag_tid=2, entry_tid=1], [object_number=32, tag_tid=2, entry_tid=0], [object_number=35, tag_tid=1, entry_tid=34707], [object_number=34, tag_tid=1, entry_tid=34706]]], state=connected] 2026-02-18T08:40:45.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=33, tag_tid=1, entry_tid=34709], [object_number=32, tag_tid=1, entry_tid=34708], [object_number=35, tag_tid=1, entry_tid=34707], [object_number=34, tag_tid=1, entry_tid=34706]]], state=connected] 2026-02-18T08:40:45.670 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_31.status 2026-02-18T08:40:45.671 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=33, tag_tid=2, entry_tid=1' 2026-02-18T08:40:45.672 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_31 2026-02-18T08:40:45.672 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:40:45.673 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:40:45.673 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_31 2026-02-18T08:40:45.673 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_31 '..*' 2026-02-18T08:40:45.673 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:40:45.673 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:40:45.673 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_31 2026-02-18T08:40:45.673 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:40:45.673 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_31.status 2026-02-18T08:40:45.674 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_31.status 2026-02-18T08:40:45.674 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:40:45.674 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_31.status 2026-02-18T08:40:45.675 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_31 2026-02-18T08:40:45.675 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_31.status 2026-02-18T08:40:45.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 8 2026-02-18T08:40:45.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 8 2026-02-18T08:40:45.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:40:45.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=33, tag_tid=2, entry_tid=1], [object_number=32, tag_tid=2, entry_tid=0], [object_number=35, tag_tid=1, entry_tid=34707], [object_number=34, tag_tid=1, entry_tid=34706]]], state=connected] 2026-02-18T08:40:45.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=33, tag_tid=1, entry_tid=34709], [object_number=32, tag_tid=1, entry_tid=34708], [object_number=35, tag_tid=1, entry_tid=34707], [object_number=34, tag_tid=1, entry_tid=34706]]], state=connected] 2026-02-18T08:40:45.714 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_31.status 2026-02-18T08:40:45.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=33, tag_tid=1, entry_tid=34709' 2026-02-18T08:40:45.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=33, tag_tid=2, entry_tid=1' -a 'object_number=33, tag_tid=2, entry_tid=1' = 'object_number=33, tag_tid=1, entry_tid=34709' 2026-02-18T08:40:45.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=33, tag_tid=1, entry_tid=34709' '!=' 'object_number=33, tag_tid=1, entry_tid=34709' 2026-02-18T08:40:45.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:40:45.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.8 2026-02-18T08:40:46.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_31 2026-02-18T08:40:46.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:40:46.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:40:46.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_31 2026-02-18T08:40:46.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:40:46.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_31 ']' 2026-02-18T08:40:46.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_31' 2026-02-18T08:40:46.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_31 2026-02-18T08:40:46.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:40:46.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:40:46.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:40:46.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:40:46.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:40:46.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:40:46.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:40:46.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:40:46.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:40:46.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:40:46.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_31 2026-02-18T08:40:46.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:40:46.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:40:46.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:40:46.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:40:46.523 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:40:46.523 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:40:46.523 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:40:46.523 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:40:46.523 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:40:46.523 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:40:46.523 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:40:46.523 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:40:46.523 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:40:46.523 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:40:46.523 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:40:46.523 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:40:46.524 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:40:46.524 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:40:46.524 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:40:46.524 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:40:46.524 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:40:46.524 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:40:46.524 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:40:46.524 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:40:46.524 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:40:46.524 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:40:46.524 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:40:46.524 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:40:46.524 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:40:46.525 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:40:46.532 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:40:46.532 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:40:46.532 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:40:46.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_31 2026-02-18T08:40:46.582 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_31'] 2026-02-18T08:40:46.582 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:40:46.582 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:40:46.585 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:40:46.585 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_31 2026-02-18T08:40:46.585 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:40:46.585 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:40:46.585 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:40:46.585 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:40:46.585 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:40:46.585 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:40:46.586 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:40:46.586 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:40:46.586 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:40:46.586 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:40:46.586 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:40:46.586 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:40:46.586 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:40:46.586 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:40:46.586 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:40:46.586 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:40:46.586 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:40:46.586 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:40:46.586 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:40:46.587 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:40:46.587 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:40:46.587 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:40:46.587 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:40:46.587 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:40:46.587 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:40:46.587 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:40:46.587 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:40:46.587 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:40:46.587 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:40:46.587 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:40:46.595 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:40:46.596 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:40:46.596 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:40:46.596 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_31 2026-02-18T08:40:46.653 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:40:46.654 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_31 2026-02-18T08:40:46.654 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:40:46.655 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:40:46.655 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_31 2026-02-18T08:40:46.655 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_31 '' 2026-02-18T08:40:46.655 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:40:46.655 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:40:46.655 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_31 2026-02-18T08:40:46.655 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:40:46.655 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_31.status 2026-02-18T08:40:46.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_31.status 2026-02-18T08:40:46.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:40:46.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_31.status 2026-02-18T08:40:46.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_31 2026-02-18T08:40:46.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_31.status 2026-02-18T08:40:46.682 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 8 2026-02-18T08:40:46.682 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 8 2026-02-18T08:40:46.683 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:40:46.683 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=33, tag_tid=2, entry_tid=1], [object_number=32, tag_tid=2, entry_tid=0], [object_number=35, tag_tid=1, entry_tid=34707], [object_number=34, tag_tid=1, entry_tid=34706]]], state=connected] 2026-02-18T08:40:46.683 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=33, tag_tid=1, entry_tid=34709], [object_number=32, tag_tid=1, entry_tid=34708], [object_number=35, tag_tid=1, entry_tid=34707], [object_number=34, tag_tid=1, entry_tid=34706]]], state=connected] 2026-02-18T08:40:46.685 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_31.status 2026-02-18T08:40:46.687 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=33, tag_tid=2, entry_tid=1' 2026-02-18T08:40:46.688 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_31 2026-02-18T08:40:46.688 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:40:46.688 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:40:46.688 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_31 2026-02-18T08:40:46.688 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_31 '..*' 2026-02-18T08:40:46.688 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:40:46.688 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:40:46.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_31 2026-02-18T08:40:46.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:40:46.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_31.status 2026-02-18T08:40:46.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_31.status 2026-02-18T08:40:46.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:40:46.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_31.status 2026-02-18T08:40:46.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_31 2026-02-18T08:40:46.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_31.status 2026-02-18T08:40:46.724 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 8 2026-02-18T08:40:46.724 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 8 2026-02-18T08:40:46.724 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:40:46.724 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=33, tag_tid=2, entry_tid=1], [object_number=32, tag_tid=2, entry_tid=0], [object_number=35, tag_tid=1, entry_tid=34707], [object_number=34, tag_tid=1, entry_tid=34706]]], state=connected] 2026-02-18T08:40:46.724 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=33, tag_tid=1, entry_tid=34709], [object_number=32, tag_tid=1, entry_tid=34708], [object_number=35, tag_tid=1, entry_tid=34707], [object_number=34, tag_tid=1, entry_tid=34706]]], state=connected] 2026-02-18T08:40:46.728 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_31.status 2026-02-18T08:40:46.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=33, tag_tid=1, entry_tid=34709' 2026-02-18T08:40:46.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=33, tag_tid=2, entry_tid=1' -a 'object_number=33, tag_tid=2, entry_tid=1' = 'object_number=33, tag_tid=1, entry_tid=34709' 2026-02-18T08:40:46.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=33, tag_tid=1, entry_tid=34709' '!=' 'object_number=33, tag_tid=1, entry_tid=34709' 2026-02-18T08:40:46.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:40:46.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1.6 2026-02-18T08:40:48.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_31 2026-02-18T08:40:48.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:40:48.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:40:48.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_31 2026-02-18T08:40:48.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:40:48.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_31 ']' 2026-02-18T08:40:48.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_31' 2026-02-18T08:40:48.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_31 2026-02-18T08:40:48.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:40:48.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:40:48.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:40:48.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:40:48.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:40:48.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:40:48.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:40:48.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:40:48.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:40:48.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:40:48.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_31 2026-02-18T08:40:48.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:40:48.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:40:48.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:40:48.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:40:48.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:40:48.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:40:48.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:40:48.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:40:48.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:40:48.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:40:48.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:40:48.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:40:48.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:40:48.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:40:48.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:40:48.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:40:48.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:40:48.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:40:48.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:40:48.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:40:48.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:40:48.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:40:48.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:40:48.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:40:48.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:40:48.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:40:48.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:40:48.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:40:48.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:40:48.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:40:48.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:40:48.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:40:48.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:40:48.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_31 2026-02-18T08:40:48.406 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_31'] 2026-02-18T08:40:48.406 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:40:48.406 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:40:48.409 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:40:48.409 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_31 2026-02-18T08:40:48.409 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:40:48.409 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:40:48.409 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:40:48.409 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:40:48.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:40:48.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:40:48.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:40:48.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:40:48.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:40:48.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:40:48.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:40:48.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:40:48.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:40:48.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:40:48.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:40:48.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:40:48.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:40:48.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:40:48.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:40:48.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:40:48.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:40:48.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:40:48.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:40:48.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:40:48.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:40:48.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:40:48.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:40:48.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:40:48.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:40:48.412 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:40:48.419 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:40:48.419 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:40:48.419 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:40:48.419 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_31 2026-02-18T08:40:48.478 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:40:48.479 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_31 2026-02-18T08:40:48.479 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:40:48.479 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:40:48.479 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_31 2026-02-18T08:40:48.479 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_31 '' 2026-02-18T08:40:48.479 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:40:48.480 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:40:48.480 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_31 2026-02-18T08:40:48.480 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:40:48.480 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_31.status 2026-02-18T08:40:48.480 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_31.status 2026-02-18T08:40:48.480 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:40:48.481 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_31.status 2026-02-18T08:40:48.481 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_31 2026-02-18T08:40:48.481 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_31.status 2026-02-18T08:40:48.508 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 8 2026-02-18T08:40:48.509 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 8 2026-02-18T08:40:48.509 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:40:48.509 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=33, tag_tid=2, entry_tid=1], [object_number=32, tag_tid=2, entry_tid=0], [object_number=35, tag_tid=1, entry_tid=34707], [object_number=34, tag_tid=1, entry_tid=34706]]], state=connected] 2026-02-18T08:40:48.509 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=33, tag_tid=1, entry_tid=34709], [object_number=32, tag_tid=1, entry_tid=34708], [object_number=35, tag_tid=1, entry_tid=34707], [object_number=34, tag_tid=1, entry_tid=34706]]], state=connected] 2026-02-18T08:40:48.511 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_31.status 2026-02-18T08:40:48.513 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=33, tag_tid=2, entry_tid=1' 2026-02-18T08:40:48.514 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_31 2026-02-18T08:40:48.514 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:40:48.514 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:40:48.514 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_31 2026-02-18T08:40:48.514 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_31 '..*' 2026-02-18T08:40:48.514 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:40:48.514 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:40:48.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_31 2026-02-18T08:40:48.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:40:48.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_31.status 2026-02-18T08:40:48.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_31.status 2026-02-18T08:40:48.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:40:48.516 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_31.status 2026-02-18T08:40:48.516 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_31 2026-02-18T08:40:48.516 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_31.status 2026-02-18T08:40:48.551 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 8 2026-02-18T08:40:48.551 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 8 2026-02-18T08:40:48.551 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:40:48.551 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=33, tag_tid=2, entry_tid=1], [object_number=32, tag_tid=2, entry_tid=0], [object_number=35, tag_tid=1, entry_tid=34707], [object_number=34, tag_tid=1, entry_tid=34706]]], state=connected] 2026-02-18T08:40:48.552 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=33, tag_tid=1, entry_tid=34709], [object_number=32, tag_tid=1, entry_tid=34708], [object_number=35, tag_tid=1, entry_tid=34707], [object_number=34, tag_tid=1, entry_tid=34706]]], state=connected] 2026-02-18T08:40:48.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_31.status 2026-02-18T08:40:48.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=33, tag_tid=1, entry_tid=34709' 2026-02-18T08:40:48.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=33, tag_tid=2, entry_tid=1' -a 'object_number=33, tag_tid=2, entry_tid=1' = 'object_number=33, tag_tid=1, entry_tid=34709' 2026-02-18T08:40:48.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=33, tag_tid=1, entry_tid=34709' '!=' 'object_number=33, tag_tid=1, entry_tid=34709' 2026-02-18T08:40:48.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:40:48.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 2 2026-02-18T08:40:48.684 INFO:tasks.rbd_mirror_thrash:kill cluster2.client.mirror.0 2026-02-18T08:40:48.685 INFO:tasks.rbd_mirror.cluster2.client.mirror.0:Sent signal 15 2026-02-18T08:40:48.685 INFO:tasks.rbd_mirror_thrash:waiting for 0.4449169156013 secs before reviving daemons 2026-02-18T08:40:48.685 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.trial163.stderr:2026-02-18T08:40:48.676+0000 7ffa6eb58640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.0 (PID: 102437) UID: 1000 2026-02-18T08:40:49.130 INFO:tasks.rbd_mirror_thrash:waiting for cluster2.client.mirror.0 2026-02-18T08:40:49.131 INFO:teuthology.orchestra.run:waiting for 600 2026-02-18T08:40:49.131 INFO:tasks.rbd_mirror_thrash:reviving cluster2.client.mirror.0 2026-02-18T08:40:49.131 INFO:tasks.rbd_mirror.cluster2.client.mirror.0:Restarting daemon 2026-02-18T08:40:49.131 DEBUG:teuthology.orchestra.run.trial163:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.0 2026-02-18T08:40:49.133 INFO:tasks.rbd_mirror.cluster2.client.mirror.0:Started 2026-02-18T08:40:49.133 INFO:tasks.rbd_mirror_thrash:waiting for 89.66129987466974 secs before thrashing 2026-02-18T08:40:50.557 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_31 2026-02-18T08:40:50.558 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:40:50.558 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:40:50.558 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_31 2026-02-18T08:40:50.558 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:40:50.558 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_31 ']' 2026-02-18T08:40:50.558 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_31' 2026-02-18T08:40:50.558 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_31 2026-02-18T08:40:50.558 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:40:50.559 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:40:50.559 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:40:50.559 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:40:50.559 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:40:50.559 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:40:50.559 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:40:50.559 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:40:50.562 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:40:50.562 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:40:50.562 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_31 2026-02-18T08:40:50.562 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:40:50.562 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:40:50.562 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:40:50.562 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:40:50.562 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:40:50.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:40:50.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:40:50.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:40:50.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:40:50.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:40:50.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:40:50.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:40:50.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:40:50.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:40:50.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:40:50.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:40:50.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:40:50.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:40:50.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:40:50.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:40:50.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:40:50.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:40:50.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:40:50.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:40:50.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:40:50.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:40:50.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:40:50.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:40:50.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:40:50.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:40:50.572 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:40:50.572 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:40:50.572 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:40:50.573 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_31 2026-02-18T08:40:50.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_31'] 2026-02-18T08:40:50.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:40:50.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:40:50.635 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:40:50.635 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_31 2026-02-18T08:40:50.636 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:40:50.636 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:40:50.636 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:40:50.636 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:40:50.636 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:40:50.636 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:40:50.636 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:40:50.636 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:40:50.636 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:40:50.636 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:40:50.636 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:40:50.637 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:40:50.637 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:40:50.637 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:40:50.637 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:40:50.637 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:40:50.637 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:40:50.637 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:40:50.637 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:40:50.637 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:40:50.637 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:40:50.637 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:40:50.637 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:40:50.638 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:40:50.638 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:40:50.638 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:40:50.638 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:40:50.638 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:40:50.638 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:40:50.638 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:40:50.646 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:40:50.647 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:40:50.647 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:40:50.647 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_31 2026-02-18T08:40:50.711 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:40:50.712 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_31 2026-02-18T08:40:50.712 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:40:50.712 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:40:50.713 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_31 2026-02-18T08:40:50.713 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_31 '' 2026-02-18T08:40:50.713 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:40:50.713 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:40:50.713 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_31 2026-02-18T08:40:50.713 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:40:50.713 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_31.status 2026-02-18T08:40:50.713 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_31.status 2026-02-18T08:40:50.713 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:40:50.714 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_31.status 2026-02-18T08:40:50.714 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_31 2026-02-18T08:40:50.714 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_31.status 2026-02-18T08:40:50.750 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 8 2026-02-18T08:40:50.750 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 8 2026-02-18T08:40:50.750 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:40:50.750 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=33, tag_tid=2, entry_tid=1], [object_number=32, tag_tid=2, entry_tid=0], [object_number=35, tag_tid=1, entry_tid=34707], [object_number=34, tag_tid=1, entry_tid=34706]]], state=connected] 2026-02-18T08:40:50.750 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=33, tag_tid=2, entry_tid=1], [object_number=32, tag_tid=2, entry_tid=0], [object_number=35, tag_tid=1, entry_tid=34707], [object_number=34, tag_tid=1, entry_tid=34706]]], state=connected] 2026-02-18T08:40:50.752 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_31.status 2026-02-18T08:40:50.754 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=33, tag_tid=2, entry_tid=1' 2026-02-18T08:40:50.755 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_31 2026-02-18T08:40:50.755 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:40:50.755 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:40:50.755 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_31 2026-02-18T08:40:50.755 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_31 '..*' 2026-02-18T08:40:50.756 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:40:50.756 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:40:50.756 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_31 2026-02-18T08:40:50.756 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:40:50.756 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_31.status 2026-02-18T08:40:50.756 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_31.status 2026-02-18T08:40:50.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:40:50.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_31.status 2026-02-18T08:40:50.758 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_31 2026-02-18T08:40:50.758 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_31.status 2026-02-18T08:40:50.784 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 8 2026-02-18T08:40:50.785 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 8 2026-02-18T08:40:50.785 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:40:50.785 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=33, tag_tid=2, entry_tid=1], [object_number=32, tag_tid=2, entry_tid=0], [object_number=35, tag_tid=1, entry_tid=34707], [object_number=34, tag_tid=1, entry_tid=34706]]], state=connected] 2026-02-18T08:40:50.785 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=33, tag_tid=2, entry_tid=1], [object_number=32, tag_tid=2, entry_tid=0], [object_number=35, tag_tid=1, entry_tid=34707], [object_number=34, tag_tid=1, entry_tid=34706]]], state=connected] 2026-02-18T08:40:50.787 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_31.status 2026-02-18T08:40:50.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=33, tag_tid=2, entry_tid=1' 2026-02-18T08:40:50.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=33, tag_tid=2, entry_tid=1' -a 'object_number=33, tag_tid=2, entry_tid=1' = 'object_number=33, tag_tid=2, entry_tid=1' 2026-02-18T08:40:50.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:40:50.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_snap_present cluster1 mirror image_31 snap 2026-02-18T08:40:50.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:40:50.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:40:50.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_31 2026-02-18T08:40:50.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap 2026-02-18T08:40:50.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-18T08:40:50.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-18T08:40:50.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1 2026-02-18T08:40:51.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster1 info mirror/image_31@snap 2026-02-18T08:40:51.819 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:rbd image 'image_31': 2026-02-18T08:40:51.819 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: size 128 MiB in 32 objects 2026-02-18T08:40:51.819 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: order 22 (4 MiB objects) 2026-02-18T08:40:51.819 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: snapshot_count: 1 2026-02-18T08:40:51.819 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: id: 11f9f294a9f5 2026-02-18T08:40:51.819 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: block_name_prefix: rbd_data.11f9f294a9f5 2026-02-18T08:40:51.819 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: format: 2 2026-02-18T08:40:51.820 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: features: layering, exclusive-lock, journaling 2026-02-18T08:40:51.820 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: op_features: 2026-02-18T08:40:51.820 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: flags: 2026-02-18T08:40:51.820 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: create_timestamp: Wed Feb 18 08:31:04 2026 2026-02-18T08:40:51.820 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: access_timestamp: Wed Feb 18 08:31:04 2026 2026-02-18T08:40:51.820 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: modify_timestamp: Wed Feb 18 08:32:05 2026 2026-02-18T08:40:51.820 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: protected: False 2026-02-18T08:40:51.820 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: journal: 11f9f294a9f5 2026-02-18T08:40:51.820 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring state: enabled 2026-02-18T08:40:51.820 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring mode: journal 2026-02-18T08:40:51.820 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring global id: 5131e3df-4984-4cc9-b7a1-61043c32f011 2026-02-18T08:40:51.821 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring primary: false 2026-02-18T08:40:51.822 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:40:51.822 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ compare_image_snaps mirror image_31 snap 2026-02-18T08:40:51.822 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:40:51.823 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_31 2026-02-18T08:40:51.823 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap 2026-02-18T08:40:51.823 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local ret=0 2026-02-18T08:40:51.823 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local rmt_export=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_31.export 2026-02-18T08:40:51.823 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loc_export=/tmp/tmp.vybh1m9yJF/cluster1-mirror-image_31.export 2026-02-18T08:40:51.823 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_31.export /tmp/tmp.vybh1m9yJF/cluster1-mirror-image_31.export 2026-02-18T08:40:51.823 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 -p mirror export image_31@snap /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_31.export 2026-02-18T08:40:51.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-18T08:40:51.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster1 -p mirror export image_31@snap /tmp/tmp.vybh1m9yJF/cluster1-mirror-image_31.export 2026-02-18T08:40:52.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-18T08:40:52.169 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmp /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_31.export /tmp/tmp.vybh1m9yJF/cluster1-mirror-image_31.export 2026-02-18T08:40:52.225 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_31.export /tmp/tmp.vybh1m9yJF/cluster1-mirror-image_31.export 2026-02-18T08:40:52.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:40:52.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T08:40:52.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_32 2026-02-18T08:40:52.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_snap cluster2 mirror image_32 snap 2026-02-18T08:40:52.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:40:52.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:40:52.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_32 2026-02-18T08:40:52.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap 2026-02-18T08:40:52.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 -p mirror snap create image_32@snap --debug-rbd=20 --debug-journaler=20 2026-02-18T08:40:55.067 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_replay_started cluster1 mirror image_32 2026-02-18T08:40:55.068 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:40:55.068 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:40:55.068 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_32 2026-02-18T08:40:55.068 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_replay_state cluster1 mirror image_32 started 2026-02-18T08:40:55.068 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:40:55.068 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:40:55.068 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_32 2026-02-18T08:40:55.068 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local state=started 2026-02-18T08:40:55.069 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-18T08:40:55.069 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:40:55.069 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1 2026-02-18T08:40:56.069 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test_image_replay_state cluster1 mirror image_32 started 2026-02-18T08:40:56.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:40:56.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:40:56.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_32 2026-02-18T08:40:56.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local test_state=started 2026-02-18T08:40:56.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local status_result 2026-02-18T08:40:56.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_state=stopped 2026-02-18T08:40:56.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_32 2026-02-18T08:40:56.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -i state 2026-02-18T08:40:56.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_instance=cluster1 2026-02-18T08:40:56.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-18T08:40:56.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:40:56.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance=cluster1 2026-02-18T08:40:56.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local loop_instance 2026-02-18T08:40:56.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:40:56.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sleep 0 2026-02-18T08:40:56.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:40:56.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ seq 0 3 2026-02-18T08:40:56.074 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:40:56.074 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_32 2026-02-18T08:40:56.074 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:0 2026-02-18T08:40:56.074 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-18T08:40:56.074 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:40:56.074 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:40:56.074 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:40:56.074 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-18T08:40:56.074 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:40:56.074 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:40:56.074 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:40:56.075 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:40:56.075 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-18T08:40:56.075 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:40:56.075 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:40:56.075 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:40:56.075 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:40:56.075 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:0 2026-02-18T08:40:56.075 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-18T08:40:56.075 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-18T08:40:56.075 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:40:56.075 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:0 2026-02-18T08:40:56.075 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T08:40:56.075 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-18T08:40:56.076 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:40:56.076 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:40:56.076 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-18T08:40:56.076 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-18T08:40:56.076 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-18T08:40:56.076 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=0 2026-02-18T08:40:56.076 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=0 2026-02-18T08:40:56.076 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:40:56.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:40:56.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:40:56.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:40:56.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_32 2026-02-18T08:40:56.137 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_32'] 2026-02-18T08:40:56.137 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status : get status for rbd mirror 2026-02-18T08:40:56.137 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:40:56.140 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:40:56.140 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_32 2026-02-18T08:40:56.140 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:1 2026-02-18T08:40:56.140 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-18T08:40:56.140 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:40:56.141 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:40:56.141 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:40:56.141 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-18T08:40:56.141 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:40:56.141 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:40:56.141 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:40:56.141 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:40:56.141 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-18T08:40:56.141 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:40:56.141 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:40:56.141 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:40:56.141 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:40:56.142 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:1 2026-02-18T08:40:56.142 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-18T08:40:56.142 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-18T08:40:56.142 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:40:56.142 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:1 2026-02-18T08:40:56.142 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T08:40:56.142 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-18T08:40:56.142 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:40:56.142 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:40:56.142 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-18T08:40:56.142 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-18T08:40:56.142 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-18T08:40:56.143 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=1 2026-02-18T08:40:56.143 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=1 2026-02-18T08:40:56.143 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:40:56.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:40:56.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:40:56.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:40:56.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_32 2026-02-18T08:40:56.219 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ return 0 2026-02-18T08:40:56.219 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ status_result=' "state": "Replaying"' 2026-02-18T08:40:56.220 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ echo ' "state": "Replaying"' 2026-02-18T08:40:56.220 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ grep -i Replaying 2026-02-18T08:40:56.221 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: "state": "Replaying" 2026-02-18T08:40:56.222 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ current_state=started 2026-02-18T08:40:56.222 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test started = started 2026-02-18T08:40:56.222 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:40:56.222 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror image_32 2026-02-18T08:40:56.223 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_cluster=cluster1 2026-02-18T08:40:56.223 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:40:56.223 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_pool=mirror 2026-02-18T08:40:56.223 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local remote_pool=mirror 2026-02-18T08:40:56.223 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_32 2026-02-18T08:40:56.223 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' journal = journal ']' 2026-02-18T08:40:56.223 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror mirror image_32 2026-02-18T08:40:56.223 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_cluster=cluster1 2026-02-18T08:40:56.223 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:40:56.223 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_pool=mirror 2026-02-18T08:40:56.223 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local remote_pool=mirror 2026-02-18T08:40:56.223 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_32 2026-02-18T08:40:56.224 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-02-18T08:40:56.224 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-02-18T08:40:56.224 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:40:56.224 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:40:56.224 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:40:56.423 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_32 2026-02-18T08:40:56.423 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:40:56.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:40:56.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_32 2026-02-18T08:40:56.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:40:56.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_32 ']' 2026-02-18T08:40:56.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_32' 2026-02-18T08:40:56.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_32 2026-02-18T08:40:56.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:40:56.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:40:56.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:40:56.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:40:56.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:40:56.425 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:40:56.425 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:40:56.425 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:40:56.427 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:40:56.427 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:40:56.427 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_32 2026-02-18T08:40:56.427 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:40:56.428 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:40:56.428 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:40:56.428 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:40:56.428 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:40:56.428 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:40:56.428 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:40:56.428 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:40:56.428 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:40:56.428 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:40:56.428 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:40:56.428 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:40:56.429 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:40:56.429 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:40:56.429 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:40:56.429 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:40:56.429 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:40:56.429 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:40:56.429 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:40:56.429 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:40:56.429 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:40:56.429 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:40:56.429 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:40:56.429 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:40:56.429 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:40:56.430 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:40:56.430 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:40:56.430 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:40:56.430 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:40:56.430 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:40:56.439 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:40:56.439 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:40:56.439 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:40:56.439 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_32 2026-02-18T08:40:56.500 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_32'] 2026-02-18T08:40:56.501 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:40:56.501 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:40:56.503 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:40:56.503 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_32 2026-02-18T08:40:56.503 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:40:56.504 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:40:56.504 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:40:56.504 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:40:56.504 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:40:56.504 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:40:56.504 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:40:56.504 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:40:56.504 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:40:56.504 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:40:56.504 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:40:56.504 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:40:56.504 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:40:56.505 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:40:56.505 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:40:56.505 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:40:56.505 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:40:56.505 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:40:56.505 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:40:56.505 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:40:56.505 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:40:56.505 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:40:56.505 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:40:56.505 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:40:56.505 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:40:56.506 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:40:56.506 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:40:56.506 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:40:56.506 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:40:56.506 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:40:56.514 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:40:56.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:40:56.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:40:56.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_32 2026-02-18T08:40:56.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:40:56.582 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_32 2026-02-18T08:40:56.582 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:40:56.582 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:40:56.582 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_32 2026-02-18T08:40:56.582 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_32 '' 2026-02-18T08:40:56.582 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:40:56.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:40:56.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_32 2026-02-18T08:40:56.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:40:56.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_32.status 2026-02-18T08:40:56.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_32.status 2026-02-18T08:40:56.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:40:56.584 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_32.status 2026-02-18T08:40:56.584 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_32 2026-02-18T08:40:56.584 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_32.status 2026-02-18T08:40:56.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 10 2026-02-18T08:40:56.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 10 2026-02-18T08:40:56.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:40:56.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=41, tag_tid=2, entry_tid=1], [object_number=40, tag_tid=2, entry_tid=0], [object_number=43, tag_tid=1, entry_tid=43387], [object_number=42, tag_tid=1, entry_tid=43386]]], state=connected] 2026-02-18T08:40:56.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=40, tag_tid=1, entry_tid=43388], [object_number=43, tag_tid=1, entry_tid=43387], [object_number=42, tag_tid=1, entry_tid=43386], [object_number=41, tag_tid=1, entry_tid=43385]]], state=connected] 2026-02-18T08:40:56.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_32.status 2026-02-18T08:40:56.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=41, tag_tid=2, entry_tid=1' 2026-02-18T08:40:56.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_32 2026-02-18T08:40:56.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:40:56.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:40:56.628 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_32 2026-02-18T08:40:56.628 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_32 '..*' 2026-02-18T08:40:56.628 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:40:56.628 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:40:56.628 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_32 2026-02-18T08:40:56.628 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:40:56.628 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_32.status 2026-02-18T08:40:56.628 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_32.status 2026-02-18T08:40:56.629 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:40:56.629 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_32.status 2026-02-18T08:40:56.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_32 2026-02-18T08:40:56.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_32.status 2026-02-18T08:40:56.664 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 10 2026-02-18T08:40:56.664 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 10 2026-02-18T08:40:56.664 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:40:56.664 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=41, tag_tid=2, entry_tid=1], [object_number=40, tag_tid=2, entry_tid=0], [object_number=43, tag_tid=1, entry_tid=43387], [object_number=42, tag_tid=1, entry_tid=43386]]], state=connected] 2026-02-18T08:40:56.664 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=40, tag_tid=1, entry_tid=43388], [object_number=43, tag_tid=1, entry_tid=43387], [object_number=42, tag_tid=1, entry_tid=43386], [object_number=41, tag_tid=1, entry_tid=43385]]], state=connected] 2026-02-18T08:40:56.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_32.status 2026-02-18T08:40:56.670 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=40, tag_tid=1, entry_tid=43388' 2026-02-18T08:40:56.670 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=41, tag_tid=2, entry_tid=1' -a 'object_number=41, tag_tid=2, entry_tid=1' = 'object_number=40, tag_tid=1, entry_tid=43388' 2026-02-18T08:40:56.670 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=40, tag_tid=1, entry_tid=43388' '!=' '' 2026-02-18T08:40:56.670 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:40:56.670 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=40, tag_tid=1, entry_tid=43388' = '' 2026-02-18T08:40:56.670 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=40, tag_tid=1, entry_tid=43388' 2026-02-18T08:40:56.671 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=41, tag_tid=2, entry_tid=1' 2026-02-18T08:40:56.671 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:40:56.671 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:40:56.672 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=2 2026-02-18T08:40:56.673 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=40, tag_tid=1, entry_tid=43388' 2026-02-18T08:40:56.673 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:40:56.673 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:40:56.674 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=1 2026-02-18T08:40:56.675 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=41, tag_tid=2, entry_tid=1' 2026-02-18T08:40:56.675 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:40:56.675 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:40:56.676 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:40:56.677 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=40, tag_tid=1, entry_tid=43388' 2026-02-18T08:40:56.677 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:40:56.677 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:40:56.678 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=43388 2026-02-18T08:40:56.678 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 2 = 1 -a 1 -le 43388 2026-02-18T08:40:56.678 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:40:56.679 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:40:56.679 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:40:56.880 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_32 2026-02-18T08:40:56.880 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:40:56.880 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:40:56.881 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_32 2026-02-18T08:40:56.881 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:40:56.881 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_32 ']' 2026-02-18T08:40:56.881 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_32' 2026-02-18T08:40:56.881 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_32 2026-02-18T08:40:56.881 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:40:56.881 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:40:56.881 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:40:56.881 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:40:56.881 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:40:56.881 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:40:56.881 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:40:56.884 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:40:56.884 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:40:56.884 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:40:56.884 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_32 2026-02-18T08:40:56.884 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:40:56.885 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:40:56.885 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:40:56.885 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:40:56.885 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:40:56.885 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:40:56.885 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:40:56.885 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:40:56.885 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:40:56.885 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:40:56.885 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:40:56.885 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:40:56.885 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:40:56.886 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:40:56.886 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:40:56.886 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:40:56.886 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:40:56.886 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:40:56.886 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:40:56.886 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:40:56.886 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:40:56.886 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:40:56.886 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:40:56.886 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:40:56.886 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:40:56.887 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:40:56.887 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:40:56.887 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:40:56.887 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:40:56.887 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:40:56.895 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:40:56.895 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:40:56.896 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:40:56.896 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_32 2026-02-18T08:40:56.954 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_32'] 2026-02-18T08:40:56.954 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:40:56.954 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:40:56.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:40:56.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_32 2026-02-18T08:40:56.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:40:56.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:40:56.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:40:56.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:40:56.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:40:56.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:40:56.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:40:56.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:40:56.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:40:56.958 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:40:56.958 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:40:56.958 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:40:56.958 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:40:56.958 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:40:56.958 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:40:56.958 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:40:56.958 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:40:56.958 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:40:56.958 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:40:56.958 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:40:56.958 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:40:56.959 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:40:56.959 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:40:56.959 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:40:56.959 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:40:56.959 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:40:56.959 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:40:56.959 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:40:56.959 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:40:56.959 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:40:56.968 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:40:56.968 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:40:56.968 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:40:56.969 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_32 2026-02-18T08:40:57.041 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:40:57.041 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_32 2026-02-18T08:40:57.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:40:57.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:40:57.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_32 2026-02-18T08:40:57.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_32 '' 2026-02-18T08:40:57.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:40:57.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:40:57.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_32 2026-02-18T08:40:57.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:40:57.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_32.status 2026-02-18T08:40:57.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_32.status 2026-02-18T08:40:57.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:40:57.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_32.status 2026-02-18T08:40:57.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_32 2026-02-18T08:40:57.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_32.status 2026-02-18T08:40:57.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 10 2026-02-18T08:40:57.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 10 2026-02-18T08:40:57.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:40:57.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=41, tag_tid=2, entry_tid=1], [object_number=40, tag_tid=2, entry_tid=0], [object_number=43, tag_tid=1, entry_tid=43387], [object_number=42, tag_tid=1, entry_tid=43386]]], state=connected] 2026-02-18T08:40:57.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=41, tag_tid=2, entry_tid=1], [object_number=40, tag_tid=2, entry_tid=0], [object_number=43, tag_tid=1, entry_tid=43387], [object_number=42, tag_tid=1, entry_tid=43386]]], state=connected] 2026-02-18T08:40:57.074 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_32.status 2026-02-18T08:40:57.076 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=41, tag_tid=2, entry_tid=1' 2026-02-18T08:40:57.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_32 2026-02-18T08:40:57.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:40:57.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:40:57.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_32 2026-02-18T08:40:57.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_32 '..*' 2026-02-18T08:40:57.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:40:57.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:40:57.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_32 2026-02-18T08:40:57.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:40:57.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_32.status 2026-02-18T08:40:57.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_32.status 2026-02-18T08:40:57.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:40:57.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_32.status 2026-02-18T08:40:57.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_32 2026-02-18T08:40:57.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_32.status 2026-02-18T08:40:57.114 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 10 2026-02-18T08:40:57.114 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 10 2026-02-18T08:40:57.114 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:40:57.114 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=41, tag_tid=2, entry_tid=1], [object_number=40, tag_tid=2, entry_tid=0], [object_number=43, tag_tid=1, entry_tid=43387], [object_number=42, tag_tid=1, entry_tid=43386]]], state=connected] 2026-02-18T08:40:57.114 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=41, tag_tid=2, entry_tid=1], [object_number=40, tag_tid=2, entry_tid=0], [object_number=43, tag_tid=1, entry_tid=43387], [object_number=42, tag_tid=1, entry_tid=43386]]], state=connected] 2026-02-18T08:40:57.118 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_32.status 2026-02-18T08:40:57.120 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=41, tag_tid=2, entry_tid=1' 2026-02-18T08:40:57.120 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=41, tag_tid=2, entry_tid=1' -a 'object_number=41, tag_tid=2, entry_tid=1' = 'object_number=41, tag_tid=2, entry_tid=1' 2026-02-18T08:40:57.120 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:40:57.120 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_snap_present cluster1 mirror image_32 snap 2026-02-18T08:40:57.120 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:40:57.120 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:40:57.121 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_32 2026-02-18T08:40:57.121 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap 2026-02-18T08:40:57.121 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-18T08:40:57.121 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-18T08:40:57.121 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1 2026-02-18T08:40:58.121 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster1 info mirror/image_32@snap 2026-02-18T08:40:58.150 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:rbd image 'image_32': 2026-02-18T08:40:58.150 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: size 128 MiB in 32 objects 2026-02-18T08:40:58.150 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: order 22 (4 MiB objects) 2026-02-18T08:40:58.150 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: snapshot_count: 1 2026-02-18T08:40:58.150 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: id: 120dee348e2a 2026-02-18T08:40:58.150 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: block_name_prefix: rbd_data.120dee348e2a 2026-02-18T08:40:58.150 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: format: 2 2026-02-18T08:40:58.150 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: features: layering, exclusive-lock, journaling 2026-02-18T08:40:58.150 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: op_features: 2026-02-18T08:40:58.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: flags: 2026-02-18T08:40:58.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: create_timestamp: Wed Feb 18 08:31:14 2026 2026-02-18T08:40:58.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: access_timestamp: Wed Feb 18 08:31:14 2026 2026-02-18T08:40:58.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: modify_timestamp: Wed Feb 18 08:32:15 2026 2026-02-18T08:40:58.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: protected: False 2026-02-18T08:40:58.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: journal: 120dee348e2a 2026-02-18T08:40:58.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring state: enabled 2026-02-18T08:40:58.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring mode: journal 2026-02-18T08:40:58.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring global id: 26772eec-fd07-4fe4-bd28-93c852efe5e1 2026-02-18T08:40:58.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring primary: false 2026-02-18T08:40:58.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:40:58.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ compare_image_snaps mirror image_32 snap 2026-02-18T08:40:58.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:40:58.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_32 2026-02-18T08:40:58.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap 2026-02-18T08:40:58.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local ret=0 2026-02-18T08:40:58.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local rmt_export=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_32.export 2026-02-18T08:40:58.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loc_export=/tmp/tmp.vybh1m9yJF/cluster1-mirror-image_32.export 2026-02-18T08:40:58.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_32.export /tmp/tmp.vybh1m9yJF/cluster1-mirror-image_32.export 2026-02-18T08:40:58.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 -p mirror export image_32@snap /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_32.export 2026-02-18T08:40:58.314 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-18T08:40:58.323 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster1 -p mirror export image_32@snap /tmp/tmp.vybh1m9yJF/cluster1-mirror-image_32.export 2026-02-18T08:40:58.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-18T08:40:58.508 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmp /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_32.export /tmp/tmp.vybh1m9yJF/cluster1-mirror-image_32.export 2026-02-18T08:40:58.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_32.export /tmp/tmp.vybh1m9yJF/cluster1-mirror-image_32.export 2026-02-18T08:40:58.598 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:40:58.598 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T08:40:58.598 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_33 2026-02-18T08:40:58.598 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_snap cluster2 mirror image_33 snap 2026-02-18T08:40:58.599 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:40:58.599 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:40:58.599 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_33 2026-02-18T08:40:58.599 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap 2026-02-18T08:40:58.599 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 -p mirror snap create image_33@snap --debug-rbd=20 --debug-journaler=20 2026-02-18T08:41:03.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_replay_started cluster1 mirror image_33 2026-02-18T08:41:03.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:41:03.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:41:03.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_33 2026-02-18T08:41:03.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_replay_state cluster1 mirror image_33 started 2026-02-18T08:41:03.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:41:03.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:41:03.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_33 2026-02-18T08:41:03.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local state=started 2026-02-18T08:41:03.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-18T08:41:03.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:41:03.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1 2026-02-18T08:41:04.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test_image_replay_state cluster1 mirror image_33 started 2026-02-18T08:41:04.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:41:04.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:41:04.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_33 2026-02-18T08:41:04.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local test_state=started 2026-02-18T08:41:04.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local status_result 2026-02-18T08:41:04.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_state=stopped 2026-02-18T08:41:04.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_33 2026-02-18T08:41:04.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -i state 2026-02-18T08:41:04.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_instance=cluster1 2026-02-18T08:41:04.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-18T08:41:04.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:41:04.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance=cluster1 2026-02-18T08:41:04.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local loop_instance 2026-02-18T08:41:04.092 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:41:04.092 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sleep 0 2026-02-18T08:41:04.092 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:41:04.092 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ seq 0 3 2026-02-18T08:41:04.094 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:41:04.094 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_33 2026-02-18T08:41:04.094 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:0 2026-02-18T08:41:04.095 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-18T08:41:04.095 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:41:04.095 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:41:04.095 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:41:04.095 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-18T08:41:04.095 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:41:04.095 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:41:04.095 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:41:04.095 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:41:04.095 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-18T08:41:04.095 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:41:04.095 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:41:04.096 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:41:04.096 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:41:04.096 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:0 2026-02-18T08:41:04.096 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-18T08:41:04.096 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-18T08:41:04.096 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:41:04.096 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:0 2026-02-18T08:41:04.096 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T08:41:04.096 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-18T08:41:04.096 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:41:04.096 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:41:04.096 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-18T08:41:04.097 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-18T08:41:04.097 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-18T08:41:04.097 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=0 2026-02-18T08:41:04.097 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=0 2026-02-18T08:41:04.097 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:41:04.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:41:04.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:41:04.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:41:04.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_33 2026-02-18T08:41:04.155 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_33'] 2026-02-18T08:41:04.155 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status : get status for rbd mirror 2026-02-18T08:41:04.155 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:41:04.158 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:41:04.158 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_33 2026-02-18T08:41:04.158 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:1 2026-02-18T08:41:04.158 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-18T08:41:04.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:41:04.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:41:04.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:41:04.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-18T08:41:04.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:41:04.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:41:04.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:41:04.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:41:04.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-18T08:41:04.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:41:04.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:41:04.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:41:04.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:41:04.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:1 2026-02-18T08:41:04.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-18T08:41:04.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-18T08:41:04.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:41:04.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:1 2026-02-18T08:41:04.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T08:41:04.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-18T08:41:04.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:41:04.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:41:04.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-18T08:41:04.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-18T08:41:04.161 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-18T08:41:04.161 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=1 2026-02-18T08:41:04.161 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=1 2026-02-18T08:41:04.161 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:41:04.169 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:41:04.170 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:41:04.170 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:41:04.170 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_33 2026-02-18T08:41:04.223 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_33'] 2026-02-18T08:41:04.223 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status : get status for rbd mirror 2026-02-18T08:41:04.223 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:41:04.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:41:04.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_33 2026-02-18T08:41:04.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:2 2026-02-18T08:41:04.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-18T08:41:04.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:41:04.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:41:04.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:41:04.227 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-18T08:41:04.227 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:41:04.227 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:41:04.227 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:41:04.227 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:41:04.227 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-18T08:41:04.227 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:41:04.227 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:41:04.227 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:41:04.227 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:41:04.227 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:2 2026-02-18T08:41:04.227 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-18T08:41:04.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-18T08:41:04.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:41:04.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:2 2026-02-18T08:41:04.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T08:41:04.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-18T08:41:04.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:41:04.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:41:04.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-18T08:41:04.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-18T08:41:04.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-18T08:41:04.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=2 2026-02-18T08:41:04.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=2 2026-02-18T08:41:04.229 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:41:04.238 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:41:04.238 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:41:04.238 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:41:04.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_33 2026-02-18T08:41:04.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ return 0 2026-02-18T08:41:04.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ status_result=' "state": "Replaying"' 2026-02-18T08:41:04.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ echo ' "state": "Replaying"' 2026-02-18T08:41:04.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ grep -i Replaying 2026-02-18T08:41:04.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: "state": "Replaying" 2026-02-18T08:41:04.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ current_state=started 2026-02-18T08:41:04.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test started = started 2026-02-18T08:41:04.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:41:04.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror image_33 2026-02-18T08:41:04.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_cluster=cluster1 2026-02-18T08:41:04.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:41:04.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_pool=mirror 2026-02-18T08:41:04.312 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local remote_pool=mirror 2026-02-18T08:41:04.312 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_33 2026-02-18T08:41:04.312 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' journal = journal ']' 2026-02-18T08:41:04.312 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror mirror image_33 2026-02-18T08:41:04.312 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_cluster=cluster1 2026-02-18T08:41:04.312 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:41:04.312 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_pool=mirror 2026-02-18T08:41:04.312 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local remote_pool=mirror 2026-02-18T08:41:04.312 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_33 2026-02-18T08:41:04.312 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-02-18T08:41:04.312 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-02-18T08:41:04.312 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:41:04.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:41:04.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:41:04.512 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_33 2026-02-18T08:41:04.512 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:41:04.512 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:41:04.512 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_33 2026-02-18T08:41:04.512 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:41:04.513 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_33 ']' 2026-02-18T08:41:04.513 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_33' 2026-02-18T08:41:04.513 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_33 2026-02-18T08:41:04.513 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:41:04.513 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:41:04.513 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:41:04.513 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:41:04.513 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:41:04.513 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:41:04.513 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:41:04.513 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:41:04.516 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:41:04.516 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:41:04.516 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_33 2026-02-18T08:41:04.516 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:41:04.516 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:41:04.516 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:41:04.516 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:41:04.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:41:04.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:41:04.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:41:04.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:41:04.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:41:04.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:41:04.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:41:04.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:41:04.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:41:04.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:41:04.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:41:04.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:41:04.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:41:04.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:41:04.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:41:04.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:41:04.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:41:04.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:41:04.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:41:04.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:41:04.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:41:04.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:41:04.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:41:04.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:41:04.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:41:04.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:41:04.527 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:41:04.527 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:41:04.528 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:41:04.528 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_33 2026-02-18T08:41:04.578 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_33'] 2026-02-18T08:41:04.578 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:41:04.578 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:41:04.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:41:04.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_33 2026-02-18T08:41:04.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:41:04.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:41:04.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:41:04.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:41:04.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:41:04.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:41:04.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:41:04.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:41:04.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:41:04.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:41:04.582 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:41:04.582 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:41:04.582 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:41:04.582 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:41:04.582 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:41:04.582 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:41:04.582 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:41:04.582 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:41:04.582 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:41:04.582 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:41:04.582 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:41:04.582 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:41:04.582 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:41:04.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:41:04.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:41:04.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:41:04.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:41:04.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:41:04.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:41:04.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:41:04.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:41:04.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:41:04.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:41:04.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_33 2026-02-18T08:41:04.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_33'] 2026-02-18T08:41:04.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:41:04.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:41:04.654 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:41:04.654 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_33 2026-02-18T08:41:04.654 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:41:04.654 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:41:04.654 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:41:04.654 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:41:04.654 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:41:04.655 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:41:04.655 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:41:04.655 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:41:04.655 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:41:04.655 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:41:04.655 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:41:04.655 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:41:04.655 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:41:04.655 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:41:04.655 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:41:04.655 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:41:04.655 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:41:04.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:41:04.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:41:04.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:41:04.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:41:04.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:41:04.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:41:04.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:41:04.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:41:04.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:41:04.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:41:04.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:41:04.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:41:04.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:41:04.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:41:04.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:41:04.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:41:04.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_33 2026-02-18T08:41:04.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:41:04.725 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_33 2026-02-18T08:41:04.725 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:41:04.725 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:41:04.725 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_33 2026-02-18T08:41:04.725 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_33 '' 2026-02-18T08:41:04.725 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:41:04.725 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:41:04.725 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_33 2026-02-18T08:41:04.725 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:41:04.725 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_33.status 2026-02-18T08:41:04.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_33.status 2026-02-18T08:41:04.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:41:04.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_33.status 2026-02-18T08:41:04.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_33 2026-02-18T08:41:04.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_33.status 2026-02-18T08:41:04.763 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 7 2026-02-18T08:41:04.763 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 7 2026-02-18T08:41:04.763 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:41:04.763 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=29, tag_tid=2, entry_tid=1], [object_number=28, tag_tid=2, entry_tid=0], [object_number=31, tag_tid=1, entry_tid=32819], [object_number=30, tag_tid=1, entry_tid=32818]]], state=connected] 2026-02-18T08:41:04.763 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=29, tag_tid=1, entry_tid=32821], [object_number=28, tag_tid=1, entry_tid=32820], [object_number=31, tag_tid=1, entry_tid=32819], [object_number=30, tag_tid=1, entry_tid=32818]]], state=connected] 2026-02-18T08:41:04.767 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_33.status 2026-02-18T08:41:04.768 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=29, tag_tid=2, entry_tid=1' 2026-02-18T08:41:04.770 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_33 2026-02-18T08:41:04.770 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:41:04.770 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:41:04.770 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_33 2026-02-18T08:41:04.770 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_33 '..*' 2026-02-18T08:41:04.770 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:41:04.770 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:41:04.770 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_33 2026-02-18T08:41:04.770 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:41:04.770 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_33.status 2026-02-18T08:41:04.771 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_33.status 2026-02-18T08:41:04.771 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:41:04.772 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_33.status 2026-02-18T08:41:04.772 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_33 2026-02-18T08:41:04.772 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_33.status 2026-02-18T08:41:04.807 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 7 2026-02-18T08:41:04.807 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 7 2026-02-18T08:41:04.807 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:41:04.807 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=29, tag_tid=2, entry_tid=1], [object_number=28, tag_tid=2, entry_tid=0], [object_number=31, tag_tid=1, entry_tid=32819], [object_number=30, tag_tid=1, entry_tid=32818]]], state=connected] 2026-02-18T08:41:04.807 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=29, tag_tid=1, entry_tid=32821], [object_number=28, tag_tid=1, entry_tid=32820], [object_number=31, tag_tid=1, entry_tid=32819], [object_number=30, tag_tid=1, entry_tid=32818]]], state=connected] 2026-02-18T08:41:04.811 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_33.status 2026-02-18T08:41:04.812 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=29, tag_tid=1, entry_tid=32821' 2026-02-18T08:41:04.812 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=29, tag_tid=2, entry_tid=1' -a 'object_number=29, tag_tid=2, entry_tid=1' = 'object_number=29, tag_tid=1, entry_tid=32821' 2026-02-18T08:41:04.812 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=29, tag_tid=1, entry_tid=32821' '!=' '' 2026-02-18T08:41:04.812 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:41:04.813 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=29, tag_tid=1, entry_tid=32821' = '' 2026-02-18T08:41:04.813 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=29, tag_tid=1, entry_tid=32821' 2026-02-18T08:41:04.813 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=29, tag_tid=2, entry_tid=1' 2026-02-18T08:41:04.813 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:41:04.814 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:41:04.814 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=2 2026-02-18T08:41:04.815 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=29, tag_tid=1, entry_tid=32821' 2026-02-18T08:41:04.815 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:41:04.815 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:41:04.817 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=1 2026-02-18T08:41:04.817 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=29, tag_tid=2, entry_tid=1' 2026-02-18T08:41:04.817 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:41:04.818 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:41:04.819 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:41:04.820 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=29, tag_tid=1, entry_tid=32821' 2026-02-18T08:41:04.820 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:41:04.820 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:41:04.821 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=32821 2026-02-18T08:41:04.822 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 2 = 1 -a 1 -le 32821 2026-02-18T08:41:04.822 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:41:04.822 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:41:04.822 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:41:05.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_33 2026-02-18T08:41:05.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:41:05.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:41:05.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_33 2026-02-18T08:41:05.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:41:05.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_33 ']' 2026-02-18T08:41:05.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_33' 2026-02-18T08:41:05.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_33 2026-02-18T08:41:05.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:41:05.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:41:05.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:41:05.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:41:05.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:41:05.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:41:05.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:41:05.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:41:05.027 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:41:05.028 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:41:05.028 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_33 2026-02-18T08:41:05.028 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:41:05.028 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:41:05.028 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:41:05.028 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:41:05.028 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:41:05.028 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:41:05.028 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:41:05.028 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:41:05.029 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:41:05.029 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:41:05.029 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:41:05.029 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:41:05.029 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:41:05.029 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:41:05.029 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:41:05.029 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:41:05.029 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:41:05.029 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:41:05.029 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:41:05.029 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:41:05.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:41:05.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:41:05.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:41:05.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:41:05.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:41:05.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:41:05.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:41:05.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:41:05.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:41:05.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:41:05.038 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:41:05.039 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:41:05.039 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:41:05.039 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_33 2026-02-18T08:41:05.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_33'] 2026-02-18T08:41:05.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:41:05.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:41:05.093 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:41:05.094 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_33 2026-02-18T08:41:05.094 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:41:05.094 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:41:05.094 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:41:05.094 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:41:05.094 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:41:05.094 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:41:05.094 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:41:05.094 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:41:05.094 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:41:05.094 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:41:05.095 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:41:05.095 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:41:05.095 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:41:05.095 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:41:05.095 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:41:05.095 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:41:05.095 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:41:05.095 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:41:05.095 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:41:05.095 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:41:05.095 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:41:05.095 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:41:05.096 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:41:05.096 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:41:05.096 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:41:05.096 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:41:05.096 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:41:05.096 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:41:05.096 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:41:05.096 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:41:05.105 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:41:05.106 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:41:05.106 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:41:05.106 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_33 2026-02-18T08:41:05.167 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_33'] 2026-02-18T08:41:05.167 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:41:05.167 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:41:05.170 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:41:05.170 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_33 2026-02-18T08:41:05.171 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:41:05.171 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:41:05.171 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:41:05.171 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:41:05.171 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:41:05.171 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:41:05.171 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:41:05.171 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:41:05.171 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:41:05.171 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:41:05.171 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:41:05.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:41:05.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:41:05.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:41:05.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:41:05.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:41:05.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:41:05.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:41:05.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:41:05.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:41:05.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:41:05.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:41:05.173 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:41:05.173 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:41:05.173 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:41:05.173 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:41:05.173 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:41:05.173 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:41:05.173 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:41:05.173 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:41:05.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:41:05.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:41:05.183 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:41:05.183 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_33 2026-02-18T08:41:05.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:41:05.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_33 2026-02-18T08:41:05.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:41:05.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:41:05.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_33 2026-02-18T08:41:05.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_33 '' 2026-02-18T08:41:05.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:41:05.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:41:05.244 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_33 2026-02-18T08:41:05.244 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:41:05.244 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_33.status 2026-02-18T08:41:05.244 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_33.status 2026-02-18T08:41:05.244 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:41:05.245 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_33.status 2026-02-18T08:41:05.245 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_33 2026-02-18T08:41:05.245 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_33.status 2026-02-18T08:41:05.273 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 7 2026-02-18T08:41:05.273 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 7 2026-02-18T08:41:05.274 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:41:05.274 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=29, tag_tid=2, entry_tid=1], [object_number=28, tag_tid=2, entry_tid=0], [object_number=31, tag_tid=1, entry_tid=32819], [object_number=30, tag_tid=1, entry_tid=32818]]], state=connected] 2026-02-18T08:41:05.274 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=29, tag_tid=1, entry_tid=32821], [object_number=28, tag_tid=1, entry_tid=32820], [object_number=31, tag_tid=1, entry_tid=32819], [object_number=30, tag_tid=1, entry_tid=32818]]], state=connected] 2026-02-18T08:41:05.276 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_33.status 2026-02-18T08:41:05.277 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=29, tag_tid=2, entry_tid=1' 2026-02-18T08:41:05.278 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_33 2026-02-18T08:41:05.279 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:41:05.279 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:41:05.279 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_33 2026-02-18T08:41:05.279 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_33 '..*' 2026-02-18T08:41:05.279 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:41:05.279 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:41:05.279 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_33 2026-02-18T08:41:05.279 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:41:05.279 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_33.status 2026-02-18T08:41:05.280 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_33.status 2026-02-18T08:41:05.280 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:41:05.280 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_33.status 2026-02-18T08:41:05.281 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_33 2026-02-18T08:41:05.281 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_33.status 2026-02-18T08:41:05.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 7 2026-02-18T08:41:05.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 7 2026-02-18T08:41:05.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:41:05.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=29, tag_tid=2, entry_tid=1], [object_number=28, tag_tid=2, entry_tid=0], [object_number=31, tag_tid=1, entry_tid=32819], [object_number=30, tag_tid=1, entry_tid=32818]]], state=connected] 2026-02-18T08:41:05.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=29, tag_tid=1, entry_tid=32821], [object_number=28, tag_tid=1, entry_tid=32820], [object_number=31, tag_tid=1, entry_tid=32819], [object_number=30, tag_tid=1, entry_tid=32818]]], state=connected] 2026-02-18T08:41:05.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_33.status 2026-02-18T08:41:05.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=29, tag_tid=1, entry_tid=32821' 2026-02-18T08:41:05.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=29, tag_tid=2, entry_tid=1' -a 'object_number=29, tag_tid=2, entry_tid=1' = 'object_number=29, tag_tid=1, entry_tid=32821' 2026-02-18T08:41:05.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=29, tag_tid=1, entry_tid=32821' '!=' 'object_number=29, tag_tid=1, entry_tid=32821' 2026-02-18T08:41:05.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:41:05.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.4 2026-02-18T08:41:05.714 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_33 2026-02-18T08:41:05.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:41:05.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:41:05.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_33 2026-02-18T08:41:05.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:41:05.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_33 ']' 2026-02-18T08:41:05.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_33' 2026-02-18T08:41:05.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_33 2026-02-18T08:41:05.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:41:05.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:41:05.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:41:05.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:41:05.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:41:05.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:41:05.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:41:05.718 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:41:05.719 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:41:05.719 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:41:05.719 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_33 2026-02-18T08:41:05.719 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:41:05.719 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:41:05.719 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:41:05.719 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:41:05.719 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:41:05.719 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:41:05.719 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:41:05.719 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:41:05.719 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:41:05.720 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:41:05.720 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:41:05.720 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:41:05.720 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:41:05.720 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:41:05.720 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:41:05.720 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:41:05.720 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:41:05.720 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:41:05.720 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:41:05.720 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:41:05.720 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:41:05.721 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:41:05.721 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:41:05.721 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:41:05.721 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:41:05.721 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:41:05.721 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:41:05.721 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:41:05.721 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:41:05.721 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:41:05.729 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:41:05.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:41:05.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:41:05.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_33 2026-02-18T08:41:05.780 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_33'] 2026-02-18T08:41:05.780 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:41:05.780 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:41:05.783 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:41:05.783 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_33 2026-02-18T08:41:05.783 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:41:05.783 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:41:05.783 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:41:05.783 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:41:05.783 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:41:05.784 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:41:05.784 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:41:05.784 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:41:05.784 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:41:05.784 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:41:05.784 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:41:05.784 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:41:05.784 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:41:05.784 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:41:05.784 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:41:05.784 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:41:05.784 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:41:05.785 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:41:05.785 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:41:05.785 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:41:05.785 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:41:05.785 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:41:05.785 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:41:05.785 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:41:05.785 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:41:05.785 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:41:05.785 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:41:05.785 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:41:05.785 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:41:05.785 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:41:05.794 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:41:05.795 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:41:05.795 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:41:05.795 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_33 2026-02-18T08:41:05.854 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_33'] 2026-02-18T08:41:05.855 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:41:05.855 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:41:05.857 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:41:05.857 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_33 2026-02-18T08:41:05.857 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:41:05.857 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:41:05.857 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:41:05.858 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:41:05.858 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:41:05.858 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:41:05.858 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:41:05.858 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:41:05.858 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:41:05.858 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:41:05.858 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:41:05.858 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:41:05.858 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:41:05.858 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:41:05.858 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:41:05.859 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:41:05.859 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:41:05.859 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:41:05.859 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:41:05.859 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:41:05.859 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:41:05.859 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:41:05.859 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:41:05.859 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:41:05.859 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:41:05.859 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:41:05.859 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:41:05.860 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:41:05.860 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:41:05.860 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:41:05.867 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:41:05.868 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:41:05.868 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:41:05.868 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_33 2026-02-18T08:41:05.933 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:41:05.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_33 2026-02-18T08:41:05.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:41:05.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:41:05.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_33 2026-02-18T08:41:05.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_33 '' 2026-02-18T08:41:05.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:41:05.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:41:05.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_33 2026-02-18T08:41:05.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:41:05.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_33.status 2026-02-18T08:41:05.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_33.status 2026-02-18T08:41:05.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:41:05.936 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_33.status 2026-02-18T08:41:05.936 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_33 2026-02-18T08:41:05.936 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_33.status 2026-02-18T08:41:06.177 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 7 2026-02-18T08:41:06.178 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 7 2026-02-18T08:41:06.178 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:41:06.178 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=29, tag_tid=2, entry_tid=1], [object_number=28, tag_tid=2, entry_tid=0], [object_number=31, tag_tid=1, entry_tid=32819], [object_number=30, tag_tid=1, entry_tid=32818]]], state=connected] 2026-02-18T08:41:06.178 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=29, tag_tid=1, entry_tid=32821], [object_number=28, tag_tid=1, entry_tid=32820], [object_number=31, tag_tid=1, entry_tid=32819], [object_number=30, tag_tid=1, entry_tid=32818]]], state=connected] 2026-02-18T08:41:06.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_33.status 2026-02-18T08:41:06.183 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=29, tag_tid=2, entry_tid=1' 2026-02-18T08:41:06.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_33 2026-02-18T08:41:06.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:41:06.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:41:06.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_33 2026-02-18T08:41:06.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_33 '..*' 2026-02-18T08:41:06.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:41:06.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:41:06.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_33 2026-02-18T08:41:06.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:41:06.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_33.status 2026-02-18T08:41:06.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_33.status 2026-02-18T08:41:06.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:41:06.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_33.status 2026-02-18T08:41:06.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_33 2026-02-18T08:41:06.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_33.status 2026-02-18T08:41:06.218 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 7 2026-02-18T08:41:06.219 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 7 2026-02-18T08:41:06.219 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:41:06.219 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=29, tag_tid=2, entry_tid=1], [object_number=28, tag_tid=2, entry_tid=0], [object_number=31, tag_tid=1, entry_tid=32819], [object_number=30, tag_tid=1, entry_tid=32818]]], state=connected] 2026-02-18T08:41:06.219 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=29, tag_tid=1, entry_tid=32821], [object_number=28, tag_tid=1, entry_tid=32820], [object_number=31, tag_tid=1, entry_tid=32819], [object_number=30, tag_tid=1, entry_tid=32818]]], state=connected] 2026-02-18T08:41:06.222 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_33.status 2026-02-18T08:41:06.224 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=29, tag_tid=1, entry_tid=32821' 2026-02-18T08:41:06.224 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=29, tag_tid=2, entry_tid=1' -a 'object_number=29, tag_tid=2, entry_tid=1' = 'object_number=29, tag_tid=1, entry_tid=32821' 2026-02-18T08:41:06.224 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=29, tag_tid=1, entry_tid=32821' '!=' 'object_number=29, tag_tid=1, entry_tid=32821' 2026-02-18T08:41:06.224 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:41:06.224 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.8 2026-02-18T08:41:07.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_33 2026-02-18T08:41:07.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:41:07.027 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:41:07.027 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_33 2026-02-18T08:41:07.027 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:41:07.027 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_33 ']' 2026-02-18T08:41:07.027 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_33' 2026-02-18T08:41:07.027 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_33 2026-02-18T08:41:07.027 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:41:07.027 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:41:07.027 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:41:07.027 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:41:07.028 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:41:07.028 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:41:07.028 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:41:07.028 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:41:07.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:41:07.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:41:07.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_33 2026-02-18T08:41:07.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:41:07.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:41:07.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:41:07.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:41:07.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:41:07.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:41:07.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:41:07.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:41:07.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:41:07.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:41:07.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:41:07.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:41:07.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:41:07.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:41:07.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:41:07.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:41:07.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:41:07.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:41:07.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:41:07.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:41:07.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:41:07.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:41:07.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:41:07.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:41:07.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:41:07.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:41:07.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:41:07.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:41:07.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:41:07.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:41:07.040 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:41:07.041 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:41:07.041 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:41:07.041 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_33 2026-02-18T08:41:07.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_33'] 2026-02-18T08:41:07.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:41:07.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:41:07.093 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:41:07.093 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_33 2026-02-18T08:41:07.093 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:41:07.093 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:41:07.093 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:41:07.093 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:41:07.093 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:41:07.093 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:41:07.093 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:41:07.093 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:41:07.094 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:41:07.094 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:41:07.094 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:41:07.094 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:41:07.094 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:41:07.094 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:41:07.094 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:41:07.094 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:41:07.094 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:41:07.094 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:41:07.094 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:41:07.094 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:41:07.095 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:41:07.095 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:41:07.095 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:41:07.095 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:41:07.095 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:41:07.095 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:41:07.095 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:41:07.095 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:41:07.095 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:41:07.095 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:41:07.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:41:07.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:41:07.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:41:07.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_33 2026-02-18T08:41:07.164 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_33'] 2026-02-18T08:41:07.164 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:41:07.164 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:41:07.166 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:41:07.167 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_33 2026-02-18T08:41:07.167 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:41:07.167 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:41:07.167 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:41:07.167 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:41:07.167 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:41:07.167 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:41:07.167 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:41:07.167 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:41:07.167 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:41:07.167 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:41:07.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:41:07.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:41:07.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:41:07.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:41:07.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:41:07.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:41:07.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:41:07.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:41:07.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:41:07.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:41:07.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:41:07.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:41:07.169 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:41:07.169 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:41:07.169 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:41:07.169 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:41:07.169 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:41:07.169 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:41:07.169 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:41:07.169 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:41:07.177 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:41:07.178 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:41:07.178 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:41:07.178 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_33 2026-02-18T08:41:07.244 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:41:07.245 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_33 2026-02-18T08:41:07.245 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:41:07.245 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:41:07.245 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_33 2026-02-18T08:41:07.245 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_33 '' 2026-02-18T08:41:07.246 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:41:07.246 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:41:07.246 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_33 2026-02-18T08:41:07.246 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:41:07.246 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_33.status 2026-02-18T08:41:07.246 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_33.status 2026-02-18T08:41:07.246 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:41:07.246 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_33.status 2026-02-18T08:41:07.247 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_33 2026-02-18T08:41:07.247 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_33.status 2026-02-18T08:41:07.283 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 7 2026-02-18T08:41:07.283 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 7 2026-02-18T08:41:07.283 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:41:07.283 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=29, tag_tid=2, entry_tid=1], [object_number=28, tag_tid=2, entry_tid=0], [object_number=31, tag_tid=1, entry_tid=32819], [object_number=30, tag_tid=1, entry_tid=32818]]], state=connected] 2026-02-18T08:41:07.283 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=29, tag_tid=1, entry_tid=32821], [object_number=28, tag_tid=1, entry_tid=32820], [object_number=31, tag_tid=1, entry_tid=32819], [object_number=30, tag_tid=1, entry_tid=32818]]], state=connected] 2026-02-18T08:41:07.286 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_33.status 2026-02-18T08:41:07.287 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=29, tag_tid=2, entry_tid=1' 2026-02-18T08:41:07.289 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_33 2026-02-18T08:41:07.289 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:41:07.289 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:41:07.289 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_33 2026-02-18T08:41:07.289 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_33 '..*' 2026-02-18T08:41:07.289 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:41:07.289 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:41:07.289 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_33 2026-02-18T08:41:07.289 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:41:07.289 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_33.status 2026-02-18T08:41:07.290 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_33.status 2026-02-18T08:41:07.290 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:41:07.290 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_33.status 2026-02-18T08:41:07.291 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_33 2026-02-18T08:41:07.291 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_33.status 2026-02-18T08:41:07.325 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 7 2026-02-18T08:41:07.326 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 7 2026-02-18T08:41:07.326 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:41:07.326 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=29, tag_tid=2, entry_tid=1], [object_number=28, tag_tid=2, entry_tid=0], [object_number=31, tag_tid=1, entry_tid=32819], [object_number=30, tag_tid=1, entry_tid=32818]]], state=connected] 2026-02-18T08:41:07.326 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=29, tag_tid=1, entry_tid=32821], [object_number=28, tag_tid=1, entry_tid=32820], [object_number=31, tag_tid=1, entry_tid=32819], [object_number=30, tag_tid=1, entry_tid=32818]]], state=connected] 2026-02-18T08:41:07.329 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_33.status 2026-02-18T08:41:07.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=29, tag_tid=1, entry_tid=32821' 2026-02-18T08:41:07.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=29, tag_tid=2, entry_tid=1' -a 'object_number=29, tag_tid=2, entry_tid=1' = 'object_number=29, tag_tid=1, entry_tid=32821' 2026-02-18T08:41:07.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=29, tag_tid=1, entry_tid=32821' '!=' 'object_number=29, tag_tid=1, entry_tid=32821' 2026-02-18T08:41:07.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:41:07.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1.6 2026-02-18T08:41:08.933 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_33 2026-02-18T08:41:08.933 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:41:08.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:41:08.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_33 2026-02-18T08:41:08.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:41:08.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_33 ']' 2026-02-18T08:41:08.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_33' 2026-02-18T08:41:08.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_33 2026-02-18T08:41:08.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:41:08.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:41:08.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:41:08.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:41:08.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:41:08.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:41:08.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:41:08.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:41:08.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:41:08.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:41:08.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_33 2026-02-18T08:41:08.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:41:08.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:41:08.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:41:08.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:41:08.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:41:08.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:41:08.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:41:08.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:41:08.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:41:08.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:41:08.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:41:08.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:41:08.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:41:08.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:41:08.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:41:08.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:41:08.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:41:08.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:41:08.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:41:08.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:41:08.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:41:08.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:41:08.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:41:08.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:41:08.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:41:08.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:41:08.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:41:08.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:41:08.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:41:08.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:41:08.948 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:41:08.948 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:41:08.948 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:41:08.948 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_33 2026-02-18T08:41:09.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_33'] 2026-02-18T08:41:09.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:41:09.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:41:09.012 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:41:09.012 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_33 2026-02-18T08:41:09.012 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:41:09.012 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:41:09.012 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:41:09.012 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:41:09.012 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:41:09.012 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:41:09.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:41:09.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:41:09.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:41:09.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:41:09.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:41:09.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:41:09.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:41:09.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:41:09.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:41:09.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:41:09.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:41:09.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:41:09.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:41:09.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:41:09.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:41:09.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:41:09.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:41:09.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:41:09.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:41:09.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:41:09.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:41:09.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:41:09.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:41:09.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:41:09.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:41:09.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:41:09.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:41:09.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_33 2026-02-18T08:41:09.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_33'] 2026-02-18T08:41:09.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:41:09.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:41:09.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:41:09.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_33 2026-02-18T08:41:09.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:41:09.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:41:09.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:41:09.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:41:09.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:41:09.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:41:09.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:41:09.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:41:09.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:41:09.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:41:09.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:41:09.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:41:09.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:41:09.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:41:09.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:41:09.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:41:09.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:41:09.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:41:09.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:41:09.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:41:09.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:41:09.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:41:09.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:41:09.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:41:09.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:41:09.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:41:09.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:41:09.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:41:09.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:41:09.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:41:09.097 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:41:09.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:41:09.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:41:09.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_33 2026-02-18T08:41:09.165 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:41:09.166 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_33 2026-02-18T08:41:09.166 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:41:09.166 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:41:09.167 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_33 2026-02-18T08:41:09.167 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_33 '' 2026-02-18T08:41:09.167 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:41:09.167 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:41:09.167 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_33 2026-02-18T08:41:09.167 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:41:09.167 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_33.status 2026-02-18T08:41:09.167 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_33.status 2026-02-18T08:41:09.167 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:41:09.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_33.status 2026-02-18T08:41:09.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_33 2026-02-18T08:41:09.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_33.status 2026-02-18T08:41:09.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 7 2026-02-18T08:41:09.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 7 2026-02-18T08:41:09.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:41:09.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=29, tag_tid=2, entry_tid=1], [object_number=28, tag_tid=2, entry_tid=0], [object_number=31, tag_tid=1, entry_tid=32819], [object_number=30, tag_tid=1, entry_tid=32818]]], state=connected] 2026-02-18T08:41:09.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=29, tag_tid=1, entry_tid=32821], [object_number=28, tag_tid=1, entry_tid=32820], [object_number=31, tag_tid=1, entry_tid=32819], [object_number=30, tag_tid=1, entry_tid=32818]]], state=connected] 2026-02-18T08:41:09.209 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_33.status 2026-02-18T08:41:09.210 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=29, tag_tid=2, entry_tid=1' 2026-02-18T08:41:09.211 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_33 2026-02-18T08:41:09.211 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:41:09.211 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:41:09.211 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_33 2026-02-18T08:41:09.211 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_33 '..*' 2026-02-18T08:41:09.211 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:41:09.211 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:41:09.212 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_33 2026-02-18T08:41:09.212 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:41:09.212 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_33.status 2026-02-18T08:41:09.212 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_33.status 2026-02-18T08:41:09.212 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:41:09.212 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_33.status 2026-02-18T08:41:09.213 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_33 2026-02-18T08:41:09.213 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_33.status 2026-02-18T08:41:09.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 7 2026-02-18T08:41:09.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 7 2026-02-18T08:41:09.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:41:09.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=29, tag_tid=2, entry_tid=1], [object_number=28, tag_tid=2, entry_tid=0], [object_number=31, tag_tid=1, entry_tid=32819], [object_number=30, tag_tid=1, entry_tid=32818]]], state=connected] 2026-02-18T08:41:09.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=29, tag_tid=1, entry_tid=32821], [object_number=28, tag_tid=1, entry_tid=32820], [object_number=31, tag_tid=1, entry_tid=32819], [object_number=30, tag_tid=1, entry_tid=32818]]], state=connected] 2026-02-18T08:41:09.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_33.status 2026-02-18T08:41:09.245 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=29, tag_tid=1, entry_tid=32821' 2026-02-18T08:41:09.245 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=29, tag_tid=2, entry_tid=1' -a 'object_number=29, tag_tid=2, entry_tid=1' = 'object_number=29, tag_tid=1, entry_tid=32821' 2026-02-18T08:41:09.245 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=29, tag_tid=1, entry_tid=32821' '!=' 'object_number=29, tag_tid=1, entry_tid=32821' 2026-02-18T08:41:09.245 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:41:09.246 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 2 2026-02-18T08:41:11.247 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_33 2026-02-18T08:41:11.248 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:41:11.248 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:41:11.248 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_33 2026-02-18T08:41:11.248 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:41:11.248 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_33 ']' 2026-02-18T08:41:11.248 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_33' 2026-02-18T08:41:11.248 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_33 2026-02-18T08:41:11.248 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:41:11.248 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:41:11.249 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:41:11.249 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:41:11.249 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:41:11.249 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:41:11.249 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:41:11.249 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:41:11.251 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:41:11.252 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:41:11.252 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_33 2026-02-18T08:41:11.252 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:41:11.252 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:41:11.252 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:41:11.252 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:41:11.252 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:41:11.252 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:41:11.252 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:41:11.252 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:41:11.252 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:41:11.253 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:41:11.253 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:41:11.253 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:41:11.253 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:41:11.253 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:41:11.253 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:41:11.253 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:41:11.253 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:41:11.253 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:41:11.253 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:41:11.253 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:41:11.253 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:41:11.254 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:41:11.254 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:41:11.254 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:41:11.254 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:41:11.254 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:41:11.254 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:41:11.254 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:41:11.254 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:41:11.254 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:41:11.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:41:11.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:41:11.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:41:11.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_33 2026-02-18T08:41:11.320 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_33'] 2026-02-18T08:41:11.321 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:41:11.321 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:41:11.323 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:41:11.323 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_33 2026-02-18T08:41:11.323 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:41:11.323 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:41:11.324 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:41:11.324 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:41:11.324 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:41:11.324 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:41:11.324 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:41:11.324 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:41:11.324 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:41:11.324 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:41:11.324 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:41:11.324 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:41:11.324 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:41:11.324 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:41:11.325 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:41:11.325 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:41:11.325 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:41:11.325 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:41:11.325 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:41:11.325 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:41:11.325 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:41:11.325 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:41:11.325 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:41:11.325 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:41:11.325 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:41:11.325 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:41:11.326 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:41:11.326 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:41:11.326 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:41:11.326 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:41:11.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:41:11.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:41:11.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:41:11.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_33 2026-02-18T08:41:11.387 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_33'] 2026-02-18T08:41:11.388 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:41:11.388 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:41:11.390 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:41:11.390 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_33 2026-02-18T08:41:11.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:41:11.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:41:11.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:41:11.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:41:11.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:41:11.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:41:11.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:41:11.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:41:11.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:41:11.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:41:11.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:41:11.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:41:11.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:41:11.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:41:11.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:41:11.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:41:11.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:41:11.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:41:11.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:41:11.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:41:11.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:41:11.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:41:11.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:41:11.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:41:11.393 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:41:11.393 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:41:11.393 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:41:11.393 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:41:11.393 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:41:11.393 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:41:11.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:41:11.403 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:41:11.403 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:41:11.403 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_33 2026-02-18T08:41:11.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:41:11.460 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_33 2026-02-18T08:41:11.460 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:41:11.460 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:41:11.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_33 2026-02-18T08:41:11.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_33 '' 2026-02-18T08:41:11.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:41:11.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:41:11.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_33 2026-02-18T08:41:11.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:41:11.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_33.status 2026-02-18T08:41:11.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_33.status 2026-02-18T08:41:11.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:41:11.462 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_33.status 2026-02-18T08:41:11.462 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_33 2026-02-18T08:41:11.462 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_33.status 2026-02-18T08:41:11.489 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 7 2026-02-18T08:41:11.489 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 7 2026-02-18T08:41:11.489 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:41:11.489 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=29, tag_tid=2, entry_tid=1], [object_number=28, tag_tid=2, entry_tid=0], [object_number=31, tag_tid=1, entry_tid=32819], [object_number=30, tag_tid=1, entry_tid=32818]]], state=connected] 2026-02-18T08:41:11.489 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=29, tag_tid=1, entry_tid=32821], [object_number=28, tag_tid=1, entry_tid=32820], [object_number=31, tag_tid=1, entry_tid=32819], [object_number=30, tag_tid=1, entry_tid=32818]]], state=connected] 2026-02-18T08:41:11.491 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_33.status 2026-02-18T08:41:11.492 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=29, tag_tid=2, entry_tid=1' 2026-02-18T08:41:11.493 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_33 2026-02-18T08:41:11.493 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:41:11.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:41:11.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_33 2026-02-18T08:41:11.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_33 '..*' 2026-02-18T08:41:11.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:41:11.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:41:11.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_33 2026-02-18T08:41:11.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:41:11.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_33.status 2026-02-18T08:41:11.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_33.status 2026-02-18T08:41:11.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:41:11.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_33.status 2026-02-18T08:41:11.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_33 2026-02-18T08:41:11.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_33.status 2026-02-18T08:41:11.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 7 2026-02-18T08:41:11.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 7 2026-02-18T08:41:11.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:41:11.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=29, tag_tid=2, entry_tid=1], [object_number=28, tag_tid=2, entry_tid=0], [object_number=31, tag_tid=1, entry_tid=32819], [object_number=30, tag_tid=1, entry_tid=32818]]], state=connected] 2026-02-18T08:41:11.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=29, tag_tid=1, entry_tid=32821], [object_number=28, tag_tid=1, entry_tid=32820], [object_number=31, tag_tid=1, entry_tid=32819], [object_number=30, tag_tid=1, entry_tid=32818]]], state=connected] 2026-02-18T08:41:11.524 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_33.status 2026-02-18T08:41:11.526 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=29, tag_tid=1, entry_tid=32821' 2026-02-18T08:41:11.526 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=29, tag_tid=2, entry_tid=1' -a 'object_number=29, tag_tid=2, entry_tid=1' = 'object_number=29, tag_tid=1, entry_tid=32821' 2026-02-18T08:41:11.526 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=29, tag_tid=1, entry_tid=32821' '!=' 'object_number=29, tag_tid=1, entry_tid=32821' 2026-02-18T08:41:11.526 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:41:11.526 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 2 2026-02-18T08:41:13.528 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_33 2026-02-18T08:41:13.528 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:41:13.528 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:41:13.528 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_33 2026-02-18T08:41:13.528 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:41:13.528 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_33 ']' 2026-02-18T08:41:13.529 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_33' 2026-02-18T08:41:13.529 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_33 2026-02-18T08:41:13.529 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:41:13.529 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:41:13.529 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:41:13.529 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:41:13.529 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:41:13.529 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:41:13.529 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:41:13.532 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:41:13.532 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:41:13.532 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:41:13.532 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_33 2026-02-18T08:41:13.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:41:13.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:41:13.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:41:13.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:41:13.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:41:13.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:41:13.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:41:13.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:41:13.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:41:13.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:41:13.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:41:13.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:41:13.534 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:41:13.534 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:41:13.534 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:41:13.534 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:41:13.534 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:41:13.534 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:41:13.534 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:41:13.534 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:41:13.534 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:41:13.534 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:41:13.534 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:41:13.534 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:41:13.534 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:41:13.535 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:41:13.535 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:41:13.535 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:41:13.535 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:41:13.535 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:41:13.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:41:13.545 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:41:13.545 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:41:13.545 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_33 2026-02-18T08:41:13.597 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_33'] 2026-02-18T08:41:13.597 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:41:13.597 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:41:13.600 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:41:13.600 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_33 2026-02-18T08:41:13.600 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:41:13.600 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:41:13.600 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:41:13.600 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:41:13.600 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:41:13.601 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:41:13.601 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:41:13.601 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:41:13.601 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:41:13.601 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:41:13.601 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:41:13.601 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:41:13.601 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:41:13.601 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:41:13.601 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:41:13.601 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:41:13.601 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:41:13.602 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:41:13.602 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:41:13.602 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:41:13.602 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:41:13.602 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:41:13.602 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:41:13.602 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:41:13.602 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:41:13.602 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:41:13.602 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:41:13.602 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:41:13.602 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:41:13.603 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:41:13.610 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:41:13.611 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:41:13.611 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:41:13.611 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_33 2026-02-18T08:41:13.671 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_33'] 2026-02-18T08:41:13.672 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:41:13.672 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:41:13.675 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:41:13.675 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_33 2026-02-18T08:41:13.675 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:41:13.675 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:41:13.675 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:41:13.675 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:41:13.675 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:41:13.675 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:41:13.675 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:41:13.675 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:41:13.676 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:41:13.676 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:41:13.676 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:41:13.676 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:41:13.676 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:41:13.676 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:41:13.676 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:41:13.676 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:41:13.676 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:41:13.676 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:41:13.676 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:41:13.676 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:41:13.677 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:41:13.677 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:41:13.677 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:41:13.677 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:41:13.677 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:41:13.677 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:41:13.677 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:41:13.677 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:41:13.677 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:41:13.677 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:41:13.685 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:41:13.685 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:41:13.686 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:41:13.686 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_33 2026-02-18T08:41:13.756 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:41:13.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_33 2026-02-18T08:41:13.758 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:41:13.758 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:41:13.758 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_33 2026-02-18T08:41:13.758 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_33 '' 2026-02-18T08:41:13.758 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:41:13.758 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:41:13.758 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_33 2026-02-18T08:41:13.758 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:41:13.758 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_33.status 2026-02-18T08:41:13.758 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_33.status 2026-02-18T08:41:13.759 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:41:13.759 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_33.status 2026-02-18T08:41:13.759 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_33 2026-02-18T08:41:13.760 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_33.status 2026-02-18T08:41:13.796 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 7 2026-02-18T08:41:13.796 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 7 2026-02-18T08:41:13.796 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:41:13.796 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=29, tag_tid=2, entry_tid=1], [object_number=28, tag_tid=2, entry_tid=0], [object_number=31, tag_tid=1, entry_tid=32819], [object_number=30, tag_tid=1, entry_tid=32818]]], state=connected] 2026-02-18T08:41:13.796 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=29, tag_tid=2, entry_tid=1], [object_number=28, tag_tid=2, entry_tid=0], [object_number=31, tag_tid=1, entry_tid=32819], [object_number=30, tag_tid=1, entry_tid=32818]]], state=connected] 2026-02-18T08:41:13.800 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_33.status 2026-02-18T08:41:13.801 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=29, tag_tid=2, entry_tid=1' 2026-02-18T08:41:13.803 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_33 2026-02-18T08:41:13.803 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:41:13.803 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:41:13.803 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_33 2026-02-18T08:41:13.803 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_33 '..*' 2026-02-18T08:41:13.803 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:41:13.803 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:41:13.804 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_33 2026-02-18T08:41:13.804 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:41:13.804 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_33.status 2026-02-18T08:41:13.804 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_33.status 2026-02-18T08:41:13.804 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:41:13.805 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_33.status 2026-02-18T08:41:13.805 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_33 2026-02-18T08:41:13.805 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_33.status 2026-02-18T08:41:13.831 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 7 2026-02-18T08:41:13.831 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 7 2026-02-18T08:41:13.831 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:41:13.832 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=29, tag_tid=2, entry_tid=1], [object_number=28, tag_tid=2, entry_tid=0], [object_number=31, tag_tid=1, entry_tid=32819], [object_number=30, tag_tid=1, entry_tid=32818]]], state=connected] 2026-02-18T08:41:13.832 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=29, tag_tid=2, entry_tid=1], [object_number=28, tag_tid=2, entry_tid=0], [object_number=31, tag_tid=1, entry_tid=32819], [object_number=30, tag_tid=1, entry_tid=32818]]], state=connected] 2026-02-18T08:41:13.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_33.status 2026-02-18T08:41:13.836 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=29, tag_tid=2, entry_tid=1' 2026-02-18T08:41:13.836 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=29, tag_tid=2, entry_tid=1' -a 'object_number=29, tag_tid=2, entry_tid=1' = 'object_number=29, tag_tid=2, entry_tid=1' 2026-02-18T08:41:13.836 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:41:13.836 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_snap_present cluster1 mirror image_33 snap 2026-02-18T08:41:13.836 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:41:13.836 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:41:13.836 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_33 2026-02-18T08:41:13.836 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap 2026-02-18T08:41:13.836 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-18T08:41:13.836 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-18T08:41:13.837 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1 2026-02-18T08:41:14.836 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster1 info mirror/image_33@snap 2026-02-18T08:41:14.867 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:rbd image 'image_33': 2026-02-18T08:41:14.867 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: size 128 MiB in 32 objects 2026-02-18T08:41:14.868 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: order 22 (4 MiB objects) 2026-02-18T08:41:14.868 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: snapshot_count: 1 2026-02-18T08:41:14.868 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: id: 10eddea34b96 2026-02-18T08:41:14.868 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: block_name_prefix: rbd_data.10eddea34b96 2026-02-18T08:41:14.868 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: format: 2 2026-02-18T08:41:14.868 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: features: layering, exclusive-lock, journaling 2026-02-18T08:41:14.868 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: op_features: 2026-02-18T08:41:14.868 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: flags: 2026-02-18T08:41:14.868 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: create_timestamp: Wed Feb 18 08:31:27 2026 2026-02-18T08:41:14.869 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: access_timestamp: Wed Feb 18 08:31:27 2026 2026-02-18T08:41:14.869 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: modify_timestamp: Wed Feb 18 08:32:28 2026 2026-02-18T08:41:14.869 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: protected: False 2026-02-18T08:41:14.869 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: journal: 10eddea34b96 2026-02-18T08:41:14.869 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring state: enabled 2026-02-18T08:41:14.869 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring mode: journal 2026-02-18T08:41:14.869 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring global id: 17e13f58-ddbf-4a3d-9ca5-317fc7000399 2026-02-18T08:41:14.869 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring primary: false 2026-02-18T08:41:14.870 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:41:14.870 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ compare_image_snaps mirror image_33 snap 2026-02-18T08:41:14.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:41:14.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_33 2026-02-18T08:41:14.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap 2026-02-18T08:41:14.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local ret=0 2026-02-18T08:41:14.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local rmt_export=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_33.export 2026-02-18T08:41:14.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loc_export=/tmp/tmp.vybh1m9yJF/cluster1-mirror-image_33.export 2026-02-18T08:41:14.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_33.export /tmp/tmp.vybh1m9yJF/cluster1-mirror-image_33.export 2026-02-18T08:41:14.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 -p mirror export image_33@snap /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_33.export 2026-02-18T08:41:15.019 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-18T08:41:15.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster1 -p mirror export image_33@snap /tmp/tmp.vybh1m9yJF/cluster1-mirror-image_33.export 2026-02-18T08:41:15.223 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-18T08:41:15.230 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmp /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_33.export /tmp/tmp.vybh1m9yJF/cluster1-mirror-image_33.export 2026-02-18T08:41:15.288 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_33.export /tmp/tmp.vybh1m9yJF/cluster1-mirror-image_33.export 2026-02-18T08:41:15.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:41:15.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T08:41:15.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_34 2026-02-18T08:41:15.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_snap cluster2 mirror image_34 snap 2026-02-18T08:41:15.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:41:15.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:41:15.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_34 2026-02-18T08:41:15.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap 2026-02-18T08:41:15.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 -p mirror snap create image_34@snap --debug-rbd=20 --debug-journaler=20 2026-02-18T08:41:18.097 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_replay_started cluster1 mirror image_34 2026-02-18T08:41:18.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:41:18.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:41:18.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_34 2026-02-18T08:41:18.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_replay_state cluster1 mirror image_34 started 2026-02-18T08:41:18.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:41:18.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:41:18.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_34 2026-02-18T08:41:18.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local state=started 2026-02-18T08:41:18.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-18T08:41:18.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:41:18.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1 2026-02-18T08:41:19.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test_image_replay_state cluster1 mirror image_34 started 2026-02-18T08:41:19.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:41:19.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:41:19.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_34 2026-02-18T08:41:19.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local test_state=started 2026-02-18T08:41:19.100 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local status_result 2026-02-18T08:41:19.100 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_state=stopped 2026-02-18T08:41:19.101 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_34 2026-02-18T08:41:19.101 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -i state 2026-02-18T08:41:19.101 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_instance=cluster1 2026-02-18T08:41:19.101 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-18T08:41:19.101 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:41:19.101 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance=cluster1 2026-02-18T08:41:19.101 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local loop_instance 2026-02-18T08:41:19.101 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:41:19.101 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sleep 0 2026-02-18T08:41:19.102 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:41:19.102 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ seq 0 3 2026-02-18T08:41:19.105 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:41:19.105 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_34 2026-02-18T08:41:19.105 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:0 2026-02-18T08:41:19.105 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-18T08:41:19.105 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:41:19.105 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:41:19.105 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:41:19.105 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-18T08:41:19.106 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:41:19.106 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:41:19.106 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:41:19.106 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:41:19.106 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-18T08:41:19.106 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:41:19.106 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:41:19.106 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:41:19.106 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:41:19.106 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:0 2026-02-18T08:41:19.106 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-18T08:41:19.106 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-18T08:41:19.106 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:41:19.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:0 2026-02-18T08:41:19.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T08:41:19.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-18T08:41:19.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:41:19.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:41:19.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-18T08:41:19.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-18T08:41:19.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-18T08:41:19.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=0 2026-02-18T08:41:19.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=0 2026-02-18T08:41:19.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:41:19.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:41:19.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:41:19.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:41:19.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_34 2026-02-18T08:41:19.167 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_34'] 2026-02-18T08:41:19.167 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status : get status for rbd mirror 2026-02-18T08:41:19.167 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:41:19.170 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:41:19.170 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_34 2026-02-18T08:41:19.170 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:1 2026-02-18T08:41:19.170 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-18T08:41:19.170 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:41:19.170 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:41:19.171 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:41:19.171 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-18T08:41:19.171 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:41:19.171 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:41:19.171 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:41:19.171 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:41:19.171 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-18T08:41:19.171 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:41:19.171 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:41:19.171 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:41:19.171 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:41:19.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:1 2026-02-18T08:41:19.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-18T08:41:19.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-18T08:41:19.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:41:19.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:1 2026-02-18T08:41:19.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T08:41:19.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-18T08:41:19.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:41:19.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:41:19.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-18T08:41:19.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-18T08:41:19.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-18T08:41:19.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=1 2026-02-18T08:41:19.173 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=1 2026-02-18T08:41:19.173 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:41:19.180 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:41:19.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:41:19.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:41:19.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_34 2026-02-18T08:41:19.234 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_34'] 2026-02-18T08:41:19.234 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status : get status for rbd mirror 2026-02-18T08:41:19.234 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:41:19.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:41:19.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_34 2026-02-18T08:41:19.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:2 2026-02-18T08:41:19.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-18T08:41:19.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:41:19.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:41:19.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:41:19.238 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-18T08:41:19.238 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:41:19.238 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:41:19.238 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:41:19.238 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:41:19.238 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-18T08:41:19.238 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:41:19.238 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:41:19.238 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:41:19.238 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:41:19.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:2 2026-02-18T08:41:19.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-18T08:41:19.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-18T08:41:19.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:41:19.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:2 2026-02-18T08:41:19.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T08:41:19.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-18T08:41:19.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:41:19.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:41:19.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-18T08:41:19.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-18T08:41:19.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-18T08:41:19.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=2 2026-02-18T08:41:19.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=2 2026-02-18T08:41:19.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:41:19.248 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:41:19.249 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:41:19.249 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:41:19.249 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_34 2026-02-18T08:41:19.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ return 0 2026-02-18T08:41:19.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ status_result=' "state": "Replaying"' 2026-02-18T08:41:19.310 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ echo ' "state": "Replaying"' 2026-02-18T08:41:19.310 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ grep -i Replaying 2026-02-18T08:41:19.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: "state": "Replaying" 2026-02-18T08:41:19.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ current_state=started 2026-02-18T08:41:19.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test started = started 2026-02-18T08:41:19.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:41:19.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror image_34 2026-02-18T08:41:19.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_cluster=cluster1 2026-02-18T08:41:19.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:41:19.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_pool=mirror 2026-02-18T08:41:19.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local remote_pool=mirror 2026-02-18T08:41:19.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_34 2026-02-18T08:41:19.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' journal = journal ']' 2026-02-18T08:41:19.314 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror mirror image_34 2026-02-18T08:41:19.314 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_cluster=cluster1 2026-02-18T08:41:19.314 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:41:19.314 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_pool=mirror 2026-02-18T08:41:19.314 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local remote_pool=mirror 2026-02-18T08:41:19.314 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_34 2026-02-18T08:41:19.314 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-02-18T08:41:19.314 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-02-18T08:41:19.314 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:41:19.314 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:41:19.314 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:41:19.514 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_34 2026-02-18T08:41:19.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:41:19.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:41:19.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_34 2026-02-18T08:41:19.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:41:19.516 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_34 ']' 2026-02-18T08:41:19.516 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_34' 2026-02-18T08:41:19.516 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_34 2026-02-18T08:41:19.516 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:41:19.516 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:41:19.516 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:41:19.516 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:41:19.516 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:41:19.516 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:41:19.516 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:41:19.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:41:19.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:41:19.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:41:19.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_34 2026-02-18T08:41:19.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:41:19.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:41:19.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:41:19.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:41:19.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:41:19.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:41:19.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:41:19.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:41:19.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:41:19.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:41:19.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:41:19.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:41:19.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:41:19.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:41:19.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:41:19.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:41:19.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:41:19.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:41:19.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:41:19.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:41:19.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:41:19.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:41:19.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:41:19.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:41:19.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:41:19.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:41:19.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:41:19.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:41:19.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:41:19.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:41:19.528 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:41:19.528 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:41:19.528 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:41:19.528 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_34 2026-02-18T08:41:19.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_34'] 2026-02-18T08:41:19.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:41:19.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:41:19.594 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:41:19.594 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_34 2026-02-18T08:41:19.594 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:41:19.594 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:41:19.594 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:41:19.594 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:41:19.594 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:41:19.595 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:41:19.595 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:41:19.595 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:41:19.595 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:41:19.595 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:41:19.595 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:41:19.595 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:41:19.595 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:41:19.595 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:41:19.595 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:41:19.595 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:41:19.595 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:41:19.596 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:41:19.596 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:41:19.596 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:41:19.596 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:41:19.596 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:41:19.596 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:41:19.596 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:41:19.596 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:41:19.596 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:41:19.596 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:41:19.596 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:41:19.596 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:41:19.597 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:41:19.605 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:41:19.606 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:41:19.606 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:41:19.606 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_34 2026-02-18T08:41:19.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_34'] 2026-02-18T08:41:19.658 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:41:19.658 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:41:19.660 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:41:19.660 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_34 2026-02-18T08:41:19.660 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:41:19.661 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:41:19.661 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:41:19.661 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:41:19.661 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:41:19.661 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:41:19.661 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:41:19.661 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:41:19.661 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:41:19.661 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:41:19.661 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:41:19.661 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:41:19.662 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:41:19.662 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:41:19.662 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:41:19.662 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:41:19.662 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:41:19.662 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:41:19.662 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:41:19.662 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:41:19.662 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:41:19.662 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:41:19.662 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:41:19.662 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:41:19.663 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:41:19.663 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:41:19.663 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:41:19.663 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:41:19.663 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:41:19.663 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:41:19.671 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:41:19.672 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:41:19.672 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:41:19.672 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_34 2026-02-18T08:41:19.737 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:41:19.738 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_34 2026-02-18T08:41:19.738 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:41:19.738 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:41:19.738 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_34 2026-02-18T08:41:19.738 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_34 '' 2026-02-18T08:41:19.738 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:41:19.738 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:41:19.738 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_34 2026-02-18T08:41:19.738 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:41:19.738 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_34.status 2026-02-18T08:41:19.739 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_34.status 2026-02-18T08:41:19.739 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:41:19.740 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_34.status 2026-02-18T08:41:19.740 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_34 2026-02-18T08:41:19.740 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_34.status 2026-02-18T08:41:19.776 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 4 2026-02-18T08:41:19.776 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 4 2026-02-18T08:41:19.777 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:41:19.777 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=2, entry_tid=1], [object_number=16, tag_tid=2, entry_tid=0], [object_number=19, tag_tid=1, entry_tid=19627], [object_number=18, tag_tid=1, entry_tid=19626]]], state=connected] 2026-02-18T08:41:19.777 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=16, tag_tid=1, entry_tid=19628], [object_number=19, tag_tid=1, entry_tid=19627], [object_number=18, tag_tid=1, entry_tid=19626], [object_number=17, tag_tid=1, entry_tid=19625]]], state=connected] 2026-02-18T08:41:19.780 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_34.status 2026-02-18T08:41:19.781 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=17, tag_tid=2, entry_tid=1' 2026-02-18T08:41:19.783 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_34 2026-02-18T08:41:19.783 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:41:19.783 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:41:19.783 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_34 2026-02-18T08:41:19.783 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_34 '..*' 2026-02-18T08:41:19.783 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:41:19.783 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:41:19.783 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_34 2026-02-18T08:41:19.783 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:41:19.783 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_34.status 2026-02-18T08:41:19.784 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_34.status 2026-02-18T08:41:19.784 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:41:19.784 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_34.status 2026-02-18T08:41:19.785 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_34 2026-02-18T08:41:19.785 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_34.status 2026-02-18T08:41:19.813 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 4 2026-02-18T08:41:19.813 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 4 2026-02-18T08:41:19.813 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:41:19.813 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=2, entry_tid=1], [object_number=16, tag_tid=2, entry_tid=0], [object_number=19, tag_tid=1, entry_tid=19627], [object_number=18, tag_tid=1, entry_tid=19626]]], state=connected] 2026-02-18T08:41:19.813 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=16, tag_tid=1, entry_tid=19628], [object_number=19, tag_tid=1, entry_tid=19627], [object_number=18, tag_tid=1, entry_tid=19626], [object_number=17, tag_tid=1, entry_tid=19625]]], state=connected] 2026-02-18T08:41:19.815 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_34.status 2026-02-18T08:41:19.817 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=16, tag_tid=1, entry_tid=19628' 2026-02-18T08:41:19.817 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=17, tag_tid=2, entry_tid=1' -a 'object_number=17, tag_tid=2, entry_tid=1' = 'object_number=16, tag_tid=1, entry_tid=19628' 2026-02-18T08:41:19.817 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=16, tag_tid=1, entry_tid=19628' '!=' '' 2026-02-18T08:41:19.817 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:41:19.817 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=16, tag_tid=1, entry_tid=19628' = '' 2026-02-18T08:41:19.817 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=16, tag_tid=1, entry_tid=19628' 2026-02-18T08:41:19.818 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=17, tag_tid=2, entry_tid=1' 2026-02-18T08:41:19.818 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:41:19.818 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:41:19.819 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=2 2026-02-18T08:41:19.820 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=16, tag_tid=1, entry_tid=19628' 2026-02-18T08:41:19.820 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:41:19.820 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:41:19.821 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=1 2026-02-18T08:41:19.822 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=17, tag_tid=2, entry_tid=1' 2026-02-18T08:41:19.822 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:41:19.822 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:41:19.824 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:41:19.824 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=16, tag_tid=1, entry_tid=19628' 2026-02-18T08:41:19.824 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:41:19.825 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:41:19.826 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=19628 2026-02-18T08:41:19.826 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 2 = 1 -a 1 -le 19628 2026-02-18T08:41:19.826 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:41:19.826 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:41:19.826 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:41:20.028 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_34 2026-02-18T08:41:20.029 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:41:20.029 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:41:20.029 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_34 2026-02-18T08:41:20.029 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:41:20.029 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_34 ']' 2026-02-18T08:41:20.029 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_34' 2026-02-18T08:41:20.029 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_34 2026-02-18T08:41:20.029 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:41:20.029 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:41:20.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:41:20.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:41:20.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:41:20.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:41:20.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:41:20.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:41:20.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:41:20.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:41:20.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_34 2026-02-18T08:41:20.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:41:20.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:41:20.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:41:20.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:41:20.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:41:20.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:41:20.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:41:20.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:41:20.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:41:20.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:41:20.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:41:20.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:41:20.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:41:20.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:41:20.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:41:20.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:41:20.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:41:20.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:41:20.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:41:20.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:41:20.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:41:20.035 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:41:20.035 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:41:20.035 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:41:20.035 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:41:20.035 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:41:20.035 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:41:20.035 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:41:20.035 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:41:20.035 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:41:20.041 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:41:20.041 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:41:20.041 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:41:20.041 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_34 2026-02-18T08:41:20.092 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_34'] 2026-02-18T08:41:20.092 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:41:20.092 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:41:20.095 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:41:20.095 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_34 2026-02-18T08:41:20.095 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:41:20.095 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:41:20.095 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:41:20.095 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:41:20.095 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:41:20.095 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:41:20.096 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:41:20.096 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:41:20.096 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:41:20.096 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:41:20.096 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:41:20.096 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:41:20.096 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:41:20.096 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:41:20.096 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:41:20.096 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:41:20.096 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:41:20.096 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:41:20.097 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:41:20.097 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:41:20.097 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:41:20.097 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:41:20.097 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:41:20.097 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:41:20.097 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:41:20.097 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:41:20.097 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:41:20.097 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:41:20.097 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:41:20.097 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:41:20.105 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:41:20.106 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:41:20.106 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:41:20.106 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_34 2026-02-18T08:41:20.167 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_34'] 2026-02-18T08:41:20.167 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:41:20.167 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:41:20.170 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:41:20.170 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_34 2026-02-18T08:41:20.170 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:41:20.170 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:41:20.170 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:41:20.170 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:41:20.170 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:41:20.170 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:41:20.170 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:41:20.171 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:41:20.171 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:41:20.171 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:41:20.171 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:41:20.171 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:41:20.171 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:41:20.171 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:41:20.171 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:41:20.171 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:41:20.171 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:41:20.171 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:41:20.171 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:41:20.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:41:20.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:41:20.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:41:20.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:41:20.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:41:20.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:41:20.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:41:20.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:41:20.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:41:20.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:41:20.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:41:20.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:41:20.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:41:20.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:41:20.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_34 2026-02-18T08:41:20.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:41:20.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_34 2026-02-18T08:41:20.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:41:20.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:41:20.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_34 2026-02-18T08:41:20.244 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_34 '' 2026-02-18T08:41:20.244 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:41:20.244 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:41:20.244 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_34 2026-02-18T08:41:20.244 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:41:20.244 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_34.status 2026-02-18T08:41:20.244 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_34.status 2026-02-18T08:41:20.244 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:41:20.245 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_34.status 2026-02-18T08:41:20.245 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_34 2026-02-18T08:41:20.245 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_34.status 2026-02-18T08:41:20.273 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 4 2026-02-18T08:41:20.273 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 4 2026-02-18T08:41:20.274 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:41:20.274 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=2, entry_tid=1], [object_number=16, tag_tid=2, entry_tid=0], [object_number=19, tag_tid=1, entry_tid=19627], [object_number=18, tag_tid=1, entry_tid=19626]]], state=connected] 2026-02-18T08:41:20.274 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=16, tag_tid=1, entry_tid=19628], [object_number=19, tag_tid=1, entry_tid=19627], [object_number=18, tag_tid=1, entry_tid=19626], [object_number=17, tag_tid=1, entry_tid=19625]]], state=connected] 2026-02-18T08:41:20.276 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_34.status 2026-02-18T08:41:20.277 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=17, tag_tid=2, entry_tid=1' 2026-02-18T08:41:20.279 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_34 2026-02-18T08:41:20.279 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:41:20.279 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:41:20.279 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_34 2026-02-18T08:41:20.279 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_34 '..*' 2026-02-18T08:41:20.279 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:41:20.279 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:41:20.279 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_34 2026-02-18T08:41:20.279 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:41:20.279 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_34.status 2026-02-18T08:41:20.280 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_34.status 2026-02-18T08:41:20.280 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:41:20.280 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_34.status 2026-02-18T08:41:20.281 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_34 2026-02-18T08:41:20.281 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_34.status 2026-02-18T08:41:20.510 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 4 2026-02-18T08:41:20.511 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 4 2026-02-18T08:41:20.511 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:41:20.511 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=2, entry_tid=1], [object_number=16, tag_tid=2, entry_tid=0], [object_number=19, tag_tid=1, entry_tid=19627], [object_number=18, tag_tid=1, entry_tid=19626]]], state=connected] 2026-02-18T08:41:20.511 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=16, tag_tid=1, entry_tid=19628], [object_number=19, tag_tid=1, entry_tid=19627], [object_number=18, tag_tid=1, entry_tid=19626], [object_number=17, tag_tid=1, entry_tid=19625]]], state=connected] 2026-02-18T08:41:20.514 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_34.status 2026-02-18T08:41:20.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=16, tag_tid=1, entry_tid=19628' 2026-02-18T08:41:20.516 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=17, tag_tid=2, entry_tid=1' -a 'object_number=17, tag_tid=2, entry_tid=1' = 'object_number=16, tag_tid=1, entry_tid=19628' 2026-02-18T08:41:20.516 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=16, tag_tid=1, entry_tid=19628' '!=' 'object_number=16, tag_tid=1, entry_tid=19628' 2026-02-18T08:41:20.516 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:41:20.516 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.4 2026-02-18T08:41:20.918 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_34 2026-02-18T08:41:20.918 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:41:20.918 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:41:20.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_34 2026-02-18T08:41:20.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:41:20.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_34 ']' 2026-02-18T08:41:20.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_34' 2026-02-18T08:41:20.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_34 2026-02-18T08:41:20.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:41:20.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:41:20.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:41:20.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:41:20.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:41:20.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:41:20.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:41:20.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:41:20.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:41:20.922 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:41:20.923 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_34 2026-02-18T08:41:20.923 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:41:20.923 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:41:20.923 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:41:20.923 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:41:20.923 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:41:20.923 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:41:20.923 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:41:20.923 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:41:20.923 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:41:20.923 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:41:20.923 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:41:20.924 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:41:20.924 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:41:20.924 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:41:20.924 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:41:20.924 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:41:20.924 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:41:20.924 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:41:20.924 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:41:20.924 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:41:20.924 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:41:20.924 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:41:20.924 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:41:20.924 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:41:20.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:41:20.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:41:20.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:41:20.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:41:20.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:41:20.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:41:20.932 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:41:20.932 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:41:20.932 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:41:20.932 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_34 2026-02-18T08:41:20.994 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_34'] 2026-02-18T08:41:20.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:41:20.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:41:20.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:41:20.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_34 2026-02-18T08:41:20.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:41:20.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:41:20.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:41:20.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:41:20.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:41:20.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:41:20.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:41:20.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:41:20.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:41:20.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:41:20.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:41:20.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:41:20.999 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:41:20.999 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:41:20.999 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:41:20.999 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:41:20.999 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:41:20.999 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:41:20.999 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:41:20.999 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:41:20.999 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:41:20.999 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:41:20.999 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:41:20.999 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:41:21.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:41:21.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:41:21.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:41:21.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:41:21.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:41:21.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:41:21.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:41:21.010 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:41:21.010 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:41:21.010 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_34 2026-02-18T08:41:21.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_34'] 2026-02-18T08:41:21.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:41:21.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:41:21.068 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:41:21.068 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_34 2026-02-18T08:41:21.068 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:41:21.068 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:41:21.069 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:41:21.069 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:41:21.069 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:41:21.069 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:41:21.069 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:41:21.069 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:41:21.069 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:41:21.069 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:41:21.069 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:41:21.069 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:41:21.069 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:41:21.069 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:41:21.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:41:21.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:41:21.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:41:21.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:41:21.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:41:21.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:41:21.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:41:21.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:41:21.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:41:21.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:41:21.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:41:21.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:41:21.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:41:21.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:41:21.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:41:21.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:41:21.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:41:21.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:41:21.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:41:21.080 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_34 2026-02-18T08:41:21.147 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:41:21.148 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_34 2026-02-18T08:41:21.148 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:41:21.148 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:41:21.148 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_34 2026-02-18T08:41:21.148 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_34 '' 2026-02-18T08:41:21.148 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:41:21.148 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:41:21.149 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_34 2026-02-18T08:41:21.149 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:41:21.149 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_34.status 2026-02-18T08:41:21.149 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_34.status 2026-02-18T08:41:21.149 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:41:21.150 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_34.status 2026-02-18T08:41:21.150 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_34 2026-02-18T08:41:21.150 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_34.status 2026-02-18T08:41:21.178 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 4 2026-02-18T08:41:21.179 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 4 2026-02-18T08:41:21.179 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:41:21.179 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=2, entry_tid=1], [object_number=16, tag_tid=2, entry_tid=0], [object_number=19, tag_tid=1, entry_tid=19627], [object_number=18, tag_tid=1, entry_tid=19626]]], state=connected] 2026-02-18T08:41:21.179 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=16, tag_tid=1, entry_tid=19628], [object_number=19, tag_tid=1, entry_tid=19627], [object_number=18, tag_tid=1, entry_tid=19626], [object_number=17, tag_tid=1, entry_tid=19625]]], state=connected] 2026-02-18T08:41:21.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_34.status 2026-02-18T08:41:21.183 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=17, tag_tid=2, entry_tid=1' 2026-02-18T08:41:21.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_34 2026-02-18T08:41:21.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:41:21.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:41:21.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_34 2026-02-18T08:41:21.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_34 '..*' 2026-02-18T08:41:21.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:41:21.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:41:21.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_34 2026-02-18T08:41:21.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:41:21.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_34.status 2026-02-18T08:41:21.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_34.status 2026-02-18T08:41:21.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:41:21.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_34.status 2026-02-18T08:41:21.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_34 2026-02-18T08:41:21.187 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_34.status 2026-02-18T08:41:21.221 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 4 2026-02-18T08:41:21.222 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 4 2026-02-18T08:41:21.222 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:41:21.222 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=2, entry_tid=1], [object_number=16, tag_tid=2, entry_tid=0], [object_number=19, tag_tid=1, entry_tid=19627], [object_number=18, tag_tid=1, entry_tid=19626]]], state=connected] 2026-02-18T08:41:21.222 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=16, tag_tid=1, entry_tid=19628], [object_number=19, tag_tid=1, entry_tid=19627], [object_number=18, tag_tid=1, entry_tid=19626], [object_number=17, tag_tid=1, entry_tid=19625]]], state=connected] 2026-02-18T08:41:21.225 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_34.status 2026-02-18T08:41:21.227 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=16, tag_tid=1, entry_tid=19628' 2026-02-18T08:41:21.227 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=17, tag_tid=2, entry_tid=1' -a 'object_number=17, tag_tid=2, entry_tid=1' = 'object_number=16, tag_tid=1, entry_tid=19628' 2026-02-18T08:41:21.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=16, tag_tid=1, entry_tid=19628' '!=' 'object_number=16, tag_tid=1, entry_tid=19628' 2026-02-18T08:41:21.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:41:21.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.8 2026-02-18T08:41:22.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_34 2026-02-18T08:41:22.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:41:22.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:41:22.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_34 2026-02-18T08:41:22.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:41:22.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_34 ']' 2026-02-18T08:41:22.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_34' 2026-02-18T08:41:22.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_34 2026-02-18T08:41:22.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:41:22.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:41:22.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:41:22.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:41:22.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:41:22.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:41:22.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:41:22.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:41:22.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:41:22.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:41:22.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_34 2026-02-18T08:41:22.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:41:22.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:41:22.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:41:22.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:41:22.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:41:22.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:41:22.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:41:22.035 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:41:22.035 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:41:22.035 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:41:22.035 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:41:22.035 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:41:22.035 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:41:22.035 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:41:22.035 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:41:22.035 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:41:22.035 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:41:22.035 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:41:22.035 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:41:22.036 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:41:22.036 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:41:22.036 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:41:22.036 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:41:22.036 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:41:22.036 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:41:22.036 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:41:22.036 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:41:22.036 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:41:22.036 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:41:22.036 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:41:22.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:41:22.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:41:22.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:41:22.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_34 2026-02-18T08:41:22.102 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_34'] 2026-02-18T08:41:22.102 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:41:22.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:41:22.105 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:41:22.105 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_34 2026-02-18T08:41:22.106 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:41:22.106 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:41:22.106 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:41:22.106 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:41:22.106 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:41:22.106 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:41:22.106 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:41:22.106 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:41:22.106 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:41:22.106 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:41:22.106 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:41:22.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:41:22.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:41:22.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:41:22.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:41:22.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:41:22.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:41:22.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:41:22.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:41:22.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:41:22.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:41:22.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:41:22.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:41:22.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:41:22.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:41:22.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:41:22.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:41:22.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:41:22.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:41:22.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:41:22.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:41:22.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:41:22.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:41:22.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_34 2026-02-18T08:41:22.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_34'] 2026-02-18T08:41:22.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:41:22.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:41:22.171 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:41:22.171 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_34 2026-02-18T08:41:22.171 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:41:22.171 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:41:22.171 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:41:22.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:41:22.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:41:22.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:41:22.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:41:22.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:41:22.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:41:22.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:41:22.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:41:22.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:41:22.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:41:22.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:41:22.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:41:22.173 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:41:22.173 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:41:22.173 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:41:22.173 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:41:22.173 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:41:22.173 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:41:22.173 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:41:22.173 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:41:22.173 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:41:22.173 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:41:22.173 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:41:22.173 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:41:22.173 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:41:22.174 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:41:22.174 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:41:22.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:41:22.183 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:41:22.183 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:41:22.183 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_34 2026-02-18T08:41:22.254 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:41:22.255 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_34 2026-02-18T08:41:22.255 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:41:22.256 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:41:22.256 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_34 2026-02-18T08:41:22.256 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_34 '' 2026-02-18T08:41:22.256 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:41:22.256 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:41:22.256 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_34 2026-02-18T08:41:22.256 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:41:22.256 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_34.status 2026-02-18T08:41:22.256 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_34.status 2026-02-18T08:41:22.256 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:41:22.257 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_34.status 2026-02-18T08:41:22.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_34 2026-02-18T08:41:22.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_34.status 2026-02-18T08:41:22.284 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 4 2026-02-18T08:41:22.284 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 4 2026-02-18T08:41:22.284 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:41:22.284 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=2, entry_tid=1], [object_number=16, tag_tid=2, entry_tid=0], [object_number=19, tag_tid=1, entry_tid=19627], [object_number=18, tag_tid=1, entry_tid=19626]]], state=connected] 2026-02-18T08:41:22.284 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=17, tag_tid=2, entry_tid=1], [object_number=16, tag_tid=2, entry_tid=0], [object_number=19, tag_tid=1, entry_tid=19627], [object_number=18, tag_tid=1, entry_tid=19626]]], state=connected] 2026-02-18T08:41:22.286 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_34.status 2026-02-18T08:41:22.288 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=17, tag_tid=2, entry_tid=1' 2026-02-18T08:41:22.290 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_34 2026-02-18T08:41:22.290 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:41:22.290 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:41:22.290 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_34 2026-02-18T08:41:22.290 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_34 '..*' 2026-02-18T08:41:22.290 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:41:22.290 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:41:22.290 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_34 2026-02-18T08:41:22.290 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:41:22.291 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_34.status 2026-02-18T08:41:22.291 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_34.status 2026-02-18T08:41:22.291 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:41:22.291 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_34.status 2026-02-18T08:41:22.292 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_34 2026-02-18T08:41:22.292 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_34.status 2026-02-18T08:41:22.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 4 2026-02-18T08:41:22.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 4 2026-02-18T08:41:22.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:41:22.329 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=2, entry_tid=1], [object_number=16, tag_tid=2, entry_tid=0], [object_number=19, tag_tid=1, entry_tid=19627], [object_number=18, tag_tid=1, entry_tid=19626]]], state=connected] 2026-02-18T08:41:22.329 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=17, tag_tid=2, entry_tid=1], [object_number=16, tag_tid=2, entry_tid=0], [object_number=19, tag_tid=1, entry_tid=19627], [object_number=18, tag_tid=1, entry_tid=19626]]], state=connected] 2026-02-18T08:41:22.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_34.status 2026-02-18T08:41:22.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=17, tag_tid=2, entry_tid=1' 2026-02-18T08:41:22.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=17, tag_tid=2, entry_tid=1' -a 'object_number=17, tag_tid=2, entry_tid=1' = 'object_number=17, tag_tid=2, entry_tid=1' 2026-02-18T08:41:22.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:41:22.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_snap_present cluster1 mirror image_34 snap 2026-02-18T08:41:22.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:41:22.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:41:22.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_34 2026-02-18T08:41:22.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap 2026-02-18T08:41:22.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-18T08:41:22.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-18T08:41:22.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1 2026-02-18T08:41:23.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster1 info mirror/image_34@snap 2026-02-18T08:41:23.570 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:rbd image 'image_34': 2026-02-18T08:41:23.570 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: size 128 MiB in 32 objects 2026-02-18T08:41:23.570 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: order 22 (4 MiB objects) 2026-02-18T08:41:23.570 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: snapshot_count: 1 2026-02-18T08:41:23.570 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: id: 1104980b516f 2026-02-18T08:41:23.570 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: block_name_prefix: rbd_data.1104980b516f 2026-02-18T08:41:23.570 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: format: 2 2026-02-18T08:41:23.570 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: features: layering, exclusive-lock, journaling 2026-02-18T08:41:23.570 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: op_features: 2026-02-18T08:41:23.571 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: flags: 2026-02-18T08:41:23.571 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: create_timestamp: Wed Feb 18 08:31:36 2026 2026-02-18T08:41:23.571 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: access_timestamp: Wed Feb 18 08:31:36 2026 2026-02-18T08:41:23.571 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: modify_timestamp: Wed Feb 18 08:31:36 2026 2026-02-18T08:41:23.571 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: protected: False 2026-02-18T08:41:23.571 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: journal: 1104980b516f 2026-02-18T08:41:23.571 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring state: enabled 2026-02-18T08:41:23.571 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring mode: journal 2026-02-18T08:41:23.571 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring global id: 6e7a6aa5-d561-44fa-9b21-9c8d70bdec18 2026-02-18T08:41:23.571 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring primary: false 2026-02-18T08:41:23.574 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:41:23.574 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ compare_image_snaps mirror image_34 snap 2026-02-18T08:41:23.574 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:41:23.574 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_34 2026-02-18T08:41:23.574 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap 2026-02-18T08:41:23.575 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local ret=0 2026-02-18T08:41:23.575 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local rmt_export=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_34.export 2026-02-18T08:41:23.575 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loc_export=/tmp/tmp.vybh1m9yJF/cluster1-mirror-image_34.export 2026-02-18T08:41:23.575 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_34.export /tmp/tmp.vybh1m9yJF/cluster1-mirror-image_34.export 2026-02-18T08:41:23.575 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 -p mirror export image_34@snap /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_34.export 2026-02-18T08:41:23.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-18T08:41:23.736 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster1 -p mirror export image_34@snap /tmp/tmp.vybh1m9yJF/cluster1-mirror-image_34.export 2026-02-18T08:41:23.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-18T08:41:23.910 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmp /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_34.export /tmp/tmp.vybh1m9yJF/cluster1-mirror-image_34.export 2026-02-18T08:41:23.966 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_34.export /tmp/tmp.vybh1m9yJF/cluster1-mirror-image_34.export 2026-02-18T08:41:24.010 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:41:24.010 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T08:41:24.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_35 2026-02-18T08:41:24.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_snap cluster2 mirror image_35 snap 2026-02-18T08:41:24.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:41:24.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:41:24.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_35 2026-02-18T08:41:24.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap 2026-02-18T08:41:24.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 -p mirror snap create image_35@snap --debug-rbd=20 --debug-journaler=20 2026-02-18T08:41:25.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_replay_started cluster1 mirror image_35 2026-02-18T08:41:25.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:41:25.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:41:25.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_35 2026-02-18T08:41:25.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_replay_state cluster1 mirror image_35 started 2026-02-18T08:41:25.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:41:25.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:41:25.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_35 2026-02-18T08:41:25.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local state=started 2026-02-18T08:41:25.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-18T08:41:25.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:41:25.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1 2026-02-18T08:41:26.110 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test_image_replay_state cluster1 mirror image_35 started 2026-02-18T08:41:26.110 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:41:26.110 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:41:26.111 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_35 2026-02-18T08:41:26.111 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local test_state=started 2026-02-18T08:41:26.111 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local status_result 2026-02-18T08:41:26.111 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_state=stopped 2026-02-18T08:41:26.111 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_35 2026-02-18T08:41:26.111 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -i state 2026-02-18T08:41:26.111 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_instance=cluster1 2026-02-18T08:41:26.111 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-18T08:41:26.111 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:41:26.111 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance=cluster1 2026-02-18T08:41:26.112 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local loop_instance 2026-02-18T08:41:26.112 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:41:26.112 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sleep 0 2026-02-18T08:41:26.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:41:26.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ seq 0 3 2026-02-18T08:41:26.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:41:26.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_35 2026-02-18T08:41:26.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:0 2026-02-18T08:41:26.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-18T08:41:26.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:41:26.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:41:26.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:41:26.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-18T08:41:26.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:41:26.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:41:26.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:41:26.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:41:26.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-18T08:41:26.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:41:26.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:41:26.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:41:26.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:41:26.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:0 2026-02-18T08:41:26.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-18T08:41:26.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-18T08:41:26.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:41:26.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:0 2026-02-18T08:41:26.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T08:41:26.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-18T08:41:26.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:41:26.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:41:26.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-18T08:41:26.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-18T08:41:26.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-18T08:41:26.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=0 2026-02-18T08:41:26.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=0 2026-02-18T08:41:26.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:41:26.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:41:26.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:41:26.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:41:26.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_35 2026-02-18T08:41:26.176 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_35'] 2026-02-18T08:41:26.176 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status : get status for rbd mirror 2026-02-18T08:41:26.177 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:41:26.179 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:41:26.179 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_35 2026-02-18T08:41:26.179 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:1 2026-02-18T08:41:26.179 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-18T08:41:26.179 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:41:26.180 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:41:26.180 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:41:26.180 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-18T08:41:26.180 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:41:26.180 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:41:26.180 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:41:26.180 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:41:26.180 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-18T08:41:26.180 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:41:26.180 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:41:26.180 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:41:26.180 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:41:26.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:1 2026-02-18T08:41:26.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-18T08:41:26.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-18T08:41:26.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:41:26.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:1 2026-02-18T08:41:26.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T08:41:26.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-18T08:41:26.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:41:26.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:41:26.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-18T08:41:26.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-18T08:41:26.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-18T08:41:26.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=1 2026-02-18T08:41:26.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=1 2026-02-18T08:41:26.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:41:26.190 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:41:26.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:41:26.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:41:26.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_35 2026-02-18T08:41:26.260 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ return 0 2026-02-18T08:41:26.261 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ status_result=' "state": "Replaying"' 2026-02-18T08:41:26.261 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ echo ' "state": "Replaying"' 2026-02-18T08:41:26.261 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ grep -i Replaying 2026-02-18T08:41:26.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: "state": "Replaying" 2026-02-18T08:41:26.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ current_state=started 2026-02-18T08:41:26.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test started = started 2026-02-18T08:41:26.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:41:26.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror image_35 2026-02-18T08:41:26.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_cluster=cluster1 2026-02-18T08:41:26.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:41:26.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_pool=mirror 2026-02-18T08:41:26.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local remote_pool=mirror 2026-02-18T08:41:26.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_35 2026-02-18T08:41:26.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' journal = journal ']' 2026-02-18T08:41:26.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror mirror image_35 2026-02-18T08:41:26.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_cluster=cluster1 2026-02-18T08:41:26.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:41:26.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_pool=mirror 2026-02-18T08:41:26.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local remote_pool=mirror 2026-02-18T08:41:26.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_35 2026-02-18T08:41:26.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-02-18T08:41:26.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-02-18T08:41:26.266 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:41:26.266 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:41:26.266 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:41:26.465 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_35 2026-02-18T08:41:26.465 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:41:26.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:41:26.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_35 2026-02-18T08:41:26.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:41:26.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_35 ']' 2026-02-18T08:41:26.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_35' 2026-02-18T08:41:26.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_35 2026-02-18T08:41:26.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:41:26.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:41:26.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:41:26.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:41:26.467 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:41:26.467 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:41:26.467 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:41:26.467 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:41:26.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:41:26.470 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:41:26.470 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_35 2026-02-18T08:41:26.470 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:41:26.470 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:41:26.470 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:41:26.470 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:41:26.470 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:41:26.470 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:41:26.470 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:41:26.470 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:41:26.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:41:26.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:41:26.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:41:26.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:41:26.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:41:26.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:41:26.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:41:26.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:41:26.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:41:26.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:41:26.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:41:26.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:41:26.472 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:41:26.472 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:41:26.472 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:41:26.472 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:41:26.472 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:41:26.472 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:41:26.472 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:41:26.472 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:41:26.472 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:41:26.472 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:41:26.478 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:41:26.479 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:41:26.479 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:41:26.479 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_35 2026-02-18T08:41:26.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_35'] 2026-02-18T08:41:26.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:41:26.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:41:26.540 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:41:26.540 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_35 2026-02-18T08:41:26.540 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:41:26.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:41:26.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:41:26.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:41:26.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:41:26.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:41:26.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:41:26.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:41:26.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:41:26.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:41:26.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:41:26.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:41:26.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:41:26.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:41:26.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:41:26.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:41:26.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:41:26.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:41:26.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:41:26.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:41:26.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:41:26.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:41:26.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:41:26.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:41:26.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:41:26.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:41:26.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:41:26.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:41:26.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:41:26.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:41:26.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:41:26.551 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:41:26.551 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:41:26.551 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_35 2026-02-18T08:41:26.618 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:41:26.619 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_35 2026-02-18T08:41:26.619 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:41:26.619 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:41:26.619 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_35 2026-02-18T08:41:26.619 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_35 '' 2026-02-18T08:41:26.619 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:41:26.620 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:41:26.620 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_35 2026-02-18T08:41:26.620 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:41:26.620 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_35.status 2026-02-18T08:41:26.620 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_35.status 2026-02-18T08:41:26.620 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:41:26.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_35.status 2026-02-18T08:41:26.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_35 2026-02-18T08:41:26.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_35.status 2026-02-18T08:41:26.649 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 12 2026-02-18T08:41:26.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 12 2026-02-18T08:41:26.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:41:26.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=49, tag_tid=2, entry_tid=1], [object_number=48, tag_tid=2, entry_tid=0], [object_number=51, tag_tid=1, entry_tid=51575], [object_number=50, tag_tid=1, entry_tid=51574]]], state=connected] 2026-02-18T08:41:26.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=49, tag_tid=1, entry_tid=51577], [object_number=48, tag_tid=1, entry_tid=51576], [object_number=51, tag_tid=1, entry_tid=51575], [object_number=50, tag_tid=1, entry_tid=51574]]], state=connected] 2026-02-18T08:41:26.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_35.status 2026-02-18T08:41:26.654 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=49, tag_tid=2, entry_tid=1' 2026-02-18T08:41:26.655 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_35 2026-02-18T08:41:26.655 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:41:26.655 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:41:26.655 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_35 2026-02-18T08:41:26.655 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_35 '..*' 2026-02-18T08:41:26.655 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:41:26.655 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:41:26.655 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_35 2026-02-18T08:41:26.655 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:41:26.655 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_35.status 2026-02-18T08:41:26.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_35.status 2026-02-18T08:41:26.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:41:26.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_35.status 2026-02-18T08:41:26.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_35 2026-02-18T08:41:26.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_35.status 2026-02-18T08:41:26.683 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 12 2026-02-18T08:41:26.684 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 12 2026-02-18T08:41:26.684 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:41:26.684 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=49, tag_tid=2, entry_tid=1], [object_number=48, tag_tid=2, entry_tid=0], [object_number=51, tag_tid=1, entry_tid=51575], [object_number=50, tag_tid=1, entry_tid=51574]]], state=connected] 2026-02-18T08:41:26.684 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=49, tag_tid=1, entry_tid=51577], [object_number=48, tag_tid=1, entry_tid=51576], [object_number=51, tag_tid=1, entry_tid=51575], [object_number=50, tag_tid=1, entry_tid=51574]]], state=connected] 2026-02-18T08:41:26.686 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_35.status 2026-02-18T08:41:26.688 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=49, tag_tid=1, entry_tid=51577' 2026-02-18T08:41:26.688 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=49, tag_tid=2, entry_tid=1' -a 'object_number=49, tag_tid=2, entry_tid=1' = 'object_number=49, tag_tid=1, entry_tid=51577' 2026-02-18T08:41:26.688 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=49, tag_tid=1, entry_tid=51577' '!=' '' 2026-02-18T08:41:26.688 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:41:26.688 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=49, tag_tid=1, entry_tid=51577' = '' 2026-02-18T08:41:26.688 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=49, tag_tid=1, entry_tid=51577' 2026-02-18T08:41:26.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=49, tag_tid=2, entry_tid=1' 2026-02-18T08:41:26.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:41:26.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:41:26.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=2 2026-02-18T08:41:26.691 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=49, tag_tid=1, entry_tid=51577' 2026-02-18T08:41:26.691 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:41:26.691 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:41:26.692 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=1 2026-02-18T08:41:26.693 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=49, tag_tid=2, entry_tid=1' 2026-02-18T08:41:26.693 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:41:26.693 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:41:26.694 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:41:26.695 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=49, tag_tid=1, entry_tid=51577' 2026-02-18T08:41:26.695 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:41:26.695 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:41:26.697 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=51577 2026-02-18T08:41:26.697 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 2 = 1 -a 1 -le 51577 2026-02-18T08:41:26.697 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:41:26.697 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:41:26.697 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:41:26.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_35 2026-02-18T08:41:26.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:41:26.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:41:26.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_35 2026-02-18T08:41:26.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:41:26.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_35 ']' 2026-02-18T08:41:26.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_35' 2026-02-18T08:41:26.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_35 2026-02-18T08:41:26.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:41:26.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:41:26.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:41:26.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:41:26.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:41:26.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:41:26.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:41:26.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:41:26.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:41:26.905 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:41:26.905 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_35 2026-02-18T08:41:26.905 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:41:26.905 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:41:26.905 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:41:26.905 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:41:26.905 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:41:26.905 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:41:26.905 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:41:26.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:41:26.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:41:26.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:41:26.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:41:26.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:41:26.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:41:26.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:41:26.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:41:26.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:41:26.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:41:26.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:41:26.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:41:26.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:41:26.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:41:26.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:41:26.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:41:26.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:41:26.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:41:26.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:41:26.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:41:26.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:41:26.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:41:26.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:41:26.914 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:41:26.914 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:41:26.914 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:41:26.915 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_35 2026-02-18T08:41:26.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_35'] 2026-02-18T08:41:26.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:41:26.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:41:26.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:41:26.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_35 2026-02-18T08:41:26.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:41:26.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:41:26.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:41:26.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:41:26.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:41:26.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:41:26.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:41:26.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:41:26.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:41:26.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:41:26.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:41:26.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:41:26.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:41:26.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:41:26.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:41:26.982 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:41:26.982 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:41:26.982 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:41:26.982 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:41:26.982 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:41:26.982 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:41:26.982 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:41:26.982 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:41:26.982 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:41:26.982 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:41:26.982 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:41:26.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:41:26.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:41:26.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:41:26.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:41:26.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:41:26.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:41:26.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:41:26.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_35 2026-02-18T08:41:27.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:41:27.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_35 2026-02-18T08:41:27.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:41:27.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:41:27.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_35 2026-02-18T08:41:27.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_35 '' 2026-02-18T08:41:27.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:41:27.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:41:27.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_35 2026-02-18T08:41:27.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:41:27.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_35.status 2026-02-18T08:41:27.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_35.status 2026-02-18T08:41:27.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:41:27.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_35.status 2026-02-18T08:41:27.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_35 2026-02-18T08:41:27.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_35.status 2026-02-18T08:41:27.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 12 2026-02-18T08:41:27.080 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 12 2026-02-18T08:41:27.080 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:41:27.080 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=49, tag_tid=2, entry_tid=1], [object_number=48, tag_tid=2, entry_tid=0], [object_number=51, tag_tid=1, entry_tid=51575], [object_number=50, tag_tid=1, entry_tid=51574]]], state=connected] 2026-02-18T08:41:27.080 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=49, tag_tid=1, entry_tid=51577], [object_number=48, tag_tid=1, entry_tid=51576], [object_number=51, tag_tid=1, entry_tid=51575], [object_number=50, tag_tid=1, entry_tid=51574]]], state=connected] 2026-02-18T08:41:27.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_35.status 2026-02-18T08:41:27.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=49, tag_tid=2, entry_tid=1' 2026-02-18T08:41:27.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_35 2026-02-18T08:41:27.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:41:27.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:41:27.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_35 2026-02-18T08:41:27.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_35 '..*' 2026-02-18T08:41:27.086 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:41:27.086 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:41:27.086 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_35 2026-02-18T08:41:27.086 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:41:27.086 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_35.status 2026-02-18T08:41:27.086 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_35.status 2026-02-18T08:41:27.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:41:27.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_35.status 2026-02-18T08:41:27.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_35 2026-02-18T08:41:27.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_35.status 2026-02-18T08:41:27.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 12 2026-02-18T08:41:27.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 12 2026-02-18T08:41:27.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:41:27.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=49, tag_tid=2, entry_tid=1], [object_number=48, tag_tid=2, entry_tid=0], [object_number=51, tag_tid=1, entry_tid=51575], [object_number=50, tag_tid=1, entry_tid=51574]]], state=connected] 2026-02-18T08:41:27.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=49, tag_tid=1, entry_tid=51577], [object_number=48, tag_tid=1, entry_tid=51576], [object_number=51, tag_tid=1, entry_tid=51575], [object_number=50, tag_tid=1, entry_tid=51574]]], state=connected] 2026-02-18T08:41:27.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_35.status 2026-02-18T08:41:27.129 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=49, tag_tid=1, entry_tid=51577' 2026-02-18T08:41:27.129 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=49, tag_tid=2, entry_tid=1' -a 'object_number=49, tag_tid=2, entry_tid=1' = 'object_number=49, tag_tid=1, entry_tid=51577' 2026-02-18T08:41:27.129 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=49, tag_tid=1, entry_tid=51577' '!=' 'object_number=49, tag_tid=1, entry_tid=51577' 2026-02-18T08:41:27.129 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:41:27.130 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.4 2026-02-18T08:41:27.467 INFO:tasks.rbd_mirror_thrash:kill cluster1.client.mirror.0 2026-02-18T08:41:27.467 INFO:tasks.rbd_mirror.cluster1.client.mirror.0:Sent signal 15 2026-02-18T08:41:27.468 INFO:tasks.rbd_mirror_thrash:waiting for 9.838930171917156 secs before reviving daemons 2026-02-18T08:41:27.468 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.trial163.stderr:2026-02-18T08:41:27.459+0000 7fa6bb839640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.0 (PID: 105119) UID: 1000 2026-02-18T08:41:27.531 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_35 2026-02-18T08:41:27.531 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:41:27.531 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:41:27.532 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_35 2026-02-18T08:41:27.532 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:41:27.532 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_35 ']' 2026-02-18T08:41:27.532 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_35' 2026-02-18T08:41:27.532 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_35 2026-02-18T08:41:27.532 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:41:27.532 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:41:27.532 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:41:27.532 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:41:27.532 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:41:27.532 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:41:27.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:41:27.535 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:41:27.535 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:41:27.535 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:41:27.536 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_35 2026-02-18T08:41:27.536 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:41:27.536 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:41:27.536 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:41:27.536 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:41:27.536 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:41:27.536 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:41:27.536 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:41:27.536 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:41:27.536 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:41:27.536 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:41:27.536 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:41:27.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:41:27.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:41:27.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:41:27.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:41:27.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:41:27.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:41:27.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:41:27.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:41:27.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:41:27.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:41:27.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:41:27.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:41:27.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:41:27.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:41:27.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:41:27.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:41:27.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:41:27.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:41:27.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:41:27.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:41:27.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:41:27.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:41:27.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_35 2026-02-18T08:41:27.601 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: exception getting command descriptions: [Errno 2] No such file or directory 2026-02-18T08:41:27.604 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:41:27.604 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_35 2026-02-18T08:41:27.604 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:41:27.604 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:41:27.604 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:41:27.604 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:41:27.604 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:41:27.604 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:41:27.604 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:41:27.604 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:41:27.605 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:41:27.605 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:41:27.605 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:41:27.605 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:41:27.605 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:41:27.605 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:41:27.605 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:41:27.605 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:41:27.605 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:41:27.605 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:41:27.605 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:41:27.605 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:41:27.605 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:41:27.606 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:41:27.606 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:41:27.606 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:41:27.606 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:41:27.606 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:41:27.606 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:41:27.606 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:41:27.606 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:41:27.606 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:41:27.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:41:27.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:41:27.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:41:27.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_35 2026-02-18T08:41:27.684 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:41:27.685 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_35 2026-02-18T08:41:27.685 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:41:27.685 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:41:27.685 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_35 2026-02-18T08:41:27.685 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_35 '' 2026-02-18T08:41:27.685 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:41:27.685 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:41:27.686 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_35 2026-02-18T08:41:27.686 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:41:27.686 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_35.status 2026-02-18T08:41:27.686 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_35.status 2026-02-18T08:41:27.686 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:41:27.687 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_35.status 2026-02-18T08:41:27.687 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_35 2026-02-18T08:41:27.687 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_35.status 2026-02-18T08:41:27.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 12 2026-02-18T08:41:27.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 12 2026-02-18T08:41:27.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:41:27.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=49, tag_tid=2, entry_tid=1], [object_number=48, tag_tid=2, entry_tid=0], [object_number=51, tag_tid=1, entry_tid=51575], [object_number=50, tag_tid=1, entry_tid=51574]]], state=connected] 2026-02-18T08:41:27.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=49, tag_tid=1, entry_tid=51577], [object_number=48, tag_tid=1, entry_tid=51576], [object_number=51, tag_tid=1, entry_tid=51575], [object_number=50, tag_tid=1, entry_tid=51574]]], state=connected] 2026-02-18T08:41:27.718 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_35.status 2026-02-18T08:41:27.719 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=49, tag_tid=2, entry_tid=1' 2026-02-18T08:41:27.720 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_35 2026-02-18T08:41:27.721 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:41:27.721 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:41:27.721 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_35 2026-02-18T08:41:27.721 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_35 '..*' 2026-02-18T08:41:27.721 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:41:27.721 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:41:27.721 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_35 2026-02-18T08:41:27.721 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:41:27.721 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_35.status 2026-02-18T08:41:27.721 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_35.status 2026-02-18T08:41:27.722 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:41:27.722 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_35.status 2026-02-18T08:41:27.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_35 2026-02-18T08:41:27.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_35.status 2026-02-18T08:41:27.759 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 12 2026-02-18T08:41:27.759 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 12 2026-02-18T08:41:27.759 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:41:27.760 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=49, tag_tid=2, entry_tid=1], [object_number=48, tag_tid=2, entry_tid=0], [object_number=51, tag_tid=1, entry_tid=51575], [object_number=50, tag_tid=1, entry_tid=51574]]], state=connected] 2026-02-18T08:41:27.760 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=49, tag_tid=1, entry_tid=51577], [object_number=48, tag_tid=1, entry_tid=51576], [object_number=51, tag_tid=1, entry_tid=51575], [object_number=50, tag_tid=1, entry_tid=51574]]], state=connected] 2026-02-18T08:41:27.762 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_35.status 2026-02-18T08:41:27.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=49, tag_tid=1, entry_tid=51577' 2026-02-18T08:41:27.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=49, tag_tid=2, entry_tid=1' -a 'object_number=49, tag_tid=2, entry_tid=1' = 'object_number=49, tag_tid=1, entry_tid=51577' 2026-02-18T08:41:27.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=49, tag_tid=1, entry_tid=51577' '!=' 'object_number=49, tag_tid=1, entry_tid=51577' 2026-02-18T08:41:27.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:41:27.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.8 2026-02-18T08:41:28.566 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_35 2026-02-18T08:41:28.566 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:41:28.566 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:41:28.566 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_35 2026-02-18T08:41:28.567 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:41:28.567 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_35 ']' 2026-02-18T08:41:28.567 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_35' 2026-02-18T08:41:28.567 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_35 2026-02-18T08:41:28.567 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:41:28.567 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:41:28.567 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:41:28.567 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:41:28.567 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:41:28.567 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:41:28.567 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:41:28.570 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:41:28.570 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:41:28.570 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:41:28.571 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_35 2026-02-18T08:41:28.571 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:41:28.571 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:41:28.571 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:41:28.571 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:41:28.571 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:41:28.571 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:41:28.571 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:41:28.571 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:41:28.571 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:41:28.571 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:41:28.571 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:41:28.572 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:41:28.572 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:41:28.572 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:41:28.572 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:41:28.572 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:41:28.572 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:41:28.572 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:41:28.572 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:41:28.572 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:41:28.572 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:41:28.572 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:41:28.572 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:41:28.572 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:41:28.573 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:41:28.573 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:41:28.573 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:41:28.573 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:41:28.573 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:41:28.573 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:41:28.582 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:41:28.582 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:41:28.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:41:28.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_35 2026-02-18T08:41:28.635 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: exception getting command descriptions: [Errno 2] No such file or directory 2026-02-18T08:41:28.638 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:41:28.638 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_35 2026-02-18T08:41:28.638 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:41:28.638 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:41:28.638 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:41:28.639 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:41:28.639 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:41:28.639 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:41:28.639 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:41:28.639 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:41:28.639 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:41:28.639 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:41:28.639 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:41:28.639 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:41:28.639 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:41:28.639 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:41:28.640 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:41:28.640 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:41:28.640 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:41:28.640 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:41:28.640 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:41:28.640 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:41:28.640 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:41:28.640 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:41:28.640 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:41:28.640 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:41:28.640 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:41:28.640 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:41:28.640 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:41:28.641 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:41:28.641 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:41:28.641 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:41:28.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:41:28.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:41:28.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:41:28.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_35 2026-02-18T08:41:28.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:41:28.717 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_35 2026-02-18T08:41:28.717 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:41:28.717 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:41:28.718 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_35 2026-02-18T08:41:28.718 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_35 '' 2026-02-18T08:41:28.718 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:41:28.718 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:41:28.718 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_35 2026-02-18T08:41:28.718 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:41:28.718 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_35.status 2026-02-18T08:41:28.718 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_35.status 2026-02-18T08:41:28.718 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:41:28.719 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_35.status 2026-02-18T08:41:28.719 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_35 2026-02-18T08:41:28.719 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_35.status 2026-02-18T08:41:28.952 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 12 2026-02-18T08:41:28.953 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 12 2026-02-18T08:41:28.953 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:41:28.953 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=49, tag_tid=2, entry_tid=1], [object_number=48, tag_tid=2, entry_tid=0], [object_number=51, tag_tid=1, entry_tid=51575], [object_number=50, tag_tid=1, entry_tid=51574]]], state=connected] 2026-02-18T08:41:28.953 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=49, tag_tid=1, entry_tid=51577], [object_number=48, tag_tid=1, entry_tid=51576], [object_number=51, tag_tid=1, entry_tid=51575], [object_number=50, tag_tid=1, entry_tid=51574]]], state=connected] 2026-02-18T08:41:28.956 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_35.status 2026-02-18T08:41:28.958 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=49, tag_tid=2, entry_tid=1' 2026-02-18T08:41:28.959 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_35 2026-02-18T08:41:28.959 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:41:28.959 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:41:28.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_35 2026-02-18T08:41:28.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_35 '..*' 2026-02-18T08:41:28.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:41:28.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:41:28.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_35 2026-02-18T08:41:28.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:41:28.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_35.status 2026-02-18T08:41:28.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_35.status 2026-02-18T08:41:28.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:41:28.961 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_35.status 2026-02-18T08:41:28.961 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_35 2026-02-18T08:41:28.961 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_35.status 2026-02-18T08:41:28.987 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 12 2026-02-18T08:41:28.988 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 12 2026-02-18T08:41:28.988 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:41:28.988 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=49, tag_tid=2, entry_tid=1], [object_number=48, tag_tid=2, entry_tid=0], [object_number=51, tag_tid=1, entry_tid=51575], [object_number=50, tag_tid=1, entry_tid=51574]]], state=connected] 2026-02-18T08:41:28.988 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=49, tag_tid=1, entry_tid=51577], [object_number=48, tag_tid=1, entry_tid=51576], [object_number=51, tag_tid=1, entry_tid=51575], [object_number=50, tag_tid=1, entry_tid=51574]]], state=connected] 2026-02-18T08:41:28.990 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_35.status 2026-02-18T08:41:28.992 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=49, tag_tid=1, entry_tid=51577' 2026-02-18T08:41:28.992 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=49, tag_tid=2, entry_tid=1' -a 'object_number=49, tag_tid=2, entry_tid=1' = 'object_number=49, tag_tid=1, entry_tid=51577' 2026-02-18T08:41:28.992 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=49, tag_tid=1, entry_tid=51577' '!=' 'object_number=49, tag_tid=1, entry_tid=51577' 2026-02-18T08:41:28.992 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:41:28.992 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1.6 2026-02-18T08:41:30.594 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_35 2026-02-18T08:41:30.594 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:41:30.594 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:41:30.594 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_35 2026-02-18T08:41:30.594 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:41:30.595 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_35 ']' 2026-02-18T08:41:30.595 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_35' 2026-02-18T08:41:30.595 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_35 2026-02-18T08:41:30.595 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:41:30.595 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:41:30.595 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:41:30.595 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:41:30.595 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:41:30.595 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:41:30.595 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:41:30.596 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:41:30.596 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:41:30.598 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:41:30.598 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_35 2026-02-18T08:41:30.598 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:41:30.599 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:41:30.599 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:41:30.599 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:41:30.599 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:41:30.599 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:41:30.599 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:41:30.599 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:41:30.599 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:41:30.599 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:41:30.599 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:41:30.599 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:41:30.599 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:41:30.600 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:41:30.600 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:41:30.600 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:41:30.600 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:41:30.600 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:41:30.600 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:41:30.600 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:41:30.600 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:41:30.600 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:41:30.600 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:41:30.601 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:41:30.601 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:41:30.601 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:41:30.601 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:41:30.601 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:41:30.601 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:41:30.601 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:41:30.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:41:30.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:41:30.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:41:30.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_35 2026-02-18T08:41:30.653 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: exception getting command descriptions: [Errno 2] No such file or directory 2026-02-18T08:41:30.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:41:30.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_35 2026-02-18T08:41:30.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:41:30.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:41:30.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:41:30.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:41:30.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:41:30.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:41:30.658 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:41:30.658 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:41:30.658 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:41:30.658 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:41:30.658 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:41:30.658 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:41:30.658 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:41:30.658 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:41:30.658 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:41:30.658 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:41:30.659 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:41:30.659 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:41:30.659 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:41:30.659 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:41:30.659 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:41:30.659 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:41:30.659 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:41:30.659 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:41:30.659 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:41:30.659 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:41:30.659 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:41:30.659 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:41:30.660 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:41:30.660 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:41:30.667 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:41:30.667 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:41:30.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:41:30.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_35 2026-02-18T08:41:30.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:41:30.728 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_35 2026-02-18T08:41:30.729 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:41:30.729 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:41:30.729 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_35 2026-02-18T08:41:30.729 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_35 '' 2026-02-18T08:41:30.729 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:41:30.729 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:41:30.729 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_35 2026-02-18T08:41:30.729 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:41:30.729 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_35.status 2026-02-18T08:41:30.729 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_35.status 2026-02-18T08:41:30.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:41:30.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_35.status 2026-02-18T08:41:30.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_35 2026-02-18T08:41:30.731 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_35.status 2026-02-18T08:41:30.760 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 12 2026-02-18T08:41:30.760 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 12 2026-02-18T08:41:30.760 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:41:30.760 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=49, tag_tid=2, entry_tid=1], [object_number=48, tag_tid=2, entry_tid=0], [object_number=51, tag_tid=1, entry_tid=51575], [object_number=50, tag_tid=1, entry_tid=51574]]], state=connected] 2026-02-18T08:41:30.760 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=49, tag_tid=1, entry_tid=51577], [object_number=48, tag_tid=1, entry_tid=51576], [object_number=51, tag_tid=1, entry_tid=51575], [object_number=50, tag_tid=1, entry_tid=51574]]], state=connected] 2026-02-18T08:41:30.763 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_35.status 2026-02-18T08:41:30.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=49, tag_tid=2, entry_tid=1' 2026-02-18T08:41:30.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_35 2026-02-18T08:41:30.766 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:41:30.766 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:41:30.766 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_35 2026-02-18T08:41:30.766 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_35 '..*' 2026-02-18T08:41:30.766 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:41:30.766 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:41:30.766 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_35 2026-02-18T08:41:30.766 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:41:30.766 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_35.status 2026-02-18T08:41:30.767 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_35.status 2026-02-18T08:41:30.767 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:41:30.767 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_35.status 2026-02-18T08:41:30.768 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_35 2026-02-18T08:41:30.768 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_35.status 2026-02-18T08:41:30.803 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 12 2026-02-18T08:41:30.803 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 12 2026-02-18T08:41:30.803 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:41:30.803 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=49, tag_tid=2, entry_tid=1], [object_number=48, tag_tid=2, entry_tid=0], [object_number=51, tag_tid=1, entry_tid=51575], [object_number=50, tag_tid=1, entry_tid=51574]]], state=connected] 2026-02-18T08:41:30.803 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=49, tag_tid=1, entry_tid=51577], [object_number=48, tag_tid=1, entry_tid=51576], [object_number=51, tag_tid=1, entry_tid=51575], [object_number=50, tag_tid=1, entry_tid=51574]]], state=connected] 2026-02-18T08:41:30.806 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_35.status 2026-02-18T08:41:30.808 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=49, tag_tid=1, entry_tid=51577' 2026-02-18T08:41:30.808 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=49, tag_tid=2, entry_tid=1' -a 'object_number=49, tag_tid=2, entry_tid=1' = 'object_number=49, tag_tid=1, entry_tid=51577' 2026-02-18T08:41:30.808 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=49, tag_tid=1, entry_tid=51577' '!=' 'object_number=49, tag_tid=1, entry_tid=51577' 2026-02-18T08:41:30.808 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:41:30.808 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 2 2026-02-18T08:41:32.809 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_35 2026-02-18T08:41:32.809 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:41:32.809 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:41:32.809 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_35 2026-02-18T08:41:32.809 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:41:32.809 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_35 ']' 2026-02-18T08:41:32.809 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_35' 2026-02-18T08:41:32.809 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_35 2026-02-18T08:41:32.809 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:41:32.809 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:41:32.809 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:41:32.809 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:41:32.809 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:41:32.809 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:41:32.809 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:41:32.810 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:41:32.811 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:41:32.812 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:41:32.812 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_35 2026-02-18T08:41:32.812 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:41:32.812 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:41:32.812 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:41:32.812 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:41:32.812 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:41:32.812 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:41:32.812 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:41:32.812 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:41:32.812 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:41:32.812 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:41:32.812 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:41:32.812 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:41:32.812 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:41:32.812 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:41:32.813 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:41:32.813 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:41:32.813 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:41:32.813 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:41:32.813 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:41:32.813 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:41:32.813 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:41:32.813 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:41:32.813 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:41:32.813 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:41:32.813 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:41:32.813 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:41:32.813 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:41:32.813 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:41:32.813 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:41:32.813 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:41:32.826 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:41:32.826 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:41:32.826 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:41:32.826 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_35 2026-02-18T08:41:32.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: exception getting command descriptions: [Errno 2] No such file or directory 2026-02-18T08:41:32.875 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:41:32.875 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_35 2026-02-18T08:41:32.876 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:41:32.876 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:41:32.876 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:41:32.876 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:41:32.876 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:41:32.876 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:41:32.876 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:41:32.876 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:41:32.876 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:41:32.876 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:41:32.876 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:41:32.877 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:41:32.877 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:41:32.877 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:41:32.877 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:41:32.877 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:41:32.877 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:41:32.877 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:41:32.877 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:41:32.877 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:41:32.877 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:41:32.878 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:41:32.878 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:41:32.878 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:41:32.878 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:41:32.878 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:41:32.878 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:41:32.878 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:41:32.878 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:41:32.878 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:41:32.887 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:41:32.887 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:41:32.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:41:32.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_35 2026-02-18T08:41:32.959 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:41:32.959 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_35 2026-02-18T08:41:32.959 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:41:32.959 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:41:32.959 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_35 2026-02-18T08:41:32.959 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_35 '' 2026-02-18T08:41:32.959 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:41:32.959 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:41:32.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_35 2026-02-18T08:41:32.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:41:32.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_35.status 2026-02-18T08:41:32.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_35.status 2026-02-18T08:41:32.961 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:41:32.961 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_35.status 2026-02-18T08:41:32.961 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_35 2026-02-18T08:41:32.961 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_35.status 2026-02-18T08:41:32.989 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 12 2026-02-18T08:41:32.989 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 12 2026-02-18T08:41:32.989 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:41:32.989 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=49, tag_tid=2, entry_tid=1], [object_number=48, tag_tid=2, entry_tid=0], [object_number=51, tag_tid=1, entry_tid=51575], [object_number=50, tag_tid=1, entry_tid=51574]]], state=connected] 2026-02-18T08:41:32.989 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=49, tag_tid=2, entry_tid=1], [object_number=48, tag_tid=2, entry_tid=0], [object_number=51, tag_tid=1, entry_tid=51575], [object_number=50, tag_tid=1, entry_tid=51574]]], state=connected] 2026-02-18T08:41:32.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_35.status 2026-02-18T08:41:32.993 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=49, tag_tid=2, entry_tid=1' 2026-02-18T08:41:32.994 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_35 2026-02-18T08:41:32.994 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:41:32.994 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:41:32.994 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_35 2026-02-18T08:41:32.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_35 '..*' 2026-02-18T08:41:32.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:41:32.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:41:32.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_35 2026-02-18T08:41:32.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:41:32.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_35.status 2026-02-18T08:41:32.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_35.status 2026-02-18T08:41:32.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:41:32.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_35.status 2026-02-18T08:41:32.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_35 2026-02-18T08:41:32.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_35.status 2026-02-18T08:41:33.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 12 2026-02-18T08:41:33.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 12 2026-02-18T08:41:33.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:41:33.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=49, tag_tid=2, entry_tid=1], [object_number=48, tag_tid=2, entry_tid=0], [object_number=51, tag_tid=1, entry_tid=51575], [object_number=50, tag_tid=1, entry_tid=51574]]], state=connected] 2026-02-18T08:41:33.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=49, tag_tid=2, entry_tid=1], [object_number=48, tag_tid=2, entry_tid=0], [object_number=51, tag_tid=1, entry_tid=51575], [object_number=50, tag_tid=1, entry_tid=51574]]], state=connected] 2026-02-18T08:41:33.036 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_35.status 2026-02-18T08:41:33.039 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=49, tag_tid=2, entry_tid=1' 2026-02-18T08:41:33.039 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=49, tag_tid=2, entry_tid=1' -a 'object_number=49, tag_tid=2, entry_tid=1' = 'object_number=49, tag_tid=2, entry_tid=1' 2026-02-18T08:41:33.039 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:41:33.039 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_snap_present cluster1 mirror image_35 snap 2026-02-18T08:41:33.039 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:41:33.039 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:41:33.039 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_35 2026-02-18T08:41:33.039 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap 2026-02-18T08:41:33.039 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-18T08:41:33.040 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-18T08:41:33.040 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1 2026-02-18T08:41:34.039 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster1 info mirror/image_35@snap 2026-02-18T08:41:34.069 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:rbd image 'image_35': 2026-02-18T08:41:34.069 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: size 128 MiB in 32 objects 2026-02-18T08:41:34.069 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: order 22 (4 MiB objects) 2026-02-18T08:41:34.069 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: snapshot_count: 1 2026-02-18T08:41:34.069 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: id: 11f948af5510 2026-02-18T08:41:34.069 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: block_name_prefix: rbd_data.11f948af5510 2026-02-18T08:41:34.069 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: format: 2 2026-02-18T08:41:34.069 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: features: layering, exclusive-lock, journaling 2026-02-18T08:41:34.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: op_features: 2026-02-18T08:41:34.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: flags: 2026-02-18T08:41:34.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: create_timestamp: Wed Feb 18 08:31:42 2026 2026-02-18T08:41:34.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: access_timestamp: Wed Feb 18 08:31:42 2026 2026-02-18T08:41:34.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: modify_timestamp: Wed Feb 18 08:33:44 2026 2026-02-18T08:41:34.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: protected: False 2026-02-18T08:41:34.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: journal: 11f948af5510 2026-02-18T08:41:34.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring state: enabled 2026-02-18T08:41:34.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring mode: journal 2026-02-18T08:41:34.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring global id: 5b5dc942-54ce-4123-a700-cec260be4057 2026-02-18T08:41:34.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring primary: false 2026-02-18T08:41:34.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:41:34.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ compare_image_snaps mirror image_35 snap 2026-02-18T08:41:34.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:41:34.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_35 2026-02-18T08:41:34.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap 2026-02-18T08:41:34.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local ret=0 2026-02-18T08:41:34.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local rmt_export=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_35.export 2026-02-18T08:41:34.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loc_export=/tmp/tmp.vybh1m9yJF/cluster1-mirror-image_35.export 2026-02-18T08:41:34.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_35.export /tmp/tmp.vybh1m9yJF/cluster1-mirror-image_35.export 2026-02-18T08:41:34.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 -p mirror export image_35@snap /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_35.export 2026-02-18T08:41:34.211 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-18T08:41:34.221 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster1 -p mirror export image_35@snap /tmp/tmp.vybh1m9yJF/cluster1-mirror-image_35.export 2026-02-18T08:41:34.437 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-18T08:41:34.449 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmp /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_35.export /tmp/tmp.vybh1m9yJF/cluster1-mirror-image_35.export 2026-02-18T08:41:34.501 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_35.export /tmp/tmp.vybh1m9yJF/cluster1-mirror-image_35.export 2026-02-18T08:41:34.545 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:41:34.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T08:41:34.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_36 2026-02-18T08:41:34.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_snap cluster2 mirror image_36 snap 2026-02-18T08:41:34.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:41:34.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:41:34.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_36 2026-02-18T08:41:34.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap 2026-02-18T08:41:34.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 -p mirror snap create image_36@snap --debug-rbd=20 --debug-journaler=20 2026-02-18T08:41:37.307 INFO:tasks.rbd_mirror_thrash:waiting for cluster1.client.mirror.0 2026-02-18T08:41:37.307 INFO:teuthology.orchestra.run:waiting for 600 2026-02-18T08:41:37.307 INFO:tasks.rbd_mirror_thrash:reviving cluster1.client.mirror.0 2026-02-18T08:41:37.308 INFO:tasks.rbd_mirror.cluster1.client.mirror.0:Restarting daemon 2026-02-18T08:41:37.308 DEBUG:teuthology.orchestra.run.trial163:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.0 2026-02-18T08:41:37.310 INFO:tasks.rbd_mirror.cluster1.client.mirror.0:Started 2026-02-18T08:41:37.310 INFO:tasks.rbd_mirror_thrash:waiting for 114.25736714004117 secs before thrashing 2026-02-18T08:41:38.037 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_replay_started cluster1 mirror image_36 2026-02-18T08:41:38.037 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:41:38.038 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:41:38.038 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_36 2026-02-18T08:41:38.038 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_replay_state cluster1 mirror image_36 started 2026-02-18T08:41:38.038 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:41:38.038 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:41:38.038 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_36 2026-02-18T08:41:38.038 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local state=started 2026-02-18T08:41:38.038 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-18T08:41:38.038 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:41:38.038 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1 2026-02-18T08:41:39.038 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test_image_replay_state cluster1 mirror image_36 started 2026-02-18T08:41:39.039 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:41:39.039 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:41:39.039 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_36 2026-02-18T08:41:39.039 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local test_state=started 2026-02-18T08:41:39.039 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local status_result 2026-02-18T08:41:39.039 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_state=stopped 2026-02-18T08:41:39.041 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_36 2026-02-18T08:41:39.041 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_instance=cluster1 2026-02-18T08:41:39.041 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-18T08:41:39.041 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:41:39.041 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance=cluster1 2026-02-18T08:41:39.041 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local loop_instance 2026-02-18T08:41:39.041 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:41:39.041 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sleep 0 2026-02-18T08:41:39.041 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -i state 2026-02-18T08:41:39.041 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:41:39.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ seq 0 3 2026-02-18T08:41:39.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:41:39.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_36 2026-02-18T08:41:39.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:0 2026-02-18T08:41:39.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-18T08:41:39.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:41:39.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:41:39.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:41:39.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-18T08:41:39.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:41:39.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:41:39.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:41:39.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:41:39.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-18T08:41:39.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:41:39.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:41:39.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:41:39.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:41:39.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:0 2026-02-18T08:41:39.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-18T08:41:39.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-18T08:41:39.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:41:39.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:0 2026-02-18T08:41:39.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T08:41:39.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-18T08:41:39.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:41:39.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:41:39.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-18T08:41:39.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-18T08:41:39.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-18T08:41:39.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=0 2026-02-18T08:41:39.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=0 2026-02-18T08:41:39.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:41:39.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:41:39.055 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:41:39.055 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:41:39.055 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_36 2026-02-18T08:41:39.114 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_36'] 2026-02-18T08:41:39.114 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status : get status for rbd mirror 2026-02-18T08:41:39.114 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:41:39.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:41:39.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_36 2026-02-18T08:41:39.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:1 2026-02-18T08:41:39.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-18T08:41:39.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:41:39.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:41:39.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:41:39.118 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-18T08:41:39.118 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:41:39.118 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:41:39.118 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:41:39.118 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:41:39.118 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-18T08:41:39.118 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:41:39.118 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:41:39.118 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:41:39.118 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:41:39.118 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:1 2026-02-18T08:41:39.118 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-18T08:41:39.118 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-18T08:41:39.119 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:41:39.119 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:1 2026-02-18T08:41:39.119 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T08:41:39.119 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-18T08:41:39.119 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:41:39.119 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:41:39.119 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-18T08:41:39.119 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-18T08:41:39.119 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-18T08:41:39.119 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=1 2026-02-18T08:41:39.119 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=1 2026-02-18T08:41:39.119 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:41:39.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:41:39.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:41:39.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:41:39.129 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_36 2026-02-18T08:41:39.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_36'] 2026-02-18T08:41:39.194 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status : get status for rbd mirror 2026-02-18T08:41:39.194 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:41:39.196 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:41:39.197 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_36 2026-02-18T08:41:39.197 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:2 2026-02-18T08:41:39.197 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-18T08:41:39.197 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:41:39.197 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:41:39.197 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:41:39.197 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-18T08:41:39.197 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:41:39.197 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:41:39.197 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:41:39.197 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:41:39.198 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-18T08:41:39.198 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:41:39.198 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:41:39.198 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:41:39.198 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:41:39.198 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:2 2026-02-18T08:41:39.198 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-18T08:41:39.198 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-18T08:41:39.198 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:41:39.198 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:2 2026-02-18T08:41:39.198 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T08:41:39.198 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-18T08:41:39.199 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:41:39.199 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:41:39.199 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-18T08:41:39.199 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-18T08:41:39.199 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-18T08:41:39.199 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=2 2026-02-18T08:41:39.199 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=2 2026-02-18T08:41:39.199 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:41:39.206 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:41:39.207 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:41:39.207 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:41:39.207 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_36 2026-02-18T08:41:39.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_36'] 2026-02-18T08:41:39.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status : get status for rbd mirror 2026-02-18T08:41:39.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:41:39.266 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:41:39.266 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/image_36 2026-02-18T08:41:39.266 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:3 2026-02-18T08:41:39.266 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-18T08:41:39.266 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:41:39.266 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:41:39.266 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:41:39.266 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-18T08:41:39.266 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:41:39.266 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:41:39.266 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:41:39.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:41:39.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-18T08:41:39.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:41:39.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:41:39.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:41:39.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:41:39.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:3 2026-02-18T08:41:39.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-18T08:41:39.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-18T08:41:39.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:41:39.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:3 2026-02-18T08:41:39.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T08:41:39.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-18T08:41:39.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:41:39.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:41:39.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-18T08:41:39.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-18T08:41:39.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-18T08:41:39.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=3 2026-02-18T08:41:39.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=3 2026-02-18T08:41:39.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:41:39.277 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:41:39.278 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:41:39.278 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:41:39.278 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/image_36 2026-02-18T08:41:39.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_36'] 2026-02-18T08:41:39.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status : get status for rbd mirror 2026-02-18T08:41:39.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:41:39.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:41:39.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sleep 1 2026-02-18T08:41:40.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:41:40.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ seq 0 3 2026-02-18T08:41:40.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:41:40.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_36 2026-02-18T08:41:40.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:0 2026-02-18T08:41:40.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-18T08:41:40.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:41:40.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:41:40.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:41:40.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-18T08:41:40.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:41:40.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:41:40.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:41:40.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:41:40.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-18T08:41:40.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:41:40.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:41:40.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:41:40.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:41:40.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:0 2026-02-18T08:41:40.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-18T08:41:40.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-18T08:41:40.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:41:40.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:0 2026-02-18T08:41:40.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T08:41:40.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-18T08:41:40.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:41:40.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:41:40.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-18T08:41:40.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-18T08:41:40.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-18T08:41:40.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=0 2026-02-18T08:41:40.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=0 2026-02-18T08:41:40.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:41:40.350 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:41:40.350 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:41:40.350 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:41:40.351 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_36 2026-02-18T08:41:40.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_36'] 2026-02-18T08:41:40.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status : get status for rbd mirror 2026-02-18T08:41:40.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:41:40.403 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:41:40.403 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_36 2026-02-18T08:41:40.403 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:1 2026-02-18T08:41:40.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-18T08:41:40.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:41:40.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:41:40.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:41:40.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-18T08:41:40.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:41:40.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:41:40.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:41:40.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:41:40.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-18T08:41:40.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:41:40.405 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:41:40.405 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:41:40.405 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:41:40.405 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:1 2026-02-18T08:41:40.405 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-18T08:41:40.405 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-18T08:41:40.405 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:41:40.405 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:1 2026-02-18T08:41:40.405 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T08:41:40.405 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-18T08:41:40.405 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:41:40.405 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:41:40.406 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-18T08:41:40.406 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-18T08:41:40.406 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-18T08:41:40.406 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=1 2026-02-18T08:41:40.406 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=1 2026-02-18T08:41:40.406 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:41:40.415 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:41:40.416 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:41:40.416 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:41:40.416 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_36 2026-02-18T08:41:40.478 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_36'] 2026-02-18T08:41:40.478 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status : get status for rbd mirror 2026-02-18T08:41:40.478 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:41:40.481 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:41:40.481 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_36 2026-02-18T08:41:40.481 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:2 2026-02-18T08:41:40.481 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-18T08:41:40.481 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:41:40.481 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:41:40.482 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:41:40.482 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-18T08:41:40.482 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:41:40.482 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:41:40.482 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:41:40.482 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:41:40.482 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-18T08:41:40.482 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:41:40.482 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:41:40.482 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:41:40.482 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:41:40.482 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:2 2026-02-18T08:41:40.483 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-18T08:41:40.483 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-18T08:41:40.483 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:41:40.483 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:2 2026-02-18T08:41:40.483 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T08:41:40.483 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-18T08:41:40.483 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:41:40.483 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:41:40.483 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-18T08:41:40.483 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-18T08:41:40.483 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-18T08:41:40.483 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=2 2026-02-18T08:41:40.484 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=2 2026-02-18T08:41:40.484 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:41:40.491 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:41:40.491 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:41:40.492 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:41:40.492 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_36 2026-02-18T08:41:40.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_36'] 2026-02-18T08:41:40.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status : get status for rbd mirror 2026-02-18T08:41:40.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:41:40.558 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:41:40.558 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/image_36 2026-02-18T08:41:40.558 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:3 2026-02-18T08:41:40.559 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-18T08:41:40.559 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:41:40.559 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:41:40.559 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:41:40.559 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-18T08:41:40.559 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:41:40.559 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:41:40.559 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:41:40.559 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:41:40.559 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-18T08:41:40.559 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:41:40.560 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:41:40.560 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:41:40.560 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:41:40.560 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:3 2026-02-18T08:41:40.560 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-18T08:41:40.560 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-18T08:41:40.560 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:41:40.560 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:3 2026-02-18T08:41:40.560 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T08:41:40.560 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-18T08:41:40.560 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:41:40.560 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:41:40.560 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-18T08:41:40.561 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-18T08:41:40.561 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-18T08:41:40.561 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=3 2026-02-18T08:41:40.561 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=3 2026-02-18T08:41:40.561 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:41:40.570 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:41:40.571 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:41:40.571 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:41:40.571 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/image_36 2026-02-18T08:41:40.635 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_36'] 2026-02-18T08:41:40.635 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status : get status for rbd mirror 2026-02-18T08:41:40.635 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:41:40.636 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:41:40.637 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sleep 2 2026-02-18T08:41:42.638 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:41:42.638 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ seq 0 3 2026-02-18T08:41:42.641 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:41:42.641 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_36 2026-02-18T08:41:42.641 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:0 2026-02-18T08:41:42.641 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-18T08:41:42.641 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:41:42.642 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:41:42.642 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:41:42.642 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-18T08:41:42.642 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:41:42.642 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:41:42.642 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:41:42.642 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:41:42.642 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-18T08:41:42.642 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:41:42.642 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:41:42.642 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:41:42.642 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:41:42.642 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:0 2026-02-18T08:41:42.643 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-18T08:41:42.643 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-18T08:41:42.643 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:41:42.643 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:0 2026-02-18T08:41:42.643 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T08:41:42.643 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-18T08:41:42.643 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:41:42.643 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:41:42.643 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-18T08:41:42.643 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-18T08:41:42.643 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-18T08:41:42.643 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=0 2026-02-18T08:41:42.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=0 2026-02-18T08:41:42.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:41:42.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:41:42.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:41:42.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:41:42.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_36 2026-02-18T08:41:42.711 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_36'] 2026-02-18T08:41:42.711 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status : get status for rbd mirror 2026-02-18T08:41:42.711 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:41:42.713 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:41:42.714 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_36 2026-02-18T08:41:42.714 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:1 2026-02-18T08:41:42.714 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-18T08:41:42.714 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:41:42.714 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:41:42.714 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:41:42.714 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-18T08:41:42.714 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:41:42.714 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:41:42.714 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:41:42.714 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:41:42.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-18T08:41:42.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:41:42.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:41:42.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:41:42.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:41:42.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:1 2026-02-18T08:41:42.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-18T08:41:42.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-18T08:41:42.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:41:42.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:1 2026-02-18T08:41:42.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T08:41:42.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-18T08:41:42.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:41:42.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:41:42.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-18T08:41:42.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-18T08:41:42.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-18T08:41:42.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=1 2026-02-18T08:41:42.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=1 2026-02-18T08:41:42.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:41:42.725 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:41:42.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:41:42.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:41:42.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_36 2026-02-18T08:41:42.788 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_36'] 2026-02-18T08:41:42.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status : get status for rbd mirror 2026-02-18T08:41:42.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:41:42.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:41:42.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_36 2026-02-18T08:41:42.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:2 2026-02-18T08:41:42.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-18T08:41:42.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:41:42.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:41:42.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:41:42.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-18T08:41:42.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:41:42.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:41:42.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:41:42.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:41:42.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-18T08:41:42.793 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:41:42.793 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:41:42.793 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:41:42.793 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:41:42.793 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:2 2026-02-18T08:41:42.793 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-18T08:41:42.793 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-18T08:41:42.793 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:41:42.793 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:2 2026-02-18T08:41:42.793 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T08:41:42.793 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-18T08:41:42.793 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:41:42.793 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:41:42.794 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-18T08:41:42.794 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-18T08:41:42.794 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-18T08:41:42.794 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=2 2026-02-18T08:41:42.794 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=2 2026-02-18T08:41:42.794 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:41:42.803 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:41:42.804 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:41:42.804 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:41:42.804 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_36 2026-02-18T08:41:42.863 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_36'] 2026-02-18T08:41:42.864 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status : get status for rbd mirror 2026-02-18T08:41:42.864 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:41:42.866 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:41:42.866 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/image_36 2026-02-18T08:41:42.866 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:3 2026-02-18T08:41:42.867 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-18T08:41:42.867 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:41:42.867 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:41:42.867 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:41:42.867 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-18T08:41:42.867 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:41:42.867 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:41:42.867 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:41:42.867 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:41:42.867 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-18T08:41:42.867 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:41:42.867 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:41:42.868 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:41:42.868 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:41:42.868 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:3 2026-02-18T08:41:42.868 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-18T08:41:42.868 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-18T08:41:42.868 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:41:42.868 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:3 2026-02-18T08:41:42.868 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T08:41:42.868 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-18T08:41:42.868 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:41:42.868 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:41:42.868 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-18T08:41:42.869 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-18T08:41:42.869 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-18T08:41:42.869 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=3 2026-02-18T08:41:42.869 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=3 2026-02-18T08:41:42.869 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:41:42.878 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:41:42.878 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:41:42.878 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:41:42.878 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/image_36 2026-02-18T08:41:42.929 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_36'] 2026-02-18T08:41:42.930 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status : get status for rbd mirror 2026-02-18T08:41:42.930 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:41:42.930 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:41:42.931 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sleep 4 2026-02-18T08:41:46.931 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:41:46.932 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ seq 0 3 2026-02-18T08:41:46.933 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:41:46.933 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_36 2026-02-18T08:41:46.933 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:0 2026-02-18T08:41:46.933 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-18T08:41:46.933 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:41:46.933 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:41:46.933 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:41:46.933 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-18T08:41:46.933 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:41:46.933 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:41:46.933 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:41:46.933 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:41:46.933 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-18T08:41:46.933 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:41:46.933 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:41:46.933 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:41:46.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:41:46.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:0 2026-02-18T08:41:46.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-18T08:41:46.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-18T08:41:46.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:41:46.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:0 2026-02-18T08:41:46.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T08:41:46.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-18T08:41:46.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:41:46.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:41:46.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-18T08:41:46.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-18T08:41:46.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-18T08:41:46.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=0 2026-02-18T08:41:46.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=0 2026-02-18T08:41:46.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:41:46.946 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:41:46.946 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:41:46.946 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:41:46.946 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_36 2026-02-18T08:41:47.008 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_36'] 2026-02-18T08:41:47.008 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status : get status for rbd mirror 2026-02-18T08:41:47.008 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:41:47.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:41:47.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_36 2026-02-18T08:41:47.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:1 2026-02-18T08:41:47.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-18T08:41:47.010 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:41:47.010 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:41:47.010 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:41:47.010 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-18T08:41:47.010 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:41:47.010 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:41:47.010 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:41:47.010 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:41:47.010 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-18T08:41:47.010 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:41:47.010 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:41:47.010 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:41:47.010 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:41:47.010 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:1 2026-02-18T08:41:47.010 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-18T08:41:47.010 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-18T08:41:47.010 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:41:47.010 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:1 2026-02-18T08:41:47.010 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T08:41:47.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-18T08:41:47.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:41:47.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:41:47.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-18T08:41:47.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-18T08:41:47.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-18T08:41:47.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=1 2026-02-18T08:41:47.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=1 2026-02-18T08:41:47.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:41:47.021 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:41:47.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:41:47.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:41:47.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_36 2026-02-18T08:41:47.086 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_36'] 2026-02-18T08:41:47.086 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status : get status for rbd mirror 2026-02-18T08:41:47.086 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:41:47.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:41:47.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_36 2026-02-18T08:41:47.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:2 2026-02-18T08:41:47.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-18T08:41:47.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:41:47.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:41:47.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:41:47.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-18T08:41:47.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:41:47.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:41:47.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:41:47.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:41:47.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-18T08:41:47.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:41:47.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:41:47.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:41:47.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:41:47.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:2 2026-02-18T08:41:47.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-18T08:41:47.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-18T08:41:47.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:41:47.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:2 2026-02-18T08:41:47.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T08:41:47.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-18T08:41:47.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:41:47.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:41:47.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-18T08:41:47.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-18T08:41:47.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-18T08:41:47.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=2 2026-02-18T08:41:47.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=2 2026-02-18T08:41:47.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:41:47.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:41:47.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:41:47.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:41:47.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_36 2026-02-18T08:41:47.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_36'] 2026-02-18T08:41:47.161 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status : get status for rbd mirror 2026-02-18T08:41:47.161 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:41:47.162 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:41:47.162 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/image_36 2026-02-18T08:41:47.162 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:3 2026-02-18T08:41:47.162 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-18T08:41:47.162 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:41:47.162 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:41:47.162 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:41:47.162 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-18T08:41:47.162 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:41:47.162 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:41:47.163 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:41:47.163 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:41:47.163 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-18T08:41:47.163 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:41:47.163 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:41:47.163 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:41:47.163 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:41:47.163 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:3 2026-02-18T08:41:47.163 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-18T08:41:47.163 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-18T08:41:47.163 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:41:47.163 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:3 2026-02-18T08:41:47.163 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T08:41:47.163 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-18T08:41:47.163 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:41:47.163 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:41:47.163 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-18T08:41:47.163 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-18T08:41:47.163 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-18T08:41:47.163 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=3 2026-02-18T08:41:47.163 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=3 2026-02-18T08:41:47.163 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:41:47.174 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:41:47.174 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:41:47.174 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:41:47.174 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/image_36 2026-02-18T08:41:47.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_36'] 2026-02-18T08:41:47.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status : get status for rbd mirror 2026-02-18T08:41:47.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:41:47.238 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:41:47.238 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sleep 8 2026-02-18T08:41:55.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:41:55.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ seq 0 3 2026-02-18T08:41:55.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:41:55.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_36 2026-02-18T08:41:55.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:0 2026-02-18T08:41:55.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-18T08:41:55.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:41:55.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:41:55.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:41:55.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-18T08:41:55.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:41:55.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:41:55.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:41:55.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:41:55.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-18T08:41:55.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:41:55.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:41:55.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:41:55.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:41:55.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:0 2026-02-18T08:41:55.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-18T08:41:55.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-18T08:41:55.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:41:55.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:0 2026-02-18T08:41:55.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T08:41:55.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-18T08:41:55.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:41:55.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:41:55.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-18T08:41:55.244 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-18T08:41:55.244 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-18T08:41:55.244 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=0 2026-02-18T08:41:55.244 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=0 2026-02-18T08:41:55.244 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:41:55.255 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:41:55.256 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:41:55.256 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:41:55.256 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_36 2026-02-18T08:41:55.318 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_36'] 2026-02-18T08:41:55.318 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status : get status for rbd mirror 2026-02-18T08:41:55.318 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:41:55.319 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:41:55.319 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_36 2026-02-18T08:41:55.319 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:1 2026-02-18T08:41:55.319 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-18T08:41:55.319 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:41:55.319 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:41:55.319 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:41:55.319 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-18T08:41:55.319 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:41:55.320 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:41:55.320 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:41:55.320 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:41:55.320 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-18T08:41:55.320 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:41:55.320 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:41:55.320 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:41:55.320 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:41:55.320 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:1 2026-02-18T08:41:55.320 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-18T08:41:55.320 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-18T08:41:55.320 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:41:55.320 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:1 2026-02-18T08:41:55.320 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T08:41:55.320 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-18T08:41:55.320 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:41:55.320 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:41:55.320 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-18T08:41:55.320 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-18T08:41:55.320 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-18T08:41:55.321 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=1 2026-02-18T08:41:55.321 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=1 2026-02-18T08:41:55.321 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:41:55.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:41:55.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:41:55.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:41:55.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_36 2026-02-18T08:41:55.394 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_36'] 2026-02-18T08:41:55.394 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status : get status for rbd mirror 2026-02-18T08:41:55.394 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:41:55.397 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:41:55.397 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_36 2026-02-18T08:41:55.397 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:2 2026-02-18T08:41:55.397 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-18T08:41:55.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:41:55.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:41:55.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:41:55.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-18T08:41:55.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:41:55.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:41:55.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:41:55.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:41:55.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-18T08:41:55.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:41:55.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:41:55.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:41:55.399 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:41:55.399 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:2 2026-02-18T08:41:55.399 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-18T08:41:55.399 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-18T08:41:55.399 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:41:55.399 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:2 2026-02-18T08:41:55.399 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T08:41:55.399 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-18T08:41:55.399 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:41:55.399 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:41:55.399 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-18T08:41:55.399 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-18T08:41:55.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-18T08:41:55.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=2 2026-02-18T08:41:55.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=2 2026-02-18T08:41:55.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:41:55.407 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:41:55.407 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:41:55.407 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:41:55.407 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_36 2026-02-18T08:41:55.467 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_36'] 2026-02-18T08:41:55.467 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status : get status for rbd mirror 2026-02-18T08:41:55.467 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:41:55.470 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:41:55.470 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/image_36 2026-02-18T08:41:55.470 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:3 2026-02-18T08:41:55.470 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-18T08:41:55.470 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:41:55.470 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:41:55.470 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:41:55.470 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-18T08:41:55.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:41:55.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:41:55.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:41:55.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:41:55.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-18T08:41:55.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:41:55.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:41:55.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:41:55.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:41:55.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:3 2026-02-18T08:41:55.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-18T08:41:55.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-18T08:41:55.472 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:41:55.472 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:3 2026-02-18T08:41:55.472 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T08:41:55.472 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-18T08:41:55.472 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:41:55.472 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:41:55.472 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-18T08:41:55.472 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-18T08:41:55.472 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-18T08:41:55.472 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=3 2026-02-18T08:41:55.472 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=3 2026-02-18T08:41:55.472 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:41:55.481 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:41:55.481 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:41:55.482 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:41:55.482 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/image_36 2026-02-18T08:41:55.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_36'] 2026-02-18T08:41:55.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status : get status for rbd mirror 2026-02-18T08:41:55.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:41:55.534 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:41:55.534 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sleep 8 2026-02-18T08:42:03.536 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:42:03.536 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ seq 0 3 2026-02-18T08:42:03.540 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:42:03.540 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_36 2026-02-18T08:42:03.540 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:0 2026-02-18T08:42:03.540 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-18T08:42:03.540 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:42:03.540 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:42:03.540 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:42:03.540 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-18T08:42:03.540 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:42:03.540 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:42:03.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:42:03.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:42:03.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-18T08:42:03.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:42:03.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:42:03.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:42:03.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:42:03.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:0 2026-02-18T08:42:03.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-18T08:42:03.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-18T08:42:03.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:42:03.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:0 2026-02-18T08:42:03.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T08:42:03.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-18T08:42:03.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:42:03.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:42:03.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-18T08:42:03.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-18T08:42:03.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-18T08:42:03.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=0 2026-02-18T08:42:03.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=0 2026-02-18T08:42:03.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:42:03.552 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:42:03.552 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:42:03.553 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:42:03.553 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_36 2026-02-18T08:42:03.623 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ return 0 2026-02-18T08:42:03.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ status_result=' "state": "Replaying"' 2026-02-18T08:42:03.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ echo ' "state": "Replaying"' 2026-02-18T08:42:03.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ grep -i Replaying 2026-02-18T08:42:03.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: "state": "Replaying" 2026-02-18T08:42:03.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ current_state=started 2026-02-18T08:42:03.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test started = started 2026-02-18T08:42:03.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:42:03.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror image_36 2026-02-18T08:42:03.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_cluster=cluster1 2026-02-18T08:42:03.628 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:42:03.628 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_pool=mirror 2026-02-18T08:42:03.628 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local remote_pool=mirror 2026-02-18T08:42:03.628 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_36 2026-02-18T08:42:03.628 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' journal = journal ']' 2026-02-18T08:42:03.628 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror mirror image_36 2026-02-18T08:42:03.628 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_cluster=cluster1 2026-02-18T08:42:03.628 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:42:03.628 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_pool=mirror 2026-02-18T08:42:03.628 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local remote_pool=mirror 2026-02-18T08:42:03.628 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_36 2026-02-18T08:42:03.628 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-02-18T08:42:03.629 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-02-18T08:42:03.629 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:42:03.629 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:42:03.629 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:42:03.828 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_36 2026-02-18T08:42:03.828 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:42:03.828 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:42:03.828 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_36 2026-02-18T08:42:03.828 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:42:03.828 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_36 ']' 2026-02-18T08:42:03.828 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_36' 2026-02-18T08:42:03.828 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_36 2026-02-18T08:42:03.829 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:42:03.829 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:42:03.829 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:42:03.829 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:42:03.829 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:42:03.829 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:42:03.829 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:42:03.832 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:42:03.832 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:42:03.832 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:42:03.832 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_36 2026-02-18T08:42:03.832 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:42:03.832 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:42:03.832 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:42:03.832 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:42:03.832 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:42:03.832 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:42:03.832 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:42:03.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:42:03.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:42:03.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:42:03.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:42:03.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:42:03.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:42:03.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:42:03.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:42:03.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:42:03.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:42:03.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:42:03.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:42:03.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:42:03.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:42:03.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:42:03.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:42:03.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:42:03.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:42:03.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:42:03.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:42:03.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:42:03.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:42:03.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:42:03.843 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:42:03.844 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:42:03.844 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:42:03.844 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_36 2026-02-18T08:42:03.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:42:03.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_36 2026-02-18T08:42:03.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:42:03.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:42:03.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_36 2026-02-18T08:42:03.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_36 '' 2026-02-18T08:42:03.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:42:03.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:42:03.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_36 2026-02-18T08:42:03.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:42:03.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_36.status 2026-02-18T08:42:03.904 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_36.status 2026-02-18T08:42:03.904 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:42:03.904 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_36.status 2026-02-18T08:42:03.905 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_36 2026-02-18T08:42:03.905 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_36.status 2026-02-18T08:42:03.933 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 7 2026-02-18T08:42:03.933 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 7 2026-02-18T08:42:03.933 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:42:03.933 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=29, tag_tid=2, entry_tid=1], [object_number=28, tag_tid=2, entry_tid=0], [object_number=31, tag_tid=1, entry_tid=30599], [object_number=30, tag_tid=1, entry_tid=30598]]], state=connected] 2026-02-18T08:42:03.933 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=29, tag_tid=2, entry_tid=1], [object_number=28, tag_tid=2, entry_tid=0], [object_number=31, tag_tid=1, entry_tid=30599], [object_number=30, tag_tid=1, entry_tid=30598]]], state=connected] 2026-02-18T08:42:03.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_36.status 2026-02-18T08:42:03.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=29, tag_tid=2, entry_tid=1' 2026-02-18T08:42:03.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_36 2026-02-18T08:42:03.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:42:03.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:42:03.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_36 2026-02-18T08:42:03.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_36 '..*' 2026-02-18T08:42:03.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:42:03.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:42:03.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_36 2026-02-18T08:42:03.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:42:03.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_36.status 2026-02-18T08:42:03.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_36.status 2026-02-18T08:42:03.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:42:03.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_36.status 2026-02-18T08:42:03.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_36 2026-02-18T08:42:03.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_36.status 2026-02-18T08:42:03.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 7 2026-02-18T08:42:03.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 7 2026-02-18T08:42:03.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:42:03.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=29, tag_tid=2, entry_tid=1], [object_number=28, tag_tid=2, entry_tid=0], [object_number=31, tag_tid=1, entry_tid=30599], [object_number=30, tag_tid=1, entry_tid=30598]]], state=connected] 2026-02-18T08:42:03.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=29, tag_tid=2, entry_tid=1], [object_number=28, tag_tid=2, entry_tid=0], [object_number=31, tag_tid=1, entry_tid=30599], [object_number=30, tag_tid=1, entry_tid=30598]]], state=connected] 2026-02-18T08:42:03.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_36.status 2026-02-18T08:42:03.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=29, tag_tid=2, entry_tid=1' 2026-02-18T08:42:03.982 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=29, tag_tid=2, entry_tid=1' -a 'object_number=29, tag_tid=2, entry_tid=1' = 'object_number=29, tag_tid=2, entry_tid=1' 2026-02-18T08:42:03.982 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:42:03.982 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_snap_present cluster1 mirror image_36 snap 2026-02-18T08:42:03.982 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:42:03.982 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:42:03.982 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_36 2026-02-18T08:42:03.982 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap 2026-02-18T08:42:03.982 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-18T08:42:03.982 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-18T08:42:03.982 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1 2026-02-18T08:42:04.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster1 info mirror/image_36@snap 2026-02-18T08:42:05.010 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:rbd image 'image_36': 2026-02-18T08:42:05.010 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: size 128 MiB in 32 objects 2026-02-18T08:42:05.010 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: order 22 (4 MiB objects) 2026-02-18T08:42:05.010 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: snapshot_count: 1 2026-02-18T08:42:05.010 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: id: 120dddf67458 2026-02-18T08:42:05.010 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: block_name_prefix: rbd_data.120dddf67458 2026-02-18T08:42:05.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: format: 2 2026-02-18T08:42:05.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: features: layering, exclusive-lock, journaling 2026-02-18T08:42:05.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: op_features: 2026-02-18T08:42:05.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: flags: 2026-02-18T08:42:05.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: create_timestamp: Wed Feb 18 08:31:56 2026 2026-02-18T08:42:05.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: access_timestamp: Wed Feb 18 08:31:56 2026 2026-02-18T08:42:05.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: modify_timestamp: Wed Feb 18 08:32:57 2026 2026-02-18T08:42:05.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: protected: False 2026-02-18T08:42:05.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: journal: 120dddf67458 2026-02-18T08:42:05.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring state: enabled 2026-02-18T08:42:05.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring mode: journal 2026-02-18T08:42:05.012 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring global id: 4fec0a63-a8eb-407d-8656-c0c86c01d97d 2026-02-18T08:42:05.012 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring primary: false 2026-02-18T08:42:05.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:42:05.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ compare_image_snaps mirror image_36 snap 2026-02-18T08:42:05.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:42:05.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_36 2026-02-18T08:42:05.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap 2026-02-18T08:42:05.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local ret=0 2026-02-18T08:42:05.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local rmt_export=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_36.export 2026-02-18T08:42:05.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loc_export=/tmp/tmp.vybh1m9yJF/cluster1-mirror-image_36.export 2026-02-18T08:42:05.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_36.export /tmp/tmp.vybh1m9yJF/cluster1-mirror-image_36.export 2026-02-18T08:42:05.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 -p mirror export image_36@snap /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_36.export 2026-02-18T08:42:05.158 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-18T08:42:05.166 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster1 -p mirror export image_36@snap /tmp/tmp.vybh1m9yJF/cluster1-mirror-image_36.export 2026-02-18T08:42:05.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-18T08:42:05.390 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmp /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_36.export /tmp/tmp.vybh1m9yJF/cluster1-mirror-image_36.export 2026-02-18T08:42:05.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_36.export /tmp/tmp.vybh1m9yJF/cluster1-mirror-image_36.export 2026-02-18T08:42:05.490 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:42:05.490 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T08:42:05.490 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_37 2026-02-18T08:42:05.490 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_snap cluster2 mirror image_37 snap 2026-02-18T08:42:05.490 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:42:05.490 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:42:05.490 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_37 2026-02-18T08:42:05.490 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap 2026-02-18T08:42:05.491 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 -p mirror snap create image_37@snap --debug-rbd=20 --debug-journaler=20 2026-02-18T08:42:07.055 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_replay_started cluster1 mirror image_37 2026-02-18T08:42:07.055 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:42:07.056 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:42:07.056 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_37 2026-02-18T08:42:07.056 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_replay_state cluster1 mirror image_37 started 2026-02-18T08:42:07.056 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:42:07.056 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:42:07.056 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_37 2026-02-18T08:42:07.056 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local state=started 2026-02-18T08:42:07.056 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-18T08:42:07.056 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:42:07.056 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1 2026-02-18T08:42:08.056 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test_image_replay_state cluster1 mirror image_37 started 2026-02-18T08:42:08.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:42:08.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:42:08.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_37 2026-02-18T08:42:08.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local test_state=started 2026-02-18T08:42:08.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local status_result 2026-02-18T08:42:08.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_state=stopped 2026-02-18T08:42:08.058 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_37 2026-02-18T08:42:08.058 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -i state 2026-02-18T08:42:08.058 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_instance=cluster1 2026-02-18T08:42:08.058 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-18T08:42:08.058 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:42:08.059 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance=cluster1 2026-02-18T08:42:08.059 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local loop_instance 2026-02-18T08:42:08.059 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:42:08.059 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sleep 0 2026-02-18T08:42:08.059 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:42:08.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ seq 0 3 2026-02-18T08:42:08.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:42:08.062 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_37 2026-02-18T08:42:08.062 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:0 2026-02-18T08:42:08.062 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-18T08:42:08.062 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:42:08.062 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:42:08.062 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:42:08.062 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-18T08:42:08.062 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:42:08.062 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:42:08.062 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:42:08.062 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:42:08.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-18T08:42:08.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:42:08.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:42:08.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:42:08.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:42:08.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:0 2026-02-18T08:42:08.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-18T08:42:08.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-18T08:42:08.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:42:08.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:0 2026-02-18T08:42:08.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T08:42:08.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-18T08:42:08.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:42:08.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:42:08.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-18T08:42:08.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-18T08:42:08.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-18T08:42:08.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=0 2026-02-18T08:42:08.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=0 2026-02-18T08:42:08.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:42:08.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:42:08.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:42:08.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:42:08.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_37 2026-02-18T08:42:08.131 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ return 0 2026-02-18T08:42:08.132 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ status_result=' "state": "Replaying"' 2026-02-18T08:42:08.132 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ echo ' "state": "Replaying"' 2026-02-18T08:42:08.132 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ grep -i Replaying 2026-02-18T08:42:08.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: "state": "Replaying" 2026-02-18T08:42:08.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ current_state=started 2026-02-18T08:42:08.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test started = started 2026-02-18T08:42:08.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:42:08.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror image_37 2026-02-18T08:42:08.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_cluster=cluster1 2026-02-18T08:42:08.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:42:08.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_pool=mirror 2026-02-18T08:42:08.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local remote_pool=mirror 2026-02-18T08:42:08.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_37 2026-02-18T08:42:08.136 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' journal = journal ']' 2026-02-18T08:42:08.136 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror mirror image_37 2026-02-18T08:42:08.136 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_cluster=cluster1 2026-02-18T08:42:08.136 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:42:08.136 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_pool=mirror 2026-02-18T08:42:08.136 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local remote_pool=mirror 2026-02-18T08:42:08.136 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_37 2026-02-18T08:42:08.136 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-02-18T08:42:08.136 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-02-18T08:42:08.136 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:42:08.136 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:42:08.136 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:42:08.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_37 2026-02-18T08:42:08.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:42:08.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:42:08.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_37 2026-02-18T08:42:08.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:42:08.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_37 ']' 2026-02-18T08:42:08.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_37' 2026-02-18T08:42:08.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_37 2026-02-18T08:42:08.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:42:08.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:42:08.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:42:08.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:42:08.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:42:08.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:42:08.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:42:08.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:42:08.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:42:08.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:42:08.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_37 2026-02-18T08:42:08.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:42:08.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:42:08.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:42:08.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:42:08.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:42:08.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:42:08.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:42:08.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:42:08.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:42:08.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:42:08.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:42:08.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:42:08.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:42:08.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:42:08.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:42:08.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:42:08.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:42:08.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:42:08.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:42:08.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:42:08.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:42:08.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:42:08.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:42:08.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:42:08.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:42:08.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:42:08.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:42:08.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:42:08.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:42:08.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:42:08.351 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:42:08.351 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:42:08.352 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:42:08.352 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_37 2026-02-18T08:42:08.413 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:42:08.414 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_37 2026-02-18T08:42:08.414 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:42:08.414 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:42:08.414 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_37 2026-02-18T08:42:08.414 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_37 '' 2026-02-18T08:42:08.414 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:42:08.414 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:42:08.414 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_37 2026-02-18T08:42:08.414 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:42:08.414 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_37.status 2026-02-18T08:42:08.415 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_37.status 2026-02-18T08:42:08.415 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:42:08.415 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_37.status 2026-02-18T08:42:08.416 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_37 2026-02-18T08:42:08.416 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_37.status 2026-02-18T08:42:08.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 8 2026-02-18T08:42:08.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 8 2026-02-18T08:42:08.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:42:08.443 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=33, tag_tid=2, entry_tid=1], [object_number=32, tag_tid=2, entry_tid=0], [object_number=35, tag_tid=1, entry_tid=36259], [object_number=34, tag_tid=1, entry_tid=36258]]], state=connected] 2026-02-18T08:42:08.443 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=33, tag_tid=2, entry_tid=1], [object_number=32, tag_tid=2, entry_tid=0], [object_number=35, tag_tid=1, entry_tid=36259], [object_number=34, tag_tid=1, entry_tid=36258]]], state=connected] 2026-02-18T08:42:08.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_37.status 2026-02-18T08:42:08.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=33, tag_tid=2, entry_tid=1' 2026-02-18T08:42:08.448 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_37 2026-02-18T08:42:08.448 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:42:08.448 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:42:08.448 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_37 2026-02-18T08:42:08.448 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_37 '..*' 2026-02-18T08:42:08.448 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:42:08.448 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:42:08.448 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_37 2026-02-18T08:42:08.448 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:42:08.449 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_37.status 2026-02-18T08:42:08.449 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_37.status 2026-02-18T08:42:08.449 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:42:08.449 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_37.status 2026-02-18T08:42:08.450 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_37 2026-02-18T08:42:08.450 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_37.status 2026-02-18T08:42:08.483 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 8 2026-02-18T08:42:08.483 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 8 2026-02-18T08:42:08.483 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:42:08.483 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=33, tag_tid=2, entry_tid=1], [object_number=32, tag_tid=2, entry_tid=0], [object_number=35, tag_tid=1, entry_tid=36259], [object_number=34, tag_tid=1, entry_tid=36258]]], state=connected] 2026-02-18T08:42:08.484 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=33, tag_tid=2, entry_tid=1], [object_number=32, tag_tid=2, entry_tid=0], [object_number=35, tag_tid=1, entry_tid=36259], [object_number=34, tag_tid=1, entry_tid=36258]]], state=connected] 2026-02-18T08:42:08.487 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_37.status 2026-02-18T08:42:08.489 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=33, tag_tid=2, entry_tid=1' 2026-02-18T08:42:08.489 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=33, tag_tid=2, entry_tid=1' -a 'object_number=33, tag_tid=2, entry_tid=1' = 'object_number=33, tag_tid=2, entry_tid=1' 2026-02-18T08:42:08.489 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:42:08.489 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_snap_present cluster1 mirror image_37 snap 2026-02-18T08:42:08.489 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:42:08.489 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:42:08.489 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_37 2026-02-18T08:42:08.489 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap 2026-02-18T08:42:08.489 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-18T08:42:08.490 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-18T08:42:08.490 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1 2026-02-18T08:42:09.490 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster1 info mirror/image_37@snap 2026-02-18T08:42:09.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:rbd image 'image_37': 2026-02-18T08:42:09.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: size 128 MiB in 32 objects 2026-02-18T08:42:09.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: order 22 (4 MiB objects) 2026-02-18T08:42:09.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: snapshot_count: 1 2026-02-18T08:42:09.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: id: 10ed27c188a 2026-02-18T08:42:09.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: block_name_prefix: rbd_data.10ed27c188a 2026-02-18T08:42:09.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: format: 2 2026-02-18T08:42:09.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: features: layering, exclusive-lock, journaling 2026-02-18T08:42:09.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: op_features: 2026-02-18T08:42:09.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: flags: 2026-02-18T08:42:09.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: create_timestamp: Wed Feb 18 08:32:04 2026 2026-02-18T08:42:09.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: access_timestamp: Wed Feb 18 08:32:04 2026 2026-02-18T08:42:09.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: modify_timestamp: Wed Feb 18 08:33:05 2026 2026-02-18T08:42:09.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: protected: False 2026-02-18T08:42:09.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: journal: 10ed27c188a 2026-02-18T08:42:09.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring state: enabled 2026-02-18T08:42:09.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring mode: journal 2026-02-18T08:42:09.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring global id: 775ba0d8-e2b0-429e-b5a4-9987c8ccedd1 2026-02-18T08:42:09.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring primary: false 2026-02-18T08:42:09.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:42:09.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ compare_image_snaps mirror image_37 snap 2026-02-18T08:42:09.523 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:42:09.523 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_37 2026-02-18T08:42:09.523 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap 2026-02-18T08:42:09.523 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local ret=0 2026-02-18T08:42:09.523 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local rmt_export=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_37.export 2026-02-18T08:42:09.523 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loc_export=/tmp/tmp.vybh1m9yJF/cluster1-mirror-image_37.export 2026-02-18T08:42:09.523 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_37.export /tmp/tmp.vybh1m9yJF/cluster1-mirror-image_37.export 2026-02-18T08:42:09.523 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 -p mirror export image_37@snap /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_37.export 2026-02-18T08:42:09.660 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-18T08:42:09.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster1 -p mirror export image_37@snap /tmp/tmp.vybh1m9yJF/cluster1-mirror-image_37.export 2026-02-18T08:42:09.842 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-18T08:42:09.850 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmp /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_37.export /tmp/tmp.vybh1m9yJF/cluster1-mirror-image_37.export 2026-02-18T08:42:09.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_37.export /tmp/tmp.vybh1m9yJF/cluster1-mirror-image_37.export 2026-02-18T08:42:09.950 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:42:09.950 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T08:42:09.950 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_38 2026-02-18T08:42:09.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_snap cluster2 mirror image_38 snap 2026-02-18T08:42:09.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:42:09.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:42:09.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_38 2026-02-18T08:42:09.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap 2026-02-18T08:42:09.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 -p mirror snap create image_38@snap --debug-rbd=20 --debug-journaler=20 2026-02-18T08:42:12.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_replay_started cluster1 mirror image_38 2026-02-18T08:42:12.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:42:12.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:42:12.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_38 2026-02-18T08:42:12.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_replay_state cluster1 mirror image_38 started 2026-02-18T08:42:12.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:42:12.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:42:12.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_38 2026-02-18T08:42:12.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local state=started 2026-02-18T08:42:12.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-18T08:42:12.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:42:12.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1 2026-02-18T08:42:13.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test_image_replay_state cluster1 mirror image_38 started 2026-02-18T08:42:13.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:42:13.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:42:13.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_38 2026-02-18T08:42:13.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local test_state=started 2026-02-18T08:42:13.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local status_result 2026-02-18T08:42:13.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_state=stopped 2026-02-18T08:42:13.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_38 2026-02-18T08:42:13.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -i state 2026-02-18T08:42:13.074 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_instance=cluster1 2026-02-18T08:42:13.074 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-18T08:42:13.074 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:42:13.074 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance=cluster1 2026-02-18T08:42:13.074 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local loop_instance 2026-02-18T08:42:13.074 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:42:13.074 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sleep 0 2026-02-18T08:42:13.074 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:42:13.076 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ seq 0 3 2026-02-18T08:42:13.076 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:42:13.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_38 2026-02-18T08:42:13.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:0 2026-02-18T08:42:13.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-18T08:42:13.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:42:13.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:42:13.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:42:13.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-18T08:42:13.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:42:13.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:42:13.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:42:13.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:42:13.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-18T08:42:13.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:42:13.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:42:13.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:42:13.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:42:13.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:0 2026-02-18T08:42:13.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-18T08:42:13.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-18T08:42:13.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:42:13.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:0 2026-02-18T08:42:13.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T08:42:13.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-18T08:42:13.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:42:13.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:42:13.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-18T08:42:13.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-18T08:42:13.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-18T08:42:13.080 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=0 2026-02-18T08:42:13.080 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=0 2026-02-18T08:42:13.080 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:42:13.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:42:13.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:42:13.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:42:13.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_38 2026-02-18T08:42:13.148 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ return 0 2026-02-18T08:42:13.149 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ status_result=' "state": "Replaying"' 2026-02-18T08:42:13.149 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ echo ' "state": "Replaying"' 2026-02-18T08:42:13.149 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ grep -i Replaying 2026-02-18T08:42:13.150 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: "state": "Replaying" 2026-02-18T08:42:13.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ current_state=started 2026-02-18T08:42:13.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test started = started 2026-02-18T08:42:13.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:42:13.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror image_38 2026-02-18T08:42:13.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_cluster=cluster1 2026-02-18T08:42:13.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:42:13.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_pool=mirror 2026-02-18T08:42:13.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local remote_pool=mirror 2026-02-18T08:42:13.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_38 2026-02-18T08:42:13.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' journal = journal ']' 2026-02-18T08:42:13.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror mirror image_38 2026-02-18T08:42:13.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_cluster=cluster1 2026-02-18T08:42:13.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:42:13.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_pool=mirror 2026-02-18T08:42:13.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local remote_pool=mirror 2026-02-18T08:42:13.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_38 2026-02-18T08:42:13.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-02-18T08:42:13.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-02-18T08:42:13.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:42:13.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:42:13.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:42:13.353 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_38 2026-02-18T08:42:13.353 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:42:13.353 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:42:13.353 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_38 2026-02-18T08:42:13.354 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:42:13.354 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_38 ']' 2026-02-18T08:42:13.354 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_38' 2026-02-18T08:42:13.354 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_38 2026-02-18T08:42:13.354 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:42:13.354 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:42:13.354 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:42:13.354 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:42:13.354 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:42:13.354 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:42:13.354 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:42:13.354 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:42:13.357 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:42:13.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:42:13.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_38 2026-02-18T08:42:13.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:42:13.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:42:13.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:42:13.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:42:13.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:42:13.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:42:13.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:42:13.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:42:13.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:42:13.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:42:13.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:42:13.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:42:13.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:42:13.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:42:13.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:42:13.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:42:13.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:42:13.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:42:13.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:42:13.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:42:13.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:42:13.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:42:13.360 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:42:13.360 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:42:13.360 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:42:13.360 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:42:13.360 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:42:13.360 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:42:13.360 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:42:13.360 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:42:13.368 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:42:13.368 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:42:13.368 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:42:13.368 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_38 2026-02-18T08:42:13.434 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:42:13.435 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_38 2026-02-18T08:42:13.436 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:42:13.436 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:42:13.436 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_38 2026-02-18T08:42:13.436 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_38 '' 2026-02-18T08:42:13.436 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:42:13.436 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:42:13.436 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_38 2026-02-18T08:42:13.436 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:42:13.436 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_38.status 2026-02-18T08:42:13.436 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_38.status 2026-02-18T08:42:13.437 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:42:13.437 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_38.status 2026-02-18T08:42:13.437 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_38 2026-02-18T08:42:13.437 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_38.status 2026-02-18T08:42:13.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 4 2026-02-18T08:42:13.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 4 2026-02-18T08:42:13.465 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:42:13.465 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=2, entry_tid=1], [object_number=16, tag_tid=2, entry_tid=0], [object_number=18, tag_tid=1, entry_tid=20142], [object_number=19, tag_tid=1, entry_tid=20139]]], state=connected] 2026-02-18T08:42:13.465 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=18, tag_tid=1, entry_tid=20142], [object_number=17, tag_tid=1, entry_tid=20141], [object_number=16, tag_tid=1, entry_tid=20140], [object_number=19, tag_tid=1, entry_tid=20139]]], state=connected] 2026-02-18T08:42:13.467 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_38.status 2026-02-18T08:42:13.468 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=17, tag_tid=2, entry_tid=1' 2026-02-18T08:42:13.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_38 2026-02-18T08:42:13.470 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:42:13.470 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:42:13.470 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_38 2026-02-18T08:42:13.470 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_38 '..*' 2026-02-18T08:42:13.470 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:42:13.470 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:42:13.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_38 2026-02-18T08:42:13.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:42:13.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_38.status 2026-02-18T08:42:13.472 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_38.status 2026-02-18T08:42:13.472 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:42:13.472 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_38.status 2026-02-18T08:42:13.472 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_38 2026-02-18T08:42:13.472 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_38.status 2026-02-18T08:42:13.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 4 2026-02-18T08:42:13.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 4 2026-02-18T08:42:13.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:42:13.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=2, entry_tid=1], [object_number=16, tag_tid=2, entry_tid=0], [object_number=18, tag_tid=1, entry_tid=20142], [object_number=19, tag_tid=1, entry_tid=20139]]], state=connected] 2026-02-18T08:42:13.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=18, tag_tid=1, entry_tid=20142], [object_number=17, tag_tid=1, entry_tid=20141], [object_number=16, tag_tid=1, entry_tid=20140], [object_number=19, tag_tid=1, entry_tid=20139]]], state=connected] 2026-02-18T08:42:13.501 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_38.status 2026-02-18T08:42:13.503 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=18, tag_tid=1, entry_tid=20142' 2026-02-18T08:42:13.504 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=17, tag_tid=2, entry_tid=1' -a 'object_number=17, tag_tid=2, entry_tid=1' = 'object_number=18, tag_tid=1, entry_tid=20142' 2026-02-18T08:42:13.504 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=18, tag_tid=1, entry_tid=20142' '!=' '' 2026-02-18T08:42:13.504 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:42:13.504 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=18, tag_tid=1, entry_tid=20142' = '' 2026-02-18T08:42:13.504 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=18, tag_tid=1, entry_tid=20142' 2026-02-18T08:42:13.505 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=17, tag_tid=2, entry_tid=1' 2026-02-18T08:42:13.505 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:42:13.505 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:42:13.506 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=2 2026-02-18T08:42:13.507 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=18, tag_tid=1, entry_tid=20142' 2026-02-18T08:42:13.507 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:42:13.507 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:42:13.508 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=1 2026-02-18T08:42:13.509 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=17, tag_tid=2, entry_tid=1' 2026-02-18T08:42:13.509 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:42:13.509 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:42:13.510 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:42:13.511 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=18, tag_tid=1, entry_tid=20142' 2026-02-18T08:42:13.512 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:42:13.512 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:42:13.513 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=20142 2026-02-18T08:42:13.513 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 2 = 1 -a 1 -le 20142 2026-02-18T08:42:13.513 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:42:13.513 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:42:13.513 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:42:13.713 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_38 2026-02-18T08:42:13.714 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:42:13.714 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:42:13.714 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_38 2026-02-18T08:42:13.714 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:42:13.714 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_38 ']' 2026-02-18T08:42:13.714 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_38' 2026-02-18T08:42:13.714 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_38 2026-02-18T08:42:13.714 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:42:13.714 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:42:13.714 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:42:13.714 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:42:13.714 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:42:13.714 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:42:13.714 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:42:13.714 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:42:13.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:42:13.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:42:13.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_38 2026-02-18T08:42:13.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:42:13.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:42:13.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:42:13.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:42:13.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:42:13.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:42:13.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:42:13.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:42:13.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:42:13.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:42:13.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:42:13.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:42:13.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:42:13.717 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:42:13.717 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:42:13.717 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:42:13.717 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:42:13.717 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:42:13.717 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:42:13.717 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:42:13.717 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:42:13.717 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:42:13.717 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:42:13.717 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:42:13.717 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:42:13.717 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:42:13.717 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:42:13.717 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:42:13.717 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:42:13.717 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:42:13.729 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:42:13.729 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:42:13.729 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:42:13.729 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_38 2026-02-18T08:42:13.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:42:13.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_38 2026-02-18T08:42:13.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:42:13.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:42:13.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_38 2026-02-18T08:42:13.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_38 '' 2026-02-18T08:42:13.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:42:13.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:42:13.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_38 2026-02-18T08:42:13.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:42:13.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_38.status 2026-02-18T08:42:13.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_38.status 2026-02-18T08:42:13.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:42:13.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_38.status 2026-02-18T08:42:13.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_38 2026-02-18T08:42:13.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_38.status 2026-02-18T08:42:13.830 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 4 2026-02-18T08:42:13.831 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 4 2026-02-18T08:42:13.831 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:42:13.831 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=2, entry_tid=1], [object_number=16, tag_tid=2, entry_tid=0], [object_number=18, tag_tid=1, entry_tid=20142], [object_number=19, tag_tid=1, entry_tid=20139]]], state=connected] 2026-02-18T08:42:13.831 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=18, tag_tid=1, entry_tid=20142], [object_number=17, tag_tid=1, entry_tid=20141], [object_number=16, tag_tid=1, entry_tid=20140], [object_number=19, tag_tid=1, entry_tid=20139]]], state=connected] 2026-02-18T08:42:13.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_38.status 2026-02-18T08:42:13.836 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=17, tag_tid=2, entry_tid=1' 2026-02-18T08:42:13.837 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_38 2026-02-18T08:42:13.837 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:42:13.837 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:42:13.837 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_38 2026-02-18T08:42:13.838 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_38 '..*' 2026-02-18T08:42:13.838 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:42:13.838 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:42:13.838 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_38 2026-02-18T08:42:13.838 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:42:13.838 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_38.status 2026-02-18T08:42:13.838 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_38.status 2026-02-18T08:42:13.838 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:42:13.840 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_38.status 2026-02-18T08:42:13.840 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_38 2026-02-18T08:42:13.840 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_38.status 2026-02-18T08:42:13.875 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 4 2026-02-18T08:42:13.876 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 4 2026-02-18T08:42:13.876 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:42:13.876 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=2, entry_tid=1], [object_number=16, tag_tid=2, entry_tid=0], [object_number=18, tag_tid=1, entry_tid=20142], [object_number=19, tag_tid=1, entry_tid=20139]]], state=connected] 2026-02-18T08:42:13.876 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=18, tag_tid=1, entry_tid=20142], [object_number=17, tag_tid=1, entry_tid=20141], [object_number=16, tag_tid=1, entry_tid=20140], [object_number=19, tag_tid=1, entry_tid=20139]]], state=connected] 2026-02-18T08:42:13.879 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_38.status 2026-02-18T08:42:13.881 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=18, tag_tid=1, entry_tid=20142' 2026-02-18T08:42:13.881 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=17, tag_tid=2, entry_tid=1' -a 'object_number=17, tag_tid=2, entry_tid=1' = 'object_number=18, tag_tid=1, entry_tid=20142' 2026-02-18T08:42:13.881 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=18, tag_tid=1, entry_tid=20142' '!=' 'object_number=18, tag_tid=1, entry_tid=20142' 2026-02-18T08:42:13.881 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:42:13.882 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.4 2026-02-18T08:42:14.283 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_38 2026-02-18T08:42:14.284 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:42:14.284 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:42:14.284 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_38 2026-02-18T08:42:14.284 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:42:14.284 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_38 ']' 2026-02-18T08:42:14.284 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_38' 2026-02-18T08:42:14.284 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_38 2026-02-18T08:42:14.284 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:42:14.285 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:42:14.285 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:42:14.285 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:42:14.285 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:42:14.285 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:42:14.285 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:42:14.285 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:42:14.287 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:42:14.288 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:42:14.288 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_38 2026-02-18T08:42:14.288 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:42:14.288 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:42:14.288 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:42:14.288 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:42:14.288 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:42:14.288 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:42:14.288 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:42:14.288 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:42:14.288 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:42:14.289 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:42:14.289 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:42:14.289 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:42:14.289 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:42:14.289 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:42:14.289 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:42:14.289 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:42:14.289 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:42:14.289 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:42:14.289 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:42:14.290 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:42:14.290 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:42:14.290 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:42:14.290 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:42:14.290 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:42:14.290 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:42:14.290 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:42:14.291 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:42:14.291 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:42:14.291 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:42:14.291 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:42:14.298 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:42:14.298 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:42:14.298 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:42:14.299 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_38 2026-02-18T08:42:14.372 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:42:14.373 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_38 2026-02-18T08:42:14.373 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:42:14.373 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:42:14.373 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_38 2026-02-18T08:42:14.373 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_38 '' 2026-02-18T08:42:14.374 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:42:14.374 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:42:14.374 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_38 2026-02-18T08:42:14.374 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:42:14.374 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_38.status 2026-02-18T08:42:14.374 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_38.status 2026-02-18T08:42:14.374 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:42:14.374 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_38.status 2026-02-18T08:42:14.374 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_38 2026-02-18T08:42:14.374 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_38.status 2026-02-18T08:42:14.612 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 4 2026-02-18T08:42:14.612 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 4 2026-02-18T08:42:14.612 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:42:14.612 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=2, entry_tid=1], [object_number=16, tag_tid=2, entry_tid=0], [object_number=18, tag_tid=1, entry_tid=20142], [object_number=19, tag_tid=1, entry_tid=20139]]], state=connected] 2026-02-18T08:42:14.613 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=18, tag_tid=1, entry_tid=20142], [object_number=17, tag_tid=1, entry_tid=20141], [object_number=16, tag_tid=1, entry_tid=20140], [object_number=19, tag_tid=1, entry_tid=20139]]], state=connected] 2026-02-18T08:42:14.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_38.status 2026-02-18T08:42:14.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=17, tag_tid=2, entry_tid=1' 2026-02-18T08:42:14.618 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_38 2026-02-18T08:42:14.618 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:42:14.619 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:42:14.619 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_38 2026-02-18T08:42:14.619 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_38 '..*' 2026-02-18T08:42:14.619 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:42:14.619 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:42:14.619 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_38 2026-02-18T08:42:14.619 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:42:14.619 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_38.status 2026-02-18T08:42:14.620 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_38.status 2026-02-18T08:42:14.620 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:42:14.620 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_38.status 2026-02-18T08:42:14.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_38 2026-02-18T08:42:14.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_38.status 2026-02-18T08:42:14.648 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 4 2026-02-18T08:42:14.648 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 4 2026-02-18T08:42:14.648 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:42:14.648 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=2, entry_tid=1], [object_number=16, tag_tid=2, entry_tid=0], [object_number=18, tag_tid=1, entry_tid=20142], [object_number=19, tag_tid=1, entry_tid=20139]]], state=connected] 2026-02-18T08:42:14.648 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=18, tag_tid=1, entry_tid=20142], [object_number=17, tag_tid=1, entry_tid=20141], [object_number=16, tag_tid=1, entry_tid=20140], [object_number=19, tag_tid=1, entry_tid=20139]]], state=connected] 2026-02-18T08:42:14.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_38.status 2026-02-18T08:42:14.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=18, tag_tid=1, entry_tid=20142' 2026-02-18T08:42:14.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=17, tag_tid=2, entry_tid=1' -a 'object_number=17, tag_tid=2, entry_tid=1' = 'object_number=18, tag_tid=1, entry_tid=20142' 2026-02-18T08:42:14.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=18, tag_tid=1, entry_tid=20142' '!=' 'object_number=18, tag_tid=1, entry_tid=20142' 2026-02-18T08:42:14.653 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:42:14.653 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.8 2026-02-18T08:42:15.454 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_38 2026-02-18T08:42:15.454 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:42:15.455 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:42:15.455 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_38 2026-02-18T08:42:15.455 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:42:15.455 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_38 ']' 2026-02-18T08:42:15.455 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_38' 2026-02-18T08:42:15.455 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_38 2026-02-18T08:42:15.455 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:42:15.455 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:42:15.455 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:42:15.455 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:42:15.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:42:15.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:42:15.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:42:15.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:42:15.458 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:42:15.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:42:15.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_38 2026-02-18T08:42:15.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:42:15.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:42:15.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:42:15.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:42:15.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:42:15.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:42:15.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:42:15.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:42:15.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:42:15.460 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:42:15.460 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:42:15.460 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:42:15.460 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:42:15.460 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:42:15.460 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:42:15.460 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:42:15.460 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:42:15.460 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:42:15.460 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:42:15.460 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:42:15.460 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:42:15.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:42:15.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:42:15.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:42:15.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:42:15.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:42:15.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:42:15.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:42:15.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:42:15.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:42:15.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:42:15.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:42:15.470 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:42:15.470 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_38 2026-02-18T08:42:15.536 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:42:15.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_38 2026-02-18T08:42:15.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:42:15.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:42:15.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_38 2026-02-18T08:42:15.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_38 '' 2026-02-18T08:42:15.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:42:15.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:42:15.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_38 2026-02-18T08:42:15.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:42:15.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_38.status 2026-02-18T08:42:15.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_38.status 2026-02-18T08:42:15.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:42:15.539 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_38.status 2026-02-18T08:42:15.539 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_38 2026-02-18T08:42:15.539 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_38.status 2026-02-18T08:42:15.566 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 4 2026-02-18T08:42:15.566 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 4 2026-02-18T08:42:15.566 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:42:15.567 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=2, entry_tid=1], [object_number=16, tag_tid=2, entry_tid=0], [object_number=18, tag_tid=1, entry_tid=20142], [object_number=19, tag_tid=1, entry_tid=20139]]], state=connected] 2026-02-18T08:42:15.567 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=18, tag_tid=1, entry_tid=20142], [object_number=17, tag_tid=1, entry_tid=20141], [object_number=16, tag_tid=1, entry_tid=20140], [object_number=19, tag_tid=1, entry_tid=20139]]], state=connected] 2026-02-18T08:42:15.569 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_38.status 2026-02-18T08:42:15.571 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=17, tag_tid=2, entry_tid=1' 2026-02-18T08:42:15.572 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_38 2026-02-18T08:42:15.572 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:42:15.572 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:42:15.572 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_38 2026-02-18T08:42:15.572 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_38 '..*' 2026-02-18T08:42:15.572 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:42:15.572 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:42:15.572 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_38 2026-02-18T08:42:15.572 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:42:15.573 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_38.status 2026-02-18T08:42:15.573 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_38.status 2026-02-18T08:42:15.573 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:42:15.574 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_38.status 2026-02-18T08:42:15.574 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_38 2026-02-18T08:42:15.574 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_38.status 2026-02-18T08:42:15.601 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 4 2026-02-18T08:42:15.601 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 4 2026-02-18T08:42:15.601 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:42:15.602 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=2, entry_tid=1], [object_number=16, tag_tid=2, entry_tid=0], [object_number=18, tag_tid=1, entry_tid=20142], [object_number=19, tag_tid=1, entry_tid=20139]]], state=connected] 2026-02-18T08:42:15.602 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=18, tag_tid=1, entry_tid=20142], [object_number=17, tag_tid=1, entry_tid=20141], [object_number=16, tag_tid=1, entry_tid=20140], [object_number=19, tag_tid=1, entry_tid=20139]]], state=connected] 2026-02-18T08:42:15.604 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_38.status 2026-02-18T08:42:15.605 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=18, tag_tid=1, entry_tid=20142' 2026-02-18T08:42:15.606 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=17, tag_tid=2, entry_tid=1' -a 'object_number=17, tag_tid=2, entry_tid=1' = 'object_number=18, tag_tid=1, entry_tid=20142' 2026-02-18T08:42:15.606 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=18, tag_tid=1, entry_tid=20142' '!=' 'object_number=18, tag_tid=1, entry_tid=20142' 2026-02-18T08:42:15.606 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:42:15.606 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1.6 2026-02-18T08:42:17.208 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_38 2026-02-18T08:42:17.208 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:42:17.208 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:42:17.208 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_38 2026-02-18T08:42:17.208 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:42:17.208 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_38 ']' 2026-02-18T08:42:17.208 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_38' 2026-02-18T08:42:17.209 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_38 2026-02-18T08:42:17.209 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:42:17.209 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:42:17.209 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:42:17.209 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:42:17.209 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:42:17.209 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:42:17.209 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:42:17.209 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:42:17.212 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:42:17.212 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:42:17.212 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_38 2026-02-18T08:42:17.212 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:42:17.212 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:42:17.212 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:42:17.212 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:42:17.213 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:42:17.213 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:42:17.213 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:42:17.213 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:42:17.213 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:42:17.213 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:42:17.213 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:42:17.213 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:42:17.213 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:42:17.213 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:42:17.213 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:42:17.213 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:42:17.214 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:42:17.214 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:42:17.214 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:42:17.214 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:42:17.214 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:42:17.214 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:42:17.214 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:42:17.214 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:42:17.214 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:42:17.214 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:42:17.214 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:42:17.214 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:42:17.215 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:42:17.215 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:42:17.222 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:42:17.223 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:42:17.223 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:42:17.223 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_38 2026-02-18T08:42:17.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:42:17.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_38 2026-02-18T08:42:17.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:42:17.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:42:17.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_38 2026-02-18T08:42:17.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_38 '' 2026-02-18T08:42:17.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:42:17.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:42:17.297 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_38 2026-02-18T08:42:17.297 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:42:17.297 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_38.status 2026-02-18T08:42:17.297 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_38.status 2026-02-18T08:42:17.297 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:42:17.297 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_38.status 2026-02-18T08:42:17.298 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_38 2026-02-18T08:42:17.298 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_38.status 2026-02-18T08:42:17.326 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 4 2026-02-18T08:42:17.326 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 4 2026-02-18T08:42:17.326 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:42:17.326 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=2, entry_tid=1], [object_number=16, tag_tid=2, entry_tid=0], [object_number=18, tag_tid=1, entry_tid=20142], [object_number=19, tag_tid=1, entry_tid=20139]]], state=connected] 2026-02-18T08:42:17.326 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=18, tag_tid=1, entry_tid=20142], [object_number=17, tag_tid=1, entry_tid=20141], [object_number=16, tag_tid=1, entry_tid=20140], [object_number=19, tag_tid=1, entry_tid=20139]]], state=connected] 2026-02-18T08:42:17.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_38.status 2026-02-18T08:42:17.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=17, tag_tid=2, entry_tid=1' 2026-02-18T08:42:17.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_38 2026-02-18T08:42:17.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:42:17.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:42:17.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_38 2026-02-18T08:42:17.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_38 '..*' 2026-02-18T08:42:17.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:42:17.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:42:17.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_38 2026-02-18T08:42:17.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:42:17.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_38.status 2026-02-18T08:42:17.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_38.status 2026-02-18T08:42:17.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:42:17.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_38.status 2026-02-18T08:42:17.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_38 2026-02-18T08:42:17.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_38.status 2026-02-18T08:42:17.365 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 4 2026-02-18T08:42:17.365 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 4 2026-02-18T08:42:17.365 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:42:17.365 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=2, entry_tid=1], [object_number=16, tag_tid=2, entry_tid=0], [object_number=18, tag_tid=1, entry_tid=20142], [object_number=19, tag_tid=1, entry_tid=20139]]], state=connected] 2026-02-18T08:42:17.366 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=18, tag_tid=1, entry_tid=20142], [object_number=17, tag_tid=1, entry_tid=20141], [object_number=16, tag_tid=1, entry_tid=20140], [object_number=19, tag_tid=1, entry_tid=20139]]], state=connected] 2026-02-18T08:42:17.368 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_38.status 2026-02-18T08:42:17.370 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=18, tag_tid=1, entry_tid=20142' 2026-02-18T08:42:17.370 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=17, tag_tid=2, entry_tid=1' -a 'object_number=17, tag_tid=2, entry_tid=1' = 'object_number=18, tag_tid=1, entry_tid=20142' 2026-02-18T08:42:17.370 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=18, tag_tid=1, entry_tid=20142' '!=' 'object_number=18, tag_tid=1, entry_tid=20142' 2026-02-18T08:42:17.370 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:42:17.370 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 2 2026-02-18T08:42:18.795 INFO:tasks.rbd_mirror_thrash:kill cluster2.client.mirror.0 2026-02-18T08:42:18.796 INFO:tasks.rbd_mirror.cluster2.client.mirror.0:Sent signal 15 2026-02-18T08:42:18.796 INFO:tasks.rbd_mirror_thrash:waiting for 6.09065203310351 secs before reviving daemons 2026-02-18T08:42:18.797 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.trial163.stderr:2026-02-18T08:42:18.790+0000 7f463a783640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.0 (PID: 113402) UID: 1000 2026-02-18T08:42:19.372 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_38 2026-02-18T08:42:19.373 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:42:19.373 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:42:19.373 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_38 2026-02-18T08:42:19.373 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:42:19.373 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_38 ']' 2026-02-18T08:42:19.373 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_38' 2026-02-18T08:42:19.373 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_38 2026-02-18T08:42:19.373 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:42:19.373 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:42:19.373 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:42:19.373 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:42:19.374 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:42:19.374 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:42:19.374 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:42:19.374 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:42:19.374 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:42:19.376 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:42:19.377 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_38 2026-02-18T08:42:19.377 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:42:19.377 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:42:19.377 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:42:19.377 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:42:19.377 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:42:19.377 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:42:19.377 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:42:19.377 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:42:19.377 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:42:19.377 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:42:19.377 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:42:19.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:42:19.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:42:19.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:42:19.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:42:19.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:42:19.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:42:19.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:42:19.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:42:19.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:42:19.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:42:19.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:42:19.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:42:19.379 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:42:19.379 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:42:19.379 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:42:19.379 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:42:19.379 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:42:19.379 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:42:19.379 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:42:19.387 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:42:19.387 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:42:19.388 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:42:19.388 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_38 2026-02-18T08:42:19.460 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:42:19.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_38 2026-02-18T08:42:19.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:42:19.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:42:19.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_38 2026-02-18T08:42:19.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_38 '' 2026-02-18T08:42:19.462 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:42:19.462 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:42:19.462 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_38 2026-02-18T08:42:19.462 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:42:19.462 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_38.status 2026-02-18T08:42:19.462 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_38.status 2026-02-18T08:42:19.462 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:42:19.462 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_38.status 2026-02-18T08:42:19.463 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_38 2026-02-18T08:42:19.463 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_38.status 2026-02-18T08:42:19.490 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 4 2026-02-18T08:42:19.491 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 4 2026-02-18T08:42:19.491 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:42:19.491 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=2, entry_tid=1], [object_number=16, tag_tid=2, entry_tid=0], [object_number=18, tag_tid=1, entry_tid=20142], [object_number=19, tag_tid=1, entry_tid=20139]]], state=connected] 2026-02-18T08:42:19.491 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=18, tag_tid=1, entry_tid=20142], [object_number=17, tag_tid=1, entry_tid=20141], [object_number=16, tag_tid=1, entry_tid=20140], [object_number=19, tag_tid=1, entry_tid=20139]]], state=connected] 2026-02-18T08:42:19.493 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_38.status 2026-02-18T08:42:19.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=17, tag_tid=2, entry_tid=1' 2026-02-18T08:42:19.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_38 2026-02-18T08:42:19.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:42:19.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:42:19.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_38 2026-02-18T08:42:19.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_38 '..*' 2026-02-18T08:42:19.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:42:19.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:42:19.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_38 2026-02-18T08:42:19.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:42:19.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_38.status 2026-02-18T08:42:19.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_38.status 2026-02-18T08:42:19.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:42:19.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_38.status 2026-02-18T08:42:19.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_38 2026-02-18T08:42:19.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_38.status 2026-02-18T08:42:19.525 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 4 2026-02-18T08:42:19.525 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 4 2026-02-18T08:42:19.526 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:42:19.526 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=2, entry_tid=1], [object_number=16, tag_tid=2, entry_tid=0], [object_number=18, tag_tid=1, entry_tid=20142], [object_number=19, tag_tid=1, entry_tid=20139]]], state=connected] 2026-02-18T08:42:19.526 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=18, tag_tid=1, entry_tid=20142], [object_number=17, tag_tid=1, entry_tid=20141], [object_number=16, tag_tid=1, entry_tid=20140], [object_number=19, tag_tid=1, entry_tid=20139]]], state=connected] 2026-02-18T08:42:19.528 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_38.status 2026-02-18T08:42:19.529 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=18, tag_tid=1, entry_tid=20142' 2026-02-18T08:42:19.530 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=17, tag_tid=2, entry_tid=1' -a 'object_number=17, tag_tid=2, entry_tid=1' = 'object_number=18, tag_tid=1, entry_tid=20142' 2026-02-18T08:42:19.530 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=18, tag_tid=1, entry_tid=20142' '!=' 'object_number=18, tag_tid=1, entry_tid=20142' 2026-02-18T08:42:19.530 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:42:19.530 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 2 2026-02-18T08:42:21.532 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_38 2026-02-18T08:42:21.532 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:42:21.532 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:42:21.532 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_38 2026-02-18T08:42:21.532 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:42:21.532 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_38 ']' 2026-02-18T08:42:21.532 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_38' 2026-02-18T08:42:21.532 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_38 2026-02-18T08:42:21.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:42:21.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:42:21.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:42:21.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:42:21.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:42:21.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:42:21.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:42:21.536 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:42:21.536 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:42:21.536 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:42:21.536 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_38 2026-02-18T08:42:21.536 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:42:21.536 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:42:21.536 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:42:21.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:42:21.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:42:21.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:42:21.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:42:21.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:42:21.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:42:21.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:42:21.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:42:21.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:42:21.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:42:21.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:42:21.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:42:21.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:42:21.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:42:21.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:42:21.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:42:21.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:42:21.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:42:21.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:42:21.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:42:21.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:42:21.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:42:21.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:42:21.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:42:21.539 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:42:21.539 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:42:21.539 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:42:21.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:42:21.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:42:21.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:42:21.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_38 2026-02-18T08:42:21.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:42:21.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_38 2026-02-18T08:42:21.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:42:21.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:42:21.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_38 2026-02-18T08:42:21.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_38 '' 2026-02-18T08:42:21.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:42:21.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:42:21.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_38 2026-02-18T08:42:21.618 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:42:21.618 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_38.status 2026-02-18T08:42:21.618 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_38.status 2026-02-18T08:42:21.618 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:42:21.618 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_38.status 2026-02-18T08:42:21.619 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_38 2026-02-18T08:42:21.619 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_38.status 2026-02-18T08:42:21.646 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 4 2026-02-18T08:42:21.646 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 4 2026-02-18T08:42:21.646 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:42:21.646 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=2, entry_tid=1], [object_number=16, tag_tid=2, entry_tid=0], [object_number=18, tag_tid=1, entry_tid=20142], [object_number=19, tag_tid=1, entry_tid=20139]]], state=connected] 2026-02-18T08:42:21.646 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=18, tag_tid=1, entry_tid=20142], [object_number=17, tag_tid=1, entry_tid=20141], [object_number=16, tag_tid=1, entry_tid=20140], [object_number=19, tag_tid=1, entry_tid=20139]]], state=connected] 2026-02-18T08:42:21.648 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_38.status 2026-02-18T08:42:21.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=17, tag_tid=2, entry_tid=1' 2026-02-18T08:42:21.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_38 2026-02-18T08:42:21.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:42:21.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:42:21.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_38 2026-02-18T08:42:21.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_38 '..*' 2026-02-18T08:42:21.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:42:21.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:42:21.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_38 2026-02-18T08:42:21.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:42:21.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_38.status 2026-02-18T08:42:21.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_38.status 2026-02-18T08:42:21.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:42:21.653 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_38.status 2026-02-18T08:42:21.653 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_38 2026-02-18T08:42:21.654 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_38.status 2026-02-18T08:42:21.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 4 2026-02-18T08:42:21.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 4 2026-02-18T08:42:21.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:42:21.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=2, entry_tid=1], [object_number=16, tag_tid=2, entry_tid=0], [object_number=18, tag_tid=1, entry_tid=20142], [object_number=19, tag_tid=1, entry_tid=20139]]], state=connected] 2026-02-18T08:42:21.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=18, tag_tid=1, entry_tid=20142], [object_number=17, tag_tid=1, entry_tid=20141], [object_number=16, tag_tid=1, entry_tid=20140], [object_number=19, tag_tid=1, entry_tid=20139]]], state=connected] 2026-02-18T08:42:21.693 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_38.status 2026-02-18T08:42:21.695 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=18, tag_tid=1, entry_tid=20142' 2026-02-18T08:42:21.695 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=17, tag_tid=2, entry_tid=1' -a 'object_number=17, tag_tid=2, entry_tid=1' = 'object_number=18, tag_tid=1, entry_tid=20142' 2026-02-18T08:42:21.695 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=18, tag_tid=1, entry_tid=20142' '!=' 'object_number=18, tag_tid=1, entry_tid=20142' 2026-02-18T08:42:21.695 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:42:21.695 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 4 2026-02-18T08:42:24.888 INFO:tasks.rbd_mirror_thrash:waiting for cluster2.client.mirror.0 2026-02-18T08:42:24.888 INFO:teuthology.orchestra.run:waiting for 600 2026-02-18T08:42:24.889 INFO:tasks.rbd_mirror_thrash:reviving cluster2.client.mirror.0 2026-02-18T08:42:24.889 INFO:tasks.rbd_mirror.cluster2.client.mirror.0:Restarting daemon 2026-02-18T08:42:24.889 DEBUG:teuthology.orchestra.run.trial163:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.0 2026-02-18T08:42:24.891 INFO:tasks.rbd_mirror.cluster2.client.mirror.0:Started 2026-02-18T08:42:24.891 INFO:tasks.rbd_mirror_thrash:waiting for 100.77741917678318 secs before thrashing 2026-02-18T08:42:25.697 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_38 2026-02-18T08:42:25.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:42:25.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:42:25.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_38 2026-02-18T08:42:25.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:42:25.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_38 ']' 2026-02-18T08:42:25.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_38' 2026-02-18T08:42:25.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_38 2026-02-18T08:42:25.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:42:25.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:42:25.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:42:25.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:42:25.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:42:25.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:42:25.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:42:25.702 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:42:25.702 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:42:25.702 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:42:25.702 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_38 2026-02-18T08:42:25.702 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:42:25.702 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:42:25.702 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:42:25.702 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:42:25.703 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:42:25.703 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:42:25.703 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:42:25.703 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:42:25.703 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:42:25.703 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:42:25.703 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:42:25.703 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:42:25.703 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:42:25.703 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:42:25.703 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:42:25.703 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:42:25.704 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:42:25.704 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:42:25.704 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:42:25.704 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:42:25.704 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:42:25.704 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:42:25.704 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:42:25.704 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:42:25.704 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:42:25.704 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:42:25.704 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:42:25.704 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:42:25.705 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:42:25.705 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:42:25.712 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:42:25.712 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:42:25.712 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:42:25.712 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_38 2026-02-18T08:42:25.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:42:25.776 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_38 2026-02-18T08:42:25.776 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:42:25.777 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:42:25.777 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_38 2026-02-18T08:42:25.777 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_38 '' 2026-02-18T08:42:25.777 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:42:25.777 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:42:25.777 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_38 2026-02-18T08:42:25.777 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:42:25.777 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_38.status 2026-02-18T08:42:25.777 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_38.status 2026-02-18T08:42:25.777 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:42:25.778 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_38.status 2026-02-18T08:42:25.778 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_38.status 2026-02-18T08:42:25.778 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_38 2026-02-18T08:42:25.808 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 4 2026-02-18T08:42:25.808 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 4 2026-02-18T08:42:25.808 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:42:25.808 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=2, entry_tid=1], [object_number=16, tag_tid=2, entry_tid=0], [object_number=18, tag_tid=1, entry_tid=20142], [object_number=19, tag_tid=1, entry_tid=20139]]], state=connected] 2026-02-18T08:42:25.808 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=17, tag_tid=2, entry_tid=1], [object_number=16, tag_tid=2, entry_tid=0], [object_number=18, tag_tid=1, entry_tid=20142], [object_number=19, tag_tid=1, entry_tid=20139]]], state=connected] 2026-02-18T08:42:25.811 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_38.status 2026-02-18T08:42:25.812 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=17, tag_tid=2, entry_tid=1' 2026-02-18T08:42:25.813 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_38 2026-02-18T08:42:25.814 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:42:25.814 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:42:25.814 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_38 2026-02-18T08:42:25.814 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_38 '..*' 2026-02-18T08:42:25.814 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:42:25.814 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:42:25.814 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_38 2026-02-18T08:42:25.814 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:42:25.814 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_38.status 2026-02-18T08:42:25.814 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_38.status 2026-02-18T08:42:25.814 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:42:25.815 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_38.status 2026-02-18T08:42:25.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_38 2026-02-18T08:42:25.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_38.status 2026-02-18T08:42:25.843 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 4 2026-02-18T08:42:25.843 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 4 2026-02-18T08:42:25.843 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:42:25.843 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=2, entry_tid=1], [object_number=16, tag_tid=2, entry_tid=0], [object_number=18, tag_tid=1, entry_tid=20142], [object_number=19, tag_tid=1, entry_tid=20139]]], state=connected] 2026-02-18T08:42:25.843 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=17, tag_tid=2, entry_tid=1], [object_number=16, tag_tid=2, entry_tid=0], [object_number=18, tag_tid=1, entry_tid=20142], [object_number=19, tag_tid=1, entry_tid=20139]]], state=connected] 2026-02-18T08:42:25.845 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_38.status 2026-02-18T08:42:25.848 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=17, tag_tid=2, entry_tid=1' 2026-02-18T08:42:25.848 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=17, tag_tid=2, entry_tid=1' -a 'object_number=17, tag_tid=2, entry_tid=1' = 'object_number=17, tag_tid=2, entry_tid=1' 2026-02-18T08:42:25.848 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:42:25.848 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_snap_present cluster1 mirror image_38 snap 2026-02-18T08:42:25.848 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:42:25.848 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:42:25.848 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_38 2026-02-18T08:42:25.848 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap 2026-02-18T08:42:25.848 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-18T08:42:25.848 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-18T08:42:25.848 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1 2026-02-18T08:42:26.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster1 info mirror/image_38@snap 2026-02-18T08:42:26.879 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:rbd image 'image_38': 2026-02-18T08:42:26.879 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: size 128 MiB in 32 objects 2026-02-18T08:42:26.879 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: order 22 (4 MiB objects) 2026-02-18T08:42:26.879 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: snapshot_count: 1 2026-02-18T08:42:26.879 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: id: 1104dcc45738 2026-02-18T08:42:26.879 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: block_name_prefix: rbd_data.1104dcc45738 2026-02-18T08:42:26.879 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: format: 2 2026-02-18T08:42:26.879 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: features: layering, exclusive-lock, journaling 2026-02-18T08:42:26.879 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: op_features: 2026-02-18T08:42:26.880 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: flags: 2026-02-18T08:42:26.880 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: create_timestamp: Wed Feb 18 08:32:14 2026 2026-02-18T08:42:26.880 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: access_timestamp: Wed Feb 18 08:32:14 2026 2026-02-18T08:42:26.880 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: modify_timestamp: Wed Feb 18 08:32:14 2026 2026-02-18T08:42:26.880 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: protected: False 2026-02-18T08:42:26.880 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: journal: 1104dcc45738 2026-02-18T08:42:26.880 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring state: enabled 2026-02-18T08:42:26.880 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring mode: journal 2026-02-18T08:42:26.880 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring global id: 3d666533-dd94-4d00-9369-1706f061e04b 2026-02-18T08:42:26.881 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring primary: false 2026-02-18T08:42:26.882 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:42:26.882 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ compare_image_snaps mirror image_38 snap 2026-02-18T08:42:26.882 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:42:26.882 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_38 2026-02-18T08:42:26.882 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap 2026-02-18T08:42:26.882 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local ret=0 2026-02-18T08:42:26.883 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local rmt_export=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_38.export 2026-02-18T08:42:26.883 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loc_export=/tmp/tmp.vybh1m9yJF/cluster1-mirror-image_38.export 2026-02-18T08:42:26.883 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_38.export /tmp/tmp.vybh1m9yJF/cluster1-mirror-image_38.export 2026-02-18T08:42:26.883 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 -p mirror export image_38@snap /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_38.export 2026-02-18T08:42:27.021 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-18T08:42:27.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster1 -p mirror export image_38@snap /tmp/tmp.vybh1m9yJF/cluster1-mirror-image_38.export 2026-02-18T08:42:27.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-18T08:42:27.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmp /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_38.export /tmp/tmp.vybh1m9yJF/cluster1-mirror-image_38.export 2026-02-18T08:42:27.294 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_38.export /tmp/tmp.vybh1m9yJF/cluster1-mirror-image_38.export 2026-02-18T08:42:27.327 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:42:27.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T08:42:27.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_39 2026-02-18T08:42:27.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_snap cluster2 mirror image_39 snap 2026-02-18T08:42:27.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:42:27.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:42:27.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_39 2026-02-18T08:42:27.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap 2026-02-18T08:42:27.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 -p mirror snap create image_39@snap --debug-rbd=20 --debug-journaler=20 2026-02-18T08:42:30.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_replay_started cluster1 mirror image_39 2026-02-18T08:42:30.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:42:30.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:42:30.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_39 2026-02-18T08:42:30.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_replay_state cluster1 mirror image_39 started 2026-02-18T08:42:30.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:42:30.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:42:30.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_39 2026-02-18T08:42:30.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local state=started 2026-02-18T08:42:30.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-18T08:42:30.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:42:30.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1 2026-02-18T08:42:31.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test_image_replay_state cluster1 mirror image_39 started 2026-02-18T08:42:31.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:42:31.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:42:31.092 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_39 2026-02-18T08:42:31.092 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local test_state=started 2026-02-18T08:42:31.092 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local status_result 2026-02-18T08:42:31.092 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_state=stopped 2026-02-18T08:42:31.093 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_39 2026-02-18T08:42:31.093 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -i state 2026-02-18T08:42:31.093 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_instance=cluster1 2026-02-18T08:42:31.093 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-18T08:42:31.093 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:42:31.094 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance=cluster1 2026-02-18T08:42:31.094 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local loop_instance 2026-02-18T08:42:31.094 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:42:31.094 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sleep 0 2026-02-18T08:42:31.094 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:42:31.094 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ seq 0 3 2026-02-18T08:42:31.096 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:42:31.096 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_39 2026-02-18T08:42:31.096 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:0 2026-02-18T08:42:31.097 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-18T08:42:31.097 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:42:31.097 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:42:31.097 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:42:31.097 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-18T08:42:31.097 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:42:31.097 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:42:31.097 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:42:31.097 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:42:31.097 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-18T08:42:31.097 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:42:31.097 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:42:31.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:42:31.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:42:31.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:0 2026-02-18T08:42:31.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-18T08:42:31.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-18T08:42:31.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:42:31.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:0 2026-02-18T08:42:31.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T08:42:31.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-18T08:42:31.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:42:31.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:42:31.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-18T08:42:31.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-18T08:42:31.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-18T08:42:31.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=0 2026-02-18T08:42:31.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=0 2026-02-18T08:42:31.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:42:31.106 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:42:31.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:42:31.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:42:31.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_39 2026-02-18T08:42:31.169 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_39'] 2026-02-18T08:42:31.170 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status : get status for rbd mirror 2026-02-18T08:42:31.170 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:42:31.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:42:31.173 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_39 2026-02-18T08:42:31.173 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:1 2026-02-18T08:42:31.173 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-18T08:42:31.173 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:42:31.173 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:42:31.173 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:42:31.173 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-18T08:42:31.173 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:42:31.173 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:42:31.173 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:42:31.174 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:42:31.174 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-18T08:42:31.174 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:42:31.174 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:42:31.174 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:42:31.174 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:42:31.174 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:1 2026-02-18T08:42:31.174 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-18T08:42:31.174 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-18T08:42:31.174 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:42:31.174 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:1 2026-02-18T08:42:31.174 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T08:42:31.174 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-18T08:42:31.175 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:42:31.175 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:42:31.175 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-18T08:42:31.175 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-18T08:42:31.175 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-18T08:42:31.175 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=1 2026-02-18T08:42:31.175 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=1 2026-02-18T08:42:31.175 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:42:31.183 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:42:31.183 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:42:31.183 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:42:31.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_39 2026-02-18T08:42:31.252 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ return 0 2026-02-18T08:42:31.253 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ status_result=' "state": "Replaying"' 2026-02-18T08:42:31.253 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ echo ' "state": "Replaying"' 2026-02-18T08:42:31.254 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ grep -i Replaying 2026-02-18T08:42:31.255 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: "state": "Replaying" 2026-02-18T08:42:31.256 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ current_state=started 2026-02-18T08:42:31.256 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test started = started 2026-02-18T08:42:31.256 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:42:31.256 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror image_39 2026-02-18T08:42:31.257 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_cluster=cluster1 2026-02-18T08:42:31.257 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:42:31.257 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_pool=mirror 2026-02-18T08:42:31.257 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local remote_pool=mirror 2026-02-18T08:42:31.257 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_39 2026-02-18T08:42:31.257 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' journal = journal ']' 2026-02-18T08:42:31.257 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror mirror image_39 2026-02-18T08:42:31.257 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_cluster=cluster1 2026-02-18T08:42:31.257 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:42:31.257 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_pool=mirror 2026-02-18T08:42:31.257 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local remote_pool=mirror 2026-02-18T08:42:31.257 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_39 2026-02-18T08:42:31.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-02-18T08:42:31.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-02-18T08:42:31.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:42:31.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:42:31.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:42:31.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_39 2026-02-18T08:42:31.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:42:31.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:42:31.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_39 2026-02-18T08:42:31.458 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:42:31.458 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_39 ']' 2026-02-18T08:42:31.458 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_39' 2026-02-18T08:42:31.458 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_39 2026-02-18T08:42:31.458 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:42:31.458 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:42:31.458 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:42:31.458 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:42:31.458 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:42:31.458 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:42:31.458 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:42:31.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:42:31.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:42:31.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:42:31.462 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_39 2026-02-18T08:42:31.462 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:42:31.462 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:42:31.462 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:42:31.462 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:42:31.462 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:42:31.462 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:42:31.462 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:42:31.462 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:42:31.462 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:42:31.462 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:42:31.462 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:42:31.463 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:42:31.463 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:42:31.463 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:42:31.463 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:42:31.463 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:42:31.463 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:42:31.463 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:42:31.463 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:42:31.463 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:42:31.463 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:42:31.463 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:42:31.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:42:31.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:42:31.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:42:31.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:42:31.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:42:31.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:42:31.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:42:31.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:42:31.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:42:31.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:42:31.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:42:31.472 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_39 2026-02-18T08:42:31.531 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_39'] 2026-02-18T08:42:31.531 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:42:31.532 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:42:31.534 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:42:31.534 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_39 2026-02-18T08:42:31.534 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:42:31.535 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:42:31.535 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:42:31.535 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:42:31.535 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:42:31.535 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:42:31.535 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:42:31.535 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:42:31.535 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:42:31.535 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:42:31.535 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:42:31.535 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:42:31.536 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:42:31.536 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:42:31.536 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:42:31.536 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:42:31.536 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:42:31.536 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:42:31.536 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:42:31.536 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:42:31.536 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:42:31.536 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:42:31.536 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:42:31.536 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:42:31.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:42:31.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:42:31.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:42:31.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:42:31.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:42:31.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:42:31.545 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:42:31.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:42:31.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:42:31.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_39 2026-02-18T08:42:31.613 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:42:31.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_39 2026-02-18T08:42:31.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:42:31.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:42:31.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_39 2026-02-18T08:42:31.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_39 '' 2026-02-18T08:42:31.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:42:31.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:42:31.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_39 2026-02-18T08:42:31.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:42:31.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_39.status 2026-02-18T08:42:31.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_39.status 2026-02-18T08:42:31.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:42:31.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_39.status 2026-02-18T08:42:31.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_39 2026-02-18T08:42:31.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_39.status 2026-02-18T08:42:31.653 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 11 2026-02-18T08:42:31.653 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 11 2026-02-18T08:42:31.653 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:42:31.653 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=2, entry_tid=1], [object_number=44, tag_tid=2, entry_tid=0], [object_number=47, tag_tid=1, entry_tid=47675], [object_number=46, tag_tid=1, entry_tid=47674]]], state=connected] 2026-02-18T08:42:31.653 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=45, tag_tid=1, entry_tid=47677], [object_number=44, tag_tid=1, entry_tid=47676], [object_number=47, tag_tid=1, entry_tid=47675], [object_number=46, tag_tid=1, entry_tid=47674]]], state=connected] 2026-02-18T08:42:31.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_39.status 2026-02-18T08:42:31.659 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=45, tag_tid=2, entry_tid=1' 2026-02-18T08:42:31.660 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_39 2026-02-18T08:42:31.660 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:42:31.660 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:42:31.660 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_39 2026-02-18T08:42:31.660 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_39 '..*' 2026-02-18T08:42:31.660 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:42:31.661 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:42:31.661 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_39 2026-02-18T08:42:31.661 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:42:31.661 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_39.status 2026-02-18T08:42:31.661 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_39.status 2026-02-18T08:42:31.661 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:42:31.662 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_39.status 2026-02-18T08:42:31.662 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_39 2026-02-18T08:42:31.662 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_39.status 2026-02-18T08:42:31.697 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 11 2026-02-18T08:42:31.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 11 2026-02-18T08:42:31.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:42:31.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=2, entry_tid=1], [object_number=44, tag_tid=2, entry_tid=0], [object_number=47, tag_tid=1, entry_tid=47675], [object_number=46, tag_tid=1, entry_tid=47674]]], state=connected] 2026-02-18T08:42:31.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=45, tag_tid=1, entry_tid=47677], [object_number=44, tag_tid=1, entry_tid=47676], [object_number=47, tag_tid=1, entry_tid=47675], [object_number=46, tag_tid=1, entry_tid=47674]]], state=connected] 2026-02-18T08:42:31.701 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_39.status 2026-02-18T08:42:31.703 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=45, tag_tid=1, entry_tid=47677' 2026-02-18T08:42:31.703 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=45, tag_tid=2, entry_tid=1' -a 'object_number=45, tag_tid=2, entry_tid=1' = 'object_number=45, tag_tid=1, entry_tid=47677' 2026-02-18T08:42:31.704 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=45, tag_tid=1, entry_tid=47677' '!=' '' 2026-02-18T08:42:31.704 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:42:31.704 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=45, tag_tid=1, entry_tid=47677' = '' 2026-02-18T08:42:31.704 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=45, tag_tid=1, entry_tid=47677' 2026-02-18T08:42:31.704 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=45, tag_tid=2, entry_tid=1' 2026-02-18T08:42:31.705 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:42:31.705 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:42:31.705 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=2 2026-02-18T08:42:31.706 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=45, tag_tid=1, entry_tid=47677' 2026-02-18T08:42:31.706 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:42:31.706 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:42:31.707 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=1 2026-02-18T08:42:31.708 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=45, tag_tid=2, entry_tid=1' 2026-02-18T08:42:31.708 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:42:31.708 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:42:31.709 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:42:31.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=45, tag_tid=1, entry_tid=47677' 2026-02-18T08:42:31.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:42:31.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:42:31.712 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=47677 2026-02-18T08:42:31.712 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 2 = 1 -a 1 -le 47677 2026-02-18T08:42:31.712 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:42:31.712 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:42:31.713 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:42:31.914 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_39 2026-02-18T08:42:31.914 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:42:31.914 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:42:31.914 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_39 2026-02-18T08:42:31.915 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:42:31.915 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_39 ']' 2026-02-18T08:42:31.915 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_39' 2026-02-18T08:42:31.915 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_39 2026-02-18T08:42:31.915 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:42:31.915 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:42:31.915 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:42:31.915 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:42:31.915 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:42:31.915 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:42:31.915 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:42:31.916 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:42:31.918 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:42:31.918 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:42:31.918 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_39 2026-02-18T08:42:31.918 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:42:31.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:42:31.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:42:31.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:42:31.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:42:31.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:42:31.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:42:31.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:42:31.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:42:31.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:42:31.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:42:31.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:42:31.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:42:31.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:42:31.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:42:31.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:42:31.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:42:31.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:42:31.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:42:31.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:42:31.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:42:31.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:42:31.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:42:31.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:42:31.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:42:31.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:42:31.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:42:31.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:42:31.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:42:31.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:42:31.928 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:42:31.928 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:42:31.928 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:42:31.928 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_39 2026-02-18T08:42:31.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_39'] 2026-02-18T08:42:31.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:42:31.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:42:31.982 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:42:31.982 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_39 2026-02-18T08:42:31.982 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:42:31.982 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:42:31.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:42:31.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:42:31.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:42:31.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:42:31.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:42:31.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:42:31.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:42:31.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:42:31.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:42:31.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:42:31.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:42:31.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:42:31.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:42:31.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:42:31.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:42:31.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:42:31.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:42:31.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:42:31.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:42:31.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:42:31.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:42:31.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:42:31.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:42:31.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:42:31.985 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:42:31.985 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:42:31.985 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:42:31.985 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:42:31.993 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:42:31.994 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:42:31.994 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:42:31.994 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_39 2026-02-18T08:42:32.060 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:42:32.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_39 2026-02-18T08:42:32.062 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:42:32.062 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:42:32.062 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_39 2026-02-18T08:42:32.062 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_39 '' 2026-02-18T08:42:32.062 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:42:32.062 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:42:32.062 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_39 2026-02-18T08:42:32.062 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:42:32.062 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_39.status 2026-02-18T08:42:32.062 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_39.status 2026-02-18T08:42:32.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:42:32.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_39.status 2026-02-18T08:42:32.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_39 2026-02-18T08:42:32.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_39.status 2026-02-18T08:42:32.101 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 11 2026-02-18T08:42:32.101 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 11 2026-02-18T08:42:32.101 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:42:32.101 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=2, entry_tid=1], [object_number=44, tag_tid=2, entry_tid=0], [object_number=47, tag_tid=1, entry_tid=47675], [object_number=46, tag_tid=1, entry_tid=47674]]], state=connected] 2026-02-18T08:42:32.101 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=45, tag_tid=1, entry_tid=47677], [object_number=44, tag_tid=1, entry_tid=47676], [object_number=47, tag_tid=1, entry_tid=47675], [object_number=46, tag_tid=1, entry_tid=47674]]], state=connected] 2026-02-18T08:42:32.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_39.status 2026-02-18T08:42:32.106 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=45, tag_tid=2, entry_tid=1' 2026-02-18T08:42:32.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_39 2026-02-18T08:42:32.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:42:32.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:42:32.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_39 2026-02-18T08:42:32.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_39 '..*' 2026-02-18T08:42:32.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:42:32.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:42:32.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_39 2026-02-18T08:42:32.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:42:32.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_39.status 2026-02-18T08:42:32.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_39.status 2026-02-18T08:42:32.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:42:32.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_39.status 2026-02-18T08:42:32.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_39 2026-02-18T08:42:32.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_39.status 2026-02-18T08:42:32.138 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 11 2026-02-18T08:42:32.138 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 11 2026-02-18T08:42:32.138 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:42:32.138 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=2, entry_tid=1], [object_number=44, tag_tid=2, entry_tid=0], [object_number=47, tag_tid=1, entry_tid=47675], [object_number=46, tag_tid=1, entry_tid=47674]]], state=connected] 2026-02-18T08:42:32.138 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=45, tag_tid=1, entry_tid=47677], [object_number=44, tag_tid=1, entry_tid=47676], [object_number=47, tag_tid=1, entry_tid=47675], [object_number=46, tag_tid=1, entry_tid=47674]]], state=connected] 2026-02-18T08:42:32.140 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_39.status 2026-02-18T08:42:32.142 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=45, tag_tid=1, entry_tid=47677' 2026-02-18T08:42:32.142 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=45, tag_tid=2, entry_tid=1' -a 'object_number=45, tag_tid=2, entry_tid=1' = 'object_number=45, tag_tid=1, entry_tid=47677' 2026-02-18T08:42:32.142 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=45, tag_tid=1, entry_tid=47677' '!=' 'object_number=45, tag_tid=1, entry_tid=47677' 2026-02-18T08:42:32.142 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:42:32.142 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.4 2026-02-18T08:42:32.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_39 2026-02-18T08:42:32.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:42:32.545 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:42:32.545 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_39 2026-02-18T08:42:32.545 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:42:32.545 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_39 ']' 2026-02-18T08:42:32.545 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_39' 2026-02-18T08:42:32.545 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_39 2026-02-18T08:42:32.545 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:42:32.545 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:42:32.545 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:42:32.545 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:42:32.545 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:42:32.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:42:32.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:42:32.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:42:32.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:42:32.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:42:32.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_39 2026-02-18T08:42:32.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:42:32.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:42:32.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:42:32.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:42:32.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:42:32.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:42:32.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:42:32.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:42:32.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:42:32.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:42:32.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:42:32.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:42:32.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:42:32.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:42:32.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:42:32.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:42:32.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:42:32.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:42:32.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:42:32.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:42:32.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:42:32.551 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:42:32.551 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:42:32.551 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:42:32.551 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:42:32.551 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:42:32.551 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:42:32.551 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:42:32.551 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:42:32.551 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:42:32.559 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:42:32.559 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:42:32.559 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:42:32.559 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_39 2026-02-18T08:42:32.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_39'] 2026-02-18T08:42:32.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:42:32.622 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:42:32.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:42:32.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_39 2026-02-18T08:42:32.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:42:32.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:42:32.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:42:32.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:42:32.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:42:32.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:42:32.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:42:32.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:42:32.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:42:32.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:42:32.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:42:32.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:42:32.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:42:32.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:42:32.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:42:32.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:42:32.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:42:32.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:42:32.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:42:32.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:42:32.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:42:32.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:42:32.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:42:32.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:42:32.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:42:32.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:42:32.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:42:32.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:42:32.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:42:32.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:42:32.635 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:42:32.635 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:42:32.636 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:42:32.636 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_39 2026-02-18T08:42:32.696 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:42:32.697 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_39 2026-02-18T08:42:32.697 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:42:32.697 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:42:32.697 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_39 2026-02-18T08:42:32.697 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_39 '' 2026-02-18T08:42:32.697 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:42:32.697 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:42:32.697 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_39 2026-02-18T08:42:32.697 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:42:32.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_39.status 2026-02-18T08:42:32.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_39.status 2026-02-18T08:42:32.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:42:32.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_39.status 2026-02-18T08:42:32.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_39 2026-02-18T08:42:32.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_39.status 2026-02-18T08:42:32.725 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 11 2026-02-18T08:42:32.725 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 11 2026-02-18T08:42:32.725 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:42:32.725 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=2, entry_tid=1], [object_number=44, tag_tid=2, entry_tid=0], [object_number=47, tag_tid=1, entry_tid=47675], [object_number=46, tag_tid=1, entry_tid=47674]]], state=connected] 2026-02-18T08:42:32.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=45, tag_tid=1, entry_tid=47677], [object_number=44, tag_tid=1, entry_tid=47676], [object_number=47, tag_tid=1, entry_tid=47675], [object_number=46, tag_tid=1, entry_tid=47674]]], state=connected] 2026-02-18T08:42:32.728 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_39.status 2026-02-18T08:42:32.729 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=45, tag_tid=2, entry_tid=1' 2026-02-18T08:42:32.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_39 2026-02-18T08:42:32.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:42:32.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:42:32.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_39 2026-02-18T08:42:32.731 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_39 '..*' 2026-02-18T08:42:32.731 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:42:32.731 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:42:32.731 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_39 2026-02-18T08:42:32.731 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:42:32.731 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_39.status 2026-02-18T08:42:32.731 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_39.status 2026-02-18T08:42:32.732 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:42:32.732 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_39.status 2026-02-18T08:42:32.733 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_39 2026-02-18T08:42:32.733 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_39.status 2026-02-18T08:42:32.759 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 11 2026-02-18T08:42:32.759 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 11 2026-02-18T08:42:32.759 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:42:32.759 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=2, entry_tid=1], [object_number=44, tag_tid=2, entry_tid=0], [object_number=47, tag_tid=1, entry_tid=47675], [object_number=46, tag_tid=1, entry_tid=47674]]], state=connected] 2026-02-18T08:42:32.760 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=45, tag_tid=1, entry_tid=47677], [object_number=44, tag_tid=1, entry_tid=47676], [object_number=47, tag_tid=1, entry_tid=47675], [object_number=46, tag_tid=1, entry_tid=47674]]], state=connected] 2026-02-18T08:42:32.762 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_39.status 2026-02-18T08:42:32.763 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=45, tag_tid=1, entry_tid=47677' 2026-02-18T08:42:32.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=45, tag_tid=2, entry_tid=1' -a 'object_number=45, tag_tid=2, entry_tid=1' = 'object_number=45, tag_tid=1, entry_tid=47677' 2026-02-18T08:42:32.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=45, tag_tid=1, entry_tid=47677' '!=' 'object_number=45, tag_tid=1, entry_tid=47677' 2026-02-18T08:42:32.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:42:32.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.8 2026-02-18T08:42:33.565 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_39 2026-02-18T08:42:33.566 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:42:33.566 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:42:33.566 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_39 2026-02-18T08:42:33.566 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:42:33.566 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_39 ']' 2026-02-18T08:42:33.566 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_39' 2026-02-18T08:42:33.566 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_39 2026-02-18T08:42:33.566 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:42:33.566 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:42:33.566 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:42:33.566 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:42:33.567 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:42:33.567 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:42:33.567 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:42:33.569 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:42:33.569 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:42:33.570 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:42:33.570 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_39 2026-02-18T08:42:33.570 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:42:33.570 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:42:33.570 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:42:33.570 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:42:33.570 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:42:33.570 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:42:33.570 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:42:33.570 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:42:33.570 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:42:33.571 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:42:33.571 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:42:33.571 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:42:33.571 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:42:33.571 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:42:33.571 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:42:33.571 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:42:33.571 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:42:33.571 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:42:33.571 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:42:33.571 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:42:33.571 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:42:33.572 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:42:33.572 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:42:33.572 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:42:33.572 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:42:33.572 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:42:33.572 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:42:33.572 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:42:33.572 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:42:33.572 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:42:33.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:42:33.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:42:33.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:42:33.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_39 2026-02-18T08:42:33.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_39'] 2026-02-18T08:42:33.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:42:33.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:42:33.637 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:42:33.637 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_39 2026-02-18T08:42:33.637 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:42:33.637 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:42:33.637 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:42:33.637 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:42:33.638 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:42:33.638 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:42:33.638 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:42:33.638 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:42:33.638 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:42:33.638 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:42:33.638 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:42:33.638 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:42:33.638 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:42:33.638 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:42:33.638 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:42:33.639 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:42:33.639 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:42:33.639 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:42:33.639 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:42:33.639 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:42:33.639 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:42:33.639 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:42:33.639 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:42:33.639 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:42:33.639 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:42:33.639 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:42:33.639 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:42:33.640 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:42:33.640 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:42:33.640 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:42:33.649 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:42:33.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:42:33.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:42:33.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_39 2026-02-18T08:42:33.707 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:42:33.708 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_39 2026-02-18T08:42:33.709 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:42:33.709 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:42:33.709 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_39 2026-02-18T08:42:33.709 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_39 '' 2026-02-18T08:42:33.709 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:42:33.709 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:42:33.709 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_39 2026-02-18T08:42:33.709 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:42:33.709 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_39.status 2026-02-18T08:42:33.709 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_39.status 2026-02-18T08:42:33.709 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:42:33.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_39.status 2026-02-18T08:42:33.711 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_39 2026-02-18T08:42:33.711 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_39.status 2026-02-18T08:42:33.745 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 11 2026-02-18T08:42:33.745 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 11 2026-02-18T08:42:33.745 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:42:33.745 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=2, entry_tid=1], [object_number=44, tag_tid=2, entry_tid=0], [object_number=47, tag_tid=1, entry_tid=47675], [object_number=46, tag_tid=1, entry_tid=47674]]], state=connected] 2026-02-18T08:42:33.746 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=45, tag_tid=1, entry_tid=47677], [object_number=44, tag_tid=1, entry_tid=47676], [object_number=47, tag_tid=1, entry_tid=47675], [object_number=46, tag_tid=1, entry_tid=47674]]], state=connected] 2026-02-18T08:42:33.749 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_39.status 2026-02-18T08:42:33.751 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=45, tag_tid=2, entry_tid=1' 2026-02-18T08:42:33.752 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_39 2026-02-18T08:42:33.752 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:42:33.752 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:42:33.752 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_39 2026-02-18T08:42:33.752 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_39 '..*' 2026-02-18T08:42:33.752 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:42:33.752 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:42:33.752 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_39 2026-02-18T08:42:33.753 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:42:33.753 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_39.status 2026-02-18T08:42:33.753 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_39.status 2026-02-18T08:42:33.753 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:42:33.753 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_39.status 2026-02-18T08:42:33.754 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_39 2026-02-18T08:42:33.754 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_39.status 2026-02-18T08:42:33.786 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 11 2026-02-18T08:42:33.786 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 11 2026-02-18T08:42:33.786 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:42:33.786 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=2, entry_tid=1], [object_number=44, tag_tid=2, entry_tid=0], [object_number=47, tag_tid=1, entry_tid=47675], [object_number=46, tag_tid=1, entry_tid=47674]]], state=connected] 2026-02-18T08:42:33.786 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=45, tag_tid=1, entry_tid=47677], [object_number=44, tag_tid=1, entry_tid=47676], [object_number=47, tag_tid=1, entry_tid=47675], [object_number=46, tag_tid=1, entry_tid=47674]]], state=connected] 2026-02-18T08:42:33.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_39.status 2026-02-18T08:42:33.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=45, tag_tid=1, entry_tid=47677' 2026-02-18T08:42:33.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=45, tag_tid=2, entry_tid=1' -a 'object_number=45, tag_tid=2, entry_tid=1' = 'object_number=45, tag_tid=1, entry_tid=47677' 2026-02-18T08:42:33.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=45, tag_tid=1, entry_tid=47677' '!=' 'object_number=45, tag_tid=1, entry_tid=47677' 2026-02-18T08:42:33.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:42:33.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1.6 2026-02-18T08:42:35.394 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_39 2026-02-18T08:42:35.394 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:42:35.394 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:42:35.395 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_39 2026-02-18T08:42:35.395 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:42:35.395 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_39 ']' 2026-02-18T08:42:35.395 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_39' 2026-02-18T08:42:35.395 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_39 2026-02-18T08:42:35.395 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:42:35.395 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:42:35.395 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:42:35.395 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:42:35.395 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:42:35.399 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:42:35.399 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:42:35.399 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:42:35.399 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:42:35.399 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:42:35.399 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_39 2026-02-18T08:42:35.399 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:42:35.399 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:42:35.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:42:35.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:42:35.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:42:35.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:42:35.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:42:35.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:42:35.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:42:35.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:42:35.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:42:35.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:42:35.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:42:35.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:42:35.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:42:35.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:42:35.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:42:35.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:42:35.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:42:35.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:42:35.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:42:35.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:42:35.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:42:35.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:42:35.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:42:35.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:42:35.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:42:35.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:42:35.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:42:35.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:42:35.409 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:42:35.409 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:42:35.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:42:35.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_39 2026-02-18T08:42:35.462 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_39'] 2026-02-18T08:42:35.463 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:42:35.463 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:42:35.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:42:35.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_39 2026-02-18T08:42:35.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:42:35.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:42:35.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:42:35.467 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:42:35.467 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:42:35.467 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:42:35.467 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:42:35.467 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:42:35.467 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:42:35.467 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:42:35.467 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:42:35.467 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:42:35.468 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:42:35.468 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:42:35.468 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:42:35.468 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:42:35.468 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:42:35.468 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:42:35.468 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:42:35.468 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:42:35.468 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:42:35.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:42:35.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:42:35.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:42:35.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:42:35.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:42:35.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:42:35.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:42:35.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:42:35.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:42:35.478 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:42:35.478 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:42:35.478 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:42:35.478 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_39 2026-02-18T08:42:35.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:42:35.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_39 2026-02-18T08:42:35.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:42:35.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:42:35.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_39 2026-02-18T08:42:35.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_39 '' 2026-02-18T08:42:35.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:42:35.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:42:35.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_39 2026-02-18T08:42:35.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:42:35.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_39.status 2026-02-18T08:42:35.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_39.status 2026-02-18T08:42:35.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:42:35.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_39.status 2026-02-18T08:42:35.551 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_39 2026-02-18T08:42:35.551 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_39.status 2026-02-18T08:42:35.577 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 11 2026-02-18T08:42:35.578 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 11 2026-02-18T08:42:35.578 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:42:35.578 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=2, entry_tid=1], [object_number=44, tag_tid=2, entry_tid=0], [object_number=47, tag_tid=1, entry_tid=47675], [object_number=46, tag_tid=1, entry_tid=47674]]], state=connected] 2026-02-18T08:42:35.578 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=45, tag_tid=1, entry_tid=47677], [object_number=44, tag_tid=1, entry_tid=47676], [object_number=47, tag_tid=1, entry_tid=47675], [object_number=46, tag_tid=1, entry_tid=47674]]], state=connected] 2026-02-18T08:42:35.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_39.status 2026-02-18T08:42:35.582 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=45, tag_tid=2, entry_tid=1' 2026-02-18T08:42:35.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_39 2026-02-18T08:42:35.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:42:35.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:42:35.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_39 2026-02-18T08:42:35.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_39 '..*' 2026-02-18T08:42:35.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:42:35.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:42:35.584 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_39 2026-02-18T08:42:35.584 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:42:35.584 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_39.status 2026-02-18T08:42:35.584 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_39.status 2026-02-18T08:42:35.584 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:42:35.585 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_39.status 2026-02-18T08:42:35.585 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_39 2026-02-18T08:42:35.585 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_39.status 2026-02-18T08:42:35.619 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 11 2026-02-18T08:42:35.619 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 11 2026-02-18T08:42:35.619 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:42:35.619 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=2, entry_tid=1], [object_number=44, tag_tid=2, entry_tid=0], [object_number=47, tag_tid=1, entry_tid=47675], [object_number=46, tag_tid=1, entry_tid=47674]]], state=connected] 2026-02-18T08:42:35.619 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=45, tag_tid=1, entry_tid=47677], [object_number=44, tag_tid=1, entry_tid=47676], [object_number=47, tag_tid=1, entry_tid=47675], [object_number=46, tag_tid=1, entry_tid=47674]]], state=connected] 2026-02-18T08:42:35.622 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_39.status 2026-02-18T08:42:35.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=45, tag_tid=1, entry_tid=47677' 2026-02-18T08:42:35.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=45, tag_tid=2, entry_tid=1' -a 'object_number=45, tag_tid=2, entry_tid=1' = 'object_number=45, tag_tid=1, entry_tid=47677' 2026-02-18T08:42:35.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=45, tag_tid=1, entry_tid=47677' '!=' 'object_number=45, tag_tid=1, entry_tid=47677' 2026-02-18T08:42:35.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:42:35.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 2 2026-02-18T08:42:37.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_39 2026-02-18T08:42:37.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:42:37.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:42:37.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_39 2026-02-18T08:42:37.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:42:37.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_39 ']' 2026-02-18T08:42:37.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_39' 2026-02-18T08:42:37.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_39 2026-02-18T08:42:37.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:42:37.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:42:37.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:42:37.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:42:37.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:42:37.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:42:37.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:42:37.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:42:37.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:42:37.628 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:42:37.628 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_39 2026-02-18T08:42:37.628 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:42:37.628 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:42:37.628 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:42:37.628 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:42:37.628 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:42:37.628 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:42:37.628 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:42:37.628 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:42:37.628 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:42:37.628 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:42:37.628 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:42:37.628 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:42:37.628 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:42:37.628 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:42:37.628 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:42:37.628 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:42:37.628 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:42:37.628 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:42:37.628 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:42:37.628 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:42:37.629 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:42:37.629 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:42:37.629 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:42:37.629 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:42:37.629 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:42:37.629 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:42:37.629 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:42:37.629 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:42:37.629 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:42:37.629 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:42:37.641 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:42:37.641 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:42:37.642 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:42:37.642 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_39 2026-02-18T08:42:37.705 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_39'] 2026-02-18T08:42:37.705 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:42:37.705 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:42:37.708 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:42:37.708 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_39 2026-02-18T08:42:37.708 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:42:37.708 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:42:37.708 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:42:37.708 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:42:37.708 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:42:37.708 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:42:37.709 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:42:37.709 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:42:37.709 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:42:37.709 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:42:37.709 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:42:37.709 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:42:37.709 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:42:37.709 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:42:37.709 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:42:37.709 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:42:37.709 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:42:37.709 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:42:37.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:42:37.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:42:37.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:42:37.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:42:37.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:42:37.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:42:37.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:42:37.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:42:37.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:42:37.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:42:37.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:42:37.711 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:42:37.719 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:42:37.719 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:42:37.719 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:42:37.720 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_39 2026-02-18T08:42:37.777 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:42:37.778 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_39 2026-02-18T08:42:37.778 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:42:37.778 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:42:37.778 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_39 2026-02-18T08:42:37.778 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_39 '' 2026-02-18T08:42:37.779 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:42:37.779 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:42:37.779 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_39 2026-02-18T08:42:37.779 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:42:37.779 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_39.status 2026-02-18T08:42:37.779 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_39.status 2026-02-18T08:42:37.779 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:42:37.780 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_39.status 2026-02-18T08:42:37.780 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_39 2026-02-18T08:42:37.780 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_39.status 2026-02-18T08:42:37.807 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 11 2026-02-18T08:42:37.807 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 11 2026-02-18T08:42:37.807 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:42:37.807 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=2, entry_tid=1], [object_number=44, tag_tid=2, entry_tid=0], [object_number=47, tag_tid=1, entry_tid=47675], [object_number=46, tag_tid=1, entry_tid=47674]]], state=connected] 2026-02-18T08:42:37.807 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=45, tag_tid=1, entry_tid=47677], [object_number=44, tag_tid=1, entry_tid=47676], [object_number=47, tag_tid=1, entry_tid=47675], [object_number=46, tag_tid=1, entry_tid=47674]]], state=connected] 2026-02-18T08:42:37.809 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_39.status 2026-02-18T08:42:37.811 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=45, tag_tid=2, entry_tid=1' 2026-02-18T08:42:37.812 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_39 2026-02-18T08:42:37.812 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:42:37.812 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:42:37.812 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_39 2026-02-18T08:42:37.813 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_39 '..*' 2026-02-18T08:42:37.813 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:42:37.813 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:42:37.813 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_39 2026-02-18T08:42:37.813 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:42:37.813 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_39.status 2026-02-18T08:42:37.813 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_39.status 2026-02-18T08:42:37.814 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:42:37.814 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_39.status 2026-02-18T08:42:37.814 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_39 2026-02-18T08:42:37.815 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_39.status 2026-02-18T08:42:37.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 11 2026-02-18T08:42:37.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 11 2026-02-18T08:42:37.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:42:37.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=2, entry_tid=1], [object_number=44, tag_tid=2, entry_tid=0], [object_number=47, tag_tid=1, entry_tid=47675], [object_number=46, tag_tid=1, entry_tid=47674]]], state=connected] 2026-02-18T08:42:37.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=45, tag_tid=1, entry_tid=47677], [object_number=44, tag_tid=1, entry_tid=47676], [object_number=47, tag_tid=1, entry_tid=47675], [object_number=46, tag_tid=1, entry_tid=47674]]], state=connected] 2026-02-18T08:42:37.853 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_39.status 2026-02-18T08:42:37.855 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=45, tag_tid=1, entry_tid=47677' 2026-02-18T08:42:37.855 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=45, tag_tid=2, entry_tid=1' -a 'object_number=45, tag_tid=2, entry_tid=1' = 'object_number=45, tag_tid=1, entry_tid=47677' 2026-02-18T08:42:37.855 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=45, tag_tid=1, entry_tid=47677' '!=' 'object_number=45, tag_tid=1, entry_tid=47677' 2026-02-18T08:42:37.855 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:42:37.855 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 2 2026-02-18T08:42:39.857 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_39 2026-02-18T08:42:39.857 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:42:39.857 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:42:39.857 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_39 2026-02-18T08:42:39.857 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:42:39.857 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_39 ']' 2026-02-18T08:42:39.857 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_39' 2026-02-18T08:42:39.858 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_39 2026-02-18T08:42:39.858 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:42:39.858 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:42:39.858 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:42:39.858 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:42:39.858 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:42:39.858 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:42:39.858 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:42:39.861 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:42:39.861 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:42:39.861 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:42:39.861 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_39 2026-02-18T08:42:39.861 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:42:39.861 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:42:39.861 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:42:39.861 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:42:39.862 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:42:39.862 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:42:39.862 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:42:39.862 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:42:39.862 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:42:39.862 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:42:39.862 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:42:39.862 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:42:39.862 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:42:39.862 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:42:39.863 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:42:39.863 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:42:39.863 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:42:39.863 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:42:39.863 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:42:39.863 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:42:39.863 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:42:39.863 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:42:39.863 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:42:39.863 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:42:39.863 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:42:39.863 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:42:39.864 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:42:39.864 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:42:39.864 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:42:39.864 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:42:39.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:42:39.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:42:39.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:42:39.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_39 2026-02-18T08:42:39.923 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_39'] 2026-02-18T08:42:39.923 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:42:39.923 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:42:39.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:42:39.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_39 2026-02-18T08:42:39.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:42:39.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:42:39.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:42:39.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:42:39.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:42:39.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:42:39.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:42:39.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:42:39.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:42:39.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:42:39.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:42:39.928 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:42:39.928 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:42:39.928 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:42:39.928 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:42:39.928 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:42:39.928 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:42:39.928 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:42:39.928 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:42:39.928 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:42:39.928 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:42:39.928 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:42:39.929 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:42:39.929 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:42:39.929 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:42:39.929 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:42:39.929 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:42:39.929 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:42:39.929 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:42:39.929 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:42:39.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:42:39.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:42:39.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:42:39.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_39 2026-02-18T08:42:40.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:42:40.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_39 2026-02-18T08:42:40.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:42:40.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:42:40.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_39 2026-02-18T08:42:40.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_39 '' 2026-02-18T08:42:40.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:42:40.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:42:40.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_39 2026-02-18T08:42:40.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:42:40.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_39.status 2026-02-18T08:42:40.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_39.status 2026-02-18T08:42:40.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:42:40.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_39.status 2026-02-18T08:42:40.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_39 2026-02-18T08:42:40.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_39.status 2026-02-18T08:42:40.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 11 2026-02-18T08:42:40.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 11 2026-02-18T08:42:40.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:42:40.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=2, entry_tid=1], [object_number=44, tag_tid=2, entry_tid=0], [object_number=47, tag_tid=1, entry_tid=47675], [object_number=46, tag_tid=1, entry_tid=47674]]], state=connected] 2026-02-18T08:42:40.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=45, tag_tid=2, entry_tid=1], [object_number=44, tag_tid=2, entry_tid=0], [object_number=47, tag_tid=1, entry_tid=47675], [object_number=46, tag_tid=1, entry_tid=47674]]], state=connected] 2026-02-18T08:42:40.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_39.status 2026-02-18T08:42:40.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=45, tag_tid=2, entry_tid=1' 2026-02-18T08:42:40.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_39 2026-02-18T08:42:40.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:42:40.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:42:40.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_39 2026-02-18T08:42:40.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_39 '..*' 2026-02-18T08:42:40.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:42:40.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:42:40.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_39 2026-02-18T08:42:40.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:42:40.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_39.status 2026-02-18T08:42:40.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_39.status 2026-02-18T08:42:40.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:42:40.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_39.status 2026-02-18T08:42:40.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_39 2026-02-18T08:42:40.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_39.status 2026-02-18T08:42:40.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 11 2026-02-18T08:42:40.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 11 2026-02-18T08:42:40.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:42:40.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=2, entry_tid=1], [object_number=44, tag_tid=2, entry_tid=0], [object_number=47, tag_tid=1, entry_tid=47675], [object_number=46, tag_tid=1, entry_tid=47674]]], state=connected] 2026-02-18T08:42:40.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=45, tag_tid=2, entry_tid=1], [object_number=44, tag_tid=2, entry_tid=0], [object_number=47, tag_tid=1, entry_tid=47675], [object_number=46, tag_tid=1, entry_tid=47674]]], state=connected] 2026-02-18T08:42:40.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_39.status 2026-02-18T08:42:40.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=45, tag_tid=2, entry_tid=1' 2026-02-18T08:42:40.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=45, tag_tid=2, entry_tid=1' -a 'object_number=45, tag_tid=2, entry_tid=1' = 'object_number=45, tag_tid=2, entry_tid=1' 2026-02-18T08:42:40.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:42:40.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_snap_present cluster1 mirror image_39 snap 2026-02-18T08:42:40.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:42:40.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:42:40.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_39 2026-02-18T08:42:40.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap 2026-02-18T08:42:40.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-18T08:42:40.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-18T08:42:40.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1 2026-02-18T08:42:41.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster1 info mirror/image_39@snap 2026-02-18T08:42:41.113 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:rbd image 'image_39': 2026-02-18T08:42:41.113 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: size 128 MiB in 32 objects 2026-02-18T08:42:41.113 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: order 22 (4 MiB objects) 2026-02-18T08:42:41.113 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: snapshot_count: 1 2026-02-18T08:42:41.113 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: id: 11f9bbdba1c3 2026-02-18T08:42:41.114 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: block_name_prefix: rbd_data.11f9bbdba1c3 2026-02-18T08:42:41.114 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: format: 2 2026-02-18T08:42:41.114 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: features: layering, exclusive-lock, journaling 2026-02-18T08:42:41.114 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: op_features: 2026-02-18T08:42:41.114 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: flags: 2026-02-18T08:42:41.114 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: create_timestamp: Wed Feb 18 08:32:20 2026 2026-02-18T08:42:41.114 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: access_timestamp: Wed Feb 18 08:32:20 2026 2026-02-18T08:42:41.114 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: modify_timestamp: Wed Feb 18 08:33:22 2026 2026-02-18T08:42:41.114 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: protected: False 2026-02-18T08:42:41.114 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: journal: 11f9bbdba1c3 2026-02-18T08:42:41.114 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring state: enabled 2026-02-18T08:42:41.114 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring mode: journal 2026-02-18T08:42:41.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring global id: 15369bca-1ff7-49f1-95a3-8f61c6942a0b 2026-02-18T08:42:41.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring primary: false 2026-02-18T08:42:41.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:42:41.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ compare_image_snaps mirror image_39 snap 2026-02-18T08:42:41.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:42:41.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_39 2026-02-18T08:42:41.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap 2026-02-18T08:42:41.118 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local ret=0 2026-02-18T08:42:41.118 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local rmt_export=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_39.export 2026-02-18T08:42:41.118 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loc_export=/tmp/tmp.vybh1m9yJF/cluster1-mirror-image_39.export 2026-02-18T08:42:41.118 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_39.export /tmp/tmp.vybh1m9yJF/cluster1-mirror-image_39.export 2026-02-18T08:42:41.118 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 -p mirror export image_39@snap /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_39.export 2026-02-18T08:42:41.276 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-18T08:42:41.288 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster1 -p mirror export image_39@snap /tmp/tmp.vybh1m9yJF/cluster1-mirror-image_39.export 2026-02-18T08:42:41.474 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-18T08:42:41.484 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmp /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_39.export /tmp/tmp.vybh1m9yJF/cluster1-mirror-image_39.export 2026-02-18T08:42:41.540 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_39.export /tmp/tmp.vybh1m9yJF/cluster1-mirror-image_39.export 2026-02-18T08:42:41.586 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:42:41.586 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T08:42:41.586 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_40 2026-02-18T08:42:41.586 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_snap cluster2 mirror image_40 snap 2026-02-18T08:42:41.586 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:42:41.586 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:42:41.586 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_40 2026-02-18T08:42:41.587 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap 2026-02-18T08:42:41.587 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 -p mirror snap create image_40@snap --debug-rbd=20 --debug-journaler=20 2026-02-18T08:42:42.509 INFO:tasks.ceph.cluster2.osd.2.trial163.stderr:2026-02-18T08:42:42.501+0000 7f2b60c87640 -1 reset not still connected to 0x55fc13917c70 2026-02-18T08:42:44.106 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_replay_started cluster1 mirror image_40 2026-02-18T08:42:44.106 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:42:44.106 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:42:44.106 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_40 2026-02-18T08:42:44.106 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_replay_state cluster1 mirror image_40 started 2026-02-18T08:42:44.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:42:44.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:42:44.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_40 2026-02-18T08:42:44.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local state=started 2026-02-18T08:42:44.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-18T08:42:44.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:42:44.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1 2026-02-18T08:42:45.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test_image_replay_state cluster1 mirror image_40 started 2026-02-18T08:42:45.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:42:45.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:42:45.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_40 2026-02-18T08:42:45.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local test_state=started 2026-02-18T08:42:45.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local status_result 2026-02-18T08:42:45.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_state=stopped 2026-02-18T08:42:45.110 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_40 2026-02-18T08:42:45.110 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -i state 2026-02-18T08:42:45.110 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_instance=cluster1 2026-02-18T08:42:45.110 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-18T08:42:45.110 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:42:45.110 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance=cluster1 2026-02-18T08:42:45.110 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local loop_instance 2026-02-18T08:42:45.110 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:42:45.111 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sleep 0 2026-02-18T08:42:45.111 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:42:45.113 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ seq 0 3 2026-02-18T08:42:45.113 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:42:45.114 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_40 2026-02-18T08:42:45.114 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:0 2026-02-18T08:42:45.114 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-18T08:42:45.114 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:42:45.114 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:42:45.114 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:42:45.114 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-18T08:42:45.114 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:42:45.114 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:42:45.114 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:42:45.114 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:42:45.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-18T08:42:45.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:42:45.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:42:45.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:42:45.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:42:45.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:0 2026-02-18T08:42:45.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-18T08:42:45.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-18T08:42:45.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:42:45.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:0 2026-02-18T08:42:45.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T08:42:45.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-18T08:42:45.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:42:45.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:42:45.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-18T08:42:45.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-18T08:42:45.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-18T08:42:45.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=0 2026-02-18T08:42:45.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=0 2026-02-18T08:42:45.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:42:45.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:42:45.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:42:45.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:42:45.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_40 2026-02-18T08:42:45.187 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_40'] 2026-02-18T08:42:45.187 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status : get status for rbd mirror 2026-02-18T08:42:45.187 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:42:45.190 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:42:45.190 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_40 2026-02-18T08:42:45.190 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:1 2026-02-18T08:42:45.190 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-18T08:42:45.190 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:42:45.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:42:45.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:42:45.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-18T08:42:45.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:42:45.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:42:45.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:42:45.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:42:45.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-18T08:42:45.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:42:45.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:42:45.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:42:45.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:42:45.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:1 2026-02-18T08:42:45.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-18T08:42:45.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-18T08:42:45.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:42:45.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:1 2026-02-18T08:42:45.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T08:42:45.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-18T08:42:45.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:42:45.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:42:45.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-18T08:42:45.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-18T08:42:45.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-18T08:42:45.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=1 2026-02-18T08:42:45.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=1 2026-02-18T08:42:45.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:42:45.200 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:42:45.201 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:42:45.201 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:42:45.201 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_40 2026-02-18T08:42:45.256 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_40'] 2026-02-18T08:42:45.256 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status : get status for rbd mirror 2026-02-18T08:42:45.256 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:42:45.260 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:42:45.260 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_40 2026-02-18T08:42:45.261 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:2 2026-02-18T08:42:45.261 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-18T08:42:45.261 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:42:45.261 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:42:45.261 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:42:45.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-18T08:42:45.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:42:45.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:42:45.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:42:45.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:42:45.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-18T08:42:45.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:42:45.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:42:45.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:42:45.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:42:45.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:2 2026-02-18T08:42:45.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-18T08:42:45.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-18T08:42:45.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:42:45.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:2 2026-02-18T08:42:45.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T08:42:45.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-18T08:42:45.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:42:45.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:42:45.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-18T08:42:45.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-18T08:42:45.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-18T08:42:45.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=2 2026-02-18T08:42:45.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=2 2026-02-18T08:42:45.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:42:45.270 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:42:45.270 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:42:45.270 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:42:45.270 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_40 2026-02-18T08:42:45.322 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_40'] 2026-02-18T08:42:45.322 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status : get status for rbd mirror 2026-02-18T08:42:45.322 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:42:45.324 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:42:45.324 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/image_40 2026-02-18T08:42:45.324 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:3 2026-02-18T08:42:45.324 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-18T08:42:45.324 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:42:45.324 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:42:45.324 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:42:45.324 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-18T08:42:45.324 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:42:45.324 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:42:45.324 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:42:45.324 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:42:45.324 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-18T08:42:45.324 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:42:45.324 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:42:45.324 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:42:45.325 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:42:45.325 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:3 2026-02-18T08:42:45.325 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-18T08:42:45.325 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-18T08:42:45.325 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:42:45.325 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:3 2026-02-18T08:42:45.325 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T08:42:45.325 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-18T08:42:45.325 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:42:45.325 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:42:45.325 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-18T08:42:45.325 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-18T08:42:45.325 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-18T08:42:45.325 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=3 2026-02-18T08:42:45.325 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=3 2026-02-18T08:42:45.325 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:42:45.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:42:45.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:42:45.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:42:45.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/image_40 2026-02-18T08:42:45.396 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ return 0 2026-02-18T08:42:45.396 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ status_result=' "state": "Replaying"' 2026-02-18T08:42:45.397 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ echo ' "state": "Replaying"' 2026-02-18T08:42:45.397 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ grep -i Replaying 2026-02-18T08:42:45.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: "state": "Replaying" 2026-02-18T08:42:45.399 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ current_state=started 2026-02-18T08:42:45.399 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test started = started 2026-02-18T08:42:45.399 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:42:45.399 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror image_40 2026-02-18T08:42:45.399 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_cluster=cluster1 2026-02-18T08:42:45.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:42:45.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_pool=mirror 2026-02-18T08:42:45.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local remote_pool=mirror 2026-02-18T08:42:45.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_40 2026-02-18T08:42:45.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' journal = journal ']' 2026-02-18T08:42:45.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror mirror image_40 2026-02-18T08:42:45.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_cluster=cluster1 2026-02-18T08:42:45.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:42:45.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_pool=mirror 2026-02-18T08:42:45.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local remote_pool=mirror 2026-02-18T08:42:45.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_40 2026-02-18T08:42:45.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-02-18T08:42:45.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-02-18T08:42:45.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:42:45.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:42:45.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:42:45.601 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_40 2026-02-18T08:42:45.601 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:42:45.601 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:42:45.602 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_40 2026-02-18T08:42:45.602 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:42:45.602 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_40 ']' 2026-02-18T08:42:45.602 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_40' 2026-02-18T08:42:45.602 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_40 2026-02-18T08:42:45.602 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:42:45.603 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:42:45.603 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:42:45.603 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:42:45.603 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:42:45.603 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:42:45.603 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:42:45.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:42:45.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:42:45.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:42:45.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_40 2026-02-18T08:42:45.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:42:45.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:42:45.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:42:45.610 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:42:45.610 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:42:45.610 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:42:45.610 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:42:45.610 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:42:45.610 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:42:45.611 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:42:45.611 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:42:45.611 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:42:45.611 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:42:45.611 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:42:45.611 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:42:45.612 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:42:45.612 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:42:45.612 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:42:45.612 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:42:45.612 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:42:45.613 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:42:45.613 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:42:45.613 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:42:45.613 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:42:45.613 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:42:45.613 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:42:45.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:42:45.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:42:45.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:42:45.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:42:45.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:42:45.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:42:45.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:42:45.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_40 2026-02-18T08:42:45.675 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_40'] 2026-02-18T08:42:45.675 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:42:45.675 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:42:45.678 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:42:45.678 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_40 2026-02-18T08:42:45.678 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:42:45.678 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:42:45.678 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:42:45.678 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:42:45.678 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:42:45.679 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:42:45.679 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:42:45.679 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:42:45.679 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:42:45.679 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:42:45.679 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:42:45.679 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:42:45.679 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:42:45.679 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:42:45.679 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:42:45.680 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:42:45.680 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:42:45.680 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:42:45.680 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:42:45.680 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:42:45.680 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:42:45.680 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:42:45.680 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:42:45.680 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:42:45.680 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:42:45.681 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:42:45.681 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:42:45.681 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:42:45.681 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:42:45.681 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:42:45.688 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:42:45.688 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:42:45.688 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:42:45.688 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_40 2026-02-18T08:42:45.738 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_40'] 2026-02-18T08:42:45.739 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:42:45.739 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:42:45.741 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:42:45.741 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_40 2026-02-18T08:42:45.742 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:42:45.742 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:42:45.742 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:42:45.742 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:42:45.742 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:42:45.742 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:42:45.742 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:42:45.742 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:42:45.742 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:42:45.742 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:42:45.742 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:42:45.742 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:42:45.743 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:42:45.743 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:42:45.743 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:42:45.743 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:42:45.743 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:42:45.743 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:42:45.743 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:42:45.743 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:42:45.743 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:42:45.743 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:42:45.743 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:42:45.744 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:42:45.744 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:42:45.744 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:42:45.744 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:42:45.744 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:42:45.744 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:42:45.744 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:42:45.751 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:42:45.752 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:42:45.752 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:42:45.752 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_40 2026-02-18T08:42:45.810 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_40'] 2026-02-18T08:42:45.811 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:42:45.811 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:42:45.814 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:42:45.814 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/image_40 2026-02-18T08:42:45.814 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:42:45.814 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:42:45.814 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:42:45.814 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:42:45.814 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:42:45.814 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:42:45.814 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:42:45.814 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:42:45.815 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:42:45.815 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:42:45.815 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:42:45.815 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:42:45.815 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:42:45.815 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:42:45.815 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:42:45.815 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:42:45.815 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:42:45.815 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:42:45.815 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:42:45.815 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:42:45.815 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:42:45.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:42:45.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:42:45.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:42:45.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:42:45.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:42:45.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:42:45.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:42:45.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:42:45.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:42:45.825 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:42:45.826 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:42:45.826 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:42:45.826 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/image_40 2026-02-18T08:42:45.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:42:45.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_40 2026-02-18T08:42:45.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:42:45.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:42:45.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_40 2026-02-18T08:42:45.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_40 '' 2026-02-18T08:42:45.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:42:45.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:42:45.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_40 2026-02-18T08:42:45.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:42:45.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_40.status 2026-02-18T08:42:45.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_40.status 2026-02-18T08:42:45.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:42:45.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_40.status 2026-02-18T08:42:45.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_40 2026-02-18T08:42:45.894 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_40.status 2026-02-18T08:42:45.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 4 2026-02-18T08:42:45.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 4 2026-02-18T08:42:45.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:42:45.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=2, entry_tid=1], [object_number=16, tag_tid=2, entry_tid=0], [object_number=19, tag_tid=1, entry_tid=20383], [object_number=18, tag_tid=1, entry_tid=20382]]], state=connected] 2026-02-18T08:42:45.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=19, tag_tid=1, entry_tid=20383], [object_number=18, tag_tid=1, entry_tid=20382], [object_number=17, tag_tid=1, entry_tid=20381], [object_number=16, tag_tid=1, entry_tid=20380]]], state=connected] 2026-02-18T08:42:45.923 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_40.status 2026-02-18T08:42:45.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=17, tag_tid=2, entry_tid=1' 2026-02-18T08:42:45.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_40 2026-02-18T08:42:45.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:42:45.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:42:45.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_40 2026-02-18T08:42:45.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_40 '..*' 2026-02-18T08:42:45.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:42:45.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:42:45.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_40 2026-02-18T08:42:45.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:42:45.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_40.status 2026-02-18T08:42:45.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_40.status 2026-02-18T08:42:45.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:42:45.928 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_40.status 2026-02-18T08:42:45.928 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_40 2026-02-18T08:42:45.928 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_40.status 2026-02-18T08:42:45.964 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 4 2026-02-18T08:42:45.964 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 4 2026-02-18T08:42:45.964 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:42:45.965 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=2, entry_tid=1], [object_number=16, tag_tid=2, entry_tid=0], [object_number=19, tag_tid=1, entry_tid=20383], [object_number=18, tag_tid=1, entry_tid=20382]]], state=connected] 2026-02-18T08:42:45.965 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=19, tag_tid=1, entry_tid=20383], [object_number=18, tag_tid=1, entry_tid=20382], [object_number=17, tag_tid=1, entry_tid=20381], [object_number=16, tag_tid=1, entry_tid=20380]]], state=connected] 2026-02-18T08:42:45.968 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_40.status 2026-02-18T08:42:45.969 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=19, tag_tid=1, entry_tid=20383' 2026-02-18T08:42:45.970 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=17, tag_tid=2, entry_tid=1' -a 'object_number=17, tag_tid=2, entry_tid=1' = 'object_number=19, tag_tid=1, entry_tid=20383' 2026-02-18T08:42:45.970 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=19, tag_tid=1, entry_tid=20383' '!=' '' 2026-02-18T08:42:45.970 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:42:45.970 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=19, tag_tid=1, entry_tid=20383' = '' 2026-02-18T08:42:45.970 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=19, tag_tid=1, entry_tid=20383' 2026-02-18T08:42:45.971 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=17, tag_tid=2, entry_tid=1' 2026-02-18T08:42:45.971 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:42:45.971 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:42:45.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=2 2026-02-18T08:42:45.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=19, tag_tid=1, entry_tid=20383' 2026-02-18T08:42:45.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:42:45.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:42:45.974 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=1 2026-02-18T08:42:45.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=17, tag_tid=2, entry_tid=1' 2026-02-18T08:42:45.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:42:45.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:42:45.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:42:45.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=19, tag_tid=1, entry_tid=20383' 2026-02-18T08:42:45.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:42:45.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:42:45.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=20383 2026-02-18T08:42:45.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 2 = 1 -a 1 -le 20383 2026-02-18T08:42:45.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:42:45.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:42:45.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:42:46.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_40 2026-02-18T08:42:46.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:42:46.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:42:46.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_40 2026-02-18T08:42:46.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:42:46.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_40 ']' 2026-02-18T08:42:46.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_40' 2026-02-18T08:42:46.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_40 2026-02-18T08:42:46.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:42:46.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:42:46.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:42:46.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:42:46.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:42:46.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:42:46.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:42:46.183 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:42:46.183 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:42:46.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:42:46.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_40 2026-02-18T08:42:46.187 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:42:46.187 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:42:46.187 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:42:46.187 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:42:46.187 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:42:46.187 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:42:46.187 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:42:46.187 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:42:46.188 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:42:46.188 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:42:46.188 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:42:46.188 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:42:46.188 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:42:46.188 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:42:46.188 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:42:46.188 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:42:46.188 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:42:46.188 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:42:46.188 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:42:46.188 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:42:46.189 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:42:46.189 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:42:46.189 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:42:46.189 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:42:46.189 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:42:46.189 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:42:46.189 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:42:46.189 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:42:46.189 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:42:46.189 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:42:46.195 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:42:46.196 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:42:46.196 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:42:46.196 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_40 2026-02-18T08:42:46.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_40'] 2026-02-18T08:42:46.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:42:46.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:42:46.260 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:42:46.261 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_40 2026-02-18T08:42:46.261 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:42:46.261 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:42:46.261 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:42:46.261 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:42:46.261 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:42:46.261 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:42:46.261 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:42:46.261 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:42:46.261 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:42:46.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:42:46.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:42:46.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:42:46.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:42:46.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:42:46.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:42:46.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:42:46.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:42:46.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:42:46.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:42:46.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:42:46.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:42:46.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:42:46.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:42:46.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:42:46.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:42:46.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:42:46.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:42:46.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:42:46.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:42:46.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:42:46.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:42:46.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:42:46.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:42:46.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_40 2026-02-18T08:42:46.323 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_40'] 2026-02-18T08:42:46.323 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:42:46.323 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:42:46.326 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:42:46.326 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_40 2026-02-18T08:42:46.326 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:42:46.326 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:42:46.326 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:42:46.326 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:42:46.326 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:42:46.326 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:42:46.327 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:42:46.327 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:42:46.327 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:42:46.327 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:42:46.327 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:42:46.327 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:42:46.327 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:42:46.327 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:42:46.327 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:42:46.327 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:42:46.327 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:42:46.327 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:42:46.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:42:46.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:42:46.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:42:46.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:42:46.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:42:46.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:42:46.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:42:46.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:42:46.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:42:46.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:42:46.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:42:46.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:42:46.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:42:46.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:42:46.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:42:46.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_40 2026-02-18T08:42:46.389 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_40'] 2026-02-18T08:42:46.389 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:42:46.389 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:42:46.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:42:46.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/image_40 2026-02-18T08:42:46.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:42:46.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:42:46.393 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:42:46.393 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:42:46.393 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:42:46.393 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:42:46.393 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:42:46.393 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:42:46.393 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:42:46.393 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:42:46.393 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:42:46.393 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:42:46.394 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:42:46.394 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:42:46.394 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:42:46.394 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:42:46.394 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:42:46.394 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:42:46.394 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:42:46.394 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:42:46.394 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:42:46.394 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:42:46.395 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:42:46.395 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:42:46.395 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:42:46.395 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:42:46.395 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:42:46.395 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:42:46.395 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:42:46.395 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:42:46.403 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:42:46.403 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:42:46.403 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:42:46.403 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/image_40 2026-02-18T08:42:46.467 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:42:46.468 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_40 2026-02-18T08:42:46.468 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:42:46.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:42:46.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_40 2026-02-18T08:42:46.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_40 '' 2026-02-18T08:42:46.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:42:46.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:42:46.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_40 2026-02-18T08:42:46.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:42:46.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_40.status 2026-02-18T08:42:46.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_40.status 2026-02-18T08:42:46.470 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:42:46.470 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_40.status 2026-02-18T08:42:46.470 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_40 2026-02-18T08:42:46.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_40.status 2026-02-18T08:42:46.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 4 2026-02-18T08:42:46.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 4 2026-02-18T08:42:46.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:42:46.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=2, entry_tid=1], [object_number=16, tag_tid=2, entry_tid=0], [object_number=19, tag_tid=1, entry_tid=20383], [object_number=18, tag_tid=1, entry_tid=20382]]], state=connected] 2026-02-18T08:42:46.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=19, tag_tid=1, entry_tid=20383], [object_number=18, tag_tid=1, entry_tid=20382], [object_number=17, tag_tid=1, entry_tid=20381], [object_number=16, tag_tid=1, entry_tid=20380]]], state=connected] 2026-02-18T08:42:46.500 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_40.status 2026-02-18T08:42:46.502 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=17, tag_tid=2, entry_tid=1' 2026-02-18T08:42:46.504 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_40 2026-02-18T08:42:46.504 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:42:46.504 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:42:46.504 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_40 2026-02-18T08:42:46.504 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_40 '..*' 2026-02-18T08:42:46.504 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:42:46.504 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:42:46.504 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_40 2026-02-18T08:42:46.504 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:42:46.505 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_40.status 2026-02-18T08:42:46.505 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_40.status 2026-02-18T08:42:46.505 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:42:46.506 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_40.status 2026-02-18T08:42:46.506 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_40 2026-02-18T08:42:46.506 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_40.status 2026-02-18T08:42:46.531 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 4 2026-02-18T08:42:46.531 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 4 2026-02-18T08:42:46.531 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:42:46.532 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=2, entry_tid=1], [object_number=16, tag_tid=2, entry_tid=0], [object_number=19, tag_tid=1, entry_tid=20383], [object_number=18, tag_tid=1, entry_tid=20382]]], state=connected] 2026-02-18T08:42:46.532 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=19, tag_tid=1, entry_tid=20383], [object_number=18, tag_tid=1, entry_tid=20382], [object_number=17, tag_tid=1, entry_tid=20381], [object_number=16, tag_tid=1, entry_tid=20380]]], state=connected] 2026-02-18T08:42:46.534 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_40.status 2026-02-18T08:42:46.536 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=19, tag_tid=1, entry_tid=20383' 2026-02-18T08:42:46.536 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=17, tag_tid=2, entry_tid=1' -a 'object_number=17, tag_tid=2, entry_tid=1' = 'object_number=19, tag_tid=1, entry_tid=20383' 2026-02-18T08:42:46.536 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=19, tag_tid=1, entry_tid=20383' '!=' 'object_number=19, tag_tid=1, entry_tid=20383' 2026-02-18T08:42:46.536 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:42:46.536 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.4 2026-02-18T08:42:46.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_40 2026-02-18T08:42:46.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:42:46.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:42:46.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_40 2026-02-18T08:42:46.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:42:46.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_40 ']' 2026-02-18T08:42:46.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_40' 2026-02-18T08:42:46.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_40 2026-02-18T08:42:46.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:42:46.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:42:46.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:42:46.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:42:46.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:42:46.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:42:46.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:42:46.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:42:46.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:42:46.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:42:46.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_40 2026-02-18T08:42:46.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:42:46.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:42:46.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:42:46.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:42:46.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:42:46.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:42:46.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:42:46.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:42:46.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:42:46.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:42:46.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:42:46.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:42:46.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:42:46.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:42:46.944 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:42:46.944 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:42:46.944 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:42:46.944 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:42:46.944 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:42:46.944 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:42:46.944 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:42:46.944 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:42:46.944 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:42:46.944 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:42:46.944 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:42:46.944 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:42:46.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:42:46.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:42:46.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:42:46.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:42:46.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:42:46.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:42:46.952 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:42:46.952 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_40 2026-02-18T08:42:47.010 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_40'] 2026-02-18T08:42:47.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:42:47.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:42:47.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:42:47.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_40 2026-02-18T08:42:47.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:42:47.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:42:47.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:42:47.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:42:47.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:42:47.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:42:47.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:42:47.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:42:47.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:42:47.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:42:47.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:42:47.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:42:47.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:42:47.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:42:47.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:42:47.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:42:47.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:42:47.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:42:47.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:42:47.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:42:47.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:42:47.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:42:47.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:42:47.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:42:47.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:42:47.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:42:47.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:42:47.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:42:47.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:42:47.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:42:47.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:42:47.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:42:47.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:42:47.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_40 2026-02-18T08:42:47.080 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_40'] 2026-02-18T08:42:47.080 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:42:47.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:42:47.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:42:47.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_40 2026-02-18T08:42:47.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:42:47.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:42:47.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:42:47.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:42:47.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:42:47.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:42:47.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:42:47.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:42:47.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:42:47.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:42:47.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:42:47.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:42:47.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:42:47.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:42:47.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:42:47.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:42:47.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:42:47.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:42:47.086 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:42:47.086 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:42:47.086 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:42:47.086 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:42:47.086 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:42:47.086 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:42:47.086 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:42:47.086 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:42:47.086 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:42:47.086 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:42:47.086 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:42:47.086 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:42:47.094 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:42:47.094 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:42:47.094 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:42:47.095 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_40 2026-02-18T08:42:47.147 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_40'] 2026-02-18T08:42:47.148 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:42:47.148 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:42:47.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:42:47.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/image_40 2026-02-18T08:42:47.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:42:47.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:42:47.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:42:47.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:42:47.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:42:47.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:42:47.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:42:47.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:42:47.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:42:47.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:42:47.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:42:47.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:42:47.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:42:47.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:42:47.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:42:47.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:42:47.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:42:47.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:42:47.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:42:47.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:42:47.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:42:47.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:42:47.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:42:47.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:42:47.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:42:47.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:42:47.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:42:47.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:42:47.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:42:47.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:42:47.161 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:42:47.162 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:42:47.162 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:42:47.162 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/image_40 2026-02-18T08:42:47.219 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:42:47.220 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_40 2026-02-18T08:42:47.220 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:42:47.221 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:42:47.221 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_40 2026-02-18T08:42:47.221 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_40 '' 2026-02-18T08:42:47.221 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:42:47.221 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:42:47.221 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_40 2026-02-18T08:42:47.221 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:42:47.221 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_40.status 2026-02-18T08:42:47.221 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_40.status 2026-02-18T08:42:47.221 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:42:47.222 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_40.status 2026-02-18T08:42:47.222 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_40 2026-02-18T08:42:47.222 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_40.status 2026-02-18T08:42:47.249 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 4 2026-02-18T08:42:47.249 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 4 2026-02-18T08:42:47.249 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:42:47.249 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=2, entry_tid=1], [object_number=16, tag_tid=2, entry_tid=0], [object_number=19, tag_tid=1, entry_tid=20383], [object_number=18, tag_tid=1, entry_tid=20382]]], state=connected] 2026-02-18T08:42:47.249 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=19, tag_tid=1, entry_tid=20383], [object_number=18, tag_tid=1, entry_tid=20382], [object_number=17, tag_tid=1, entry_tid=20381], [object_number=16, tag_tid=1, entry_tid=20380]]], state=connected] 2026-02-18T08:42:47.251 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_40.status 2026-02-18T08:42:47.253 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=17, tag_tid=2, entry_tid=1' 2026-02-18T08:42:47.254 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_40 2026-02-18T08:42:47.254 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:42:47.254 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:42:47.254 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_40 2026-02-18T08:42:47.255 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_40 '..*' 2026-02-18T08:42:47.255 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:42:47.255 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:42:47.255 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_40 2026-02-18T08:42:47.255 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:42:47.255 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_40.status 2026-02-18T08:42:47.256 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_40.status 2026-02-18T08:42:47.256 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:42:47.256 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_40.status 2026-02-18T08:42:47.257 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_40 2026-02-18T08:42:47.257 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_40.status 2026-02-18T08:42:47.292 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 4 2026-02-18T08:42:47.292 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 4 2026-02-18T08:42:47.292 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:42:47.292 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=2, entry_tid=1], [object_number=16, tag_tid=2, entry_tid=0], [object_number=19, tag_tid=1, entry_tid=20383], [object_number=18, tag_tid=1, entry_tid=20382]]], state=connected] 2026-02-18T08:42:47.292 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=19, tag_tid=1, entry_tid=20383], [object_number=18, tag_tid=1, entry_tid=20382], [object_number=17, tag_tid=1, entry_tid=20381], [object_number=16, tag_tid=1, entry_tid=20380]]], state=connected] 2026-02-18T08:42:47.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_40.status 2026-02-18T08:42:47.297 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=19, tag_tid=1, entry_tid=20383' 2026-02-18T08:42:47.297 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=17, tag_tid=2, entry_tid=1' -a 'object_number=17, tag_tid=2, entry_tid=1' = 'object_number=19, tag_tid=1, entry_tid=20383' 2026-02-18T08:42:47.297 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=19, tag_tid=1, entry_tid=20383' '!=' 'object_number=19, tag_tid=1, entry_tid=20383' 2026-02-18T08:42:47.297 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:42:47.297 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.8 2026-02-18T08:42:48.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_40 2026-02-18T08:42:48.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:42:48.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:42:48.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_40 2026-02-18T08:42:48.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:42:48.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_40 ']' 2026-02-18T08:42:48.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_40' 2026-02-18T08:42:48.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_40 2026-02-18T08:42:48.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:42:48.100 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:42:48.100 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:42:48.100 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:42:48.100 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:42:48.100 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:42:48.100 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:42:48.100 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:42:48.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:42:48.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:42:48.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_40 2026-02-18T08:42:48.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:42:48.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:42:48.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:42:48.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:42:48.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:42:48.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:42:48.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:42:48.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:42:48.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:42:48.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:42:48.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:42:48.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:42:48.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:42:48.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:42:48.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:42:48.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:42:48.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:42:48.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:42:48.105 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:42:48.105 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:42:48.105 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:42:48.105 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:42:48.105 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:42:48.105 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:42:48.105 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:42:48.105 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:42:48.105 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:42:48.105 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:42:48.106 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:42:48.106 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:42:48.114 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:42:48.114 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:42:48.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:42:48.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_40 2026-02-18T08:42:48.165 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_40'] 2026-02-18T08:42:48.166 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:42:48.166 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:42:48.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:42:48.169 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_40 2026-02-18T08:42:48.169 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:42:48.169 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:42:48.169 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:42:48.169 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:42:48.169 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:42:48.169 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:42:48.169 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:42:48.169 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:42:48.169 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:42:48.169 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:42:48.170 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:42:48.170 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:42:48.170 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:42:48.170 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:42:48.170 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:42:48.170 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:42:48.170 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:42:48.170 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:42:48.170 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:42:48.171 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:42:48.171 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:42:48.171 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:42:48.171 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:42:48.171 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:42:48.171 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:42:48.171 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:42:48.171 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:42:48.171 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:42:48.171 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:42:48.171 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:42:48.179 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:42:48.179 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:42:48.179 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:42:48.179 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_40 2026-02-18T08:42:48.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_40'] 2026-02-18T08:42:48.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:42:48.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:42:48.244 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:42:48.245 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_40 2026-02-18T08:42:48.245 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:42:48.245 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:42:48.245 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:42:48.245 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:42:48.245 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:42:48.245 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:42:48.245 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:42:48.245 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:42:48.245 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:42:48.245 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:42:48.246 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:42:48.246 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:42:48.246 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:42:48.246 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:42:48.246 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:42:48.246 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:42:48.246 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:42:48.246 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:42:48.246 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:42:48.246 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:42:48.246 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:42:48.246 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:42:48.247 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:42:48.247 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:42:48.247 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:42:48.247 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:42:48.247 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:42:48.247 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:42:48.247 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:42:48.247 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:42:48.255 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:42:48.255 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:42:48.256 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:42:48.256 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_40 2026-02-18T08:42:48.315 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_40'] 2026-02-18T08:42:48.315 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:42:48.315 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:42:48.318 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:42:48.318 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/image_40 2026-02-18T08:42:48.318 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:42:48.319 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:42:48.319 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:42:48.319 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:42:48.319 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:42:48.319 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:42:48.319 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:42:48.319 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:42:48.319 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:42:48.319 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:42:48.319 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:42:48.320 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:42:48.320 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:42:48.320 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:42:48.320 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:42:48.320 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:42:48.320 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:42:48.320 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:42:48.320 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:42:48.320 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:42:48.320 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:42:48.320 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:42:48.320 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:42:48.321 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:42:48.321 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:42:48.321 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:42:48.321 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:42:48.321 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:42:48.321 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:42:48.321 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:42:48.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:42:48.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:42:48.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:42:48.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/image_40 2026-02-18T08:42:48.397 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:42:48.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_40 2026-02-18T08:42:48.399 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:42:48.399 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:42:48.399 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_40 2026-02-18T08:42:48.399 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_40 '' 2026-02-18T08:42:48.399 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:42:48.399 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:42:48.399 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_40 2026-02-18T08:42:48.399 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:42:48.399 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_40.status 2026-02-18T08:42:48.399 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_40.status 2026-02-18T08:42:48.399 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:42:48.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_40.status 2026-02-18T08:42:48.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_40 2026-02-18T08:42:48.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_40.status 2026-02-18T08:42:48.437 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 4 2026-02-18T08:42:48.437 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 4 2026-02-18T08:42:48.437 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:42:48.437 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=2, entry_tid=1], [object_number=16, tag_tid=2, entry_tid=0], [object_number=19, tag_tid=1, entry_tid=20383], [object_number=18, tag_tid=1, entry_tid=20382]]], state=connected] 2026-02-18T08:42:48.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=19, tag_tid=1, entry_tid=20383], [object_number=18, tag_tid=1, entry_tid=20382], [object_number=17, tag_tid=1, entry_tid=20381], [object_number=16, tag_tid=1, entry_tid=20380]]], state=connected] 2026-02-18T08:42:48.440 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_40.status 2026-02-18T08:42:48.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=17, tag_tid=2, entry_tid=1' 2026-02-18T08:42:48.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_40 2026-02-18T08:42:48.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:42:48.443 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:42:48.443 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_40 2026-02-18T08:42:48.443 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_40 '..*' 2026-02-18T08:42:48.443 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:42:48.443 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:42:48.443 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_40 2026-02-18T08:42:48.443 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:42:48.443 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_40.status 2026-02-18T08:42:48.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_40.status 2026-02-18T08:42:48.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:42:48.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_40.status 2026-02-18T08:42:48.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_40 2026-02-18T08:42:48.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_40.status 2026-02-18T08:42:48.472 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 4 2026-02-18T08:42:48.472 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 4 2026-02-18T08:42:48.472 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:42:48.472 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=2, entry_tid=1], [object_number=16, tag_tid=2, entry_tid=0], [object_number=19, tag_tid=1, entry_tid=20383], [object_number=18, tag_tid=1, entry_tid=20382]]], state=connected] 2026-02-18T08:42:48.472 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=19, tag_tid=1, entry_tid=20383], [object_number=18, tag_tid=1, entry_tid=20382], [object_number=17, tag_tid=1, entry_tid=20381], [object_number=16, tag_tid=1, entry_tid=20380]]], state=connected] 2026-02-18T08:42:48.474 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_40.status 2026-02-18T08:42:48.476 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=19, tag_tid=1, entry_tid=20383' 2026-02-18T08:42:48.476 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=17, tag_tid=2, entry_tid=1' -a 'object_number=17, tag_tid=2, entry_tid=1' = 'object_number=19, tag_tid=1, entry_tid=20383' 2026-02-18T08:42:48.476 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=19, tag_tid=1, entry_tid=20383' '!=' 'object_number=19, tag_tid=1, entry_tid=20383' 2026-02-18T08:42:48.476 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:42:48.477 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1.6 2026-02-18T08:42:50.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_40 2026-02-18T08:42:50.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:42:50.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:42:50.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_40 2026-02-18T08:42:50.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:42:50.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_40 ']' 2026-02-18T08:42:50.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_40' 2026-02-18T08:42:50.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_40 2026-02-18T08:42:50.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:42:50.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:42:50.080 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:42:50.080 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:42:50.080 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:42:50.080 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:42:50.080 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:42:50.080 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:42:50.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:42:50.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:42:50.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_40 2026-02-18T08:42:50.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:42:50.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:42:50.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:42:50.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:42:50.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:42:50.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:42:50.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:42:50.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:42:50.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:42:50.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:42:50.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:42:50.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:42:50.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:42:50.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:42:50.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:42:50.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:42:50.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:42:50.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:42:50.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:42:50.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:42:50.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:42:50.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:42:50.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:42:50.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:42:50.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:42:50.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:42:50.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:42:50.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:42:50.086 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:42:50.086 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:42:50.094 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:42:50.094 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:42:50.094 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:42:50.094 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_40 2026-02-18T08:42:50.145 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_40'] 2026-02-18T08:42:50.146 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:42:50.146 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:42:50.149 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:42:50.149 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_40 2026-02-18T08:42:50.149 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:42:50.149 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:42:50.149 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:42:50.149 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:42:50.149 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:42:50.149 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:42:50.149 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:42:50.150 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:42:50.150 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:42:50.150 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:42:50.150 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:42:50.150 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:42:50.150 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:42:50.150 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:42:50.150 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:42:50.150 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:42:50.150 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:42:50.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:42:50.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:42:50.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:42:50.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:42:50.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:42:50.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:42:50.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:42:50.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:42:50.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:42:50.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:42:50.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:42:50.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:42:50.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:42:50.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:42:50.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:42:50.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:42:50.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_40 2026-02-18T08:42:50.211 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_40'] 2026-02-18T08:42:50.211 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:42:50.211 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:42:50.214 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:42:50.214 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_40 2026-02-18T08:42:50.214 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:42:50.214 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:42:50.215 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:42:50.215 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:42:50.215 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:42:50.215 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:42:50.215 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:42:50.215 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:42:50.215 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:42:50.215 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:42:50.215 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:42:50.215 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:42:50.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:42:50.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:42:50.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:42:50.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:42:50.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:42:50.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:42:50.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:42:50.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:42:50.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:42:50.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:42:50.217 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:42:50.217 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:42:50.217 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:42:50.217 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:42:50.217 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:42:50.217 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:42:50.217 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:42:50.217 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:42:50.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:42:50.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:42:50.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:42:50.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_40 2026-02-18T08:42:50.276 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_40'] 2026-02-18T08:42:50.277 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:42:50.277 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:42:50.279 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:42:50.279 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/image_40 2026-02-18T08:42:50.279 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:42:50.280 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:42:50.280 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:42:50.280 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:42:50.280 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:42:50.280 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:42:50.280 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:42:50.280 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:42:50.280 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:42:50.280 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:42:50.280 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:42:50.280 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:42:50.280 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:42:50.281 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:42:50.281 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:42:50.281 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:42:50.281 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:42:50.281 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:42:50.281 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:42:50.281 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:42:50.281 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:42:50.281 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:42:50.281 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:42:50.281 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:42:50.281 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:42:50.282 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:42:50.282 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:42:50.282 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:42:50.282 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:42:50.282 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:42:50.290 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:42:50.290 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:42:50.291 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:42:50.291 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/image_40 2026-02-18T08:42:50.360 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:42:50.360 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_40 2026-02-18T08:42:50.360 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:42:50.360 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:42:50.360 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_40 2026-02-18T08:42:50.361 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_40 '' 2026-02-18T08:42:50.361 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:42:50.361 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:42:50.361 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_40 2026-02-18T08:42:50.361 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:42:50.361 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_40.status 2026-02-18T08:42:50.361 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_40.status 2026-02-18T08:42:50.362 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:42:50.362 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_40.status 2026-02-18T08:42:50.362 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_40 2026-02-18T08:42:50.363 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_40.status 2026-02-18T08:42:50.390 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 4 2026-02-18T08:42:50.390 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 4 2026-02-18T08:42:50.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:42:50.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=2, entry_tid=1], [object_number=16, tag_tid=2, entry_tid=0], [object_number=19, tag_tid=1, entry_tid=20383], [object_number=18, tag_tid=1, entry_tid=20382]]], state=connected] 2026-02-18T08:42:50.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=17, tag_tid=2, entry_tid=1], [object_number=16, tag_tid=2, entry_tid=0], [object_number=19, tag_tid=1, entry_tid=20383], [object_number=18, tag_tid=1, entry_tid=20382]]], state=connected] 2026-02-18T08:42:50.393 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_40.status 2026-02-18T08:42:50.394 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=17, tag_tid=2, entry_tid=1' 2026-02-18T08:42:50.396 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_40 2026-02-18T08:42:50.396 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:42:50.396 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:42:50.396 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_40 2026-02-18T08:42:50.396 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_40 '..*' 2026-02-18T08:42:50.397 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:42:50.397 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:42:50.397 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_40 2026-02-18T08:42:50.397 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:42:50.397 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_40.status 2026-02-18T08:42:50.397 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_40.status 2026-02-18T08:42:50.397 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:42:50.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_40.status 2026-02-18T08:42:50.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_40 2026-02-18T08:42:50.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_40.status 2026-02-18T08:42:50.425 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 4 2026-02-18T08:42:50.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 4 2026-02-18T08:42:50.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:42:50.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=2, entry_tid=1], [object_number=16, tag_tid=2, entry_tid=0], [object_number=19, tag_tid=1, entry_tid=20383], [object_number=18, tag_tid=1, entry_tid=20382]]], state=connected] 2026-02-18T08:42:50.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=17, tag_tid=2, entry_tid=1], [object_number=16, tag_tid=2, entry_tid=0], [object_number=19, tag_tid=1, entry_tid=20383], [object_number=18, tag_tid=1, entry_tid=20382]]], state=connected] 2026-02-18T08:42:50.428 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_40.status 2026-02-18T08:42:50.430 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=17, tag_tid=2, entry_tid=1' 2026-02-18T08:42:50.430 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=17, tag_tid=2, entry_tid=1' -a 'object_number=17, tag_tid=2, entry_tid=1' = 'object_number=17, tag_tid=2, entry_tid=1' 2026-02-18T08:42:50.430 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:42:50.430 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_snap_present cluster1 mirror image_40 snap 2026-02-18T08:42:50.430 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:42:50.430 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:42:50.431 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_40 2026-02-18T08:42:50.431 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap 2026-02-18T08:42:50.431 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-18T08:42:50.431 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-18T08:42:50.431 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1 2026-02-18T08:42:51.431 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster1 info mirror/image_40@snap 2026-02-18T08:42:51.458 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:rbd image 'image_40': 2026-02-18T08:42:51.458 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: size 128 MiB in 32 objects 2026-02-18T08:42:51.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: order 22 (4 MiB objects) 2026-02-18T08:42:51.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: snapshot_count: 1 2026-02-18T08:42:51.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: id: 120d1645510c 2026-02-18T08:42:51.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: block_name_prefix: rbd_data.120d1645510c 2026-02-18T08:42:51.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: format: 2 2026-02-18T08:42:51.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: features: layering, exclusive-lock, journaling 2026-02-18T08:42:51.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: op_features: 2026-02-18T08:42:51.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: flags: 2026-02-18T08:42:51.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: create_timestamp: Wed Feb 18 08:32:34 2026 2026-02-18T08:42:51.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: access_timestamp: Wed Feb 18 08:32:34 2026 2026-02-18T08:42:51.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: modify_timestamp: Wed Feb 18 08:32:34 2026 2026-02-18T08:42:51.460 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: protected: False 2026-02-18T08:42:51.460 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: journal: 120d1645510c 2026-02-18T08:42:51.460 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring state: enabled 2026-02-18T08:42:51.460 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring mode: journal 2026-02-18T08:42:51.460 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring global id: 9d0922d7-afc0-4eb6-a47c-ea3804570c8f 2026-02-18T08:42:51.460 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring primary: false 2026-02-18T08:42:51.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:42:51.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ compare_image_snaps mirror image_40 snap 2026-02-18T08:42:51.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:42:51.462 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_40 2026-02-18T08:42:51.462 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap 2026-02-18T08:42:51.462 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local ret=0 2026-02-18T08:42:51.462 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local rmt_export=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_40.export 2026-02-18T08:42:51.462 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loc_export=/tmp/tmp.vybh1m9yJF/cluster1-mirror-image_40.export 2026-02-18T08:42:51.462 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_40.export /tmp/tmp.vybh1m9yJF/cluster1-mirror-image_40.export 2026-02-18T08:42:51.462 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 -p mirror export image_40@snap /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_40.export 2026-02-18T08:42:51.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-18T08:42:51.619 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster1 -p mirror export image_40@snap /tmp/tmp.vybh1m9yJF/cluster1-mirror-image_40.export 2026-02-18T08:42:51.783 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-18T08:42:51.793 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmp /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_40.export /tmp/tmp.vybh1m9yJF/cluster1-mirror-image_40.export 2026-02-18T08:42:51.848 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_40.export /tmp/tmp.vybh1m9yJF/cluster1-mirror-image_40.export 2026-02-18T08:42:51.882 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:42:51.882 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T08:42:51.883 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_41 2026-02-18T08:42:51.883 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_snap cluster2 mirror image_41 snap 2026-02-18T08:42:51.883 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:42:51.883 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:42:51.883 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_41 2026-02-18T08:42:51.883 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap 2026-02-18T08:42:51.883 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 -p mirror snap create image_41@snap --debug-rbd=20 --debug-journaler=20 2026-02-18T08:42:54.121 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_replay_started cluster1 mirror image_41 2026-02-18T08:42:54.121 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:42:54.121 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:42:54.121 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_41 2026-02-18T08:42:54.122 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_replay_state cluster1 mirror image_41 started 2026-02-18T08:42:54.122 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:42:54.122 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:42:54.122 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_41 2026-02-18T08:42:54.122 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local state=started 2026-02-18T08:42:54.122 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-18T08:42:54.122 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:42:54.122 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1 2026-02-18T08:42:55.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test_image_replay_state cluster1 mirror image_41 started 2026-02-18T08:42:55.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:42:55.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:42:55.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_41 2026-02-18T08:42:55.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local test_state=started 2026-02-18T08:42:55.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local status_result 2026-02-18T08:42:55.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_state=stopped 2026-02-18T08:42:55.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_41 2026-02-18T08:42:55.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -i state 2026-02-18T08:42:55.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_instance=cluster1 2026-02-18T08:42:55.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-18T08:42:55.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:42:55.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance=cluster1 2026-02-18T08:42:55.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local loop_instance 2026-02-18T08:42:55.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:42:55.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sleep 0 2026-02-18T08:42:55.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:42:55.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ seq 0 3 2026-02-18T08:42:55.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:42:55.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_41 2026-02-18T08:42:55.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:0 2026-02-18T08:42:55.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-18T08:42:55.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:42:55.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:42:55.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:42:55.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-18T08:42:55.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:42:55.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:42:55.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:42:55.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:42:55.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-18T08:42:55.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:42:55.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:42:55.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:42:55.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:42:55.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:0 2026-02-18T08:42:55.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-18T08:42:55.129 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-18T08:42:55.129 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:42:55.129 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:0 2026-02-18T08:42:55.129 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T08:42:55.129 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-18T08:42:55.129 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:42:55.129 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:42:55.129 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-18T08:42:55.129 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-18T08:42:55.129 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-18T08:42:55.129 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=0 2026-02-18T08:42:55.129 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=0 2026-02-18T08:42:55.130 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:42:55.138 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:42:55.138 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:42:55.138 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:42:55.138 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_41 2026-02-18T08:42:55.199 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_41'] 2026-02-18T08:42:55.200 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status : get status for rbd mirror 2026-02-18T08:42:55.200 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:42:55.202 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:42:55.203 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_41 2026-02-18T08:42:55.203 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:1 2026-02-18T08:42:55.203 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-18T08:42:55.203 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:42:55.203 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:42:55.203 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:42:55.203 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-18T08:42:55.203 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:42:55.203 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:42:55.203 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:42:55.203 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:42:55.204 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-18T08:42:55.204 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:42:55.204 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:42:55.204 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:42:55.204 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:42:55.204 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:1 2026-02-18T08:42:55.204 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-18T08:42:55.204 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-18T08:42:55.204 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:42:55.204 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:1 2026-02-18T08:42:55.204 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T08:42:55.204 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-18T08:42:55.204 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:42:55.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:42:55.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-18T08:42:55.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-18T08:42:55.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-18T08:42:55.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=1 2026-02-18T08:42:55.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=1 2026-02-18T08:42:55.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:42:55.212 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:42:55.213 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:42:55.213 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:42:55.213 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_41 2026-02-18T08:42:55.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_41'] 2026-02-18T08:42:55.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status : get status for rbd mirror 2026-02-18T08:42:55.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:42:55.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:42:55.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_41 2026-02-18T08:42:55.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:2 2026-02-18T08:42:55.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-18T08:42:55.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:42:55.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:42:55.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:42:55.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-18T08:42:55.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:42:55.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:42:55.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:42:55.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:42:55.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-18T08:42:55.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:42:55.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:42:55.273 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:42:55.273 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:42:55.273 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:2 2026-02-18T08:42:55.273 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-18T08:42:55.273 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-18T08:42:55.273 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:42:55.273 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:2 2026-02-18T08:42:55.273 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T08:42:55.273 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-18T08:42:55.273 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:42:55.273 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:42:55.274 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-18T08:42:55.274 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-18T08:42:55.274 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-18T08:42:55.274 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=2 2026-02-18T08:42:55.274 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=2 2026-02-18T08:42:55.274 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:42:55.282 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:42:55.283 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:42:55.283 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:42:55.283 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_41 2026-02-18T08:42:55.344 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ return 0 2026-02-18T08:42:55.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ status_result=' "state": "Replaying"' 2026-02-18T08:42:55.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ echo ' "state": "Replaying"' 2026-02-18T08:42:55.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ grep -i Replaying 2026-02-18T08:42:55.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: "state": "Replaying" 2026-02-18T08:42:55.348 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ current_state=started 2026-02-18T08:42:55.348 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test started = started 2026-02-18T08:42:55.348 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:42:55.348 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror image_41 2026-02-18T08:42:55.348 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_cluster=cluster1 2026-02-18T08:42:55.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:42:55.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_pool=mirror 2026-02-18T08:42:55.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local remote_pool=mirror 2026-02-18T08:42:55.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_41 2026-02-18T08:42:55.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' journal = journal ']' 2026-02-18T08:42:55.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror mirror image_41 2026-02-18T08:42:55.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_cluster=cluster1 2026-02-18T08:42:55.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:42:55.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_pool=mirror 2026-02-18T08:42:55.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local remote_pool=mirror 2026-02-18T08:42:55.350 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_41 2026-02-18T08:42:55.350 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-02-18T08:42:55.350 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-02-18T08:42:55.350 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:42:55.350 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:42:55.350 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:42:55.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_41 2026-02-18T08:42:55.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:42:55.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:42:55.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_41 2026-02-18T08:42:55.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:42:55.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_41 ']' 2026-02-18T08:42:55.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_41' 2026-02-18T08:42:55.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_41 2026-02-18T08:42:55.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:42:55.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:42:55.551 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:42:55.551 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:42:55.551 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:42:55.551 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:42:55.551 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:42:55.551 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:42:55.551 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:42:55.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:42:55.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_41 2026-02-18T08:42:55.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:42:55.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:42:55.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:42:55.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:42:55.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:42:55.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:42:55.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:42:55.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:42:55.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:42:55.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:42:55.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:42:55.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:42:55.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:42:55.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:42:55.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:42:55.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:42:55.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:42:55.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:42:55.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:42:55.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:42:55.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:42:55.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:42:55.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:42:55.557 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:42:55.557 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:42:55.557 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:42:55.557 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:42:55.557 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:42:55.557 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:42:55.557 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:42:55.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:42:55.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:42:55.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:42:55.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_41 2026-02-18T08:42:55.623 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_41'] 2026-02-18T08:42:55.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:42:55.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:42:55.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:42:55.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_41 2026-02-18T08:42:55.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:42:55.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:42:55.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:42:55.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:42:55.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:42:55.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:42:55.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:42:55.628 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:42:55.628 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:42:55.628 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:42:55.628 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:42:55.628 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:42:55.628 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:42:55.628 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:42:55.628 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:42:55.628 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:42:55.628 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:42:55.629 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:42:55.629 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:42:55.629 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:42:55.629 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:42:55.629 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:42:55.629 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:42:55.629 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:42:55.629 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:42:55.629 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:42:55.629 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:42:55.629 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:42:55.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:42:55.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:42:55.638 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:42:55.639 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:42:55.639 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:42:55.639 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_41 2026-02-18T08:42:55.700 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_41'] 2026-02-18T08:42:55.700 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:42:55.700 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:42:55.703 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:42:55.703 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_41 2026-02-18T08:42:55.703 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:42:55.703 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:42:55.704 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:42:55.704 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:42:55.704 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:42:55.704 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:42:55.704 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:42:55.704 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:42:55.704 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:42:55.704 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:42:55.704 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:42:55.704 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:42:55.704 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:42:55.705 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:42:55.705 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:42:55.705 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:42:55.705 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:42:55.705 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:42:55.705 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:42:55.705 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:42:55.705 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:42:55.705 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:42:55.705 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:42:55.706 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:42:55.706 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:42:55.706 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:42:55.706 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:42:55.706 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:42:55.706 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:42:55.706 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:42:55.714 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:42:55.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:42:55.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:42:55.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_41 2026-02-18T08:42:55.780 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:42:55.781 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_41 2026-02-18T08:42:55.781 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:42:55.782 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:42:55.782 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_41 2026-02-18T08:42:55.782 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_41 '' 2026-02-18T08:42:55.782 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:42:55.782 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:42:55.782 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_41 2026-02-18T08:42:55.782 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:42:55.782 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_41.status 2026-02-18T08:42:55.782 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_41.status 2026-02-18T08:42:55.782 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:42:55.783 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_41.status 2026-02-18T08:42:55.783 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_41 2026-02-18T08:42:55.784 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_41.status 2026-02-18T08:42:55.809 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 5 2026-02-18T08:42:55.809 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 5 2026-02-18T08:42:55.809 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:42:55.810 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=21, tag_tid=2, entry_tid=1], [object_number=20, tag_tid=2, entry_tid=0], [object_number=23, tag_tid=1, entry_tid=24527], [object_number=22, tag_tid=1, entry_tid=24526]]], state=connected] 2026-02-18T08:42:55.810 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=20, tag_tid=1, entry_tid=24528], [object_number=23, tag_tid=1, entry_tid=24527], [object_number=22, tag_tid=1, entry_tid=24526], [object_number=21, tag_tid=1, entry_tid=24525]]], state=connected] 2026-02-18T08:42:55.812 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_41.status 2026-02-18T08:42:55.813 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=21, tag_tid=2, entry_tid=1' 2026-02-18T08:42:55.815 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_41 2026-02-18T08:42:55.815 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:42:55.815 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:42:55.815 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_41 2026-02-18T08:42:55.815 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_41 '..*' 2026-02-18T08:42:55.815 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:42:55.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:42:55.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_41 2026-02-18T08:42:55.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:42:55.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_41.status 2026-02-18T08:42:55.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_41.status 2026-02-18T08:42:55.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:42:55.817 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_41.status 2026-02-18T08:42:55.817 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_41 2026-02-18T08:42:55.817 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_41.status 2026-02-18T08:42:55.844 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 5 2026-02-18T08:42:55.844 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 5 2026-02-18T08:42:55.844 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:42:55.844 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=21, tag_tid=2, entry_tid=1], [object_number=20, tag_tid=2, entry_tid=0], [object_number=23, tag_tid=1, entry_tid=24527], [object_number=22, tag_tid=1, entry_tid=24526]]], state=connected] 2026-02-18T08:42:55.844 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=20, tag_tid=1, entry_tid=24528], [object_number=23, tag_tid=1, entry_tid=24527], [object_number=22, tag_tid=1, entry_tid=24526], [object_number=21, tag_tid=1, entry_tid=24525]]], state=connected] 2026-02-18T08:42:55.847 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_41.status 2026-02-18T08:42:55.848 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=20, tag_tid=1, entry_tid=24528' 2026-02-18T08:42:55.848 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=21, tag_tid=2, entry_tid=1' -a 'object_number=21, tag_tid=2, entry_tid=1' = 'object_number=20, tag_tid=1, entry_tid=24528' 2026-02-18T08:42:55.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=20, tag_tid=1, entry_tid=24528' '!=' '' 2026-02-18T08:42:55.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:42:55.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=20, tag_tid=1, entry_tid=24528' = '' 2026-02-18T08:42:55.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=20, tag_tid=1, entry_tid=24528' 2026-02-18T08:42:55.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=21, tag_tid=2, entry_tid=1' 2026-02-18T08:42:55.850 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:42:55.850 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:42:55.850 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=2 2026-02-18T08:42:55.851 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=20, tag_tid=1, entry_tid=24528' 2026-02-18T08:42:55.851 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:42:55.852 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:42:55.853 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=1 2026-02-18T08:42:55.854 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=21, tag_tid=2, entry_tid=1' 2026-02-18T08:42:55.854 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:42:55.854 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:42:55.855 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:42:55.856 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=20, tag_tid=1, entry_tid=24528' 2026-02-18T08:42:55.856 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:42:55.856 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:42:55.858 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=24528 2026-02-18T08:42:55.858 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 2 = 1 -a 1 -le 24528 2026-02-18T08:42:55.858 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:42:55.858 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:42:55.858 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:42:56.060 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_41 2026-02-18T08:42:56.060 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:42:56.060 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:42:56.060 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_41 2026-02-18T08:42:56.060 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:42:56.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_41 ']' 2026-02-18T08:42:56.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_41' 2026-02-18T08:42:56.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_41 2026-02-18T08:42:56.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:42:56.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:42:56.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:42:56.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:42:56.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:42:56.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:42:56.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:42:56.062 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:42:56.062 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:42:56.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:42:56.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_41 2026-02-18T08:42:56.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:42:56.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:42:56.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:42:56.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:42:56.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:42:56.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:42:56.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:42:56.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:42:56.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:42:56.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:42:56.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:42:56.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:42:56.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:42:56.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:42:56.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:42:56.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:42:56.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:42:56.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:42:56.067 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:42:56.067 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:42:56.067 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:42:56.067 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:42:56.067 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:42:56.067 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:42:56.067 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:42:56.067 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:42:56.067 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:42:56.067 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:42:56.067 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:42:56.068 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:42:56.075 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:42:56.075 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:42:56.075 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:42:56.075 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_41 2026-02-18T08:42:56.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_41'] 2026-02-18T08:42:56.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:42:56.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:42:56.138 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:42:56.138 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_41 2026-02-18T08:42:56.139 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:42:56.139 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:42:56.139 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:42:56.139 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:42:56.139 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:42:56.139 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:42:56.139 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:42:56.139 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:42:56.139 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:42:56.139 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:42:56.139 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:42:56.139 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:42:56.140 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:42:56.140 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:42:56.140 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:42:56.140 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:42:56.140 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:42:56.140 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:42:56.140 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:42:56.140 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:42:56.140 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:42:56.140 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:42:56.140 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:42:56.140 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:42:56.141 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:42:56.141 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:42:56.141 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:42:56.141 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:42:56.141 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:42:56.141 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:42:56.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:42:56.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:42:56.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:42:56.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_41 2026-02-18T08:42:56.203 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_41'] 2026-02-18T08:42:56.203 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:42:56.203 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:42:56.206 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:42:56.206 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_41 2026-02-18T08:42:56.206 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:42:56.206 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:42:56.206 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:42:56.206 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:42:56.206 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:42:56.207 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:42:56.207 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:42:56.207 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:42:56.207 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:42:56.207 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:42:56.207 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:42:56.207 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:42:56.207 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:42:56.207 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:42:56.207 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:42:56.207 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:42:56.207 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:42:56.207 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:42:56.208 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:42:56.208 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:42:56.208 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:42:56.208 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:42:56.208 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:42:56.208 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:42:56.208 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:42:56.208 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:42:56.208 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:42:56.208 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:42:56.208 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:42:56.208 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:42:56.217 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:42:56.218 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:42:56.218 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:42:56.218 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_41 2026-02-18T08:42:56.286 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:42:56.288 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_41 2026-02-18T08:42:56.288 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:42:56.288 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:42:56.288 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_41 2026-02-18T08:42:56.288 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_41 '' 2026-02-18T08:42:56.288 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:42:56.288 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:42:56.288 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_41 2026-02-18T08:42:56.288 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:42:56.288 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_41.status 2026-02-18T08:42:56.289 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_41.status 2026-02-18T08:42:56.289 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:42:56.289 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_41.status 2026-02-18T08:42:56.290 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_41.status 2026-02-18T08:42:56.290 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_41 2026-02-18T08:42:56.318 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 5 2026-02-18T08:42:56.318 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 5 2026-02-18T08:42:56.318 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:42:56.318 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=21, tag_tid=2, entry_tid=1], [object_number=20, tag_tid=2, entry_tid=0], [object_number=23, tag_tid=1, entry_tid=24527], [object_number=22, tag_tid=1, entry_tid=24526]]], state=connected] 2026-02-18T08:42:56.318 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=21, tag_tid=2, entry_tid=1], [object_number=20, tag_tid=2, entry_tid=0], [object_number=23, tag_tid=1, entry_tid=24527], [object_number=22, tag_tid=1, entry_tid=24526]]], state=connected] 2026-02-18T08:42:56.320 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_41.status 2026-02-18T08:42:56.321 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=21, tag_tid=2, entry_tid=1' 2026-02-18T08:42:56.323 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_41 2026-02-18T08:42:56.323 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:42:56.323 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:42:56.323 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_41 2026-02-18T08:42:56.323 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_41 '..*' 2026-02-18T08:42:56.323 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:42:56.323 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:42:56.323 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_41 2026-02-18T08:42:56.324 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:42:56.324 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_41.status 2026-02-18T08:42:56.324 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_41.status 2026-02-18T08:42:56.324 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:42:56.325 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_41.status 2026-02-18T08:42:56.325 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_41 2026-02-18T08:42:56.325 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_41.status 2026-02-18T08:42:56.350 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 5 2026-02-18T08:42:56.351 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 5 2026-02-18T08:42:56.351 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:42:56.351 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=21, tag_tid=2, entry_tid=1], [object_number=20, tag_tid=2, entry_tid=0], [object_number=23, tag_tid=1, entry_tid=24527], [object_number=22, tag_tid=1, entry_tid=24526]]], state=connected] 2026-02-18T08:42:56.351 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=21, tag_tid=2, entry_tid=1], [object_number=20, tag_tid=2, entry_tid=0], [object_number=23, tag_tid=1, entry_tid=24527], [object_number=22, tag_tid=1, entry_tid=24526]]], state=connected] 2026-02-18T08:42:56.353 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_41.status 2026-02-18T08:42:56.355 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=21, tag_tid=2, entry_tid=1' 2026-02-18T08:42:56.355 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=21, tag_tid=2, entry_tid=1' -a 'object_number=21, tag_tid=2, entry_tid=1' = 'object_number=21, tag_tid=2, entry_tid=1' 2026-02-18T08:42:56.355 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:42:56.355 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_snap_present cluster1 mirror image_41 snap 2026-02-18T08:42:56.355 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:42:56.355 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:42:56.355 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_41 2026-02-18T08:42:56.356 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap 2026-02-18T08:42:56.356 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-18T08:42:56.356 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-18T08:42:56.356 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1 2026-02-18T08:42:57.355 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster1 info mirror/image_41@snap 2026-02-18T08:42:57.383 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:rbd image 'image_41': 2026-02-18T08:42:57.383 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: size 128 MiB in 32 objects 2026-02-18T08:42:57.383 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: order 22 (4 MiB objects) 2026-02-18T08:42:57.383 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: snapshot_count: 1 2026-02-18T08:42:57.383 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: id: 10ed70cae9d2 2026-02-18T08:42:57.384 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: block_name_prefix: rbd_data.10ed70cae9d2 2026-02-18T08:42:57.384 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: format: 2 2026-02-18T08:42:57.384 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: features: layering, exclusive-lock, journaling 2026-02-18T08:42:57.384 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: op_features: 2026-02-18T08:42:57.384 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: flags: 2026-02-18T08:42:57.384 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: create_timestamp: Wed Feb 18 08:32:40 2026 2026-02-18T08:42:57.384 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: access_timestamp: Wed Feb 18 08:32:40 2026 2026-02-18T08:42:57.384 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: modify_timestamp: Wed Feb 18 08:33:41 2026 2026-02-18T08:42:57.384 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: protected: False 2026-02-18T08:42:57.384 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: journal: 10ed70cae9d2 2026-02-18T08:42:57.384 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring state: enabled 2026-02-18T08:42:57.384 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring mode: journal 2026-02-18T08:42:57.385 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring global id: d67be488-7d7b-436f-b603-2448522f2062 2026-02-18T08:42:57.385 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring primary: false 2026-02-18T08:42:57.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:42:57.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ compare_image_snaps mirror image_41 snap 2026-02-18T08:42:57.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:42:57.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_41 2026-02-18T08:42:57.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap 2026-02-18T08:42:57.387 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local ret=0 2026-02-18T08:42:57.387 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local rmt_export=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_41.export 2026-02-18T08:42:57.387 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loc_export=/tmp/tmp.vybh1m9yJF/cluster1-mirror-image_41.export 2026-02-18T08:42:57.387 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_41.export /tmp/tmp.vybh1m9yJF/cluster1-mirror-image_41.export 2026-02-18T08:42:57.387 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 -p mirror export image_41@snap /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_41.export 2026-02-18T08:42:57.539 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-18T08:42:57.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster1 -p mirror export image_41@snap /tmp/tmp.vybh1m9yJF/cluster1-mirror-image_41.export 2026-02-18T08:42:57.708 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-18T08:42:57.717 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmp /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_41.export /tmp/tmp.vybh1m9yJF/cluster1-mirror-image_41.export 2026-02-18T08:42:57.772 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_41.export /tmp/tmp.vybh1m9yJF/cluster1-mirror-image_41.export 2026-02-18T08:42:57.818 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:42:57.818 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T08:42:57.818 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_42 2026-02-18T08:42:57.818 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_snap cluster2 mirror image_42 snap 2026-02-18T08:42:57.818 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:42:57.818 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:42:57.818 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_42 2026-02-18T08:42:57.819 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap 2026-02-18T08:42:57.819 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 -p mirror snap create image_42@snap --debug-rbd=20 --debug-journaler=20 2026-02-18T08:42:59.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_replay_started cluster1 mirror image_42 2026-02-18T08:42:59.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:42:59.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:42:59.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_42 2026-02-18T08:42:59.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_replay_state cluster1 mirror image_42 started 2026-02-18T08:42:59.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:42:59.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:42:59.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_42 2026-02-18T08:42:59.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local state=started 2026-02-18T08:42:59.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-18T08:42:59.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:42:59.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1 2026-02-18T08:43:00.129 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test_image_replay_state cluster1 mirror image_42 started 2026-02-18T08:43:00.129 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:43:00.129 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:43:00.129 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_42 2026-02-18T08:43:00.129 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local test_state=started 2026-02-18T08:43:00.129 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local status_result 2026-02-18T08:43:00.129 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_state=stopped 2026-02-18T08:43:00.130 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_42 2026-02-18T08:43:00.130 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -i state 2026-02-18T08:43:00.130 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_instance=cluster1 2026-02-18T08:43:00.130 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-18T08:43:00.130 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:43:00.130 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance=cluster1 2026-02-18T08:43:00.130 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local loop_instance 2026-02-18T08:43:00.130 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:43:00.130 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sleep 0 2026-02-18T08:43:00.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:43:00.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ seq 0 3 2026-02-18T08:43:00.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:43:00.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_42 2026-02-18T08:43:00.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:0 2026-02-18T08:43:00.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-18T08:43:00.134 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:43:00.134 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:43:00.134 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:43:00.134 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-18T08:43:00.134 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:43:00.134 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:43:00.134 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:43:00.134 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:43:00.134 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-18T08:43:00.134 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:43:00.134 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:43:00.134 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:43:00.134 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:43:00.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:0 2026-02-18T08:43:00.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-18T08:43:00.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-18T08:43:00.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:43:00.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:0 2026-02-18T08:43:00.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T08:43:00.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-18T08:43:00.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:43:00.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:43:00.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-18T08:43:00.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-18T08:43:00.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-18T08:43:00.136 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=0 2026-02-18T08:43:00.136 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=0 2026-02-18T08:43:00.136 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:43:00.142 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:43:00.142 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:43:00.142 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:43:00.143 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_42 2026-02-18T08:43:00.203 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ return 0 2026-02-18T08:43:00.204 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ status_result=' "state": "Replaying"' 2026-02-18T08:43:00.204 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ echo ' "state": "Replaying"' 2026-02-18T08:43:00.204 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ grep -i Replaying 2026-02-18T08:43:00.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: "state": "Replaying" 2026-02-18T08:43:00.207 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ current_state=started 2026-02-18T08:43:00.207 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test started = started 2026-02-18T08:43:00.207 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:43:00.207 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror image_42 2026-02-18T08:43:00.207 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_cluster=cluster1 2026-02-18T08:43:00.207 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:43:00.207 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_pool=mirror 2026-02-18T08:43:00.207 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local remote_pool=mirror 2026-02-18T08:43:00.208 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_42 2026-02-18T08:43:00.208 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' journal = journal ']' 2026-02-18T08:43:00.208 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror mirror image_42 2026-02-18T08:43:00.208 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_cluster=cluster1 2026-02-18T08:43:00.208 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:43:00.208 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_pool=mirror 2026-02-18T08:43:00.208 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local remote_pool=mirror 2026-02-18T08:43:00.208 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_42 2026-02-18T08:43:00.208 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-02-18T08:43:00.208 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-02-18T08:43:00.208 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:43:00.208 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:43:00.209 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:43:00.408 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_42 2026-02-18T08:43:00.408 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:43:00.408 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:43:00.408 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_42 2026-02-18T08:43:00.408 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:43:00.408 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_42 ']' 2026-02-18T08:43:00.408 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_42' 2026-02-18T08:43:00.409 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_42 2026-02-18T08:43:00.409 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:43:00.409 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:43:00.409 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:43:00.409 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:43:00.409 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:43:00.409 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:43:00.409 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:43:00.409 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:43:00.412 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:43:00.412 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:43:00.412 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_42 2026-02-18T08:43:00.413 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:43:00.413 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:43:00.413 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:43:00.413 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:43:00.413 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:43:00.413 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:43:00.413 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:43:00.413 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:43:00.413 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:43:00.413 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:43:00.413 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:43:00.413 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:43:00.414 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:43:00.414 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:43:00.414 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:43:00.414 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:43:00.414 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:43:00.414 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:43:00.414 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:43:00.414 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:43:00.414 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:43:00.414 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:43:00.414 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:43:00.414 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:43:00.415 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:43:00.415 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:43:00.415 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:43:00.415 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:43:00.415 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:43:00.415 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:43:00.423 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:43:00.423 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:43:00.423 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:43:00.423 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_42 2026-02-18T08:43:00.479 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:43:00.481 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_42 2026-02-18T08:43:00.481 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:43:00.481 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:43:00.481 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_42 2026-02-18T08:43:00.481 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_42 '' 2026-02-18T08:43:00.481 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:43:00.481 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:43:00.481 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_42 2026-02-18T08:43:00.481 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:43:00.482 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_42.status 2026-02-18T08:43:00.482 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_42.status 2026-02-18T08:43:00.482 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:43:00.482 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_42.status 2026-02-18T08:43:00.483 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_42 2026-02-18T08:43:00.483 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_42.status 2026-02-18T08:43:00.516 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 4 2026-02-18T08:43:00.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 4 2026-02-18T08:43:00.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:43:00.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=2, entry_tid=1], [object_number=16, tag_tid=2, entry_tid=0], [object_number=19, tag_tid=1, entry_tid=17099], [object_number=18, tag_tid=1, entry_tid=17098]]], state=connected] 2026-02-18T08:43:00.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=17, tag_tid=1, entry_tid=17101], [object_number=16, tag_tid=1, entry_tid=17100], [object_number=19, tag_tid=1, entry_tid=17099], [object_number=18, tag_tid=1, entry_tid=17098]]], state=connected] 2026-02-18T08:43:00.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_42.status 2026-02-18T08:43:00.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=17, tag_tid=2, entry_tid=1' 2026-02-18T08:43:00.523 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_42 2026-02-18T08:43:00.523 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:43:00.523 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:43:00.523 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_42 2026-02-18T08:43:00.523 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_42 '..*' 2026-02-18T08:43:00.523 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:43:00.523 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:43:00.523 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_42 2026-02-18T08:43:00.523 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:43:00.523 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_42.status 2026-02-18T08:43:00.524 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_42.status 2026-02-18T08:43:00.524 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:43:00.525 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_42.status 2026-02-18T08:43:00.525 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_42 2026-02-18T08:43:00.525 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_42.status 2026-02-18T08:43:00.558 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 4 2026-02-18T08:43:00.558 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 4 2026-02-18T08:43:00.559 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:43:00.559 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=2, entry_tid=1], [object_number=16, tag_tid=2, entry_tid=0], [object_number=19, tag_tid=1, entry_tid=17099], [object_number=18, tag_tid=1, entry_tid=17098]]], state=connected] 2026-02-18T08:43:00.559 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=17, tag_tid=1, entry_tid=17101], [object_number=16, tag_tid=1, entry_tid=17100], [object_number=19, tag_tid=1, entry_tid=17099], [object_number=18, tag_tid=1, entry_tid=17098]]], state=connected] 2026-02-18T08:43:00.561 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_42.status 2026-02-18T08:43:00.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=17, tag_tid=1, entry_tid=17101' 2026-02-18T08:43:00.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=17, tag_tid=2, entry_tid=1' -a 'object_number=17, tag_tid=2, entry_tid=1' = 'object_number=17, tag_tid=1, entry_tid=17101' 2026-02-18T08:43:00.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=17, tag_tid=1, entry_tid=17101' '!=' '' 2026-02-18T08:43:00.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:43:00.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=17, tag_tid=1, entry_tid=17101' = '' 2026-02-18T08:43:00.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=17, tag_tid=1, entry_tid=17101' 2026-02-18T08:43:00.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=17, tag_tid=2, entry_tid=1' 2026-02-18T08:43:00.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:43:00.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:43:00.565 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=2 2026-02-18T08:43:00.566 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=17, tag_tid=1, entry_tid=17101' 2026-02-18T08:43:00.566 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:43:00.566 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:43:00.567 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=1 2026-02-18T08:43:00.568 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=17, tag_tid=2, entry_tid=1' 2026-02-18T08:43:00.568 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:43:00.568 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:43:00.569 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:43:00.570 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=17, tag_tid=1, entry_tid=17101' 2026-02-18T08:43:00.570 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:43:00.570 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:43:00.572 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=17101 2026-02-18T08:43:00.572 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 2 = 1 -a 1 -le 17101 2026-02-18T08:43:00.572 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:43:00.572 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:43:00.572 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:43:00.774 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_42 2026-02-18T08:43:00.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:43:00.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:43:00.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_42 2026-02-18T08:43:00.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:43:00.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_42 ']' 2026-02-18T08:43:00.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_42' 2026-02-18T08:43:00.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_42 2026-02-18T08:43:00.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:43:00.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:43:00.776 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:43:00.776 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:43:00.776 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:43:00.776 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:43:00.776 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:43:00.776 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:43:00.776 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:43:00.779 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:43:00.779 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_42 2026-02-18T08:43:00.779 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:43:00.779 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:43:00.779 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:43:00.779 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:43:00.779 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:43:00.779 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:43:00.779 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:43:00.779 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:43:00.780 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:43:00.780 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:43:00.780 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:43:00.780 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:43:00.780 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:43:00.780 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:43:00.780 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:43:00.780 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:43:00.780 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:43:00.780 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:43:00.780 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:43:00.780 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:43:00.780 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:43:00.781 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:43:00.781 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:43:00.781 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:43:00.781 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:43:00.781 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:43:00.781 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:43:00.781 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:43:00.781 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:43:00.781 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:43:00.787 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:43:00.788 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:43:00.788 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:43:00.788 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_42 2026-02-18T08:43:00.856 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:43:00.858 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_42 2026-02-18T08:43:00.858 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:43:00.858 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:43:00.858 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_42 2026-02-18T08:43:00.858 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_42 '' 2026-02-18T08:43:00.858 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:43:00.858 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:43:00.859 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_42 2026-02-18T08:43:00.859 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:43:00.859 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_42.status 2026-02-18T08:43:00.859 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_42.status 2026-02-18T08:43:00.859 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:43:00.859 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_42.status 2026-02-18T08:43:00.860 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_42 2026-02-18T08:43:00.860 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_42.status 2026-02-18T08:43:00.895 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 4 2026-02-18T08:43:00.896 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 4 2026-02-18T08:43:00.896 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:43:00.896 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=2, entry_tid=1], [object_number=16, tag_tid=2, entry_tid=0], [object_number=19, tag_tid=1, entry_tid=17099], [object_number=18, tag_tid=1, entry_tid=17098]]], state=connected] 2026-02-18T08:43:00.896 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=17, tag_tid=1, entry_tid=17101], [object_number=16, tag_tid=1, entry_tid=17100], [object_number=19, tag_tid=1, entry_tid=17099], [object_number=18, tag_tid=1, entry_tid=17098]]], state=connected] 2026-02-18T08:43:00.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_42.status 2026-02-18T08:43:00.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=17, tag_tid=2, entry_tid=1' 2026-02-18T08:43:00.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_42 2026-02-18T08:43:00.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:43:00.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:43:00.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_42 2026-02-18T08:43:00.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_42 '..*' 2026-02-18T08:43:00.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:43:00.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:43:00.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_42 2026-02-18T08:43:00.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:43:00.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_42.status 2026-02-18T08:43:00.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_42.status 2026-02-18T08:43:00.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:43:00.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_42.status 2026-02-18T08:43:00.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_42 2026-02-18T08:43:00.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_42.status 2026-02-18T08:43:00.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 4 2026-02-18T08:43:00.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 4 2026-02-18T08:43:00.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:43:00.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=2, entry_tid=1], [object_number=16, tag_tid=2, entry_tid=0], [object_number=19, tag_tid=1, entry_tid=17099], [object_number=18, tag_tid=1, entry_tid=17098]]], state=connected] 2026-02-18T08:43:00.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=17, tag_tid=1, entry_tid=17101], [object_number=16, tag_tid=1, entry_tid=17100], [object_number=19, tag_tid=1, entry_tid=17099], [object_number=18, tag_tid=1, entry_tid=17098]]], state=connected] 2026-02-18T08:43:00.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_42.status 2026-02-18T08:43:00.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=17, tag_tid=1, entry_tid=17101' 2026-02-18T08:43:00.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=17, tag_tid=2, entry_tid=1' -a 'object_number=17, tag_tid=2, entry_tid=1' = 'object_number=17, tag_tid=1, entry_tid=17101' 2026-02-18T08:43:00.944 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=17, tag_tid=1, entry_tid=17101' '!=' 'object_number=17, tag_tid=1, entry_tid=17101' 2026-02-18T08:43:00.944 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:43:00.944 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.4 2026-02-18T08:43:01.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_42 2026-02-18T08:43:01.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:43:01.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:43:01.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_42 2026-02-18T08:43:01.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:43:01.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_42 ']' 2026-02-18T08:43:01.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_42' 2026-02-18T08:43:01.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_42 2026-02-18T08:43:01.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:43:01.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:43:01.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:43:01.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:43:01.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:43:01.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:43:01.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:43:01.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:43:01.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:43:01.348 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:43:01.348 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_42 2026-02-18T08:43:01.348 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:43:01.348 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:43:01.348 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:43:01.348 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:43:01.348 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:43:01.348 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:43:01.348 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:43:01.348 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:43:01.348 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:43:01.348 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:43:01.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:43:01.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:43:01.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:43:01.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:43:01.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:43:01.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:43:01.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:43:01.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:43:01.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:43:01.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:43:01.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:43:01.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:43:01.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:43:01.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:43:01.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:43:01.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:43:01.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:43:01.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:43:01.350 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:43:01.350 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:43:01.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:43:01.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:43:01.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:43:01.360 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_42 2026-02-18T08:43:01.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:43:01.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_42 2026-02-18T08:43:01.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:43:01.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:43:01.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_42 2026-02-18T08:43:01.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_42 '' 2026-02-18T08:43:01.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:43:01.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:43:01.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_42 2026-02-18T08:43:01.427 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:43:01.427 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_42.status 2026-02-18T08:43:01.427 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_42.status 2026-02-18T08:43:01.427 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:43:01.428 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_42.status 2026-02-18T08:43:01.428 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_42 2026-02-18T08:43:01.428 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_42.status 2026-02-18T08:43:01.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 4 2026-02-18T08:43:01.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 4 2026-02-18T08:43:01.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:43:01.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=2, entry_tid=1], [object_number=16, tag_tid=2, entry_tid=0], [object_number=19, tag_tid=1, entry_tid=17099], [object_number=18, tag_tid=1, entry_tid=17098]]], state=connected] 2026-02-18T08:43:01.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=17, tag_tid=1, entry_tid=17101], [object_number=16, tag_tid=1, entry_tid=17100], [object_number=19, tag_tid=1, entry_tid=17099], [object_number=18, tag_tid=1, entry_tid=17098]]], state=connected] 2026-02-18T08:43:01.468 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_42.status 2026-02-18T08:43:01.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=17, tag_tid=2, entry_tid=1' 2026-02-18T08:43:01.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_42 2026-02-18T08:43:01.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:43:01.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:43:01.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_42 2026-02-18T08:43:01.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_42 '..*' 2026-02-18T08:43:01.472 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:43:01.472 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:43:01.472 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_42 2026-02-18T08:43:01.472 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:43:01.472 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_42.status 2026-02-18T08:43:01.472 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_42.status 2026-02-18T08:43:01.472 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:43:01.473 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_42.status 2026-02-18T08:43:01.474 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_42 2026-02-18T08:43:01.474 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_42.status 2026-02-18T08:43:01.508 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 4 2026-02-18T08:43:01.509 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 4 2026-02-18T08:43:01.509 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:43:01.509 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=2, entry_tid=1], [object_number=16, tag_tid=2, entry_tid=0], [object_number=19, tag_tid=1, entry_tid=17099], [object_number=18, tag_tid=1, entry_tid=17098]]], state=connected] 2026-02-18T08:43:01.509 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=17, tag_tid=1, entry_tid=17101], [object_number=16, tag_tid=1, entry_tid=17100], [object_number=19, tag_tid=1, entry_tid=17099], [object_number=18, tag_tid=1, entry_tid=17098]]], state=connected] 2026-02-18T08:43:01.512 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_42.status 2026-02-18T08:43:01.513 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=17, tag_tid=1, entry_tid=17101' 2026-02-18T08:43:01.514 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=17, tag_tid=2, entry_tid=1' -a 'object_number=17, tag_tid=2, entry_tid=1' = 'object_number=17, tag_tid=1, entry_tid=17101' 2026-02-18T08:43:01.514 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=17, tag_tid=1, entry_tid=17101' '!=' 'object_number=17, tag_tid=1, entry_tid=17101' 2026-02-18T08:43:01.514 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:43:01.514 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.8 2026-02-18T08:43:02.315 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_42 2026-02-18T08:43:02.316 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:43:02.316 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:43:02.316 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_42 2026-02-18T08:43:02.316 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:43:02.316 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_42 ']' 2026-02-18T08:43:02.316 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_42' 2026-02-18T08:43:02.316 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_42 2026-02-18T08:43:02.316 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:43:02.317 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:43:02.317 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:43:02.317 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:43:02.317 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:43:02.317 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:43:02.317 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:43:02.317 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:43:02.320 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:43:02.320 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:43:02.320 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_42 2026-02-18T08:43:02.320 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:43:02.320 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:43:02.320 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:43:02.320 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:43:02.320 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:43:02.321 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:43:02.321 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:43:02.321 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:43:02.321 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:43:02.321 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:43:02.321 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:43:02.321 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:43:02.321 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:43:02.321 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:43:02.321 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:43:02.321 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:43:02.322 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:43:02.322 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:43:02.322 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:43:02.322 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:43:02.322 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:43:02.322 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:43:02.322 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:43:02.322 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:43:02.322 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:43:02.322 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:43:02.322 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:43:02.322 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:43:02.323 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:43:02.323 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:43:02.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:43:02.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:43:02.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:43:02.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_42 2026-02-18T08:43:02.396 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:43:02.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_42 2026-02-18T08:43:02.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:43:02.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:43:02.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_42 2026-02-18T08:43:02.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_42 '' 2026-02-18T08:43:02.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:43:02.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:43:02.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_42 2026-02-18T08:43:02.399 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:43:02.399 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_42.status 2026-02-18T08:43:02.399 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_42.status 2026-02-18T08:43:02.399 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:43:02.399 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_42.status 2026-02-18T08:43:02.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_42 2026-02-18T08:43:02.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_42.status 2026-02-18T08:43:02.436 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 4 2026-02-18T08:43:02.436 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 4 2026-02-18T08:43:02.436 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:43:02.436 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=2, entry_tid=1], [object_number=16, tag_tid=2, entry_tid=0], [object_number=19, tag_tid=1, entry_tid=17099], [object_number=18, tag_tid=1, entry_tid=17098]]], state=connected] 2026-02-18T08:43:02.436 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=17, tag_tid=1, entry_tid=17101], [object_number=16, tag_tid=1, entry_tid=17100], [object_number=19, tag_tid=1, entry_tid=17099], [object_number=18, tag_tid=1, entry_tid=17098]]], state=connected] 2026-02-18T08:43:02.439 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_42.status 2026-02-18T08:43:02.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=17, tag_tid=2, entry_tid=1' 2026-02-18T08:43:02.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_42 2026-02-18T08:43:02.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:43:02.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:43:02.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_42 2026-02-18T08:43:02.443 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_42 '..*' 2026-02-18T08:43:02.443 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:43:02.443 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:43:02.443 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_42 2026-02-18T08:43:02.443 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:43:02.443 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_42.status 2026-02-18T08:43:02.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_42.status 2026-02-18T08:43:02.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:43:02.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_42.status 2026-02-18T08:43:02.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_42 2026-02-18T08:43:02.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_42.status 2026-02-18T08:43:02.480 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 4 2026-02-18T08:43:02.480 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 4 2026-02-18T08:43:02.480 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:43:02.480 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=2, entry_tid=1], [object_number=16, tag_tid=2, entry_tid=0], [object_number=19, tag_tid=1, entry_tid=17099], [object_number=18, tag_tid=1, entry_tid=17098]]], state=connected] 2026-02-18T08:43:02.480 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=17, tag_tid=1, entry_tid=17101], [object_number=16, tag_tid=1, entry_tid=17100], [object_number=19, tag_tid=1, entry_tid=17099], [object_number=18, tag_tid=1, entry_tid=17098]]], state=connected] 2026-02-18T08:43:02.483 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_42.status 2026-02-18T08:43:02.485 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=17, tag_tid=1, entry_tid=17101' 2026-02-18T08:43:02.485 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=17, tag_tid=2, entry_tid=1' -a 'object_number=17, tag_tid=2, entry_tid=1' = 'object_number=17, tag_tid=1, entry_tid=17101' 2026-02-18T08:43:02.485 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=17, tag_tid=1, entry_tid=17101' '!=' 'object_number=17, tag_tid=1, entry_tid=17101' 2026-02-18T08:43:02.485 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:43:02.485 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1.6 2026-02-18T08:43:04.086 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_42 2026-02-18T08:43:04.086 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:43:04.086 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:43:04.086 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_42 2026-02-18T08:43:04.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:43:04.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_42 ']' 2026-02-18T08:43:04.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_42' 2026-02-18T08:43:04.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_42 2026-02-18T08:43:04.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:43:04.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:43:04.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:43:04.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:43:04.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:43:04.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:43:04.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:43:04.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:43:04.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:43:04.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:43:04.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_42 2026-02-18T08:43:04.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:43:04.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:43:04.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:43:04.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:43:04.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:43:04.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:43:04.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:43:04.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:43:04.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:43:04.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:43:04.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:43:04.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:43:04.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:43:04.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:43:04.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:43:04.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:43:04.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:43:04.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:43:04.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:43:04.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:43:04.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:43:04.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:43:04.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:43:04.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:43:04.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:43:04.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:43:04.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:43:04.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:43:04.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:43:04.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:43:04.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:43:04.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:43:04.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:43:04.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_42 2026-02-18T08:43:04.163 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:43:04.163 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_42 2026-02-18T08:43:04.163 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:43:04.163 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:43:04.163 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_42 2026-02-18T08:43:04.163 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_42 '' 2026-02-18T08:43:04.163 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:43:04.164 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:43:04.164 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_42 2026-02-18T08:43:04.164 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:43:04.164 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_42.status 2026-02-18T08:43:04.165 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_42.status 2026-02-18T08:43:04.165 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:43:04.165 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_42.status 2026-02-18T08:43:04.165 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_42 2026-02-18T08:43:04.165 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_42.status 2026-02-18T08:43:04.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 4 2026-02-18T08:43:04.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 4 2026-02-18T08:43:04.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:43:04.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=2, entry_tid=1], [object_number=16, tag_tid=2, entry_tid=0], [object_number=19, tag_tid=1, entry_tid=17099], [object_number=18, tag_tid=1, entry_tid=17098]]], state=connected] 2026-02-18T08:43:04.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=17, tag_tid=1, entry_tid=17101], [object_number=16, tag_tid=1, entry_tid=17100], [object_number=19, tag_tid=1, entry_tid=17099], [object_number=18, tag_tid=1, entry_tid=17098]]], state=connected] 2026-02-18T08:43:04.195 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_42.status 2026-02-18T08:43:04.196 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=17, tag_tid=2, entry_tid=1' 2026-02-18T08:43:04.197 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_42 2026-02-18T08:43:04.197 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:43:04.197 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:43:04.197 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_42 2026-02-18T08:43:04.197 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_42 '..*' 2026-02-18T08:43:04.197 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:43:04.197 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:43:04.197 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_42 2026-02-18T08:43:04.197 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:43:04.197 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_42.status 2026-02-18T08:43:04.198 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_42.status 2026-02-18T08:43:04.198 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:43:04.199 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_42.status 2026-02-18T08:43:04.199 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_42 2026-02-18T08:43:04.200 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_42.status 2026-02-18T08:43:04.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 4 2026-02-18T08:43:04.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 4 2026-02-18T08:43:04.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:43:04.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=2, entry_tid=1], [object_number=16, tag_tid=2, entry_tid=0], [object_number=19, tag_tid=1, entry_tid=17099], [object_number=18, tag_tid=1, entry_tid=17098]]], state=connected] 2026-02-18T08:43:04.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=17, tag_tid=1, entry_tid=17101], [object_number=16, tag_tid=1, entry_tid=17100], [object_number=19, tag_tid=1, entry_tid=17099], [object_number=18, tag_tid=1, entry_tid=17098]]], state=connected] 2026-02-18T08:43:04.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_42.status 2026-02-18T08:43:04.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=17, tag_tid=1, entry_tid=17101' 2026-02-18T08:43:04.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=17, tag_tid=2, entry_tid=1' -a 'object_number=17, tag_tid=2, entry_tid=1' = 'object_number=17, tag_tid=1, entry_tid=17101' 2026-02-18T08:43:04.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=17, tag_tid=1, entry_tid=17101' '!=' 'object_number=17, tag_tid=1, entry_tid=17101' 2026-02-18T08:43:04.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:43:04.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 2 2026-02-18T08:43:06.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_42 2026-02-18T08:43:06.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:43:06.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:43:06.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_42 2026-02-18T08:43:06.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:43:06.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_42 ']' 2026-02-18T08:43:06.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_42' 2026-02-18T08:43:06.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_42 2026-02-18T08:43:06.244 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:43:06.244 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:43:06.244 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:43:06.244 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:43:06.244 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:43:06.244 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:43:06.244 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:43:06.244 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:43:06.247 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:43:06.247 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:43:06.247 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_42 2026-02-18T08:43:06.247 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:43:06.247 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:43:06.247 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:43:06.247 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:43:06.248 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:43:06.248 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:43:06.248 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:43:06.248 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:43:06.248 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:43:06.248 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:43:06.248 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:43:06.248 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:43:06.248 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:43:06.248 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:43:06.248 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:43:06.248 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:43:06.249 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:43:06.249 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:43:06.249 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:43:06.249 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:43:06.249 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:43:06.249 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:43:06.249 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:43:06.249 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:43:06.249 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:43:06.249 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:43:06.249 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:43:06.249 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:43:06.250 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:43:06.250 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:43:06.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:43:06.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:43:06.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:43:06.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_42 2026-02-18T08:43:06.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:43:06.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_42 2026-02-18T08:43:06.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:43:06.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:43:06.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_42 2026-02-18T08:43:06.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_42 '' 2026-02-18T08:43:06.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:43:06.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:43:06.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_42 2026-02-18T08:43:06.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:43:06.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_42.status 2026-02-18T08:43:06.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_42.status 2026-02-18T08:43:06.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:43:06.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_42.status 2026-02-18T08:43:06.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_42 2026-02-18T08:43:06.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_42.status 2026-02-18T08:43:06.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 4 2026-02-18T08:43:06.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 4 2026-02-18T08:43:06.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:43:06.370 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=2, entry_tid=1], [object_number=16, tag_tid=2, entry_tid=0], [object_number=19, tag_tid=1, entry_tid=17099], [object_number=18, tag_tid=1, entry_tid=17098]]], state=connected] 2026-02-18T08:43:06.370 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=17, tag_tid=1, entry_tid=17101], [object_number=16, tag_tid=1, entry_tid=17100], [object_number=19, tag_tid=1, entry_tid=17099], [object_number=18, tag_tid=1, entry_tid=17098]]], state=connected] 2026-02-18T08:43:06.373 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_42.status 2026-02-18T08:43:06.375 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=17, tag_tid=2, entry_tid=1' 2026-02-18T08:43:06.376 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_42 2026-02-18T08:43:06.377 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:43:06.377 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:43:06.377 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_42 2026-02-18T08:43:06.377 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_42 '..*' 2026-02-18T08:43:06.377 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:43:06.377 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:43:06.377 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_42 2026-02-18T08:43:06.377 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:43:06.377 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_42.status 2026-02-18T08:43:06.377 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_42.status 2026-02-18T08:43:06.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:43:06.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_42.status 2026-02-18T08:43:06.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_42 2026-02-18T08:43:06.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_42.status 2026-02-18T08:43:06.415 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 4 2026-02-18T08:43:06.415 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 4 2026-02-18T08:43:06.415 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:43:06.415 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=2, entry_tid=1], [object_number=16, tag_tid=2, entry_tid=0], [object_number=19, tag_tid=1, entry_tid=17099], [object_number=18, tag_tid=1, entry_tid=17098]]], state=connected] 2026-02-18T08:43:06.415 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=17, tag_tid=1, entry_tid=17101], [object_number=16, tag_tid=1, entry_tid=17100], [object_number=19, tag_tid=1, entry_tid=17099], [object_number=18, tag_tid=1, entry_tid=17098]]], state=connected] 2026-02-18T08:43:06.419 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_42.status 2026-02-18T08:43:06.420 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=17, tag_tid=1, entry_tid=17101' 2026-02-18T08:43:06.421 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=17, tag_tid=2, entry_tid=1' -a 'object_number=17, tag_tid=2, entry_tid=1' = 'object_number=17, tag_tid=1, entry_tid=17101' 2026-02-18T08:43:06.421 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=17, tag_tid=1, entry_tid=17101' '!=' 'object_number=17, tag_tid=1, entry_tid=17101' 2026-02-18T08:43:06.421 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:43:06.421 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 2 2026-02-18T08:43:08.423 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_42 2026-02-18T08:43:08.423 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:43:08.423 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:43:08.423 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_42 2026-02-18T08:43:08.423 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:43:08.423 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_42 ']' 2026-02-18T08:43:08.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_42' 2026-02-18T08:43:08.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_42 2026-02-18T08:43:08.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:43:08.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:43:08.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:43:08.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:43:08.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:43:08.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:43:08.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:43:08.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:43:08.427 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:43:08.427 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:43:08.427 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_42 2026-02-18T08:43:08.427 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:43:08.427 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:43:08.427 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:43:08.428 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:43:08.428 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:43:08.428 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:43:08.428 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:43:08.428 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:43:08.428 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:43:08.428 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:43:08.428 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:43:08.428 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:43:08.428 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:43:08.428 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:43:08.428 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:43:08.428 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:43:08.429 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:43:08.429 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:43:08.429 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:43:08.429 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:43:08.429 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:43:08.429 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:43:08.429 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:43:08.429 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:43:08.429 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:43:08.429 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:43:08.429 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:43:08.429 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:43:08.430 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:43:08.430 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:43:08.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:43:08.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:43:08.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:43:08.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_42 2026-02-18T08:43:08.507 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:43:08.508 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_42 2026-02-18T08:43:08.509 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:43:08.509 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:43:08.509 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_42 2026-02-18T08:43:08.509 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_42 '' 2026-02-18T08:43:08.509 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:43:08.509 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:43:08.509 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_42 2026-02-18T08:43:08.509 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:43:08.509 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_42.status 2026-02-18T08:43:08.509 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_42.status 2026-02-18T08:43:08.510 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:43:08.510 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_42.status 2026-02-18T08:43:08.510 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_42 2026-02-18T08:43:08.510 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_42.status 2026-02-18T08:43:08.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 4 2026-02-18T08:43:08.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 4 2026-02-18T08:43:08.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:43:08.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=2, entry_tid=1], [object_number=16, tag_tid=2, entry_tid=0], [object_number=19, tag_tid=1, entry_tid=17099], [object_number=18, tag_tid=1, entry_tid=17098]]], state=connected] 2026-02-18T08:43:08.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=17, tag_tid=2, entry_tid=1], [object_number=16, tag_tid=2, entry_tid=0], [object_number=19, tag_tid=1, entry_tid=17099], [object_number=18, tag_tid=1, entry_tid=17098]]], state=connected] 2026-02-18T08:43:08.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_42.status 2026-02-18T08:43:08.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=17, tag_tid=2, entry_tid=1' 2026-02-18T08:43:08.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_42 2026-02-18T08:43:08.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:43:08.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:43:08.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_42 2026-02-18T08:43:08.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_42 '..*' 2026-02-18T08:43:08.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:43:08.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:43:08.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_42 2026-02-18T08:43:08.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:43:08.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_42.status 2026-02-18T08:43:08.545 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_42.status 2026-02-18T08:43:08.545 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:43:08.545 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_42.status 2026-02-18T08:43:08.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_42 2026-02-18T08:43:08.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_42.status 2026-02-18T08:43:08.577 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 4 2026-02-18T08:43:08.577 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 4 2026-02-18T08:43:08.577 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:43:08.577 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=2, entry_tid=1], [object_number=16, tag_tid=2, entry_tid=0], [object_number=19, tag_tid=1, entry_tid=17099], [object_number=18, tag_tid=1, entry_tid=17098]]], state=connected] 2026-02-18T08:43:08.577 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=17, tag_tid=2, entry_tid=1], [object_number=16, tag_tid=2, entry_tid=0], [object_number=19, tag_tid=1, entry_tid=17099], [object_number=18, tag_tid=1, entry_tid=17098]]], state=connected] 2026-02-18T08:43:08.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_42.status 2026-02-18T08:43:08.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=17, tag_tid=2, entry_tid=1' 2026-02-18T08:43:08.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=17, tag_tid=2, entry_tid=1' -a 'object_number=17, tag_tid=2, entry_tid=1' = 'object_number=17, tag_tid=2, entry_tid=1' 2026-02-18T08:43:08.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:43:08.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_snap_present cluster1 mirror image_42 snap 2026-02-18T08:43:08.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:43:08.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:43:08.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_42 2026-02-18T08:43:08.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap 2026-02-18T08:43:08.584 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-18T08:43:08.584 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-18T08:43:08.584 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1 2026-02-18T08:43:09.584 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster1 info mirror/image_42@snap 2026-02-18T08:43:09.612 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:rbd image 'image_42': 2026-02-18T08:43:09.612 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: size 128 MiB in 32 objects 2026-02-18T08:43:09.612 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: order 22 (4 MiB objects) 2026-02-18T08:43:09.612 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: snapshot_count: 1 2026-02-18T08:43:09.612 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: id: 1104b7369751 2026-02-18T08:43:09.612 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: block_name_prefix: rbd_data.1104b7369751 2026-02-18T08:43:09.612 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: format: 2 2026-02-18T08:43:09.613 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: features: layering, exclusive-lock, journaling 2026-02-18T08:43:09.613 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: op_features: 2026-02-18T08:43:09.613 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: flags: 2026-02-18T08:43:09.613 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: create_timestamp: Wed Feb 18 08:32:47 2026 2026-02-18T08:43:09.613 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: access_timestamp: Wed Feb 18 08:32:47 2026 2026-02-18T08:43:09.613 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: modify_timestamp: Wed Feb 18 08:32:47 2026 2026-02-18T08:43:09.613 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: protected: False 2026-02-18T08:43:09.613 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: journal: 1104b7369751 2026-02-18T08:43:09.613 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring state: enabled 2026-02-18T08:43:09.613 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring mode: journal 2026-02-18T08:43:09.613 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring global id: 11615c58-c988-4db3-b081-10039f9c28c7 2026-02-18T08:43:09.613 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring primary: false 2026-02-18T08:43:09.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:43:09.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ compare_image_snaps mirror image_42 snap 2026-02-18T08:43:09.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:43:09.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_42 2026-02-18T08:43:09.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap 2026-02-18T08:43:09.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local ret=0 2026-02-18T08:43:09.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local rmt_export=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_42.export 2026-02-18T08:43:09.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loc_export=/tmp/tmp.vybh1m9yJF/cluster1-mirror-image_42.export 2026-02-18T08:43:09.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_42.export /tmp/tmp.vybh1m9yJF/cluster1-mirror-image_42.export 2026-02-18T08:43:09.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 -p mirror export image_42@snap /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_42.export 2026-02-18T08:43:09.754 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-18T08:43:09.763 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster1 -p mirror export image_42@snap /tmp/tmp.vybh1m9yJF/cluster1-mirror-image_42.export 2026-02-18T08:43:09.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-18T08:43:09.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmp /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_42.export /tmp/tmp.vybh1m9yJF/cluster1-mirror-image_42.export 2026-02-18T08:43:09.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_42.export /tmp/tmp.vybh1m9yJF/cluster1-mirror-image_42.export 2026-02-18T08:43:10.019 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:43:10.019 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T08:43:10.019 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_43 2026-02-18T08:43:10.019 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_snap cluster2 mirror image_43 snap 2026-02-18T08:43:10.019 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:43:10.020 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:43:10.020 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_43 2026-02-18T08:43:10.020 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap 2026-02-18T08:43:10.020 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 -p mirror snap create image_43@snap --debug-rbd=20 --debug-journaler=20 2026-02-18T08:43:11.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_replay_started cluster1 mirror image_43 2026-02-18T08:43:11.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:43:11.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:43:11.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_43 2026-02-18T08:43:11.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_replay_state cluster1 mirror image_43 started 2026-02-18T08:43:11.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:43:11.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:43:11.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_43 2026-02-18T08:43:11.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local state=started 2026-02-18T08:43:11.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-18T08:43:11.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:43:11.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1 2026-02-18T08:43:12.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test_image_replay_state cluster1 mirror image_43 started 2026-02-18T08:43:12.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:43:12.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:43:12.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_43 2026-02-18T08:43:12.904 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local test_state=started 2026-02-18T08:43:12.904 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local status_result 2026-02-18T08:43:12.904 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_state=stopped 2026-02-18T08:43:12.904 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_43 2026-02-18T08:43:12.904 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -i state 2026-02-18T08:43:12.904 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_instance=cluster1 2026-02-18T08:43:12.904 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-18T08:43:12.904 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:43:12.904 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance=cluster1 2026-02-18T08:43:12.905 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local loop_instance 2026-02-18T08:43:12.905 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:43:12.905 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sleep 0 2026-02-18T08:43:12.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:43:12.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ seq 0 3 2026-02-18T08:43:12.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:43:12.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_43 2026-02-18T08:43:12.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:0 2026-02-18T08:43:12.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-18T08:43:12.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:43:12.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:43:12.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:43:12.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-18T08:43:12.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:43:12.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:43:12.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:43:12.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:43:12.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-18T08:43:12.909 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:43:12.909 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:43:12.909 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:43:12.909 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:43:12.909 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:0 2026-02-18T08:43:12.909 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-18T08:43:12.909 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-18T08:43:12.909 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:43:12.909 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:0 2026-02-18T08:43:12.909 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T08:43:12.909 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-18T08:43:12.909 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:43:12.910 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:43:12.910 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-18T08:43:12.910 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-18T08:43:12.910 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-18T08:43:12.910 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=0 2026-02-18T08:43:12.910 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=0 2026-02-18T08:43:12.910 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:43:12.917 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:43:12.917 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:43:12.918 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:43:12.918 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_43 2026-02-18T08:43:12.971 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_43'] 2026-02-18T08:43:12.971 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status : get status for rbd mirror 2026-02-18T08:43:12.971 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:43:12.974 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:43:12.974 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_43 2026-02-18T08:43:12.974 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:1 2026-02-18T08:43:12.974 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-18T08:43:12.974 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:43:12.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:43:12.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:43:12.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-18T08:43:12.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:43:12.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:43:12.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:43:12.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:43:12.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-18T08:43:12.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:43:12.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:43:12.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:43:12.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:43:12.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:1 2026-02-18T08:43:12.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-18T08:43:12.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-18T08:43:12.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:43:12.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:1 2026-02-18T08:43:12.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T08:43:12.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-18T08:43:12.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:43:12.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:43:12.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-18T08:43:12.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-18T08:43:12.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-18T08:43:12.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=1 2026-02-18T08:43:12.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=1 2026-02-18T08:43:12.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:43:12.986 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:43:12.986 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:43:12.986 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:43:12.987 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_43 2026-02-18T08:43:13.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_43'] 2026-02-18T08:43:13.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status : get status for rbd mirror 2026-02-18T08:43:13.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:43:13.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:43:13.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_43 2026-02-18T08:43:13.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:2 2026-02-18T08:43:13.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-18T08:43:13.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:43:13.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:43:13.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:43:13.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-18T08:43:13.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:43:13.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:43:13.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:43:13.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:43:13.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-18T08:43:13.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:43:13.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:43:13.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:43:13.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:43:13.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:2 2026-02-18T08:43:13.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-18T08:43:13.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-18T08:43:13.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:43:13.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:2 2026-02-18T08:43:13.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T08:43:13.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-18T08:43:13.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:43:13.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:43:13.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-18T08:43:13.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-18T08:43:13.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-18T08:43:13.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=2 2026-02-18T08:43:13.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=2 2026-02-18T08:43:13.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:43:13.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:43:13.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:43:13.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:43:13.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_43 2026-02-18T08:43:13.129 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ return 0 2026-02-18T08:43:13.130 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ status_result=' "state": "Replaying"' 2026-02-18T08:43:13.131 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ echo ' "state": "Replaying"' 2026-02-18T08:43:13.131 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ grep -i Replaying 2026-02-18T08:43:13.132 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: "state": "Replaying" 2026-02-18T08:43:13.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ current_state=started 2026-02-18T08:43:13.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test started = started 2026-02-18T08:43:13.134 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:43:13.134 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror image_43 2026-02-18T08:43:13.134 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_cluster=cluster1 2026-02-18T08:43:13.134 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:43:13.134 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_pool=mirror 2026-02-18T08:43:13.134 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local remote_pool=mirror 2026-02-18T08:43:13.134 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_43 2026-02-18T08:43:13.134 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' journal = journal ']' 2026-02-18T08:43:13.134 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror mirror image_43 2026-02-18T08:43:13.134 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_cluster=cluster1 2026-02-18T08:43:13.134 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:43:13.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_pool=mirror 2026-02-18T08:43:13.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local remote_pool=mirror 2026-02-18T08:43:13.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_43 2026-02-18T08:43:13.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-02-18T08:43:13.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-02-18T08:43:13.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:43:13.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:43:13.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:43:13.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_43 2026-02-18T08:43:13.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:43:13.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:43:13.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_43 2026-02-18T08:43:13.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:43:13.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_43 ']' 2026-02-18T08:43:13.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_43' 2026-02-18T08:43:13.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_43 2026-02-18T08:43:13.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:43:13.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:43:13.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:43:13.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:43:13.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:43:13.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:43:13.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:43:13.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:43:13.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:43:13.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:43:13.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_43 2026-02-18T08:43:13.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:43:13.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:43:13.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:43:13.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:43:13.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:43:13.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:43:13.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:43:13.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:43:13.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:43:13.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:43:13.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:43:13.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:43:13.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:43:13.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:43:13.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:43:13.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:43:13.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:43:13.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:43:13.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:43:13.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:43:13.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:43:13.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:43:13.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:43:13.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:43:13.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:43:13.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:43:13.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:43:13.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:43:13.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:43:13.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:43:13.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:43:13.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:43:13.350 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:43:13.350 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_43 2026-02-18T08:43:13.409 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_43'] 2026-02-18T08:43:13.409 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:43:13.409 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:43:13.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:43:13.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_43 2026-02-18T08:43:13.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:43:13.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:43:13.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:43:13.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:43:13.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:43:13.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:43:13.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:43:13.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:43:13.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:43:13.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:43:13.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:43:13.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:43:13.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:43:13.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:43:13.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:43:13.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:43:13.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:43:13.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:43:13.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:43:13.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:43:13.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:43:13.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:43:13.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:43:13.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:43:13.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:43:13.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:43:13.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:43:13.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:43:13.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:43:13.412 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:43:13.422 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:43:13.422 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:43:13.422 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:43:13.422 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_43 2026-02-18T08:43:13.482 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_43'] 2026-02-18T08:43:13.482 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:43:13.482 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:43:13.485 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:43:13.485 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_43 2026-02-18T08:43:13.485 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:43:13.485 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:43:13.485 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:43:13.485 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:43:13.485 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:43:13.486 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:43:13.486 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:43:13.486 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:43:13.486 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:43:13.486 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:43:13.486 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:43:13.486 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:43:13.486 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:43:13.486 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:43:13.486 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:43:13.486 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:43:13.487 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:43:13.487 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:43:13.487 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:43:13.487 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:43:13.487 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:43:13.487 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:43:13.487 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:43:13.487 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:43:13.487 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:43:13.487 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:43:13.487 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:43:13.487 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:43:13.488 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:43:13.488 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:43:13.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:43:13.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:43:13.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:43:13.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_43 2026-02-18T08:43:13.559 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:43:13.560 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_43 2026-02-18T08:43:13.561 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:43:13.561 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:43:13.561 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_43 2026-02-18T08:43:13.561 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_43 '' 2026-02-18T08:43:13.561 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:43:13.561 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:43:13.561 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_43 2026-02-18T08:43:13.561 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:43:13.561 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_43.status 2026-02-18T08:43:13.561 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_43.status 2026-02-18T08:43:13.561 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:43:13.562 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_43.status 2026-02-18T08:43:13.562 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_43 2026-02-18T08:43:13.562 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_43.status 2026-02-18T08:43:13.597 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 4 2026-02-18T08:43:13.597 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 4 2026-02-18T08:43:13.597 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:43:13.597 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=2, entry_tid=1], [object_number=16, tag_tid=2, entry_tid=0], [object_number=19, tag_tid=1, entry_tid=19799], [object_number=18, tag_tid=1, entry_tid=19798]]], state=connected] 2026-02-18T08:43:13.597 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=16, tag_tid=1, entry_tid=19800], [object_number=19, tag_tid=1, entry_tid=19799], [object_number=18, tag_tid=1, entry_tid=19798], [object_number=17, tag_tid=1, entry_tid=19797]]], state=connected] 2026-02-18T08:43:13.601 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_43.status 2026-02-18T08:43:13.602 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=17, tag_tid=2, entry_tid=1' 2026-02-18T08:43:13.603 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_43 2026-02-18T08:43:13.604 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:43:13.604 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:43:13.604 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_43 2026-02-18T08:43:13.604 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_43 '..*' 2026-02-18T08:43:13.604 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:43:13.604 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:43:13.604 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_43 2026-02-18T08:43:13.604 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:43:13.604 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_43.status 2026-02-18T08:43:13.605 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_43.status 2026-02-18T08:43:13.605 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:43:13.605 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_43.status 2026-02-18T08:43:13.606 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_43 2026-02-18T08:43:13.606 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_43.status 2026-02-18T08:43:13.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 4 2026-02-18T08:43:13.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 4 2026-02-18T08:43:13.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:43:13.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=2, entry_tid=1], [object_number=16, tag_tid=2, entry_tid=0], [object_number=19, tag_tid=1, entry_tid=19799], [object_number=18, tag_tid=1, entry_tid=19798]]], state=connected] 2026-02-18T08:43:13.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=16, tag_tid=1, entry_tid=19800], [object_number=19, tag_tid=1, entry_tid=19799], [object_number=18, tag_tid=1, entry_tid=19798], [object_number=17, tag_tid=1, entry_tid=19797]]], state=connected] 2026-02-18T08:43:13.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_43.status 2026-02-18T08:43:13.636 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=16, tag_tid=1, entry_tid=19800' 2026-02-18T08:43:13.637 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=17, tag_tid=2, entry_tid=1' -a 'object_number=17, tag_tid=2, entry_tid=1' = 'object_number=16, tag_tid=1, entry_tid=19800' 2026-02-18T08:43:13.637 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=16, tag_tid=1, entry_tid=19800' '!=' '' 2026-02-18T08:43:13.637 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:43:13.637 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=16, tag_tid=1, entry_tid=19800' = '' 2026-02-18T08:43:13.637 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=16, tag_tid=1, entry_tid=19800' 2026-02-18T08:43:13.638 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=17, tag_tid=2, entry_tid=1' 2026-02-18T08:43:13.638 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:43:13.638 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:43:13.639 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=2 2026-02-18T08:43:13.640 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=16, tag_tid=1, entry_tid=19800' 2026-02-18T08:43:13.640 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:43:13.640 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:43:13.641 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=1 2026-02-18T08:43:13.642 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=17, tag_tid=2, entry_tid=1' 2026-02-18T08:43:13.642 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:43:13.642 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:43:13.643 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:43:13.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=16, tag_tid=1, entry_tid=19800' 2026-02-18T08:43:13.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:43:13.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:43:13.646 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=19800 2026-02-18T08:43:13.646 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 2 = 1 -a 1 -le 19800 2026-02-18T08:43:13.646 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:43:13.646 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:43:13.646 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:43:13.848 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_43 2026-02-18T08:43:13.848 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:43:13.848 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:43:13.848 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_43 2026-02-18T08:43:13.848 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:43:13.848 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_43 ']' 2026-02-18T08:43:13.848 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_43' 2026-02-18T08:43:13.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_43 2026-02-18T08:43:13.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:43:13.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:43:13.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:43:13.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:43:13.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:43:13.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:43:13.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:43:13.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:43:13.852 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:43:13.852 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:43:13.852 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_43 2026-02-18T08:43:13.852 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:43:13.852 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:43:13.852 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:43:13.852 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:43:13.853 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:43:13.853 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:43:13.853 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:43:13.853 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:43:13.853 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:43:13.853 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:43:13.853 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:43:13.853 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:43:13.853 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:43:13.853 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:43:13.853 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:43:13.853 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:43:13.854 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:43:13.854 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:43:13.854 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:43:13.854 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:43:13.854 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:43:13.854 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:43:13.854 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:43:13.854 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:43:13.854 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:43:13.854 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:43:13.854 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:43:13.854 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:43:13.855 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:43:13.855 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:43:13.862 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:43:13.863 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:43:13.863 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:43:13.863 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_43 2026-02-18T08:43:13.928 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_43'] 2026-02-18T08:43:13.928 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:43:13.929 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:43:13.931 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:43:13.931 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_43 2026-02-18T08:43:13.931 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:43:13.931 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:43:13.931 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:43:13.932 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:43:13.932 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:43:13.932 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:43:13.932 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:43:13.932 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:43:13.932 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:43:13.932 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:43:13.932 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:43:13.932 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:43:13.932 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:43:13.932 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:43:13.932 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:43:13.932 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:43:13.933 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:43:13.933 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:43:13.933 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:43:13.933 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:43:13.933 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:43:13.933 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:43:13.933 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:43:13.933 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:43:13.933 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:43:13.933 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:43:13.933 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:43:13.933 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:43:13.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:43:13.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:43:13.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:43:13.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:43:13.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:43:13.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_43 2026-02-18T08:43:14.004 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_43'] 2026-02-18T08:43:14.004 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:43:14.004 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:43:14.007 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:43:14.007 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_43 2026-02-18T08:43:14.007 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:43:14.007 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:43:14.007 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:43:14.007 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:43:14.007 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:43:14.007 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:43:14.007 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:43:14.007 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:43:14.008 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:43:14.008 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:43:14.008 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:43:14.008 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:43:14.008 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:43:14.008 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:43:14.008 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:43:14.008 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:43:14.008 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:43:14.008 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:43:14.008 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:43:14.008 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:43:14.008 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:43:14.008 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:43:14.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:43:14.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:43:14.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:43:14.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:43:14.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:43:14.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:43:14.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:43:14.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:43:14.018 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:43:14.018 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:43:14.018 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:43:14.018 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_43 2026-02-18T08:43:14.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:43:14.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_43 2026-02-18T08:43:14.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:43:14.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:43:14.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_43 2026-02-18T08:43:14.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_43 '' 2026-02-18T08:43:14.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:43:14.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:43:14.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_43 2026-02-18T08:43:14.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:43:14.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_43.status 2026-02-18T08:43:14.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_43.status 2026-02-18T08:43:14.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:43:14.086 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_43.status 2026-02-18T08:43:14.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_43 2026-02-18T08:43:14.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_43.status 2026-02-18T08:43:14.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 4 2026-02-18T08:43:14.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 4 2026-02-18T08:43:14.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:43:14.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=2, entry_tid=1], [object_number=16, tag_tid=2, entry_tid=0], [object_number=19, tag_tid=1, entry_tid=19799], [object_number=18, tag_tid=1, entry_tid=19798]]], state=connected] 2026-02-18T08:43:14.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=16, tag_tid=1, entry_tid=19800], [object_number=19, tag_tid=1, entry_tid=19799], [object_number=18, tag_tid=1, entry_tid=19798], [object_number=17, tag_tid=1, entry_tid=19797]]], state=connected] 2026-02-18T08:43:14.129 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_43.status 2026-02-18T08:43:14.130 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=17, tag_tid=2, entry_tid=1' 2026-02-18T08:43:14.132 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_43 2026-02-18T08:43:14.132 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:43:14.132 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:43:14.132 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_43 2026-02-18T08:43:14.132 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_43 '..*' 2026-02-18T08:43:14.132 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:43:14.132 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:43:14.132 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_43 2026-02-18T08:43:14.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:43:14.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_43.status 2026-02-18T08:43:14.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_43.status 2026-02-18T08:43:14.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:43:14.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_43.status 2026-02-18T08:43:14.134 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_43 2026-02-18T08:43:14.134 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_43.status 2026-02-18T08:43:14.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 4 2026-02-18T08:43:14.161 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 4 2026-02-18T08:43:14.161 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:43:14.161 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=2, entry_tid=1], [object_number=16, tag_tid=2, entry_tid=0], [object_number=19, tag_tid=1, entry_tid=19799], [object_number=18, tag_tid=1, entry_tid=19798]]], state=connected] 2026-02-18T08:43:14.161 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=16, tag_tid=1, entry_tid=19800], [object_number=19, tag_tid=1, entry_tid=19799], [object_number=18, tag_tid=1, entry_tid=19798], [object_number=17, tag_tid=1, entry_tid=19797]]], state=connected] 2026-02-18T08:43:14.163 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_43.status 2026-02-18T08:43:14.165 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=16, tag_tid=1, entry_tid=19800' 2026-02-18T08:43:14.165 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=17, tag_tid=2, entry_tid=1' -a 'object_number=17, tag_tid=2, entry_tid=1' = 'object_number=16, tag_tid=1, entry_tid=19800' 2026-02-18T08:43:14.165 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=16, tag_tid=1, entry_tid=19800' '!=' 'object_number=16, tag_tid=1, entry_tid=19800' 2026-02-18T08:43:14.165 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:43:14.166 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.4 2026-02-18T08:43:14.567 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_43 2026-02-18T08:43:14.567 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:43:14.567 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:43:14.567 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_43 2026-02-18T08:43:14.567 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:43:14.568 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_43 ']' 2026-02-18T08:43:14.568 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_43' 2026-02-18T08:43:14.568 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_43 2026-02-18T08:43:14.568 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:43:14.568 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:43:14.568 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:43:14.568 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:43:14.568 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:43:14.568 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:43:14.568 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:43:14.568 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:43:14.571 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:43:14.571 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:43:14.571 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_43 2026-02-18T08:43:14.571 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:43:14.571 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:43:14.572 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:43:14.572 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:43:14.572 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:43:14.572 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:43:14.572 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:43:14.572 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:43:14.572 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:43:14.572 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:43:14.572 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:43:14.572 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:43:14.572 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:43:14.572 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:43:14.573 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:43:14.573 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:43:14.573 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:43:14.573 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:43:14.573 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:43:14.573 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:43:14.573 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:43:14.573 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:43:14.573 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:43:14.573 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:43:14.573 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:43:14.574 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:43:14.574 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:43:14.574 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:43:14.574 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:43:14.574 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:43:14.582 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:43:14.582 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:43:14.582 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:43:14.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_43 2026-02-18T08:43:14.641 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_43'] 2026-02-18T08:43:14.642 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:43:14.642 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:43:14.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:43:14.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_43 2026-02-18T08:43:14.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:43:14.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:43:14.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:43:14.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:43:14.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:43:14.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:43:14.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:43:14.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:43:14.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:43:14.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:43:14.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:43:14.646 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:43:14.646 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:43:14.646 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:43:14.646 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:43:14.646 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:43:14.646 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:43:14.646 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:43:14.646 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:43:14.646 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:43:14.646 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:43:14.646 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:43:14.646 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:43:14.647 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:43:14.647 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:43:14.647 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:43:14.647 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:43:14.647 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:43:14.647 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:43:14.647 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:43:14.655 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:43:14.655 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:43:14.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:43:14.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_43 2026-02-18T08:43:14.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_43'] 2026-02-18T08:43:14.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:43:14.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:43:14.713 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:43:14.713 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_43 2026-02-18T08:43:14.713 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:43:14.713 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:43:14.713 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:43:14.713 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:43:14.714 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:43:14.714 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:43:14.714 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:43:14.714 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:43:14.714 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:43:14.714 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:43:14.714 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:43:14.714 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:43:14.714 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:43:14.714 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:43:14.714 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:43:14.714 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:43:14.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:43:14.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:43:14.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:43:14.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:43:14.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:43:14.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:43:14.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:43:14.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:43:14.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:43:14.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:43:14.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:43:14.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:43:14.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:43:14.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:43:14.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:43:14.724 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:43:14.724 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:43:14.724 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_43 2026-02-18T08:43:14.793 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:43:14.794 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_43 2026-02-18T08:43:14.795 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:43:14.795 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:43:14.795 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_43 2026-02-18T08:43:14.795 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_43 '' 2026-02-18T08:43:14.795 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:43:14.795 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:43:14.795 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_43 2026-02-18T08:43:14.795 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:43:14.795 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_43.status 2026-02-18T08:43:14.795 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_43.status 2026-02-18T08:43:14.795 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:43:14.796 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_43.status 2026-02-18T08:43:14.796 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_43 2026-02-18T08:43:14.796 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_43.status 2026-02-18T08:43:14.832 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 4 2026-02-18T08:43:14.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 4 2026-02-18T08:43:14.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:43:14.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=2, entry_tid=1], [object_number=16, tag_tid=2, entry_tid=0], [object_number=19, tag_tid=1, entry_tid=19799], [object_number=18, tag_tid=1, entry_tid=19798]]], state=connected] 2026-02-18T08:43:14.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=17, tag_tid=2, entry_tid=1], [object_number=16, tag_tid=2, entry_tid=0], [object_number=19, tag_tid=1, entry_tid=19799], [object_number=18, tag_tid=1, entry_tid=19798]]], state=connected] 2026-02-18T08:43:14.836 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_43.status 2026-02-18T08:43:14.838 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=17, tag_tid=2, entry_tid=1' 2026-02-18T08:43:14.839 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_43 2026-02-18T08:43:14.839 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:43:14.839 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:43:14.839 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_43 2026-02-18T08:43:14.839 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_43 '..*' 2026-02-18T08:43:14.839 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:43:14.840 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:43:14.840 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_43 2026-02-18T08:43:14.840 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:43:14.840 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_43.status 2026-02-18T08:43:14.840 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_43.status 2026-02-18T08:43:14.840 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:43:14.841 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_43.status 2026-02-18T08:43:14.841 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_43 2026-02-18T08:43:14.841 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_43.status 2026-02-18T08:43:14.867 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 4 2026-02-18T08:43:14.868 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 4 2026-02-18T08:43:14.868 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:43:14.868 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=2, entry_tid=1], [object_number=16, tag_tid=2, entry_tid=0], [object_number=19, tag_tid=1, entry_tid=19799], [object_number=18, tag_tid=1, entry_tid=19798]]], state=connected] 2026-02-18T08:43:14.868 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=17, tag_tid=2, entry_tid=1], [object_number=16, tag_tid=2, entry_tid=0], [object_number=19, tag_tid=1, entry_tid=19799], [object_number=18, tag_tid=1, entry_tid=19798]]], state=connected] 2026-02-18T08:43:14.870 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_43.status 2026-02-18T08:43:14.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=17, tag_tid=2, entry_tid=1' 2026-02-18T08:43:14.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=17, tag_tid=2, entry_tid=1' -a 'object_number=17, tag_tid=2, entry_tid=1' = 'object_number=17, tag_tid=2, entry_tid=1' 2026-02-18T08:43:14.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:43:14.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_snap_present cluster1 mirror image_43 snap 2026-02-18T08:43:14.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:43:14.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:43:14.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_43 2026-02-18T08:43:14.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap 2026-02-18T08:43:14.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-18T08:43:14.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-18T08:43:14.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1 2026-02-18T08:43:15.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster1 info mirror/image_43@snap 2026-02-18T08:43:15.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:rbd image 'image_43': 2026-02-18T08:43:15.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: size 128 MiB in 32 objects 2026-02-18T08:43:15.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: order 22 (4 MiB objects) 2026-02-18T08:43:15.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: snapshot_count: 1 2026-02-18T08:43:15.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: id: 11f9ff32041e 2026-02-18T08:43:15.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: block_name_prefix: rbd_data.11f9ff32041e 2026-02-18T08:43:15.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: format: 2 2026-02-18T08:43:15.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: features: layering, exclusive-lock, journaling 2026-02-18T08:43:15.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: op_features: 2026-02-18T08:43:15.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: flags: 2026-02-18T08:43:15.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: create_timestamp: Wed Feb 18 08:32:54 2026 2026-02-18T08:43:15.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: access_timestamp: Wed Feb 18 08:32:54 2026 2026-02-18T08:43:15.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: modify_timestamp: Wed Feb 18 08:32:54 2026 2026-02-18T08:43:15.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: protected: False 2026-02-18T08:43:15.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: journal: 11f9ff32041e 2026-02-18T08:43:15.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring state: enabled 2026-02-18T08:43:15.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring mode: journal 2026-02-18T08:43:15.904 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring global id: 4f165d98-9745-45da-be08-63c20a5a1e2c 2026-02-18T08:43:15.904 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring primary: false 2026-02-18T08:43:15.905 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:43:15.905 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ compare_image_snaps mirror image_43 snap 2026-02-18T08:43:15.905 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:43:15.905 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_43 2026-02-18T08:43:15.905 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap 2026-02-18T08:43:15.905 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local ret=0 2026-02-18T08:43:15.905 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local rmt_export=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_43.export 2026-02-18T08:43:15.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loc_export=/tmp/tmp.vybh1m9yJF/cluster1-mirror-image_43.export 2026-02-18T08:43:15.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_43.export /tmp/tmp.vybh1m9yJF/cluster1-mirror-image_43.export 2026-02-18T08:43:15.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 -p mirror export image_43@snap /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_43.export 2026-02-18T08:43:16.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-18T08:43:16.060 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster1 -p mirror export image_43@snap /tmp/tmp.vybh1m9yJF/cluster1-mirror-image_43.export 2026-02-18T08:43:16.229 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-18T08:43:16.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmp /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_43.export /tmp/tmp.vybh1m9yJF/cluster1-mirror-image_43.export 2026-02-18T08:43:16.291 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_43.export /tmp/tmp.vybh1m9yJF/cluster1-mirror-image_43.export 2026-02-18T08:43:16.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:43:16.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T08:43:16.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_44 2026-02-18T08:43:16.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_snap cluster2 mirror image_44 snap 2026-02-18T08:43:16.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:43:16.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:43:16.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_44 2026-02-18T08:43:16.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap 2026-02-18T08:43:16.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 -p mirror snap create image_44@snap --debug-rbd=20 --debug-journaler=20 2026-02-18T08:43:18.913 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_replay_started cluster1 mirror image_44 2026-02-18T08:43:18.913 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:43:18.913 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:43:18.913 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_44 2026-02-18T08:43:18.913 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_replay_state cluster1 mirror image_44 started 2026-02-18T08:43:18.913 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:43:18.914 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:43:18.914 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_44 2026-02-18T08:43:18.914 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local state=started 2026-02-18T08:43:18.914 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-18T08:43:18.914 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:43:18.914 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1 2026-02-18T08:43:19.914 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test_image_replay_state cluster1 mirror image_44 started 2026-02-18T08:43:19.914 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:43:19.914 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:43:19.914 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_44 2026-02-18T08:43:19.914 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local test_state=started 2026-02-18T08:43:19.915 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local status_result 2026-02-18T08:43:19.915 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_state=stopped 2026-02-18T08:43:19.916 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_44 2026-02-18T08:43:19.916 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -i state 2026-02-18T08:43:19.916 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_instance=cluster1 2026-02-18T08:43:19.916 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-18T08:43:19.916 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:43:19.916 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance=cluster1 2026-02-18T08:43:19.916 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local loop_instance 2026-02-18T08:43:19.916 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:43:19.916 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sleep 0 2026-02-18T08:43:19.916 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:43:19.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ seq 0 3 2026-02-18T08:43:19.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:43:19.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_44 2026-02-18T08:43:19.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:0 2026-02-18T08:43:19.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-18T08:43:19.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:43:19.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:43:19.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:43:19.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-18T08:43:19.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:43:19.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:43:19.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:43:19.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:43:19.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-18T08:43:19.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:43:19.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:43:19.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:43:19.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:43:19.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:0 2026-02-18T08:43:19.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-18T08:43:19.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-18T08:43:19.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:43:19.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:0 2026-02-18T08:43:19.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T08:43:19.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-18T08:43:19.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:43:19.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:43:19.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-18T08:43:19.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-18T08:43:19.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-18T08:43:19.922 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=0 2026-02-18T08:43:19.922 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=0 2026-02-18T08:43:19.922 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:43:19.930 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:43:19.930 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:43:19.931 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:43:19.931 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_44 2026-02-18T08:43:19.992 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_44'] 2026-02-18T08:43:19.992 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status : get status for rbd mirror 2026-02-18T08:43:19.992 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:43:19.994 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:43:19.994 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_44 2026-02-18T08:43:19.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:1 2026-02-18T08:43:19.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-18T08:43:19.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:43:19.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:43:19.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:43:19.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-18T08:43:19.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:43:19.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:43:19.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:43:19.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:43:19.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-18T08:43:19.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:43:19.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:43:19.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:43:19.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:43:19.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:1 2026-02-18T08:43:19.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-18T08:43:19.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-18T08:43:19.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:43:19.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:1 2026-02-18T08:43:19.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T08:43:19.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-18T08:43:19.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:43:19.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:43:19.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-18T08:43:19.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-18T08:43:19.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-18T08:43:19.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=1 2026-02-18T08:43:19.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=1 2026-02-18T08:43:19.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:43:20.004 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:43:20.005 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:43:20.005 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:43:20.005 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_44 2026-02-18T08:43:20.060 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_44'] 2026-02-18T08:43:20.060 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status : get status for rbd mirror 2026-02-18T08:43:20.060 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:43:20.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:43:20.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_44 2026-02-18T08:43:20.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:2 2026-02-18T08:43:20.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-18T08:43:20.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:43:20.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:43:20.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:43:20.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-18T08:43:20.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:43:20.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:43:20.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:43:20.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:43:20.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-18T08:43:20.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:43:20.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:43:20.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:43:20.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:43:20.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:2 2026-02-18T08:43:20.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-18T08:43:20.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-18T08:43:20.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:43:20.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:2 2026-02-18T08:43:20.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T08:43:20.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-18T08:43:20.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:43:20.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:43:20.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-18T08:43:20.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-18T08:43:20.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-18T08:43:20.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=2 2026-02-18T08:43:20.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=2 2026-02-18T08:43:20.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:43:20.074 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:43:20.075 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:43:20.075 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:43:20.075 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_44 2026-02-18T08:43:20.137 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_44'] 2026-02-18T08:43:20.137 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status : get status for rbd mirror 2026-02-18T08:43:20.137 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:43:20.140 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:43:20.140 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/image_44 2026-02-18T08:43:20.140 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:3 2026-02-18T08:43:20.140 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-18T08:43:20.140 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:43:20.140 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:43:20.140 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:43:20.140 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-18T08:43:20.141 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:43:20.141 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:43:20.141 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:43:20.141 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:43:20.141 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-18T08:43:20.141 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:43:20.141 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:43:20.141 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:43:20.141 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:43:20.141 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:3 2026-02-18T08:43:20.141 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-18T08:43:20.141 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-18T08:43:20.142 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:43:20.142 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:3 2026-02-18T08:43:20.142 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T08:43:20.142 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-18T08:43:20.142 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:43:20.142 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:43:20.142 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-18T08:43:20.142 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-18T08:43:20.142 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-18T08:43:20.142 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=3 2026-02-18T08:43:20.142 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=3 2026-02-18T08:43:20.142 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:43:20.150 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:43:20.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:43:20.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:43:20.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/image_44 2026-02-18T08:43:20.218 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ return 0 2026-02-18T08:43:20.219 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ status_result=' "state": "Replaying"' 2026-02-18T08:43:20.219 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ echo ' "state": "Replaying"' 2026-02-18T08:43:20.220 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ grep -i Replaying 2026-02-18T08:43:20.221 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: "state": "Replaying" 2026-02-18T08:43:20.222 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ current_state=started 2026-02-18T08:43:20.222 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test started = started 2026-02-18T08:43:20.223 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:43:20.223 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror image_44 2026-02-18T08:43:20.223 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_cluster=cluster1 2026-02-18T08:43:20.223 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:43:20.223 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_pool=mirror 2026-02-18T08:43:20.223 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local remote_pool=mirror 2026-02-18T08:43:20.223 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_44 2026-02-18T08:43:20.223 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' journal = journal ']' 2026-02-18T08:43:20.223 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror mirror image_44 2026-02-18T08:43:20.223 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_cluster=cluster1 2026-02-18T08:43:20.223 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:43:20.223 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_pool=mirror 2026-02-18T08:43:20.224 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local remote_pool=mirror 2026-02-18T08:43:20.224 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_44 2026-02-18T08:43:20.224 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-02-18T08:43:20.224 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-02-18T08:43:20.224 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:43:20.224 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:43:20.224 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:43:20.423 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_44 2026-02-18T08:43:20.423 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:43:20.423 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:43:20.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_44 2026-02-18T08:43:20.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:43:20.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_44 ']' 2026-02-18T08:43:20.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_44' 2026-02-18T08:43:20.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_44 2026-02-18T08:43:20.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:43:20.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:43:20.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:43:20.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:43:20.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:43:20.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:43:20.425 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:43:20.427 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:43:20.427 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:43:20.428 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:43:20.428 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_44 2026-02-18T08:43:20.428 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:43:20.428 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:43:20.428 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:43:20.428 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:43:20.428 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:43:20.428 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:43:20.428 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:43:20.428 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:43:20.429 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:43:20.429 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:43:20.429 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:43:20.429 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:43:20.429 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:43:20.429 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:43:20.429 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:43:20.429 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:43:20.429 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:43:20.429 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:43:20.429 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:43:20.429 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:43:20.429 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:43:20.430 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:43:20.430 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:43:20.430 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:43:20.430 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:43:20.430 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:43:20.430 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:43:20.430 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:43:20.430 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:43:20.430 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:43:20.439 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:43:20.439 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:43:20.439 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:43:20.439 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_44 2026-02-18T08:43:20.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_44'] 2026-02-18T08:43:20.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:43:20.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:43:20.501 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:43:20.502 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_44 2026-02-18T08:43:20.502 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:43:20.502 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:43:20.502 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:43:20.502 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:43:20.502 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:43:20.502 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:43:20.502 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:43:20.502 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:43:20.502 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:43:20.502 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:43:20.503 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:43:20.503 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:43:20.503 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:43:20.503 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:43:20.503 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:43:20.503 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:43:20.503 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:43:20.503 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:43:20.503 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:43:20.503 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:43:20.503 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:43:20.503 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:43:20.504 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:43:20.504 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:43:20.504 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:43:20.504 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:43:20.504 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:43:20.504 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:43:20.504 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:43:20.504 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:43:20.514 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:43:20.514 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:43:20.514 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:43:20.514 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_44 2026-02-18T08:43:20.575 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_44'] 2026-02-18T08:43:20.575 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:43:20.575 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:43:20.577 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:43:20.577 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_44 2026-02-18T08:43:20.578 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:43:20.578 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:43:20.578 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:43:20.578 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:43:20.578 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:43:20.578 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:43:20.578 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:43:20.578 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:43:20.578 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:43:20.578 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:43:20.578 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:43:20.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:43:20.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:43:20.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:43:20.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:43:20.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:43:20.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:43:20.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:43:20.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:43:20.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:43:20.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:43:20.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:43:20.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:43:20.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:43:20.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:43:20.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:43:20.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:43:20.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:43:20.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:43:20.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:43:20.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:43:20.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:43:20.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:43:20.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_44 2026-02-18T08:43:20.643 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_44'] 2026-02-18T08:43:20.643 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:43:20.643 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:43:20.646 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:43:20.646 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/image_44 2026-02-18T08:43:20.646 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:43:20.646 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:43:20.646 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:43:20.646 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:43:20.646 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:43:20.646 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:43:20.646 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:43:20.646 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:43:20.647 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:43:20.647 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:43:20.647 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:43:20.647 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:43:20.647 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:43:20.647 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:43:20.647 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:43:20.647 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:43:20.647 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:43:20.647 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:43:20.647 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:43:20.647 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:43:20.648 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:43:20.648 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:43:20.648 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:43:20.648 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:43:20.648 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:43:20.648 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:43:20.648 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:43:20.648 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:43:20.648 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:43:20.648 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:43:20.655 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:43:20.655 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:43:20.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:43:20.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/image_44 2026-02-18T08:43:20.721 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:43:20.722 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_44 2026-02-18T08:43:20.722 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:43:20.722 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:43:20.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_44 2026-02-18T08:43:20.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_44 '' 2026-02-18T08:43:20.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:43:20.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:43:20.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_44 2026-02-18T08:43:20.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:43:20.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_44.status 2026-02-18T08:43:20.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_44.status 2026-02-18T08:43:20.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:43:20.724 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_44.status 2026-02-18T08:43:20.724 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_44 2026-02-18T08:43:20.724 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_44.status 2026-02-18T08:43:20.760 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 8 2026-02-18T08:43:20.760 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 8 2026-02-18T08:43:20.760 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:43:20.760 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=33, tag_tid=2, entry_tid=1], [object_number=32, tag_tid=2, entry_tid=0], [object_number=35, tag_tid=1, entry_tid=37499], [object_number=34, tag_tid=1, entry_tid=37498]]], state=connected] 2026-02-18T08:43:20.760 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=33, tag_tid=1, entry_tid=37501], [object_number=32, tag_tid=1, entry_tid=37500], [object_number=35, tag_tid=1, entry_tid=37499], [object_number=34, tag_tid=1, entry_tid=37498]]], state=connected] 2026-02-18T08:43:20.763 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_44.status 2026-02-18T08:43:20.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=33, tag_tid=2, entry_tid=1' 2026-02-18T08:43:20.766 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_44 2026-02-18T08:43:20.766 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:43:20.766 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:43:20.766 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_44 2026-02-18T08:43:20.766 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_44 '..*' 2026-02-18T08:43:20.766 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:43:20.766 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:43:20.766 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_44 2026-02-18T08:43:20.766 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:43:20.767 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_44.status 2026-02-18T08:43:20.767 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_44.status 2026-02-18T08:43:20.767 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:43:20.768 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_44.status 2026-02-18T08:43:20.768 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_44 2026-02-18T08:43:20.768 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_44.status 2026-02-18T08:43:20.794 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 8 2026-02-18T08:43:20.794 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 8 2026-02-18T08:43:20.794 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:43:20.794 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=33, tag_tid=2, entry_tid=1], [object_number=32, tag_tid=2, entry_tid=0], [object_number=35, tag_tid=1, entry_tid=37499], [object_number=34, tag_tid=1, entry_tid=37498]]], state=connected] 2026-02-18T08:43:20.794 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=33, tag_tid=1, entry_tid=37501], [object_number=32, tag_tid=1, entry_tid=37500], [object_number=35, tag_tid=1, entry_tid=37499], [object_number=34, tag_tid=1, entry_tid=37498]]], state=connected] 2026-02-18T08:43:20.796 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_44.status 2026-02-18T08:43:20.798 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=33, tag_tid=1, entry_tid=37501' 2026-02-18T08:43:20.798 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=33, tag_tid=2, entry_tid=1' -a 'object_number=33, tag_tid=2, entry_tid=1' = 'object_number=33, tag_tid=1, entry_tid=37501' 2026-02-18T08:43:20.798 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=33, tag_tid=1, entry_tid=37501' '!=' '' 2026-02-18T08:43:20.798 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:43:20.798 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=33, tag_tid=1, entry_tid=37501' = '' 2026-02-18T08:43:20.798 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=33, tag_tid=1, entry_tid=37501' 2026-02-18T08:43:20.799 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=33, tag_tid=2, entry_tid=1' 2026-02-18T08:43:20.799 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:43:20.799 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:43:20.800 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=2 2026-02-18T08:43:20.801 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=33, tag_tid=1, entry_tid=37501' 2026-02-18T08:43:20.801 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:43:20.801 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:43:20.803 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=1 2026-02-18T08:43:20.803 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=33, tag_tid=2, entry_tid=1' 2026-02-18T08:43:20.804 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:43:20.804 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:43:20.805 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:43:20.806 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=33, tag_tid=1, entry_tid=37501' 2026-02-18T08:43:20.806 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:43:20.806 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:43:20.808 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=37501 2026-02-18T08:43:20.808 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 2 = 1 -a 1 -le 37501 2026-02-18T08:43:20.808 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:43:20.808 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:43:20.808 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:43:21.010 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_44 2026-02-18T08:43:21.010 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:43:21.010 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:43:21.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_44 2026-02-18T08:43:21.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:43:21.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_44 ']' 2026-02-18T08:43:21.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_44' 2026-02-18T08:43:21.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_44 2026-02-18T08:43:21.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:43:21.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:43:21.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:43:21.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:43:21.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:43:21.012 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:43:21.012 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:43:21.012 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:43:21.012 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:43:21.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:43:21.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_44 2026-02-18T08:43:21.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:43:21.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:43:21.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:43:21.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:43:21.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:43:21.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:43:21.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:43:21.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:43:21.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:43:21.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:43:21.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:43:21.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:43:21.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:43:21.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:43:21.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:43:21.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:43:21.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:43:21.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:43:21.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:43:21.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:43:21.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:43:21.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:43:21.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:43:21.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:43:21.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:43:21.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:43:21.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:43:21.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:43:21.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:43:21.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:43:21.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:43:21.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:43:21.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:43:21.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_44 2026-02-18T08:43:21.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_44'] 2026-02-18T08:43:21.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:43:21.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:43:21.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:43:21.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_44 2026-02-18T08:43:21.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:43:21.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:43:21.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:43:21.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:43:21.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:43:21.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:43:21.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:43:21.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:43:21.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:43:21.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:43:21.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:43:21.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:43:21.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:43:21.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:43:21.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:43:21.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:43:21.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:43:21.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:43:21.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:43:21.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:43:21.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:43:21.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:43:21.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:43:21.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:43:21.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:43:21.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:43:21.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:43:21.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:43:21.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:43:21.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:43:21.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:43:21.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:43:21.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:43:21.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_44 2026-02-18T08:43:21.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_44'] 2026-02-18T08:43:21.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:43:21.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:43:21.155 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:43:21.155 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_44 2026-02-18T08:43:21.155 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:43:21.155 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:43:21.155 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:43:21.155 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:43:21.155 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:43:21.155 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:43:21.155 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:43:21.155 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:43:21.156 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:43:21.156 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:43:21.156 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:43:21.156 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:43:21.156 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:43:21.156 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:43:21.156 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:43:21.156 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:43:21.156 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:43:21.156 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:43:21.156 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:43:21.157 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:43:21.157 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:43:21.157 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:43:21.157 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:43:21.157 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:43:21.157 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:43:21.157 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:43:21.157 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:43:21.157 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:43:21.157 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:43:21.157 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:43:21.165 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:43:21.166 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:43:21.166 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:43:21.166 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_44 2026-02-18T08:43:21.227 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_44'] 2026-02-18T08:43:21.227 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:43:21.227 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:43:21.230 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:43:21.230 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/image_44 2026-02-18T08:43:21.231 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:43:21.231 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:43:21.231 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:43:21.231 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:43:21.231 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:43:21.231 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:43:21.231 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:43:21.231 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:43:21.231 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:43:21.231 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:43:21.231 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:43:21.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:43:21.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:43:21.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:43:21.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:43:21.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:43:21.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:43:21.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:43:21.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:43:21.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:43:21.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:43:21.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:43:21.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:43:21.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:43:21.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:43:21.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:43:21.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:43:21.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:43:21.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:43:21.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:43:21.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:43:21.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:43:21.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:43:21.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/image_44 2026-02-18T08:43:21.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:43:21.310 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_44 2026-02-18T08:43:21.310 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:43:21.310 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:43:21.310 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_44 2026-02-18T08:43:21.310 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_44 '' 2026-02-18T08:43:21.310 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:43:21.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:43:21.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_44 2026-02-18T08:43:21.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:43:21.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_44.status 2026-02-18T08:43:21.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_44.status 2026-02-18T08:43:21.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:43:21.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_44.status 2026-02-18T08:43:21.312 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_44 2026-02-18T08:43:21.312 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_44.status 2026-02-18T08:43:21.551 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 8 2026-02-18T08:43:21.552 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 8 2026-02-18T08:43:21.552 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:43:21.552 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=33, tag_tid=2, entry_tid=1], [object_number=32, tag_tid=2, entry_tid=0], [object_number=35, tag_tid=1, entry_tid=37499], [object_number=34, tag_tid=1, entry_tid=37498]]], state=connected] 2026-02-18T08:43:21.552 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=33, tag_tid=1, entry_tid=37501], [object_number=32, tag_tid=1, entry_tid=37500], [object_number=35, tag_tid=1, entry_tid=37499], [object_number=34, tag_tid=1, entry_tid=37498]]], state=connected] 2026-02-18T08:43:21.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_44.status 2026-02-18T08:43:21.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=33, tag_tid=2, entry_tid=1' 2026-02-18T08:43:21.558 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_44 2026-02-18T08:43:21.558 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:43:21.558 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:43:21.558 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_44 2026-02-18T08:43:21.558 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_44 '..*' 2026-02-18T08:43:21.558 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:43:21.558 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:43:21.558 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_44 2026-02-18T08:43:21.558 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:43:21.559 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_44.status 2026-02-18T08:43:21.559 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_44.status 2026-02-18T08:43:21.559 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:43:21.559 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_44.status 2026-02-18T08:43:21.560 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_44 2026-02-18T08:43:21.560 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_44.status 2026-02-18T08:43:21.595 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 8 2026-02-18T08:43:21.595 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 8 2026-02-18T08:43:21.595 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:43:21.596 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=33, tag_tid=2, entry_tid=1], [object_number=32, tag_tid=2, entry_tid=0], [object_number=35, tag_tid=1, entry_tid=37499], [object_number=34, tag_tid=1, entry_tid=37498]]], state=connected] 2026-02-18T08:43:21.596 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=33, tag_tid=1, entry_tid=37501], [object_number=32, tag_tid=1, entry_tid=37500], [object_number=35, tag_tid=1, entry_tid=37499], [object_number=34, tag_tid=1, entry_tid=37498]]], state=connected] 2026-02-18T08:43:21.599 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_44.status 2026-02-18T08:43:21.600 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=33, tag_tid=1, entry_tid=37501' 2026-02-18T08:43:21.601 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=33, tag_tid=2, entry_tid=1' -a 'object_number=33, tag_tid=2, entry_tid=1' = 'object_number=33, tag_tid=1, entry_tid=37501' 2026-02-18T08:43:21.601 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=33, tag_tid=1, entry_tid=37501' '!=' 'object_number=33, tag_tid=1, entry_tid=37501' 2026-02-18T08:43:21.601 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:43:21.601 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.4 2026-02-18T08:43:22.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_44 2026-02-18T08:43:22.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:43:22.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:43:22.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_44 2026-02-18T08:43:22.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:43:22.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_44 ']' 2026-02-18T08:43:22.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_44' 2026-02-18T08:43:22.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_44 2026-02-18T08:43:22.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:43:22.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:43:22.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:43:22.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:43:22.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:43:22.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:43:22.003 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:43:22.003 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:43:22.003 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:43:22.006 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:43:22.006 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_44 2026-02-18T08:43:22.006 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:43:22.006 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:43:22.006 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:43:22.006 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:43:22.006 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:43:22.006 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:43:22.006 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:43:22.007 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:43:22.007 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:43:22.007 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:43:22.007 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:43:22.007 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:43:22.007 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:43:22.007 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:43:22.007 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:43:22.007 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:43:22.007 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:43:22.007 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:43:22.007 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:43:22.007 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:43:22.007 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:43:22.007 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:43:22.007 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:43:22.008 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:43:22.008 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:43:22.008 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:43:22.008 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:43:22.008 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:43:22.008 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:43:22.008 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:43:22.019 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:43:22.019 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:43:22.019 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:43:22.019 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_44 2026-02-18T08:43:22.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_44'] 2026-02-18T08:43:22.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:43:22.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:43:22.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:43:22.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_44 2026-02-18T08:43:22.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:43:22.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:43:22.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:43:22.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:43:22.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:43:22.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:43:22.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:43:22.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:43:22.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:43:22.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:43:22.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:43:22.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:43:22.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:43:22.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:43:22.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:43:22.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:43:22.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:43:22.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:43:22.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:43:22.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:43:22.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:43:22.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:43:22.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:43:22.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:43:22.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:43:22.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:43:22.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:43:22.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:43:22.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:43:22.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:43:22.093 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:43:22.094 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:43:22.094 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:43:22.094 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_44 2026-02-18T08:43:22.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_44'] 2026-02-18T08:43:22.155 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:43:22.155 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:43:22.157 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:43:22.157 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_44 2026-02-18T08:43:22.157 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:43:22.157 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:43:22.157 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:43:22.157 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:43:22.157 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:43:22.157 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:43:22.157 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:43:22.157 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:43:22.157 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:43:22.157 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:43:22.157 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:43:22.157 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:43:22.157 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:43:22.157 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:43:22.157 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:43:22.157 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:43:22.157 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:43:22.157 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:43:22.158 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:43:22.158 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:43:22.158 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:43:22.158 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:43:22.158 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:43:22.158 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:43:22.158 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:43:22.158 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:43:22.158 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:43:22.158 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:43:22.158 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:43:22.158 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:43:22.169 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:43:22.170 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:43:22.170 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:43:22.170 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_44 2026-02-18T08:43:22.229 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_44'] 2026-02-18T08:43:22.230 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:43:22.230 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:43:22.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:43:22.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/image_44 2026-02-18T08:43:22.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:43:22.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:43:22.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:43:22.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:43:22.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:43:22.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:43:22.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:43:22.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:43:22.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:43:22.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:43:22.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:43:22.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:43:22.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:43:22.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:43:22.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:43:22.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:43:22.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:43:22.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:43:22.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:43:22.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:43:22.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:43:22.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:43:22.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:43:22.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:43:22.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:43:22.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:43:22.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:43:22.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:43:22.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:43:22.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:43:22.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:43:22.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:43:22.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:43:22.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/image_44 2026-02-18T08:43:22.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:43:22.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_44 2026-02-18T08:43:22.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:43:22.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:43:22.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_44 2026-02-18T08:43:22.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_44 '' 2026-02-18T08:43:22.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:43:22.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:43:22.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_44 2026-02-18T08:43:22.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:43:22.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_44.status 2026-02-18T08:43:22.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_44.status 2026-02-18T08:43:22.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:43:22.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_44.status 2026-02-18T08:43:22.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_44 2026-02-18T08:43:22.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_44.status 2026-02-18T08:43:22.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 8 2026-02-18T08:43:22.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 8 2026-02-18T08:43:22.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:43:22.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=33, tag_tid=2, entry_tid=1], [object_number=32, tag_tid=2, entry_tid=0], [object_number=35, tag_tid=1, entry_tid=37499], [object_number=34, tag_tid=1, entry_tid=37498]]], state=connected] 2026-02-18T08:43:22.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=33, tag_tid=1, entry_tid=37501], [object_number=32, tag_tid=1, entry_tid=37500], [object_number=35, tag_tid=1, entry_tid=37499], [object_number=34, tag_tid=1, entry_tid=37498]]], state=connected] 2026-02-18T08:43:22.350 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_44.status 2026-02-18T08:43:22.352 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=33, tag_tid=2, entry_tid=1' 2026-02-18T08:43:22.353 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_44 2026-02-18T08:43:22.353 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:43:22.353 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:43:22.353 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_44 2026-02-18T08:43:22.353 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_44 '..*' 2026-02-18T08:43:22.353 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:43:22.353 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:43:22.353 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_44 2026-02-18T08:43:22.353 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:43:22.354 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_44.status 2026-02-18T08:43:22.354 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_44.status 2026-02-18T08:43:22.354 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:43:22.355 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_44.status 2026-02-18T08:43:22.355 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_44 2026-02-18T08:43:22.355 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_44.status 2026-02-18T08:43:22.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 8 2026-02-18T08:43:22.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 8 2026-02-18T08:43:22.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:43:22.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=33, tag_tid=2, entry_tid=1], [object_number=32, tag_tid=2, entry_tid=0], [object_number=35, tag_tid=1, entry_tid=37499], [object_number=34, tag_tid=1, entry_tid=37498]]], state=connected] 2026-02-18T08:43:22.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=33, tag_tid=1, entry_tid=37501], [object_number=32, tag_tid=1, entry_tid=37500], [object_number=35, tag_tid=1, entry_tid=37499], [object_number=34, tag_tid=1, entry_tid=37498]]], state=connected] 2026-02-18T08:43:22.395 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_44.status 2026-02-18T08:43:22.396 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=33, tag_tid=1, entry_tid=37501' 2026-02-18T08:43:22.396 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=33, tag_tid=2, entry_tid=1' -a 'object_number=33, tag_tid=2, entry_tid=1' = 'object_number=33, tag_tid=1, entry_tid=37501' 2026-02-18T08:43:22.396 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=33, tag_tid=1, entry_tid=37501' '!=' 'object_number=33, tag_tid=1, entry_tid=37501' 2026-02-18T08:43:22.396 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:43:22.397 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.8 2026-02-18T08:43:23.199 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_44 2026-02-18T08:43:23.199 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:43:23.199 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:43:23.199 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_44 2026-02-18T08:43:23.199 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:43:23.200 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_44 ']' 2026-02-18T08:43:23.200 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_44' 2026-02-18T08:43:23.200 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_44 2026-02-18T08:43:23.200 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:43:23.200 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:43:23.200 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:43:23.200 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:43:23.200 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:43:23.200 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:43:23.200 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:43:23.200 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:43:23.203 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:43:23.203 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:43:23.203 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_44 2026-02-18T08:43:23.203 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:43:23.203 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:43:23.204 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:43:23.204 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:43:23.204 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:43:23.204 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:43:23.204 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:43:23.204 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:43:23.204 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:43:23.204 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:43:23.204 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:43:23.204 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:43:23.204 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:43:23.204 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:43:23.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:43:23.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:43:23.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:43:23.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:43:23.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:43:23.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:43:23.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:43:23.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:43:23.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:43:23.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:43:23.206 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:43:23.206 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:43:23.206 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:43:23.206 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:43:23.206 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:43:23.206 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:43:23.215 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:43:23.215 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:43:23.215 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:43:23.215 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_44 2026-02-18T08:43:23.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_44'] 2026-02-18T08:43:23.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:43:23.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:43:23.270 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:43:23.270 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_44 2026-02-18T08:43:23.270 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:43:23.270 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:43:23.270 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:43:23.270 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:43:23.270 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:43:23.270 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:43:23.270 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:43:23.270 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:43:23.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:43:23.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:43:23.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:43:23.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:43:23.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:43:23.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:43:23.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:43:23.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:43:23.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:43:23.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:43:23.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:43:23.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:43:23.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:43:23.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:43:23.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:43:23.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:43:23.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:43:23.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:43:23.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:43:23.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:43:23.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:43:23.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:43:23.281 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:43:23.281 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:43:23.282 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:43:23.282 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_44 2026-02-18T08:43:23.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_44'] 2026-02-18T08:43:23.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:43:23.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:43:23.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:43:23.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_44 2026-02-18T08:43:23.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:43:23.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:43:23.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:43:23.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:43:23.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:43:23.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:43:23.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:43:23.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:43:23.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:43:23.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:43:23.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:43:23.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:43:23.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:43:23.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:43:23.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:43:23.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:43:23.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:43:23.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:43:23.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:43:23.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:43:23.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:43:23.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:43:23.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:43:23.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:43:23.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:43:23.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:43:23.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:43:23.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:43:23.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:43:23.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:43:23.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:43:23.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:43:23.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:43:23.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_44 2026-02-18T08:43:23.406 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_44'] 2026-02-18T08:43:23.406 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:43:23.406 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:43:23.409 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:43:23.409 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/image_44 2026-02-18T08:43:23.409 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:43:23.409 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:43:23.409 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:43:23.409 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:43:23.409 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:43:23.409 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:43:23.409 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:43:23.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:43:23.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:43:23.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:43:23.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:43:23.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:43:23.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:43:23.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:43:23.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:43:23.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:43:23.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:43:23.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:43:23.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:43:23.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:43:23.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:43:23.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:43:23.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:43:23.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:43:23.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:43:23.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:43:23.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:43:23.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:43:23.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:43:23.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:43:23.419 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:43:23.419 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:43:23.419 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:43:23.419 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/image_44 2026-02-18T08:43:23.478 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:43:23.479 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_44 2026-02-18T08:43:23.479 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:43:23.479 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:43:23.479 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_44 2026-02-18T08:43:23.479 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_44 '' 2026-02-18T08:43:23.480 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:43:23.480 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:43:23.480 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_44 2026-02-18T08:43:23.480 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:43:23.480 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_44.status 2026-02-18T08:43:23.480 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_44.status 2026-02-18T08:43:23.480 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:43:23.481 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_44.status 2026-02-18T08:43:23.481 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_44 2026-02-18T08:43:23.482 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_44.status 2026-02-18T08:43:23.510 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 8 2026-02-18T08:43:23.510 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 8 2026-02-18T08:43:23.510 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:43:23.510 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=33, tag_tid=2, entry_tid=1], [object_number=32, tag_tid=2, entry_tid=0], [object_number=35, tag_tid=1, entry_tid=37499], [object_number=34, tag_tid=1, entry_tid=37498]]], state=connected] 2026-02-18T08:43:23.510 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=33, tag_tid=1, entry_tid=37501], [object_number=32, tag_tid=1, entry_tid=37500], [object_number=35, tag_tid=1, entry_tid=37499], [object_number=34, tag_tid=1, entry_tid=37498]]], state=connected] 2026-02-18T08:43:23.512 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_44.status 2026-02-18T08:43:23.514 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=33, tag_tid=2, entry_tid=1' 2026-02-18T08:43:23.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_44 2026-02-18T08:43:23.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:43:23.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:43:23.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_44 2026-02-18T08:43:23.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_44 '..*' 2026-02-18T08:43:23.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:43:23.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:43:23.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_44 2026-02-18T08:43:23.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:43:23.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_44.status 2026-02-18T08:43:23.516 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_44.status 2026-02-18T08:43:23.516 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:43:23.516 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_44.status 2026-02-18T08:43:23.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_44 2026-02-18T08:43:23.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_44.status 2026-02-18T08:43:23.545 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 8 2026-02-18T08:43:23.545 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 8 2026-02-18T08:43:23.545 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:43:23.545 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=33, tag_tid=2, entry_tid=1], [object_number=32, tag_tid=2, entry_tid=0], [object_number=35, tag_tid=1, entry_tid=37499], [object_number=34, tag_tid=1, entry_tid=37498]]], state=connected] 2026-02-18T08:43:23.545 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=33, tag_tid=1, entry_tid=37501], [object_number=32, tag_tid=1, entry_tid=37500], [object_number=35, tag_tid=1, entry_tid=37499], [object_number=34, tag_tid=1, entry_tid=37498]]], state=connected] 2026-02-18T08:43:23.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_44.status 2026-02-18T08:43:23.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=33, tag_tid=1, entry_tid=37501' 2026-02-18T08:43:23.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=33, tag_tid=2, entry_tid=1' -a 'object_number=33, tag_tid=2, entry_tid=1' = 'object_number=33, tag_tid=1, entry_tid=37501' 2026-02-18T08:43:23.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=33, tag_tid=1, entry_tid=37501' '!=' 'object_number=33, tag_tid=1, entry_tid=37501' 2026-02-18T08:43:23.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:43:23.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1.6 2026-02-18T08:43:25.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_44 2026-02-18T08:43:25.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:43:25.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:43:25.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_44 2026-02-18T08:43:25.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:43:25.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_44 ']' 2026-02-18T08:43:25.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_44' 2026-02-18T08:43:25.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_44 2026-02-18T08:43:25.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:43:25.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:43:25.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:43:25.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:43:25.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:43:25.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:43:25.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:43:25.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:43:25.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:43:25.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:43:25.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_44 2026-02-18T08:43:25.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:43:25.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:43:25.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:43:25.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:43:25.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:43:25.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:43:25.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:43:25.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:43:25.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:43:25.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:43:25.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:43:25.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:43:25.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:43:25.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:43:25.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:43:25.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:43:25.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:43:25.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:43:25.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:43:25.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:43:25.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:43:25.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:43:25.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:43:25.155 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:43:25.155 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:43:25.155 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:43:25.155 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:43:25.155 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:43:25.155 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:43:25.155 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:43:25.166 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:43:25.167 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:43:25.167 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:43:25.167 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_44 2026-02-18T08:43:25.218 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_44'] 2026-02-18T08:43:25.219 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:43:25.219 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:43:25.221 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:43:25.221 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_44 2026-02-18T08:43:25.222 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:43:25.222 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:43:25.222 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:43:25.222 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:43:25.222 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:43:25.222 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:43:25.222 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:43:25.222 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:43:25.222 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:43:25.222 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:43:25.222 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:43:25.223 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:43:25.223 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:43:25.223 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:43:25.223 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:43:25.223 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:43:25.223 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:43:25.223 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:43:25.223 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:43:25.223 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:43:25.223 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:43:25.223 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:43:25.223 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:43:25.223 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:43:25.224 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:43:25.224 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:43:25.224 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:43:25.224 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:43:25.224 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:43:25.224 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:43:25.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:43:25.234 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:43:25.234 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:43:25.234 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_44 2026-02-18T08:43:25.294 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_44'] 2026-02-18T08:43:25.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:43:25.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:43:25.298 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:43:25.298 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_44 2026-02-18T08:43:25.298 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:43:25.298 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:43:25.298 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:43:25.298 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:43:25.298 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:43:25.298 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:43:25.298 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:43:25.298 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:43:25.299 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:43:25.299 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:43:25.299 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:43:25.299 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:43:25.299 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:43:25.299 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:43:25.299 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:43:25.299 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:43:25.299 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:43:25.299 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:43:25.299 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:43:25.299 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:43:25.299 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:43:25.300 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:43:25.300 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:43:25.300 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:43:25.300 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:43:25.300 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:43:25.300 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:43:25.300 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:43:25.300 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:43:25.300 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:43:25.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:43:25.310 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:43:25.310 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:43:25.310 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_44 2026-02-18T08:43:25.370 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_44'] 2026-02-18T08:43:25.370 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:43:25.370 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:43:25.373 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:43:25.373 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/image_44 2026-02-18T08:43:25.373 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:43:25.373 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:43:25.373 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:43:25.373 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:43:25.373 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:43:25.374 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:43:25.374 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:43:25.374 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:43:25.374 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:43:25.374 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:43:25.374 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:43:25.374 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:43:25.374 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:43:25.374 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:43:25.374 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:43:25.374 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:43:25.374 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:43:25.375 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:43:25.375 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:43:25.375 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:43:25.375 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:43:25.375 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:43:25.375 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:43:25.375 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:43:25.375 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:43:25.375 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:43:25.375 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:43:25.375 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:43:25.376 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:43:25.376 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:43:25.384 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:43:25.385 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:43:25.385 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:43:25.385 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/image_44 2026-02-18T08:43:25.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:43:25.443 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_44 2026-02-18T08:43:25.443 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:43:25.443 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:43:25.443 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_44 2026-02-18T08:43:25.443 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_44 '' 2026-02-18T08:43:25.443 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:43:25.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:43:25.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_44 2026-02-18T08:43:25.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:43:25.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_44.status 2026-02-18T08:43:25.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_44.status 2026-02-18T08:43:25.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:43:25.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_44.status 2026-02-18T08:43:25.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_44 2026-02-18T08:43:25.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_44.status 2026-02-18T08:43:25.472 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 8 2026-02-18T08:43:25.473 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 8 2026-02-18T08:43:25.473 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:43:25.473 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=33, tag_tid=2, entry_tid=1], [object_number=32, tag_tid=2, entry_tid=0], [object_number=35, tag_tid=1, entry_tid=37499], [object_number=34, tag_tid=1, entry_tid=37498]]], state=connected] 2026-02-18T08:43:25.473 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=33, tag_tid=1, entry_tid=37501], [object_number=32, tag_tid=1, entry_tid=37500], [object_number=35, tag_tid=1, entry_tid=37499], [object_number=34, tag_tid=1, entry_tid=37498]]], state=connected] 2026-02-18T08:43:25.475 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_44.status 2026-02-18T08:43:25.477 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=33, tag_tid=2, entry_tid=1' 2026-02-18T08:43:25.478 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_44 2026-02-18T08:43:25.478 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:43:25.478 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:43:25.478 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_44 2026-02-18T08:43:25.478 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_44 '..*' 2026-02-18T08:43:25.479 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:43:25.479 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:43:25.479 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_44 2026-02-18T08:43:25.479 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:43:25.479 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_44.status 2026-02-18T08:43:25.479 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_44.status 2026-02-18T08:43:25.479 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:43:25.480 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_44.status 2026-02-18T08:43:25.480 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_44 2026-02-18T08:43:25.480 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_44.status 2026-02-18T08:43:25.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 8 2026-02-18T08:43:25.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 8 2026-02-18T08:43:25.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:43:25.516 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=33, tag_tid=2, entry_tid=1], [object_number=32, tag_tid=2, entry_tid=0], [object_number=35, tag_tid=1, entry_tid=37499], [object_number=34, tag_tid=1, entry_tid=37498]]], state=connected] 2026-02-18T08:43:25.516 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=33, tag_tid=1, entry_tid=37501], [object_number=32, tag_tid=1, entry_tid=37500], [object_number=35, tag_tid=1, entry_tid=37499], [object_number=34, tag_tid=1, entry_tid=37498]]], state=connected] 2026-02-18T08:43:25.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_44.status 2026-02-18T08:43:25.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=33, tag_tid=1, entry_tid=37501' 2026-02-18T08:43:25.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=33, tag_tid=2, entry_tid=1' -a 'object_number=33, tag_tid=2, entry_tid=1' = 'object_number=33, tag_tid=1, entry_tid=37501' 2026-02-18T08:43:25.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=33, tag_tid=1, entry_tid=37501' '!=' 'object_number=33, tag_tid=1, entry_tid=37501' 2026-02-18T08:43:25.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:43:25.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 2 2026-02-18T08:43:27.523 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_44 2026-02-18T08:43:27.523 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:43:27.523 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:43:27.523 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_44 2026-02-18T08:43:27.523 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:43:27.523 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_44 ']' 2026-02-18T08:43:27.523 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_44' 2026-02-18T08:43:27.523 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_44 2026-02-18T08:43:27.524 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:43:27.524 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:43:27.524 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:43:27.524 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:43:27.524 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:43:27.524 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:43:27.524 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:43:27.527 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:43:27.527 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:43:27.527 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:43:27.527 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_44 2026-02-18T08:43:27.527 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:43:27.527 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:43:27.528 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:43:27.528 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:43:27.528 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:43:27.528 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:43:27.528 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:43:27.528 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:43:27.528 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:43:27.528 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:43:27.528 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:43:27.528 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:43:27.528 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:43:27.528 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:43:27.529 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:43:27.529 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:43:27.529 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:43:27.529 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:43:27.529 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:43:27.529 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:43:27.529 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:43:27.529 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:43:27.529 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:43:27.529 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:43:27.529 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:43:27.530 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:43:27.530 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:43:27.530 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:43:27.530 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:43:27.530 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:43:27.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:43:27.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:43:27.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:43:27.539 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_44 2026-02-18T08:43:27.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_44'] 2026-02-18T08:43:27.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:43:27.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:43:27.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:43:27.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_44 2026-02-18T08:43:27.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:43:27.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:43:27.594 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:43:27.594 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:43:27.594 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:43:27.594 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:43:27.594 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:43:27.594 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:43:27.594 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:43:27.594 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:43:27.594 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:43:27.594 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:43:27.594 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:43:27.594 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:43:27.595 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:43:27.595 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:43:27.595 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:43:27.595 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:43:27.595 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:43:27.595 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:43:27.595 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:43:27.595 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:43:27.595 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:43:27.595 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:43:27.595 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:43:27.595 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:43:27.596 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:43:27.596 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:43:27.596 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:43:27.596 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:43:27.603 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:43:27.604 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:43:27.604 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:43:27.604 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_44 2026-02-18T08:43:27.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_44'] 2026-02-18T08:43:27.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:43:27.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:43:27.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:43:27.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_44 2026-02-18T08:43:27.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:43:27.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:43:27.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:43:27.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:43:27.670 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:43:27.670 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:43:27.670 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:43:27.670 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:43:27.670 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:43:27.670 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:43:27.670 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:43:27.670 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:43:27.670 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:43:27.670 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:43:27.670 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:43:27.670 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:43:27.670 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:43:27.671 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:43:27.671 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:43:27.671 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:43:27.671 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:43:27.671 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:43:27.671 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:43:27.671 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:43:27.671 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:43:27.671 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:43:27.671 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:43:27.671 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:43:27.671 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:43:27.672 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:43:27.679 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:43:27.680 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:43:27.680 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:43:27.680 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_44 2026-02-18T08:43:27.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_44'] 2026-02-18T08:43:27.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:43:27.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:43:27.732 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:43:27.732 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/image_44 2026-02-18T08:43:27.732 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:43:27.732 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:43:27.732 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:43:27.732 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:43:27.732 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:43:27.732 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:43:27.732 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:43:27.732 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:43:27.732 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:43:27.732 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:43:27.732 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:43:27.732 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:43:27.732 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:43:27.732 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:43:27.732 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:43:27.733 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:43:27.733 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:43:27.733 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:43:27.733 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:43:27.733 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:43:27.733 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:43:27.733 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:43:27.733 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:43:27.733 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:43:27.733 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:43:27.733 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:43:27.733 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:43:27.733 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:43:27.733 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:43:27.733 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:43:27.744 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:43:27.745 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:43:27.745 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:43:27.745 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/image_44 2026-02-18T08:43:27.807 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:43:27.808 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_44 2026-02-18T08:43:27.808 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:43:27.808 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:43:27.808 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_44 2026-02-18T08:43:27.808 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_44 '' 2026-02-18T08:43:27.809 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:43:27.809 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:43:27.809 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_44 2026-02-18T08:43:27.809 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:43:27.809 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_44.status 2026-02-18T08:43:27.809 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_44.status 2026-02-18T08:43:27.809 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:43:27.810 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_44.status 2026-02-18T08:43:27.810 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_44 2026-02-18T08:43:27.810 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_44.status 2026-02-18T08:43:27.837 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 8 2026-02-18T08:43:27.838 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 8 2026-02-18T08:43:27.838 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:43:27.838 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=33, tag_tid=2, entry_tid=1], [object_number=32, tag_tid=2, entry_tid=0], [object_number=35, tag_tid=1, entry_tid=37499], [object_number=34, tag_tid=1, entry_tid=37498]]], state=connected] 2026-02-18T08:43:27.838 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=33, tag_tid=2, entry_tid=1], [object_number=32, tag_tid=2, entry_tid=0], [object_number=35, tag_tid=1, entry_tid=37499], [object_number=34, tag_tid=1, entry_tid=37498]]], state=connected] 2026-02-18T08:43:27.840 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_44.status 2026-02-18T08:43:27.842 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=33, tag_tid=2, entry_tid=1' 2026-02-18T08:43:27.843 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_44 2026-02-18T08:43:27.843 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:43:27.843 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:43:27.843 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_44 2026-02-18T08:43:27.843 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_44 '..*' 2026-02-18T08:43:27.844 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:43:27.844 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:43:27.844 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_44 2026-02-18T08:43:27.844 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:43:27.844 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_44.status 2026-02-18T08:43:27.844 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_44.status 2026-02-18T08:43:27.844 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:43:27.844 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_44.status 2026-02-18T08:43:27.845 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_44 2026-02-18T08:43:27.845 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_44.status 2026-02-18T08:43:27.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 8 2026-02-18T08:43:27.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 8 2026-02-18T08:43:27.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:43:27.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=33, tag_tid=2, entry_tid=1], [object_number=32, tag_tid=2, entry_tid=0], [object_number=35, tag_tid=1, entry_tid=37499], [object_number=34, tag_tid=1, entry_tid=37498]]], state=connected] 2026-02-18T08:43:27.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=33, tag_tid=2, entry_tid=1], [object_number=32, tag_tid=2, entry_tid=0], [object_number=35, tag_tid=1, entry_tid=37499], [object_number=34, tag_tid=1, entry_tid=37498]]], state=connected] 2026-02-18T08:43:27.874 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_44.status 2026-02-18T08:43:27.876 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=33, tag_tid=2, entry_tid=1' 2026-02-18T08:43:27.876 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=33, tag_tid=2, entry_tid=1' -a 'object_number=33, tag_tid=2, entry_tid=1' = 'object_number=33, tag_tid=2, entry_tid=1' 2026-02-18T08:43:27.876 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:43:27.876 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_snap_present cluster1 mirror image_44 snap 2026-02-18T08:43:27.877 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:43:27.877 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:43:27.877 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_44 2026-02-18T08:43:27.877 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap 2026-02-18T08:43:27.877 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-18T08:43:27.877 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-18T08:43:27.877 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1 2026-02-18T08:43:28.877 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster1 info mirror/image_44@snap 2026-02-18T08:43:28.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:rbd image 'image_44': 2026-02-18T08:43:28.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: size 128 MiB in 32 objects 2026-02-18T08:43:28.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: order 22 (4 MiB objects) 2026-02-18T08:43:28.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: snapshot_count: 1 2026-02-18T08:43:28.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: id: 120d9a218c60 2026-02-18T08:43:28.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: block_name_prefix: rbd_data.120d9a218c60 2026-02-18T08:43:28.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: format: 2 2026-02-18T08:43:28.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: features: layering, exclusive-lock, journaling 2026-02-18T08:43:28.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: op_features: 2026-02-18T08:43:28.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: flags: 2026-02-18T08:43:28.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: create_timestamp: Wed Feb 18 08:33:00 2026 2026-02-18T08:43:28.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: access_timestamp: Wed Feb 18 08:33:00 2026 2026-02-18T08:43:28.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: modify_timestamp: Wed Feb 18 08:34:01 2026 2026-02-18T08:43:28.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: protected: False 2026-02-18T08:43:28.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: journal: 120d9a218c60 2026-02-18T08:43:28.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring state: enabled 2026-02-18T08:43:28.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring mode: journal 2026-02-18T08:43:28.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring global id: f80281bd-536b-45ea-b19e-c37472d52a4b 2026-02-18T08:43:28.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring primary: false 2026-02-18T08:43:28.911 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:43:28.911 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ compare_image_snaps mirror image_44 snap 2026-02-18T08:43:28.911 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:43:28.911 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_44 2026-02-18T08:43:28.911 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap 2026-02-18T08:43:28.911 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local ret=0 2026-02-18T08:43:28.911 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local rmt_export=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_44.export 2026-02-18T08:43:28.911 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loc_export=/tmp/tmp.vybh1m9yJF/cluster1-mirror-image_44.export 2026-02-18T08:43:28.912 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_44.export /tmp/tmp.vybh1m9yJF/cluster1-mirror-image_44.export 2026-02-18T08:43:28.912 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 -p mirror export image_44@snap /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_44.export 2026-02-18T08:43:29.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-18T08:43:29.059 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster1 -p mirror export image_44@snap /tmp/tmp.vybh1m9yJF/cluster1-mirror-image_44.export 2026-02-18T08:43:29.259 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-18T08:43:29.269 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmp /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_44.export /tmp/tmp.vybh1m9yJF/cluster1-mirror-image_44.export 2026-02-18T08:43:29.323 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_44.export /tmp/tmp.vybh1m9yJF/cluster1-mirror-image_44.export 2026-02-18T08:43:29.368 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:43:29.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T08:43:29.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_45 2026-02-18T08:43:29.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_snap cluster2 mirror image_45 snap 2026-02-18T08:43:29.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:43:29.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:43:29.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_45 2026-02-18T08:43:29.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap 2026-02-18T08:43:29.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 -p mirror snap create image_45@snap --debug-rbd=20 --debug-journaler=20 2026-02-18T08:43:31.568 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.0 with skip (0.5104866985809193) > weight (0.25) 2026-02-18T08:43:31.569 INFO:tasks.rbd_mirror_thrash:kill cluster1.client.mirror.1 2026-02-18T08:43:31.569 INFO:tasks.rbd_mirror.cluster1.client.mirror.1:Sent signal 15 2026-02-18T08:43:31.569 INFO:tasks.rbd_mirror_thrash:waiting for 4.020515451736451 secs before reviving daemons 2026-02-18T08:43:31.570 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.trial163.stderr:2026-02-18T08:43:31.561+0000 7fc8a5ed6640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.1 (PID: 99954) UID: 1000 2026-02-18T08:43:31.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_replay_started cluster1 mirror image_45 2026-02-18T08:43:31.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:43:31.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:43:31.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_45 2026-02-18T08:43:31.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_replay_state cluster1 mirror image_45 started 2026-02-18T08:43:31.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:43:31.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:43:31.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_45 2026-02-18T08:43:31.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local state=started 2026-02-18T08:43:31.936 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-18T08:43:31.936 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:43:31.936 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1 2026-02-18T08:43:32.936 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test_image_replay_state cluster1 mirror image_45 started 2026-02-18T08:43:32.936 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:43:32.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:43:32.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_45 2026-02-18T08:43:32.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local test_state=started 2026-02-18T08:43:32.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local status_result 2026-02-18T08:43:32.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_state=stopped 2026-02-18T08:43:32.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_45 2026-02-18T08:43:32.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -i state 2026-02-18T08:43:32.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_instance=cluster1 2026-02-18T08:43:32.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-18T08:43:32.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:43:32.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance=cluster1 2026-02-18T08:43:32.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local loop_instance 2026-02-18T08:43:32.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:43:32.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sleep 0 2026-02-18T08:43:32.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:43:32.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ seq 0 3 2026-02-18T08:43:32.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:43:32.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_45 2026-02-18T08:43:32.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:0 2026-02-18T08:43:32.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-18T08:43:32.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:43:32.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:43:32.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:43:32.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-18T08:43:32.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:43:32.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:43:32.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:43:32.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:43:32.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-18T08:43:32.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:43:32.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:43:32.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:43:32.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:43:32.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:0 2026-02-18T08:43:32.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-18T08:43:32.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-18T08:43:32.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:43:32.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:0 2026-02-18T08:43:32.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T08:43:32.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-18T08:43:32.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:43:32.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:43:32.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-18T08:43:32.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-18T08:43:32.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-18T08:43:32.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=0 2026-02-18T08:43:32.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=0 2026-02-18T08:43:32.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:43:32.950 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:43:32.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:43:32.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:43:32.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_45 2026-02-18T08:43:33.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ return 0 2026-02-18T08:43:33.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ status_result=' "state": "Replaying"' 2026-02-18T08:43:33.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ echo ' "state": "Replaying"' 2026-02-18T08:43:33.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ grep -i Replaying 2026-02-18T08:43:33.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: "state": "Replaying" 2026-02-18T08:43:33.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ current_state=started 2026-02-18T08:43:33.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test started = started 2026-02-18T08:43:33.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:43:33.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror image_45 2026-02-18T08:43:33.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_cluster=cluster1 2026-02-18T08:43:33.027 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:43:33.027 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_pool=mirror 2026-02-18T08:43:33.027 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local remote_pool=mirror 2026-02-18T08:43:33.027 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_45 2026-02-18T08:43:33.027 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' journal = journal ']' 2026-02-18T08:43:33.027 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror mirror image_45 2026-02-18T08:43:33.027 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_cluster=cluster1 2026-02-18T08:43:33.027 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:43:33.027 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_pool=mirror 2026-02-18T08:43:33.027 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local remote_pool=mirror 2026-02-18T08:43:33.027 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_45 2026-02-18T08:43:33.027 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-02-18T08:43:33.028 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-02-18T08:43:33.028 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:43:33.028 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:43:33.028 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:43:33.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_45 2026-02-18T08:43:33.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:43:33.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:43:33.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_45 2026-02-18T08:43:33.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:43:33.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_45 ']' 2026-02-18T08:43:33.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_45' 2026-02-18T08:43:33.229 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_45 2026-02-18T08:43:33.229 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:43:33.229 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:43:33.229 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:43:33.229 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:43:33.229 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:43:33.229 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:43:33.229 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:43:33.229 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:43:33.229 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:43:33.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:43:33.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_45 2026-02-18T08:43:33.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:43:33.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:43:33.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:43:33.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:43:33.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:43:33.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:43:33.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:43:33.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:43:33.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:43:33.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:43:33.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:43:33.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:43:33.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:43:33.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:43:33.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:43:33.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:43:33.234 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:43:33.234 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:43:33.234 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:43:33.234 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:43:33.234 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:43:33.234 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:43:33.234 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:43:33.234 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:43:33.234 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:43:33.234 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:43:33.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:43:33.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:43:33.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:43:33.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:43:33.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:43:33.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:43:33.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:43:33.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_45 2026-02-18T08:43:33.312 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:43:33.314 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_45 2026-02-18T08:43:33.314 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:43:33.314 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:43:33.314 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_45 2026-02-18T08:43:33.314 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_45 '' 2026-02-18T08:43:33.315 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:43:33.315 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:43:33.315 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_45 2026-02-18T08:43:33.315 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:43:33.315 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_45.status 2026-02-18T08:43:33.315 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_45.status 2026-02-18T08:43:33.315 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:43:33.316 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_45.status 2026-02-18T08:43:33.316 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_45 2026-02-18T08:43:33.316 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_45.status 2026-02-18T08:43:33.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 10 2026-02-18T08:43:33.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 10 2026-02-18T08:43:33.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:43:33.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=41, tag_tid=2, entry_tid=1], [object_number=40, tag_tid=2, entry_tid=0], [object_number=43, tag_tid=1, entry_tid=42863], [object_number=42, tag_tid=1, entry_tid=42862]]], state=connected] 2026-02-18T08:43:33.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=41, tag_tid=1, entry_tid=42865], [object_number=40, tag_tid=1, entry_tid=42864], [object_number=43, tag_tid=1, entry_tid=42863], [object_number=42, tag_tid=1, entry_tid=42862]]], state=connected] 2026-02-18T08:43:33.559 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_45.status 2026-02-18T08:43:33.560 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=41, tag_tid=2, entry_tid=1' 2026-02-18T08:43:33.562 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_45 2026-02-18T08:43:33.562 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:43:33.562 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:43:33.562 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_45 2026-02-18T08:43:33.562 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_45 '..*' 2026-02-18T08:43:33.562 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:43:33.562 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:43:33.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_45 2026-02-18T08:43:33.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:43:33.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_45.status 2026-02-18T08:43:33.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_45.status 2026-02-18T08:43:33.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:43:33.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_45.status 2026-02-18T08:43:33.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_45 2026-02-18T08:43:33.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_45.status 2026-02-18T08:43:33.598 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 10 2026-02-18T08:43:33.598 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 10 2026-02-18T08:43:33.598 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:43:33.599 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=41, tag_tid=2, entry_tid=1], [object_number=40, tag_tid=2, entry_tid=0], [object_number=43, tag_tid=1, entry_tid=42863], [object_number=42, tag_tid=1, entry_tid=42862]]], state=connected] 2026-02-18T08:43:33.599 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=41, tag_tid=1, entry_tid=42865], [object_number=40, tag_tid=1, entry_tid=42864], [object_number=43, tag_tid=1, entry_tid=42863], [object_number=42, tag_tid=1, entry_tid=42862]]], state=connected] 2026-02-18T08:43:33.601 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_45.status 2026-02-18T08:43:33.603 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=41, tag_tid=1, entry_tid=42865' 2026-02-18T08:43:33.603 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=41, tag_tid=2, entry_tid=1' -a 'object_number=41, tag_tid=2, entry_tid=1' = 'object_number=41, tag_tid=1, entry_tid=42865' 2026-02-18T08:43:33.603 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=41, tag_tid=1, entry_tid=42865' '!=' '' 2026-02-18T08:43:33.604 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:43:33.604 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=41, tag_tid=1, entry_tid=42865' = '' 2026-02-18T08:43:33.604 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=41, tag_tid=1, entry_tid=42865' 2026-02-18T08:43:33.604 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=41, tag_tid=2, entry_tid=1' 2026-02-18T08:43:33.604 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:43:33.605 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:43:33.606 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=2 2026-02-18T08:43:33.607 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=41, tag_tid=1, entry_tid=42865' 2026-02-18T08:43:33.607 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:43:33.607 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:43:33.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=1 2026-02-18T08:43:33.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=41, tag_tid=2, entry_tid=1' 2026-02-18T08:43:33.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:43:33.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:43:33.610 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:43:33.611 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=41, tag_tid=1, entry_tid=42865' 2026-02-18T08:43:33.611 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:43:33.611 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:43:33.613 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=42865 2026-02-18T08:43:33.613 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 2 = 1 -a 1 -le 42865 2026-02-18T08:43:33.613 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:43:33.613 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:43:33.613 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:43:33.815 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_45 2026-02-18T08:43:33.815 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:43:33.815 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:43:33.815 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_45 2026-02-18T08:43:33.815 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:43:33.815 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_45 ']' 2026-02-18T08:43:33.815 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_45' 2026-02-18T08:43:33.815 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_45 2026-02-18T08:43:33.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:43:33.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:43:33.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:43:33.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:43:33.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:43:33.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:43:33.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:43:33.819 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:43:33.819 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:43:33.819 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:43:33.819 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_45 2026-02-18T08:43:33.819 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:43:33.819 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:43:33.819 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:43:33.819 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:43:33.820 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:43:33.820 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:43:33.820 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:43:33.820 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:43:33.820 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:43:33.820 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:43:33.820 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:43:33.820 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:43:33.820 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:43:33.820 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:43:33.820 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:43:33.820 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:43:33.821 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:43:33.821 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:43:33.821 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:43:33.821 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:43:33.821 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:43:33.821 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:43:33.821 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:43:33.821 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:43:33.821 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:43:33.821 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:43:33.821 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:43:33.821 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:43:33.821 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:43:33.822 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:43:33.829 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:43:33.830 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:43:33.830 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:43:33.830 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_45 2026-02-18T08:43:33.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:43:33.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_45 2026-02-18T08:43:33.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:43:33.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:43:33.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_45 2026-02-18T08:43:33.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_45 '' 2026-02-18T08:43:33.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:43:33.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:43:33.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_45 2026-02-18T08:43:33.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:43:33.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_45.status 2026-02-18T08:43:33.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_45.status 2026-02-18T08:43:33.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:43:33.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_45.status 2026-02-18T08:43:33.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_45 2026-02-18T08:43:33.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_45.status 2026-02-18T08:43:33.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 10 2026-02-18T08:43:33.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 10 2026-02-18T08:43:33.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:43:33.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=41, tag_tid=2, entry_tid=1], [object_number=40, tag_tid=2, entry_tid=0], [object_number=43, tag_tid=1, entry_tid=42863], [object_number=42, tag_tid=1, entry_tid=42862]]], state=connected] 2026-02-18T08:43:33.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=41, tag_tid=1, entry_tid=42865], [object_number=40, tag_tid=1, entry_tid=42864], [object_number=43, tag_tid=1, entry_tid=42863], [object_number=42, tag_tid=1, entry_tid=42862]]], state=connected] 2026-02-18T08:43:33.923 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_45.status 2026-02-18T08:43:33.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=41, tag_tid=2, entry_tid=1' 2026-02-18T08:43:33.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_45 2026-02-18T08:43:33.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:43:33.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:43:33.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_45 2026-02-18T08:43:33.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_45 '..*' 2026-02-18T08:43:33.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:43:33.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:43:33.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_45 2026-02-18T08:43:33.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:43:33.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_45.status 2026-02-18T08:43:33.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_45.status 2026-02-18T08:43:33.928 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:43:33.928 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_45.status 2026-02-18T08:43:33.928 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_45 2026-02-18T08:43:33.929 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_45.status 2026-02-18T08:43:33.964 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 10 2026-02-18T08:43:33.964 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 10 2026-02-18T08:43:33.964 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:43:33.964 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=41, tag_tid=2, entry_tid=1], [object_number=40, tag_tid=2, entry_tid=0], [object_number=43, tag_tid=1, entry_tid=42863], [object_number=42, tag_tid=1, entry_tid=42862]]], state=connected] 2026-02-18T08:43:33.965 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=41, tag_tid=1, entry_tid=42865], [object_number=40, tag_tid=1, entry_tid=42864], [object_number=43, tag_tid=1, entry_tid=42863], [object_number=42, tag_tid=1, entry_tid=42862]]], state=connected] 2026-02-18T08:43:33.968 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_45.status 2026-02-18T08:43:33.970 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=41, tag_tid=1, entry_tid=42865' 2026-02-18T08:43:33.970 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=41, tag_tid=2, entry_tid=1' -a 'object_number=41, tag_tid=2, entry_tid=1' = 'object_number=41, tag_tid=1, entry_tid=42865' 2026-02-18T08:43:33.970 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=41, tag_tid=1, entry_tid=42865' '!=' 'object_number=41, tag_tid=1, entry_tid=42865' 2026-02-18T08:43:33.970 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:43:33.970 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.4 2026-02-18T08:43:34.372 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_45 2026-02-18T08:43:34.373 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:43:34.373 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:43:34.373 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_45 2026-02-18T08:43:34.373 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:43:34.373 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_45 ']' 2026-02-18T08:43:34.373 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_45' 2026-02-18T08:43:34.373 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_45 2026-02-18T08:43:34.373 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:43:34.373 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:43:34.373 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:43:34.373 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:43:34.374 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:43:34.374 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:43:34.374 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:43:34.374 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:43:34.376 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:43:34.377 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:43:34.377 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_45 2026-02-18T08:43:34.377 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:43:34.377 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:43:34.377 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:43:34.377 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:43:34.377 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:43:34.377 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:43:34.377 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:43:34.377 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:43:34.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:43:34.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:43:34.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:43:34.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:43:34.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:43:34.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:43:34.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:43:34.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:43:34.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:43:34.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:43:34.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:43:34.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:43:34.379 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:43:34.379 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:43:34.379 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:43:34.379 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:43:34.379 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:43:34.379 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:43:34.379 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:43:34.379 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:43:34.379 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:43:34.379 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:43:34.387 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:43:34.387 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:43:34.387 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:43:34.388 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_45 2026-02-18T08:43:34.453 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:43:34.454 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_45 2026-02-18T08:43:34.454 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:43:34.455 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:43:34.455 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_45 2026-02-18T08:43:34.455 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_45 '' 2026-02-18T08:43:34.455 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:43:34.455 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:43:34.455 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_45 2026-02-18T08:43:34.455 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:43:34.455 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_45.status 2026-02-18T08:43:34.455 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_45.status 2026-02-18T08:43:34.455 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:43:34.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_45.status 2026-02-18T08:43:34.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_45 2026-02-18T08:43:34.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_45.status 2026-02-18T08:43:34.483 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 10 2026-02-18T08:43:34.483 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 10 2026-02-18T08:43:34.483 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:43:34.483 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=41, tag_tid=2, entry_tid=1], [object_number=40, tag_tid=2, entry_tid=0], [object_number=43, tag_tid=1, entry_tid=42863], [object_number=42, tag_tid=1, entry_tid=42862]]], state=connected] 2026-02-18T08:43:34.484 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=41, tag_tid=1, entry_tid=42865], [object_number=40, tag_tid=1, entry_tid=42864], [object_number=43, tag_tid=1, entry_tid=42863], [object_number=42, tag_tid=1, entry_tid=42862]]], state=connected] 2026-02-18T08:43:34.486 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_45.status 2026-02-18T08:43:34.487 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=41, tag_tid=2, entry_tid=1' 2026-02-18T08:43:34.488 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_45 2026-02-18T08:43:34.488 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:43:34.488 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:43:34.489 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_45 2026-02-18T08:43:34.489 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_45 '..*' 2026-02-18T08:43:34.489 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:43:34.489 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:43:34.489 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_45 2026-02-18T08:43:34.489 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:43:34.489 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_45.status 2026-02-18T08:43:34.490 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_45.status 2026-02-18T08:43:34.490 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:43:34.490 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_45.status 2026-02-18T08:43:34.491 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_45 2026-02-18T08:43:34.491 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_45.status 2026-02-18T08:43:34.526 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 10 2026-02-18T08:43:34.526 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 10 2026-02-18T08:43:34.527 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:43:34.527 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=41, tag_tid=2, entry_tid=1], [object_number=40, tag_tid=2, entry_tid=0], [object_number=43, tag_tid=1, entry_tid=42863], [object_number=42, tag_tid=1, entry_tid=42862]]], state=connected] 2026-02-18T08:43:34.527 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=41, tag_tid=1, entry_tid=42865], [object_number=40, tag_tid=1, entry_tid=42864], [object_number=43, tag_tid=1, entry_tid=42863], [object_number=42, tag_tid=1, entry_tid=42862]]], state=connected] 2026-02-18T08:43:34.530 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_45.status 2026-02-18T08:43:34.532 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=41, tag_tid=1, entry_tid=42865' 2026-02-18T08:43:34.532 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=41, tag_tid=2, entry_tid=1' -a 'object_number=41, tag_tid=2, entry_tid=1' = 'object_number=41, tag_tid=1, entry_tid=42865' 2026-02-18T08:43:34.532 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=41, tag_tid=1, entry_tid=42865' '!=' 'object_number=41, tag_tid=1, entry_tid=42865' 2026-02-18T08:43:34.532 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:43:34.532 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.8 2026-02-18T08:43:35.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_45 2026-02-18T08:43:35.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:43:35.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:43:35.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_45 2026-02-18T08:43:35.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:43:35.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_45 ']' 2026-02-18T08:43:35.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_45' 2026-02-18T08:43:35.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_45 2026-02-18T08:43:35.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:43:35.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:43:35.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:43:35.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:43:35.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:43:35.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:43:35.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:43:35.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:43:35.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:43:35.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:43:35.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_45 2026-02-18T08:43:35.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:43:35.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:43:35.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:43:35.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:43:35.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:43:35.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:43:35.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:43:35.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:43:35.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:43:35.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:43:35.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:43:35.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:43:35.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:43:35.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:43:35.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:43:35.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:43:35.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:43:35.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:43:35.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:43:35.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:43:35.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:43:35.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:43:35.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:43:35.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:43:35.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:43:35.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:43:35.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:43:35.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:43:35.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:43:35.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:43:35.348 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:43:35.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:43:35.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:43:35.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_45 2026-02-18T08:43:35.412 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:43:35.412 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_45 2026-02-18T08:43:35.412 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:43:35.412 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:43:35.412 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_45 2026-02-18T08:43:35.412 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_45 '' 2026-02-18T08:43:35.412 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:43:35.413 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:43:35.413 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_45 2026-02-18T08:43:35.413 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:43:35.413 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_45.status 2026-02-18T08:43:35.413 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_45.status 2026-02-18T08:43:35.413 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:43:35.415 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_45.status 2026-02-18T08:43:35.415 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_45 2026-02-18T08:43:35.415 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_45.status 2026-02-18T08:43:35.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 10 2026-02-18T08:43:35.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 10 2026-02-18T08:43:35.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:43:35.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=41, tag_tid=2, entry_tid=1], [object_number=40, tag_tid=2, entry_tid=0], [object_number=43, tag_tid=1, entry_tid=42863], [object_number=42, tag_tid=1, entry_tid=42862]]], state=connected] 2026-02-18T08:43:35.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=41, tag_tid=1, entry_tid=42865], [object_number=40, tag_tid=1, entry_tid=42864], [object_number=43, tag_tid=1, entry_tid=42863], [object_number=42, tag_tid=1, entry_tid=42862]]], state=connected] 2026-02-18T08:43:35.451 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_45.status 2026-02-18T08:43:35.452 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=41, tag_tid=2, entry_tid=1' 2026-02-18T08:43:35.453 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_45 2026-02-18T08:43:35.454 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:43:35.454 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:43:35.454 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_45 2026-02-18T08:43:35.454 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_45 '..*' 2026-02-18T08:43:35.454 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:43:35.454 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:43:35.454 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_45 2026-02-18T08:43:35.455 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:43:35.455 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_45.status 2026-02-18T08:43:35.455 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_45.status 2026-02-18T08:43:35.455 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:43:35.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_45.status 2026-02-18T08:43:35.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_45 2026-02-18T08:43:35.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_45.status 2026-02-18T08:43:35.492 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 10 2026-02-18T08:43:35.492 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 10 2026-02-18T08:43:35.492 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:43:35.492 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=41, tag_tid=2, entry_tid=1], [object_number=40, tag_tid=2, entry_tid=0], [object_number=43, tag_tid=1, entry_tid=42863], [object_number=42, tag_tid=1, entry_tid=42862]]], state=connected] 2026-02-18T08:43:35.492 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=41, tag_tid=1, entry_tid=42865], [object_number=40, tag_tid=1, entry_tid=42864], [object_number=43, tag_tid=1, entry_tid=42863], [object_number=42, tag_tid=1, entry_tid=42862]]], state=connected] 2026-02-18T08:43:35.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_45.status 2026-02-18T08:43:35.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=41, tag_tid=1, entry_tid=42865' 2026-02-18T08:43:35.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=41, tag_tid=2, entry_tid=1' -a 'object_number=41, tag_tid=2, entry_tid=1' = 'object_number=41, tag_tid=1, entry_tid=42865' 2026-02-18T08:43:35.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=41, tag_tid=1, entry_tid=42865' '!=' 'object_number=41, tag_tid=1, entry_tid=42865' 2026-02-18T08:43:35.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:43:35.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1.6 2026-02-18T08:43:35.591 INFO:tasks.rbd_mirror_thrash:waiting for cluster1.client.mirror.1 2026-02-18T08:43:35.591 INFO:teuthology.orchestra.run:waiting for 600 2026-02-18T08:43:35.592 INFO:tasks.rbd_mirror_thrash:reviving cluster1.client.mirror.1 2026-02-18T08:43:35.592 INFO:tasks.rbd_mirror.cluster1.client.mirror.1:Restarting daemon 2026-02-18T08:43:35.592 DEBUG:teuthology.orchestra.run.trial163:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.1 2026-02-18T08:43:35.594 INFO:tasks.rbd_mirror.cluster1.client.mirror.1:Started 2026-02-18T08:43:35.594 INFO:tasks.rbd_mirror_thrash:waiting for 106.26556164415607 secs before thrashing 2026-02-18T08:43:37.100 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_45 2026-02-18T08:43:37.100 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:43:37.101 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:43:37.101 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_45 2026-02-18T08:43:37.101 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:43:37.101 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_45 ']' 2026-02-18T08:43:37.101 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_45' 2026-02-18T08:43:37.101 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_45 2026-02-18T08:43:37.101 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:43:37.101 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:43:37.102 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:43:37.102 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:43:37.102 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:43:37.102 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:43:37.102 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:43:37.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:43:37.105 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:43:37.105 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:43:37.105 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_45 2026-02-18T08:43:37.105 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:43:37.105 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:43:37.105 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:43:37.105 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:43:37.105 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:43:37.105 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:43:37.106 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:43:37.106 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:43:37.106 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:43:37.106 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:43:37.106 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:43:37.106 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:43:37.106 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:43:37.106 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:43:37.106 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:43:37.106 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:43:37.106 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:43:37.106 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:43:37.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:43:37.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:43:37.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:43:37.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:43:37.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:43:37.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:43:37.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:43:37.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:43:37.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:43:37.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:43:37.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:43:37.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:43:37.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:43:37.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:43:37.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:43:37.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_45 2026-02-18T08:43:37.177 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:43:37.178 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_45 2026-02-18T08:43:37.178 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:43:37.178 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:43:37.178 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_45 2026-02-18T08:43:37.178 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_45 '' 2026-02-18T08:43:37.178 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:43:37.178 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:43:37.178 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_45 2026-02-18T08:43:37.178 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:43:37.178 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_45.status 2026-02-18T08:43:37.179 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_45.status 2026-02-18T08:43:37.179 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:43:37.180 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_45.status 2026-02-18T08:43:37.180 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_45 2026-02-18T08:43:37.180 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_45.status 2026-02-18T08:43:37.217 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 10 2026-02-18T08:43:37.217 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 10 2026-02-18T08:43:37.217 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:43:37.217 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=41, tag_tid=2, entry_tid=1], [object_number=40, tag_tid=2, entry_tid=0], [object_number=43, tag_tid=1, entry_tid=42863], [object_number=42, tag_tid=1, entry_tid=42862]]], state=connected] 2026-02-18T08:43:37.218 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=41, tag_tid=1, entry_tid=42865], [object_number=40, tag_tid=1, entry_tid=42864], [object_number=43, tag_tid=1, entry_tid=42863], [object_number=42, tag_tid=1, entry_tid=42862]]], state=connected] 2026-02-18T08:43:37.221 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_45.status 2026-02-18T08:43:37.223 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=41, tag_tid=2, entry_tid=1' 2026-02-18T08:43:37.224 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_45 2026-02-18T08:43:37.224 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:43:37.224 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:43:37.224 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_45 2026-02-18T08:43:37.224 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_45 '..*' 2026-02-18T08:43:37.225 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:43:37.225 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:43:37.225 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_45 2026-02-18T08:43:37.225 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:43:37.225 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_45.status 2026-02-18T08:43:37.225 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_45.status 2026-02-18T08:43:37.225 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:43:37.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_45.status 2026-02-18T08:43:37.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_45.status 2026-02-18T08:43:37.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_45 2026-02-18T08:43:37.252 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 10 2026-02-18T08:43:37.252 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 10 2026-02-18T08:43:37.252 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:43:37.253 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=41, tag_tid=2, entry_tid=1], [object_number=40, tag_tid=2, entry_tid=0], [object_number=43, tag_tid=1, entry_tid=42863], [object_number=42, tag_tid=1, entry_tid=42862]]], state=connected] 2026-02-18T08:43:37.253 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=41, tag_tid=1, entry_tid=42865], [object_number=40, tag_tid=1, entry_tid=42864], [object_number=43, tag_tid=1, entry_tid=42863], [object_number=42, tag_tid=1, entry_tid=42862]]], state=connected] 2026-02-18T08:43:37.255 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_45.status 2026-02-18T08:43:37.257 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=41, tag_tid=1, entry_tid=42865' 2026-02-18T08:43:37.257 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=41, tag_tid=2, entry_tid=1' -a 'object_number=41, tag_tid=2, entry_tid=1' = 'object_number=41, tag_tid=1, entry_tid=42865' 2026-02-18T08:43:37.257 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=41, tag_tid=1, entry_tid=42865' '!=' 'object_number=41, tag_tid=1, entry_tid=42865' 2026-02-18T08:43:37.257 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:43:37.257 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 2 2026-02-18T08:43:39.259 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_45 2026-02-18T08:43:39.259 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:43:39.260 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:43:39.260 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_45 2026-02-18T08:43:39.260 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:43:39.260 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_45 ']' 2026-02-18T08:43:39.260 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_45' 2026-02-18T08:43:39.260 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_45 2026-02-18T08:43:39.260 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:43:39.260 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:43:39.260 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:43:39.261 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:43:39.261 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:43:39.261 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:43:39.261 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:43:39.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:43:39.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:43:39.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:43:39.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_45 2026-02-18T08:43:39.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:43:39.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:43:39.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:43:39.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:43:39.266 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:43:39.266 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:43:39.266 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:43:39.266 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:43:39.266 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:43:39.266 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:43:39.266 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:43:39.266 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:43:39.266 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:43:39.266 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:43:39.266 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:43:39.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:43:39.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:43:39.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:43:39.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:43:39.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:43:39.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:43:39.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:43:39.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:43:39.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:43:39.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:43:39.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:43:39.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:43:39.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:43:39.269 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:43:39.269 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:43:39.275 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:43:39.276 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:43:39.276 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:43:39.277 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_45 2026-02-18T08:43:39.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:43:39.348 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_45 2026-02-18T08:43:39.348 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:43:39.348 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:43:39.348 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_45 2026-02-18T08:43:39.348 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_45 '' 2026-02-18T08:43:39.348 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:43:39.348 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:43:39.348 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_45 2026-02-18T08:43:39.348 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:43:39.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_45.status 2026-02-18T08:43:39.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_45.status 2026-02-18T08:43:39.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:43:39.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_45.status 2026-02-18T08:43:39.350 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_45 2026-02-18T08:43:39.350 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_45.status 2026-02-18T08:43:39.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 10 2026-02-18T08:43:39.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 10 2026-02-18T08:43:39.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:43:39.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=41, tag_tid=2, entry_tid=1], [object_number=40, tag_tid=2, entry_tid=0], [object_number=43, tag_tid=1, entry_tid=42863], [object_number=42, tag_tid=1, entry_tid=42862]]], state=connected] 2026-02-18T08:43:39.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=41, tag_tid=2, entry_tid=1], [object_number=40, tag_tid=2, entry_tid=0], [object_number=43, tag_tid=1, entry_tid=42863], [object_number=42, tag_tid=1, entry_tid=42862]]], state=connected] 2026-02-18T08:43:39.380 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_45.status 2026-02-18T08:43:39.382 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=41, tag_tid=2, entry_tid=1' 2026-02-18T08:43:39.383 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_45 2026-02-18T08:43:39.383 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:43:39.383 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:43:39.383 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_45 2026-02-18T08:43:39.383 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_45 '..*' 2026-02-18T08:43:39.384 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:43:39.384 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:43:39.384 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_45 2026-02-18T08:43:39.384 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:43:39.384 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_45.status 2026-02-18T08:43:39.384 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_45.status 2026-02-18T08:43:39.385 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:43:39.385 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_45.status 2026-02-18T08:43:39.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_45 2026-02-18T08:43:39.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_45.status 2026-02-18T08:43:39.412 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 10 2026-02-18T08:43:39.413 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 10 2026-02-18T08:43:39.413 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:43:39.413 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=41, tag_tid=2, entry_tid=1], [object_number=40, tag_tid=2, entry_tid=0], [object_number=43, tag_tid=1, entry_tid=42863], [object_number=42, tag_tid=1, entry_tid=42862]]], state=connected] 2026-02-18T08:43:39.413 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=41, tag_tid=2, entry_tid=1], [object_number=40, tag_tid=2, entry_tid=0], [object_number=43, tag_tid=1, entry_tid=42863], [object_number=42, tag_tid=1, entry_tid=42862]]], state=connected] 2026-02-18T08:43:39.415 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_45.status 2026-02-18T08:43:39.417 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=41, tag_tid=2, entry_tid=1' 2026-02-18T08:43:39.417 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=41, tag_tid=2, entry_tid=1' -a 'object_number=41, tag_tid=2, entry_tid=1' = 'object_number=41, tag_tid=2, entry_tid=1' 2026-02-18T08:43:39.417 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:43:39.417 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_snap_present cluster1 mirror image_45 snap 2026-02-18T08:43:39.417 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:43:39.417 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:43:39.417 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_45 2026-02-18T08:43:39.417 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap 2026-02-18T08:43:39.417 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-18T08:43:39.417 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-18T08:43:39.418 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1 2026-02-18T08:43:40.418 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster1 info mirror/image_45@snap 2026-02-18T08:43:40.446 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:rbd image 'image_45': 2026-02-18T08:43:40.446 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: size 128 MiB in 32 objects 2026-02-18T08:43:40.446 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: order 22 (4 MiB objects) 2026-02-18T08:43:40.446 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: snapshot_count: 1 2026-02-18T08:43:40.446 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: id: 10edeefecfc1 2026-02-18T08:43:40.446 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: block_name_prefix: rbd_data.10edeefecfc1 2026-02-18T08:43:40.446 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: format: 2 2026-02-18T08:43:40.446 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: features: layering, exclusive-lock, journaling 2026-02-18T08:43:40.446 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: op_features: 2026-02-18T08:43:40.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: flags: 2026-02-18T08:43:40.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: create_timestamp: Wed Feb 18 08:33:11 2026 2026-02-18T08:43:40.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: access_timestamp: Wed Feb 18 08:33:11 2026 2026-02-18T08:43:40.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: modify_timestamp: Wed Feb 18 08:35:13 2026 2026-02-18T08:43:40.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: protected: False 2026-02-18T08:43:40.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: journal: 10edeefecfc1 2026-02-18T08:43:40.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring state: enabled 2026-02-18T08:43:40.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring mode: journal 2026-02-18T08:43:40.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring global id: 715214c1-af51-4c8b-899a-cfed67f38c0b 2026-02-18T08:43:40.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring primary: false 2026-02-18T08:43:40.449 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:43:40.449 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ compare_image_snaps mirror image_45 snap 2026-02-18T08:43:40.449 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:43:40.449 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_45 2026-02-18T08:43:40.449 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap 2026-02-18T08:43:40.450 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local ret=0 2026-02-18T08:43:40.450 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local rmt_export=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_45.export 2026-02-18T08:43:40.450 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loc_export=/tmp/tmp.vybh1m9yJF/cluster1-mirror-image_45.export 2026-02-18T08:43:40.450 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_45.export /tmp/tmp.vybh1m9yJF/cluster1-mirror-image_45.export 2026-02-18T08:43:40.450 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 -p mirror export image_45@snap /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_45.export 2026-02-18T08:43:40.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-18T08:43:40.600 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster1 -p mirror export image_45@snap /tmp/tmp.vybh1m9yJF/cluster1-mirror-image_45.export 2026-02-18T08:43:40.780 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-18T08:43:40.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmp /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_45.export /tmp/tmp.vybh1m9yJF/cluster1-mirror-image_45.export 2026-02-18T08:43:40.844 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_45.export /tmp/tmp.vybh1m9yJF/cluster1-mirror-image_45.export 2026-02-18T08:43:40.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:43:40.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T08:43:40.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_46 2026-02-18T08:43:40.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_snap cluster2 mirror image_46 snap 2026-02-18T08:43:40.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:43:40.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:43:40.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_46 2026-02-18T08:43:40.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap 2026-02-18T08:43:40.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 -p mirror snap create image_46@snap --debug-rbd=20 --debug-journaler=20 2026-02-18T08:43:43.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_replay_started cluster1 mirror image_46 2026-02-18T08:43:43.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:43:43.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:43:43.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_46 2026-02-18T08:43:43.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_replay_state cluster1 mirror image_46 started 2026-02-18T08:43:43.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:43:43.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:43:43.952 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_46 2026-02-18T08:43:43.952 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local state=started 2026-02-18T08:43:43.952 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-18T08:43:43.952 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:43:43.952 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1 2026-02-18T08:43:44.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test_image_replay_state cluster1 mirror image_46 started 2026-02-18T08:43:44.952 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:43:44.952 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:43:44.952 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_46 2026-02-18T08:43:44.952 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local test_state=started 2026-02-18T08:43:44.952 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local status_result 2026-02-18T08:43:44.952 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_state=stopped 2026-02-18T08:43:44.952 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_46 2026-02-18T08:43:44.952 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -i state 2026-02-18T08:43:44.952 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_instance=cluster1 2026-02-18T08:43:44.953 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-18T08:43:44.953 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:43:44.953 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance=cluster1 2026-02-18T08:43:44.953 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local loop_instance 2026-02-18T08:43:44.953 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:43:44.953 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sleep 0 2026-02-18T08:43:44.953 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:43:44.953 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ seq 0 3 2026-02-18T08:43:44.955 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:43:44.956 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_46 2026-02-18T08:43:44.956 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:0 2026-02-18T08:43:44.956 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-18T08:43:44.956 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:43:44.956 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:43:44.956 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:43:44.956 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-18T08:43:44.956 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:43:44.956 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:43:44.956 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:43:44.956 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:43:44.956 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-18T08:43:44.956 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:43:44.956 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:43:44.956 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:43:44.956 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:43:44.956 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:0 2026-02-18T08:43:44.956 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-18T08:43:44.956 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-18T08:43:44.956 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:43:44.956 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:0 2026-02-18T08:43:44.956 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T08:43:44.956 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-18T08:43:44.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:43:44.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:43:44.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-18T08:43:44.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-18T08:43:44.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-18T08:43:44.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=0 2026-02-18T08:43:44.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=0 2026-02-18T08:43:44.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:43:44.967 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:43:44.967 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:43:44.967 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:43:44.967 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_46 2026-02-18T08:43:45.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_46'] 2026-02-18T08:43:45.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status : get status for rbd mirror 2026-02-18T08:43:45.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:43:45.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:43:45.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_46 2026-02-18T08:43:45.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:1 2026-02-18T08:43:45.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-18T08:43:45.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:43:45.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:43:45.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:43:45.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-18T08:43:45.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:43:45.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:43:45.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:43:45.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:43:45.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-18T08:43:45.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:43:45.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:43:45.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:43:45.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:43:45.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:1 2026-02-18T08:43:45.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-18T08:43:45.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-18T08:43:45.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:43:45.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:1 2026-02-18T08:43:45.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T08:43:45.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-18T08:43:45.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:43:45.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:43:45.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-18T08:43:45.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-18T08:43:45.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-18T08:43:45.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=1 2026-02-18T08:43:45.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=1 2026-02-18T08:43:45.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:43:45.035 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:43:45.036 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:43:45.036 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:43:45.036 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_46 2026-02-18T08:43:45.101 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ return 0 2026-02-18T08:43:45.102 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ status_result=' "state": "Replaying"' 2026-02-18T08:43:45.102 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ echo ' "state": "Replaying"' 2026-02-18T08:43:45.102 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ grep -i Replaying 2026-02-18T08:43:45.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: "state": "Replaying" 2026-02-18T08:43:45.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ current_state=started 2026-02-18T08:43:45.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test started = started 2026-02-18T08:43:45.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:43:45.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror image_46 2026-02-18T08:43:45.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_cluster=cluster1 2026-02-18T08:43:45.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:43:45.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_pool=mirror 2026-02-18T08:43:45.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local remote_pool=mirror 2026-02-18T08:43:45.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_46 2026-02-18T08:43:45.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' journal = journal ']' 2026-02-18T08:43:45.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror mirror image_46 2026-02-18T08:43:45.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_cluster=cluster1 2026-02-18T08:43:45.105 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:43:45.105 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_pool=mirror 2026-02-18T08:43:45.105 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local remote_pool=mirror 2026-02-18T08:43:45.105 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_46 2026-02-18T08:43:45.105 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-02-18T08:43:45.105 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-02-18T08:43:45.105 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:43:45.105 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:43:45.105 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:43:45.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_46 2026-02-18T08:43:45.312 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:43:45.312 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:43:45.312 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_46 2026-02-18T08:43:45.312 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:43:45.312 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_46 ']' 2026-02-18T08:43:45.312 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_46' 2026-02-18T08:43:45.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_46 2026-02-18T08:43:45.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:43:45.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:43:45.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:43:45.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:43:45.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:43:45.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:43:45.314 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:43:45.314 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:43:45.314 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:43:45.314 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:43:45.314 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_46 2026-02-18T08:43:45.314 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:43:45.314 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:43:45.314 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:43:45.315 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:43:45.315 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:43:45.315 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:43:45.315 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:43:45.315 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:43:45.315 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:43:45.315 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:43:45.316 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:43:45.316 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:43:45.316 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:43:45.316 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:43:45.316 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:43:45.316 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:43:45.316 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:43:45.316 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:43:45.317 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:43:45.317 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:43:45.317 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:43:45.317 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:43:45.317 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:43:45.317 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:43:45.317 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:43:45.317 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:43:45.318 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:43:45.318 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:43:45.318 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:43:45.318 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:43:45.322 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:43:45.322 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:43:45.323 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:43:45.323 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_46 2026-02-18T08:43:45.382 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_46'] 2026-02-18T08:43:45.383 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:43:45.383 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:43:45.385 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:43:45.385 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_46 2026-02-18T08:43:45.385 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:43:45.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:43:45.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:43:45.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:43:45.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:43:45.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:43:45.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:43:45.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:43:45.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:43:45.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:43:45.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:43:45.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:43:45.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:43:45.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:43:45.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:43:45.387 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:43:45.387 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:43:45.387 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:43:45.387 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:43:45.387 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:43:45.387 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:43:45.387 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:43:45.387 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:43:45.387 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:43:45.387 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:43:45.387 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:43:45.387 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:43:45.387 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:43:45.387 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:43:45.387 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:43:45.397 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:43:45.397 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:43:45.397 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:43:45.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_46 2026-02-18T08:43:45.453 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:43:45.453 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_46 2026-02-18T08:43:45.453 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:43:45.453 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:43:45.454 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_46 2026-02-18T08:43:45.454 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_46 '' 2026-02-18T08:43:45.454 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:43:45.454 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:43:45.454 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_46 2026-02-18T08:43:45.454 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:43:45.454 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_46.status 2026-02-18T08:43:45.454 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_46.status 2026-02-18T08:43:45.455 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:43:45.455 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_46.status 2026-02-18T08:43:45.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_46 2026-02-18T08:43:45.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_46.status 2026-02-18T08:43:45.482 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 4 2026-02-18T08:43:45.482 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 4 2026-02-18T08:43:45.482 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:43:45.483 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=2, entry_tid=1], [object_number=16, tag_tid=2, entry_tid=0], [object_number=19, tag_tid=1, entry_tid=18523], [object_number=18, tag_tid=1, entry_tid=18522]]], state=connected] 2026-02-18T08:43:45.483 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=19, tag_tid=1, entry_tid=18523], [object_number=18, tag_tid=1, entry_tid=18522], [object_number=17, tag_tid=1, entry_tid=18521], [object_number=16, tag_tid=1, entry_tid=18520]]], state=connected] 2026-02-18T08:43:45.485 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_46.status 2026-02-18T08:43:45.487 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=17, tag_tid=2, entry_tid=1' 2026-02-18T08:43:45.488 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_46 2026-02-18T08:43:45.488 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:43:45.488 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:43:45.488 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_46 2026-02-18T08:43:45.488 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_46 '..*' 2026-02-18T08:43:45.488 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:43:45.488 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:43:45.488 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_46 2026-02-18T08:43:45.488 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:43:45.489 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_46.status 2026-02-18T08:43:45.489 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_46.status 2026-02-18T08:43:45.489 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:43:45.490 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_46.status 2026-02-18T08:43:45.490 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_46 2026-02-18T08:43:45.490 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_46.status 2026-02-18T08:43:45.523 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 4 2026-02-18T08:43:45.523 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 4 2026-02-18T08:43:45.523 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:43:45.523 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=2, entry_tid=1], [object_number=16, tag_tid=2, entry_tid=0], [object_number=19, tag_tid=1, entry_tid=18523], [object_number=18, tag_tid=1, entry_tid=18522]]], state=connected] 2026-02-18T08:43:45.523 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=19, tag_tid=1, entry_tid=18523], [object_number=18, tag_tid=1, entry_tid=18522], [object_number=17, tag_tid=1, entry_tid=18521], [object_number=16, tag_tid=1, entry_tid=18520]]], state=connected] 2026-02-18T08:43:45.526 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_46.status 2026-02-18T08:43:45.528 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=19, tag_tid=1, entry_tid=18523' 2026-02-18T08:43:45.528 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=17, tag_tid=2, entry_tid=1' -a 'object_number=17, tag_tid=2, entry_tid=1' = 'object_number=19, tag_tid=1, entry_tid=18523' 2026-02-18T08:43:45.528 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=19, tag_tid=1, entry_tid=18523' '!=' '' 2026-02-18T08:43:45.528 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:43:45.528 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=19, tag_tid=1, entry_tid=18523' = '' 2026-02-18T08:43:45.528 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=19, tag_tid=1, entry_tid=18523' 2026-02-18T08:43:45.529 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=17, tag_tid=2, entry_tid=1' 2026-02-18T08:43:45.529 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:43:45.529 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:43:45.530 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=2 2026-02-18T08:43:45.531 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=19, tag_tid=1, entry_tid=18523' 2026-02-18T08:43:45.531 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:43:45.531 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:43:45.532 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=1 2026-02-18T08:43:45.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=17, tag_tid=2, entry_tid=1' 2026-02-18T08:43:45.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:43:45.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:43:45.534 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:43:45.535 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=19, tag_tid=1, entry_tid=18523' 2026-02-18T08:43:45.535 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:43:45.535 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:43:45.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=18523 2026-02-18T08:43:45.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 2 = 1 -a 1 -le 18523 2026-02-18T08:43:45.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:43:45.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:43:45.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:43:45.739 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_46 2026-02-18T08:43:45.739 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:43:45.739 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:43:45.739 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_46 2026-02-18T08:43:45.739 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:43:45.740 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_46 ']' 2026-02-18T08:43:45.740 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_46' 2026-02-18T08:43:45.740 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_46 2026-02-18T08:43:45.740 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:43:45.740 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:43:45.740 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:43:45.740 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:43:45.740 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:43:45.740 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:43:45.740 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:43:45.741 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:43:45.743 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:43:45.743 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:43:45.744 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_46 2026-02-18T08:43:45.744 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:43:45.744 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:43:45.744 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:43:45.744 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:43:45.744 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:43:45.744 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:43:45.744 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:43:45.744 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:43:45.744 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:43:45.744 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:43:45.745 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:43:45.745 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:43:45.745 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:43:45.745 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:43:45.745 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:43:45.745 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:43:45.745 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:43:45.745 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:43:45.745 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:43:45.745 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:43:45.745 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:43:45.746 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:43:45.746 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:43:45.746 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:43:45.746 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:43:45.746 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:43:45.746 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:43:45.746 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:43:45.746 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:43:45.746 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:43:45.754 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:43:45.755 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:43:45.755 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:43:45.755 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_46 2026-02-18T08:43:45.807 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_46'] 2026-02-18T08:43:45.808 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:43:45.808 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:43:45.810 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:43:45.810 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_46 2026-02-18T08:43:45.810 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:43:45.811 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:43:45.811 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:43:45.811 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:43:45.811 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:43:45.811 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:43:45.811 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:43:45.811 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:43:45.811 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:43:45.811 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:43:45.811 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:43:45.811 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:43:45.812 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:43:45.812 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:43:45.812 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:43:45.812 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:43:45.812 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:43:45.812 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:43:45.812 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:43:45.812 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:43:45.812 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:43:45.812 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:43:45.812 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:43:45.813 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:43:45.813 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:43:45.813 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:43:45.813 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:43:45.813 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:43:45.813 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:43:45.813 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:43:45.822 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:43:45.822 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:43:45.822 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:43:45.823 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_46 2026-02-18T08:43:45.879 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:43:45.880 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_46 2026-02-18T08:43:45.880 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:43:45.880 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:43:45.880 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_46 2026-02-18T08:43:45.880 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_46 '' 2026-02-18T08:43:45.880 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:43:45.881 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:43:45.881 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_46 2026-02-18T08:43:45.881 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:43:45.881 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_46.status 2026-02-18T08:43:45.881 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_46.status 2026-02-18T08:43:45.881 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:43:45.881 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_46.status 2026-02-18T08:43:45.882 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_46 2026-02-18T08:43:45.882 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_46.status 2026-02-18T08:43:45.910 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 4 2026-02-18T08:43:45.910 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 4 2026-02-18T08:43:45.910 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:43:45.910 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=2, entry_tid=1], [object_number=16, tag_tid=2, entry_tid=0], [object_number=19, tag_tid=1, entry_tid=18523], [object_number=18, tag_tid=1, entry_tid=18522]]], state=connected] 2026-02-18T08:43:45.910 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=19, tag_tid=1, entry_tid=18523], [object_number=18, tag_tid=1, entry_tid=18522], [object_number=17, tag_tid=1, entry_tid=18521], [object_number=16, tag_tid=1, entry_tid=18520]]], state=connected] 2026-02-18T08:43:45.913 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_46.status 2026-02-18T08:43:45.915 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=17, tag_tid=2, entry_tid=1' 2026-02-18T08:43:45.916 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_46 2026-02-18T08:43:45.916 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:43:45.916 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:43:45.916 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_46 2026-02-18T08:43:45.916 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_46 '..*' 2026-02-18T08:43:45.917 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:43:45.917 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:43:45.917 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_46 2026-02-18T08:43:45.917 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:43:45.917 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_46.status 2026-02-18T08:43:45.918 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_46.status 2026-02-18T08:43:45.918 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:43:45.918 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_46.status 2026-02-18T08:43:45.918 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_46.status 2026-02-18T08:43:45.918 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_46 2026-02-18T08:43:45.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 4 2026-02-18T08:43:45.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 4 2026-02-18T08:43:45.946 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:43:45.946 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=2, entry_tid=1], [object_number=16, tag_tid=2, entry_tid=0], [object_number=19, tag_tid=1, entry_tid=18523], [object_number=18, tag_tid=1, entry_tid=18522]]], state=connected] 2026-02-18T08:43:45.946 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=19, tag_tid=1, entry_tid=18523], [object_number=18, tag_tid=1, entry_tid=18522], [object_number=17, tag_tid=1, entry_tid=18521], [object_number=16, tag_tid=1, entry_tid=18520]]], state=connected] 2026-02-18T08:43:45.948 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_46.status 2026-02-18T08:43:45.950 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=19, tag_tid=1, entry_tid=18523' 2026-02-18T08:43:45.950 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=17, tag_tid=2, entry_tid=1' -a 'object_number=17, tag_tid=2, entry_tid=1' = 'object_number=19, tag_tid=1, entry_tid=18523' 2026-02-18T08:43:45.950 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=19, tag_tid=1, entry_tid=18523' '!=' 'object_number=19, tag_tid=1, entry_tid=18523' 2026-02-18T08:43:45.950 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:43:45.950 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.4 2026-02-18T08:43:46.351 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_46 2026-02-18T08:43:46.351 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:43:46.351 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:43:46.351 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_46 2026-02-18T08:43:46.351 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:43:46.351 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_46 ']' 2026-02-18T08:43:46.351 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_46' 2026-02-18T08:43:46.351 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_46 2026-02-18T08:43:46.351 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:43:46.351 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:43:46.351 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:43:46.351 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:43:46.351 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:43:46.351 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:43:46.351 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:43:46.351 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:43:46.352 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:43:46.353 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:43:46.353 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_46 2026-02-18T08:43:46.353 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:43:46.353 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:43:46.354 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:43:46.354 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:43:46.354 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:43:46.354 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:43:46.354 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:43:46.354 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:43:46.354 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:43:46.354 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:43:46.354 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:43:46.354 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:43:46.354 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:43:46.354 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:43:46.354 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:43:46.354 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:43:46.354 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:43:46.354 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:43:46.355 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:43:46.355 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:43:46.355 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:43:46.355 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:43:46.355 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:43:46.355 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:43:46.355 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:43:46.355 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:43:46.355 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:43:46.355 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:43:46.355 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:43:46.355 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:43:46.367 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:43:46.367 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:43:46.367 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:43:46.367 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_46 2026-02-18T08:43:46.428 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_46'] 2026-02-18T08:43:46.428 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:43:46.428 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:43:46.431 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:43:46.431 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_46 2026-02-18T08:43:46.431 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:43:46.431 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:43:46.431 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:43:46.431 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:43:46.431 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:43:46.432 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:43:46.432 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:43:46.432 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:43:46.432 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:43:46.432 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:43:46.432 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:43:46.432 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:43:46.432 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:43:46.432 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:43:46.432 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:43:46.432 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:43:46.432 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:43:46.433 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:43:46.433 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:43:46.433 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:43:46.433 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:43:46.433 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:43:46.433 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:43:46.433 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:43:46.433 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:43:46.433 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:43:46.433 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:43:46.433 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:43:46.433 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:43:46.434 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:43:46.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:43:46.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:43:46.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:43:46.443 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_46 2026-02-18T08:43:46.506 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:43:46.506 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_46 2026-02-18T08:43:46.506 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:43:46.506 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:43:46.506 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_46 2026-02-18T08:43:46.507 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_46 '' 2026-02-18T08:43:46.507 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:43:46.507 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:43:46.507 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_46 2026-02-18T08:43:46.507 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:43:46.507 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_46.status 2026-02-18T08:43:46.507 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_46.status 2026-02-18T08:43:46.508 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:43:46.508 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_46.status 2026-02-18T08:43:46.508 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_46 2026-02-18T08:43:46.508 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_46.status 2026-02-18T08:43:46.534 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 4 2026-02-18T08:43:46.534 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 4 2026-02-18T08:43:46.534 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:43:46.534 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=2, entry_tid=1], [object_number=16, tag_tid=2, entry_tid=0], [object_number=19, tag_tid=1, entry_tid=18523], [object_number=18, tag_tid=1, entry_tid=18522]]], state=connected] 2026-02-18T08:43:46.534 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=19, tag_tid=1, entry_tid=18523], [object_number=18, tag_tid=1, entry_tid=18522], [object_number=17, tag_tid=1, entry_tid=18521], [object_number=16, tag_tid=1, entry_tid=18520]]], state=connected] 2026-02-18T08:43:46.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_46.status 2026-02-18T08:43:46.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=17, tag_tid=2, entry_tid=1' 2026-02-18T08:43:46.540 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_46 2026-02-18T08:43:46.540 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:43:46.540 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:43:46.540 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_46 2026-02-18T08:43:46.540 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_46 '..*' 2026-02-18T08:43:46.540 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:43:46.540 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:43:46.540 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_46 2026-02-18T08:43:46.540 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:43:46.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_46.status 2026-02-18T08:43:46.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_46.status 2026-02-18T08:43:46.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:43:46.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_46.status 2026-02-18T08:43:46.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_46 2026-02-18T08:43:46.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_46.status 2026-02-18T08:43:46.568 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 4 2026-02-18T08:43:46.568 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 4 2026-02-18T08:43:46.568 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:43:46.568 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=2, entry_tid=1], [object_number=16, tag_tid=2, entry_tid=0], [object_number=19, tag_tid=1, entry_tid=18523], [object_number=18, tag_tid=1, entry_tid=18522]]], state=connected] 2026-02-18T08:43:46.569 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=19, tag_tid=1, entry_tid=18523], [object_number=18, tag_tid=1, entry_tid=18522], [object_number=17, tag_tid=1, entry_tid=18521], [object_number=16, tag_tid=1, entry_tid=18520]]], state=connected] 2026-02-18T08:43:46.570 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_46.status 2026-02-18T08:43:46.572 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=19, tag_tid=1, entry_tid=18523' 2026-02-18T08:43:46.573 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=17, tag_tid=2, entry_tid=1' -a 'object_number=17, tag_tid=2, entry_tid=1' = 'object_number=19, tag_tid=1, entry_tid=18523' 2026-02-18T08:43:46.573 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=19, tag_tid=1, entry_tid=18523' '!=' 'object_number=19, tag_tid=1, entry_tid=18523' 2026-02-18T08:43:46.573 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:43:46.573 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.8 2026-02-18T08:43:47.374 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_46 2026-02-18T08:43:47.374 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:43:47.375 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:43:47.375 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_46 2026-02-18T08:43:47.375 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:43:47.375 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_46 ']' 2026-02-18T08:43:47.375 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_46' 2026-02-18T08:43:47.375 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_46 2026-02-18T08:43:47.375 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:43:47.375 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:43:47.375 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:43:47.376 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:43:47.376 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:43:47.376 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:43:47.376 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:43:47.376 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:43:47.377 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:43:47.377 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:43:47.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_46 2026-02-18T08:43:47.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:43:47.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:43:47.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:43:47.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:43:47.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:43:47.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:43:47.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:43:47.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:43:47.380 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:43:47.380 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:43:47.380 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:43:47.380 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:43:47.381 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:43:47.381 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:43:47.381 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:43:47.381 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:43:47.381 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:43:47.381 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:43:47.381 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:43:47.381 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:43:47.381 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:43:47.381 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:43:47.381 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:43:47.382 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:43:47.382 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:43:47.382 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:43:47.382 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:43:47.382 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:43:47.382 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:43:47.382 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:43:47.390 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:43:47.390 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:43:47.390 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:43:47.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_46 2026-02-18T08:43:47.450 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_46'] 2026-02-18T08:43:47.450 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:43:47.450 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:43:47.452 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:43:47.452 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_46 2026-02-18T08:43:47.452 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:43:47.452 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:43:47.452 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:43:47.452 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:43:47.452 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:43:47.452 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:43:47.452 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:43:47.452 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:43:47.453 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:43:47.453 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:43:47.453 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:43:47.453 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:43:47.453 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:43:47.453 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:43:47.453 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:43:47.453 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:43:47.453 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:43:47.453 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:43:47.453 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:43:47.453 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:43:47.453 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:43:47.453 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:43:47.453 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:43:47.453 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:43:47.453 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:43:47.453 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:43:47.453 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:43:47.453 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:43:47.453 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:43:47.453 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:43:47.465 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:43:47.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:43:47.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:43:47.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_46 2026-02-18T08:43:47.526 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:43:47.526 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_46 2026-02-18T08:43:47.526 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:43:47.526 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:43:47.527 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_46 2026-02-18T08:43:47.527 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_46 '' 2026-02-18T08:43:47.527 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:43:47.527 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:43:47.527 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_46 2026-02-18T08:43:47.527 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:43:47.527 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_46.status 2026-02-18T08:43:47.527 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_46.status 2026-02-18T08:43:47.527 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:43:47.528 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_46.status 2026-02-18T08:43:47.529 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_46 2026-02-18T08:43:47.529 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_46.status 2026-02-18T08:43:47.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 4 2026-02-18T08:43:47.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 4 2026-02-18T08:43:47.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:43:47.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=2, entry_tid=1], [object_number=16, tag_tid=2, entry_tid=0], [object_number=19, tag_tid=1, entry_tid=18523], [object_number=18, tag_tid=1, entry_tid=18522]]], state=connected] 2026-02-18T08:43:47.557 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=19, tag_tid=1, entry_tid=18523], [object_number=18, tag_tid=1, entry_tid=18522], [object_number=17, tag_tid=1, entry_tid=18521], [object_number=16, tag_tid=1, entry_tid=18520]]], state=connected] 2026-02-18T08:43:47.559 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_46.status 2026-02-18T08:43:47.560 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=17, tag_tid=2, entry_tid=1' 2026-02-18T08:43:47.562 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_46 2026-02-18T08:43:47.562 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:43:47.562 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:43:47.562 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_46 2026-02-18T08:43:47.562 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_46 '..*' 2026-02-18T08:43:47.562 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:43:47.562 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:43:47.562 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_46 2026-02-18T08:43:47.562 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:43:47.562 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_46.status 2026-02-18T08:43:47.562 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_46.status 2026-02-18T08:43:47.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:43:47.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_46.status 2026-02-18T08:43:47.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_46 2026-02-18T08:43:47.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_46.status 2026-02-18T08:43:47.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 4 2026-02-18T08:43:47.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 4 2026-02-18T08:43:47.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:43:47.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=2, entry_tid=1], [object_number=16, tag_tid=2, entry_tid=0], [object_number=19, tag_tid=1, entry_tid=18523], [object_number=18, tag_tid=1, entry_tid=18522]]], state=connected] 2026-02-18T08:43:47.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=19, tag_tid=1, entry_tid=18523], [object_number=18, tag_tid=1, entry_tid=18522], [object_number=17, tag_tid=1, entry_tid=18521], [object_number=16, tag_tid=1, entry_tid=18520]]], state=connected] 2026-02-18T08:43:47.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_46.status 2026-02-18T08:43:47.595 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=19, tag_tid=1, entry_tid=18523' 2026-02-18T08:43:47.595 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=17, tag_tid=2, entry_tid=1' -a 'object_number=17, tag_tid=2, entry_tid=1' = 'object_number=19, tag_tid=1, entry_tid=18523' 2026-02-18T08:43:47.595 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=19, tag_tid=1, entry_tid=18523' '!=' 'object_number=19, tag_tid=1, entry_tid=18523' 2026-02-18T08:43:47.595 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:43:47.595 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1.6 2026-02-18T08:43:49.197 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_46 2026-02-18T08:43:49.198 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:43:49.198 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:43:49.198 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_46 2026-02-18T08:43:49.198 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:43:49.198 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_46 ']' 2026-02-18T08:43:49.198 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_46' 2026-02-18T08:43:49.198 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_46 2026-02-18T08:43:49.198 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:43:49.198 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:43:49.199 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:43:49.199 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:43:49.199 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:43:49.199 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:43:49.199 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:43:49.199 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:43:49.199 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:43:49.202 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:43:49.202 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_46 2026-02-18T08:43:49.202 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:43:49.202 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:43:49.202 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:43:49.203 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:43:49.203 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:43:49.203 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:43:49.203 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:43:49.203 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:43:49.203 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:43:49.203 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:43:49.203 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:43:49.203 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:43:49.204 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:43:49.204 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:43:49.204 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:43:49.204 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:43:49.204 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:43:49.204 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:43:49.204 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:43:49.204 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:43:49.204 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:43:49.204 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:43:49.204 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:43:49.204 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:43:49.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:43:49.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:43:49.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:43:49.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:43:49.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:43:49.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:43:49.211 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:43:49.211 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:43:49.211 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:43:49.212 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_46 2026-02-18T08:43:49.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_46'] 2026-02-18T08:43:49.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:43:49.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:43:49.274 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:43:49.274 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_46 2026-02-18T08:43:49.274 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:43:49.274 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:43:49.274 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:43:49.274 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:43:49.274 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:43:49.274 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:43:49.274 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:43:49.274 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:43:49.274 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:43:49.274 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:43:49.274 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:43:49.274 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:43:49.274 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:43:49.274 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:43:49.274 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:43:49.274 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:43:49.274 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:43:49.274 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:43:49.274 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:43:49.274 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:43:49.274 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:43:49.274 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:43:49.274 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:43:49.274 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:43:49.274 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:43:49.274 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:43:49.274 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:43:49.274 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:43:49.274 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:43:49.275 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:43:49.285 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:43:49.285 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:43:49.286 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:43:49.286 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_46 2026-02-18T08:43:49.354 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:43:49.356 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_46 2026-02-18T08:43:49.356 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:43:49.356 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:43:49.356 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_46 2026-02-18T08:43:49.356 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_46 '' 2026-02-18T08:43:49.356 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:43:49.356 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:43:49.356 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_46 2026-02-18T08:43:49.357 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:43:49.357 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_46.status 2026-02-18T08:43:49.357 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_46.status 2026-02-18T08:43:49.357 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:43:49.357 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_46.status 2026-02-18T08:43:49.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_46 2026-02-18T08:43:49.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_46.status 2026-02-18T08:43:49.384 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 4 2026-02-18T08:43:49.384 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 4 2026-02-18T08:43:49.384 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:43:49.385 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=2, entry_tid=1], [object_number=16, tag_tid=2, entry_tid=0], [object_number=19, tag_tid=1, entry_tid=18523], [object_number=18, tag_tid=1, entry_tid=18522]]], state=connected] 2026-02-18T08:43:49.385 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=17, tag_tid=2, entry_tid=1], [object_number=16, tag_tid=2, entry_tid=0], [object_number=19, tag_tid=1, entry_tid=18523], [object_number=18, tag_tid=1, entry_tid=18522]]], state=connected] 2026-02-18T08:43:49.387 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_46.status 2026-02-18T08:43:49.388 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=17, tag_tid=2, entry_tid=1' 2026-02-18T08:43:49.389 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_46 2026-02-18T08:43:49.390 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:43:49.390 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:43:49.390 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_46 2026-02-18T08:43:49.390 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_46 '..*' 2026-02-18T08:43:49.390 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:43:49.390 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:43:49.390 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_46 2026-02-18T08:43:49.390 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:43:49.390 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_46.status 2026-02-18T08:43:49.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_46.status 2026-02-18T08:43:49.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:43:49.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_46.status 2026-02-18T08:43:49.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_46 2026-02-18T08:43:49.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_46.status 2026-02-18T08:43:49.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 4 2026-02-18T08:43:49.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 4 2026-02-18T08:43:49.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:43:49.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=2, entry_tid=1], [object_number=16, tag_tid=2, entry_tid=0], [object_number=19, tag_tid=1, entry_tid=18523], [object_number=18, tag_tid=1, entry_tid=18522]]], state=connected] 2026-02-18T08:43:49.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=17, tag_tid=2, entry_tid=1], [object_number=16, tag_tid=2, entry_tid=0], [object_number=19, tag_tid=1, entry_tid=18523], [object_number=18, tag_tid=1, entry_tid=18522]]], state=connected] 2026-02-18T08:43:49.430 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_46.status 2026-02-18T08:43:49.432 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=17, tag_tid=2, entry_tid=1' 2026-02-18T08:43:49.432 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=17, tag_tid=2, entry_tid=1' -a 'object_number=17, tag_tid=2, entry_tid=1' = 'object_number=17, tag_tid=2, entry_tid=1' 2026-02-18T08:43:49.432 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:43:49.432 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_snap_present cluster1 mirror image_46 snap 2026-02-18T08:43:49.432 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:43:49.432 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:43:49.432 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_46 2026-02-18T08:43:49.432 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap 2026-02-18T08:43:49.433 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-18T08:43:49.433 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-18T08:43:49.433 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1 2026-02-18T08:43:50.433 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster1 info mirror/image_46@snap 2026-02-18T08:43:50.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:rbd image 'image_46': 2026-02-18T08:43:50.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: size 128 MiB in 32 objects 2026-02-18T08:43:50.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: order 22 (4 MiB objects) 2026-02-18T08:43:50.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: snapshot_count: 1 2026-02-18T08:43:50.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: id: 11f93645cda5 2026-02-18T08:43:50.462 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: block_name_prefix: rbd_data.11f93645cda5 2026-02-18T08:43:50.462 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: format: 2 2026-02-18T08:43:50.462 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: features: layering, exclusive-lock, journaling 2026-02-18T08:43:50.462 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: op_features: 2026-02-18T08:43:50.462 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: flags: 2026-02-18T08:43:50.462 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: create_timestamp: Wed Feb 18 08:33:23 2026 2026-02-18T08:43:50.462 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: access_timestamp: Wed Feb 18 08:33:23 2026 2026-02-18T08:43:50.462 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: modify_timestamp: Wed Feb 18 08:33:23 2026 2026-02-18T08:43:50.462 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: protected: False 2026-02-18T08:43:50.462 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: journal: 11f93645cda5 2026-02-18T08:43:50.462 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring state: enabled 2026-02-18T08:43:50.462 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring mode: journal 2026-02-18T08:43:50.463 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring global id: 74a9499f-cf82-41d4-ab3b-46ca72fd84f9 2026-02-18T08:43:50.463 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring primary: false 2026-02-18T08:43:50.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:43:50.465 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ compare_image_snaps mirror image_46 snap 2026-02-18T08:43:50.465 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:43:50.465 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_46 2026-02-18T08:43:50.465 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap 2026-02-18T08:43:50.465 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local ret=0 2026-02-18T08:43:50.465 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local rmt_export=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_46.export 2026-02-18T08:43:50.465 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loc_export=/tmp/tmp.vybh1m9yJF/cluster1-mirror-image_46.export 2026-02-18T08:43:50.465 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_46.export /tmp/tmp.vybh1m9yJF/cluster1-mirror-image_46.export 2026-02-18T08:43:50.465 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 -p mirror export image_46@snap /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_46.export 2026-02-18T08:43:50.601 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-18T08:43:50.610 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster1 -p mirror export image_46@snap /tmp/tmp.vybh1m9yJF/cluster1-mirror-image_46.export 2026-02-18T08:43:50.788 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-18T08:43:50.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmp /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_46.export /tmp/tmp.vybh1m9yJF/cluster1-mirror-image_46.export 2026-02-18T08:43:50.853 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_46.export /tmp/tmp.vybh1m9yJF/cluster1-mirror-image_46.export 2026-02-18T08:43:50.897 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:43:50.897 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T08:43:50.897 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_47 2026-02-18T08:43:50.897 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_snap cluster2 mirror image_47 snap 2026-02-18T08:43:50.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:43:50.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:43:50.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_47 2026-02-18T08:43:50.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap 2026-02-18T08:43:50.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 -p mirror snap create image_47@snap --debug-rbd=20 --debug-journaler=20 2026-02-18T08:43:54.962 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_replay_started cluster1 mirror image_47 2026-02-18T08:43:54.962 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:43:54.963 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:43:54.963 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_47 2026-02-18T08:43:54.963 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_replay_state cluster1 mirror image_47 started 2026-02-18T08:43:54.963 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:43:54.963 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:43:54.963 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_47 2026-02-18T08:43:54.963 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local state=started 2026-02-18T08:43:54.963 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-18T08:43:54.963 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:43:54.963 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1 2026-02-18T08:43:55.964 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test_image_replay_state cluster1 mirror image_47 started 2026-02-18T08:43:55.964 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:43:55.964 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:43:55.964 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_47 2026-02-18T08:43:55.964 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local test_state=started 2026-02-18T08:43:55.964 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local status_result 2026-02-18T08:43:55.964 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_state=stopped 2026-02-18T08:43:55.964 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_47 2026-02-18T08:43:55.964 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -i state 2026-02-18T08:43:55.964 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_instance=cluster1 2026-02-18T08:43:55.964 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-18T08:43:55.964 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:43:55.965 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance=cluster1 2026-02-18T08:43:55.965 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local loop_instance 2026-02-18T08:43:55.965 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:43:55.965 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sleep 0 2026-02-18T08:43:55.965 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:43:55.965 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ seq 0 3 2026-02-18T08:43:55.968 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:43:55.968 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_47 2026-02-18T08:43:55.968 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:0 2026-02-18T08:43:55.968 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-18T08:43:55.968 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:43:55.968 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:43:55.968 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:43:55.968 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-18T08:43:55.968 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:43:55.968 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:43:55.968 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:43:55.968 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:43:55.968 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-18T08:43:55.968 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:43:55.968 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:43:55.969 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:43:55.969 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:43:55.969 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:0 2026-02-18T08:43:55.969 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-18T08:43:55.969 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-18T08:43:55.969 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:43:55.969 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:0 2026-02-18T08:43:55.969 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T08:43:55.969 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-18T08:43:55.969 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:43:55.969 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:43:55.969 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-18T08:43:55.969 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-18T08:43:55.969 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-18T08:43:55.969 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=0 2026-02-18T08:43:55.969 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=0 2026-02-18T08:43:55.969 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:43:55.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:43:55.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:43:55.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:43:55.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_47 2026-02-18T08:43:56.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_47'] 2026-02-18T08:43:56.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status : get status for rbd mirror 2026-02-18T08:43:56.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:43:56.036 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:43:56.036 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_47 2026-02-18T08:43:56.036 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:1 2026-02-18T08:43:56.036 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-18T08:43:56.036 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:43:56.036 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:43:56.036 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:43:56.036 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-18T08:43:56.036 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:43:56.036 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:43:56.036 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:43:56.037 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:43:56.037 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-18T08:43:56.037 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:43:56.037 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:43:56.037 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:43:56.037 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:43:56.037 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:1 2026-02-18T08:43:56.037 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-18T08:43:56.037 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-18T08:43:56.037 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:43:56.038 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:1 2026-02-18T08:43:56.038 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T08:43:56.038 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-18T08:43:56.038 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:43:56.038 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:43:56.038 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-18T08:43:56.038 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-18T08:43:56.038 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-18T08:43:56.038 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=1 2026-02-18T08:43:56.038 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=1 2026-02-18T08:43:56.039 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:43:56.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:43:56.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:43:56.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:43:56.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_47 2026-02-18T08:43:56.097 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_47'] 2026-02-18T08:43:56.097 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status : get status for rbd mirror 2026-02-18T08:43:56.097 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:43:56.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:43:56.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_47 2026-02-18T08:43:56.100 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:2 2026-02-18T08:43:56.100 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-18T08:43:56.100 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:43:56.100 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:43:56.100 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:43:56.100 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-18T08:43:56.100 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:43:56.100 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:43:56.100 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:43:56.100 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:43:56.100 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-18T08:43:56.100 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:43:56.100 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:43:56.100 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:43:56.100 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:43:56.100 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:2 2026-02-18T08:43:56.100 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-18T08:43:56.100 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-18T08:43:56.100 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:43:56.101 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:2 2026-02-18T08:43:56.101 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T08:43:56.101 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-18T08:43:56.101 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:43:56.101 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:43:56.101 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-18T08:43:56.101 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-18T08:43:56.101 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-18T08:43:56.101 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=2 2026-02-18T08:43:56.101 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=2 2026-02-18T08:43:56.101 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:43:56.110 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:43:56.110 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:43:56.111 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:43:56.111 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_47 2026-02-18T08:43:56.164 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_47'] 2026-02-18T08:43:56.164 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status : get status for rbd mirror 2026-02-18T08:43:56.164 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:43:56.166 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:43:56.166 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/image_47 2026-02-18T08:43:56.166 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:3 2026-02-18T08:43:56.167 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-18T08:43:56.167 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:43:56.167 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:43:56.167 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:43:56.167 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-18T08:43:56.167 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:43:56.167 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:43:56.167 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:43:56.167 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:43:56.167 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-18T08:43:56.167 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:43:56.167 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:43:56.167 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:43:56.167 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:43:56.167 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:3 2026-02-18T08:43:56.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-18T08:43:56.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-18T08:43:56.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:43:56.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:3 2026-02-18T08:43:56.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T08:43:56.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-18T08:43:56.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:43:56.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:43:56.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-18T08:43:56.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-18T08:43:56.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-18T08:43:56.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=3 2026-02-18T08:43:56.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=3 2026-02-18T08:43:56.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:43:56.177 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:43:56.178 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:43:56.178 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:43:56.178 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/image_47 2026-02-18T08:43:56.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ return 0 2026-02-18T08:43:56.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ status_result=' "state": "Replaying"' 2026-02-18T08:43:56.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ echo ' "state": "Replaying"' 2026-02-18T08:43:56.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ grep -i Replaying 2026-02-18T08:43:56.238 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: "state": "Replaying" 2026-02-18T08:43:56.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ current_state=started 2026-02-18T08:43:56.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test started = started 2026-02-18T08:43:56.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:43:56.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror image_47 2026-02-18T08:43:56.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_cluster=cluster1 2026-02-18T08:43:56.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:43:56.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_pool=mirror 2026-02-18T08:43:56.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local remote_pool=mirror 2026-02-18T08:43:56.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_47 2026-02-18T08:43:56.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' journal = journal ']' 2026-02-18T08:43:56.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror mirror image_47 2026-02-18T08:43:56.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_cluster=cluster1 2026-02-18T08:43:56.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:43:56.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_pool=mirror 2026-02-18T08:43:56.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local remote_pool=mirror 2026-02-18T08:43:56.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_47 2026-02-18T08:43:56.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-02-18T08:43:56.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-02-18T08:43:56.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:43:56.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:43:56.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:43:56.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_47 2026-02-18T08:43:56.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:43:56.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:43:56.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_47 2026-02-18T08:43:56.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:43:56.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_47 ']' 2026-02-18T08:43:56.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_47' 2026-02-18T08:43:56.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_47 2026-02-18T08:43:56.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:43:56.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:43:56.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:43:56.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:43:56.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:43:56.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:43:56.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:43:56.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:43:56.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:43:56.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:43:56.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_47 2026-02-18T08:43:56.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:43:56.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:43:56.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:43:56.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:43:56.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:43:56.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:43:56.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:43:56.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:43:56.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:43:56.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:43:56.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:43:56.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:43:56.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:43:56.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:43:56.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:43:56.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:43:56.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:43:56.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:43:56.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:43:56.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:43:56.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:43:56.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:43:56.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:43:56.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:43:56.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:43:56.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:43:56.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:43:56.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:43:56.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:43:56.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:43:56.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:43:56.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:43:56.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:43:56.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_47 2026-02-18T08:43:56.516 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_47'] 2026-02-18T08:43:56.516 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:43:56.516 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:43:56.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:43:56.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_47 2026-02-18T08:43:56.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:43:56.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:43:56.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:43:56.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:43:56.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:43:56.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:43:56.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:43:56.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:43:56.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:43:56.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:43:56.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:43:56.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:43:56.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:43:56.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:43:56.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:43:56.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:43:56.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:43:56.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:43:56.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:43:56.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:43:56.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:43:56.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:43:56.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:43:56.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:43:56.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:43:56.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:43:56.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:43:56.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:43:56.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:43:56.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:43:56.530 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:43:56.530 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:43:56.530 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:43:56.530 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_47 2026-02-18T08:43:56.578 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_47'] 2026-02-18T08:43:56.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:43:56.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:43:56.582 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:43:56.582 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_47 2026-02-18T08:43:56.582 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:43:56.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:43:56.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:43:56.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:43:56.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:43:56.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:43:56.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:43:56.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:43:56.584 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:43:56.584 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:43:56.584 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:43:56.584 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:43:56.584 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:43:56.584 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:43:56.584 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:43:56.585 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:43:56.585 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:43:56.585 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:43:56.585 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:43:56.585 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:43:56.585 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:43:56.585 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:43:56.585 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:43:56.586 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:43:56.586 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:43:56.586 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:43:56.586 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:43:56.586 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:43:56.586 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:43:56.586 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:43:56.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:43:56.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:43:56.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:43:56.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_47 2026-02-18T08:43:56.643 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_47'] 2026-02-18T08:43:56.643 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:43:56.643 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:43:56.646 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:43:56.647 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/image_47 2026-02-18T08:43:56.647 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:43:56.647 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:43:56.647 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:43:56.647 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:43:56.647 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:43:56.647 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:43:56.647 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:43:56.648 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:43:56.648 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:43:56.648 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:43:56.648 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:43:56.648 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:43:56.648 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:43:56.648 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:43:56.648 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:43:56.648 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:43:56.648 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:43:56.649 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:43:56.649 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:43:56.649 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:43:56.649 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:43:56.649 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:43:56.649 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:43:56.649 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:43:56.649 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:43:56.649 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:43:56.649 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:43:56.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:43:56.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:43:56.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:43:56.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:43:56.658 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:43:56.658 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:43:56.658 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/image_47 2026-02-18T08:43:56.722 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:43:56.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_47 2026-02-18T08:43:56.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:43:56.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:43:56.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_47 2026-02-18T08:43:56.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_47 '' 2026-02-18T08:43:56.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:43:56.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:43:56.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_47 2026-02-18T08:43:56.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:43:56.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_47.status 2026-02-18T08:43:56.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_47.status 2026-02-18T08:43:56.724 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:43:56.725 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_47.status 2026-02-18T08:43:56.725 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_47 2026-02-18T08:43:56.725 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_47.status 2026-02-18T08:43:56.752 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 6 2026-02-18T08:43:56.752 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 6 2026-02-18T08:43:56.752 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:43:56.752 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=25, tag_tid=2, entry_tid=1], [object_number=24, tag_tid=2, entry_tid=0], [object_number=27, tag_tid=1, entry_tid=26259], [object_number=26, tag_tid=1, entry_tid=26258]]], state=connected] 2026-02-18T08:43:56.752 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=24, tag_tid=1, entry_tid=26260], [object_number=27, tag_tid=1, entry_tid=26259], [object_number=26, tag_tid=1, entry_tid=26258], [object_number=25, tag_tid=1, entry_tid=26257]]], state=connected] 2026-02-18T08:43:56.755 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_47.status 2026-02-18T08:43:56.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=25, tag_tid=2, entry_tid=1' 2026-02-18T08:43:56.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_47 2026-02-18T08:43:56.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:43:56.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:43:56.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_47 2026-02-18T08:43:56.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_47 '..*' 2026-02-18T08:43:56.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:43:56.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:43:56.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_47 2026-02-18T08:43:56.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:43:56.758 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_47.status 2026-02-18T08:43:56.758 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_47.status 2026-02-18T08:43:56.758 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:43:56.760 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_47.status 2026-02-18T08:43:56.760 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_47 2026-02-18T08:43:56.760 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_47.status 2026-02-18T08:43:56.796 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 6 2026-02-18T08:43:56.796 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 6 2026-02-18T08:43:56.796 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:43:56.796 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=25, tag_tid=2, entry_tid=1], [object_number=24, tag_tid=2, entry_tid=0], [object_number=27, tag_tid=1, entry_tid=26259], [object_number=26, tag_tid=1, entry_tid=26258]]], state=connected] 2026-02-18T08:43:56.796 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=24, tag_tid=1, entry_tid=26260], [object_number=27, tag_tid=1, entry_tid=26259], [object_number=26, tag_tid=1, entry_tid=26258], [object_number=25, tag_tid=1, entry_tid=26257]]], state=connected] 2026-02-18T08:43:56.800 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_47.status 2026-02-18T08:43:56.801 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=24, tag_tid=1, entry_tid=26260' 2026-02-18T08:43:56.801 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=25, tag_tid=2, entry_tid=1' -a 'object_number=25, tag_tid=2, entry_tid=1' = 'object_number=24, tag_tid=1, entry_tid=26260' 2026-02-18T08:43:56.801 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=24, tag_tid=1, entry_tid=26260' '!=' '' 2026-02-18T08:43:56.801 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:43:56.801 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=24, tag_tid=1, entry_tid=26260' = '' 2026-02-18T08:43:56.801 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=24, tag_tid=1, entry_tid=26260' 2026-02-18T08:43:56.802 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=25, tag_tid=2, entry_tid=1' 2026-02-18T08:43:56.802 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:43:56.802 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:43:56.803 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=2 2026-02-18T08:43:56.804 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=24, tag_tid=1, entry_tid=26260' 2026-02-18T08:43:56.804 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:43:56.804 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:43:56.806 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=1 2026-02-18T08:43:56.806 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=25, tag_tid=2, entry_tid=1' 2026-02-18T08:43:56.806 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:43:56.807 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:43:56.808 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:43:56.809 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=24, tag_tid=1, entry_tid=26260' 2026-02-18T08:43:56.809 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:43:56.809 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:43:56.810 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=26260 2026-02-18T08:43:56.811 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 2 = 1 -a 1 -le 26260 2026-02-18T08:43:56.811 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:43:56.811 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:43:56.811 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:43:57.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_47 2026-02-18T08:43:57.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:43:57.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:43:57.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_47 2026-02-18T08:43:57.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:43:57.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_47 ']' 2026-02-18T08:43:57.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_47' 2026-02-18T08:43:57.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_47 2026-02-18T08:43:57.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:43:57.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:43:57.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:43:57.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:43:57.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:43:57.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:43:57.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:43:57.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:43:57.018 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:43:57.018 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:43:57.018 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_47 2026-02-18T08:43:57.018 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:43:57.018 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:43:57.018 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:43:57.018 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:43:57.018 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:43:57.018 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:43:57.019 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:43:57.019 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:43:57.019 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:43:57.019 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:43:57.019 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:43:57.019 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:43:57.019 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:43:57.019 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:43:57.019 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:43:57.019 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:43:57.019 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:43:57.019 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:43:57.020 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:43:57.020 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:43:57.020 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:43:57.020 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:43:57.020 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:43:57.020 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:43:57.020 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:43:57.020 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:43:57.020 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:43:57.020 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:43:57.020 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:43:57.020 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:43:57.028 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:43:57.029 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:43:57.029 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:43:57.029 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_47 2026-02-18T08:43:57.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_47'] 2026-02-18T08:43:57.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:43:57.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:43:57.093 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:43:57.093 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_47 2026-02-18T08:43:57.093 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:43:57.093 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:43:57.093 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:43:57.093 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:43:57.094 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:43:57.094 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:43:57.094 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:43:57.094 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:43:57.094 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:43:57.094 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:43:57.094 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:43:57.094 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:43:57.094 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:43:57.094 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:43:57.094 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:43:57.094 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:43:57.095 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:43:57.095 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:43:57.095 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:43:57.095 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:43:57.095 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:43:57.095 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:43:57.095 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:43:57.095 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:43:57.095 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:43:57.095 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:43:57.095 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:43:57.095 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:43:57.096 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:43:57.096 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:43:57.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:43:57.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:43:57.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:43:57.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_47 2026-02-18T08:43:57.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_47'] 2026-02-18T08:43:57.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:43:57.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:43:57.156 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:43:57.156 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_47 2026-02-18T08:43:57.156 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:43:57.156 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:43:57.157 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:43:57.157 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:43:57.157 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:43:57.157 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:43:57.157 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:43:57.157 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:43:57.157 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:43:57.157 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:43:57.158 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:43:57.158 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:43:57.158 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:43:57.158 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:43:57.158 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:43:57.158 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:43:57.158 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:43:57.158 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:43:57.158 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:43:57.158 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:43:57.158 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:43:57.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:43:57.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:43:57.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:43:57.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:43:57.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:43:57.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:43:57.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:43:57.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:43:57.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:43:57.167 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:43:57.167 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:43:57.167 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:43:57.167 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_47 2026-02-18T08:43:57.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_47'] 2026-02-18T08:43:57.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:43:57.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:43:57.231 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:43:57.231 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/image_47 2026-02-18T08:43:57.231 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:43:57.231 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:43:57.231 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:43:57.231 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:43:57.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:43:57.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:43:57.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:43:57.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:43:57.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:43:57.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:43:57.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:43:57.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:43:57.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:43:57.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:43:57.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:43:57.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:43:57.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:43:57.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:43:57.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:43:57.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:43:57.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:43:57.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:43:57.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:43:57.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:43:57.234 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:43:57.234 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:43:57.234 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:43:57.234 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:43:57.234 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:43:57.234 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:43:57.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:43:57.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:43:57.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:43:57.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/image_47 2026-02-18T08:43:57.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:43:57.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_47 2026-02-18T08:43:57.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:43:57.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:43:57.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_47 2026-02-18T08:43:57.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_47 '' 2026-02-18T08:43:57.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:43:57.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:43:57.310 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_47 2026-02-18T08:43:57.310 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:43:57.310 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_47.status 2026-02-18T08:43:57.310 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_47.status 2026-02-18T08:43:57.310 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:43:57.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_47.status 2026-02-18T08:43:57.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_47 2026-02-18T08:43:57.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_47.status 2026-02-18T08:43:57.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 6 2026-02-18T08:43:57.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 6 2026-02-18T08:43:57.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:43:57.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=25, tag_tid=2, entry_tid=1], [object_number=24, tag_tid=2, entry_tid=0], [object_number=27, tag_tid=1, entry_tid=26259], [object_number=26, tag_tid=1, entry_tid=26258]]], state=connected] 2026-02-18T08:43:57.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=24, tag_tid=1, entry_tid=26260], [object_number=27, tag_tid=1, entry_tid=26259], [object_number=26, tag_tid=1, entry_tid=26258], [object_number=25, tag_tid=1, entry_tid=26257]]], state=connected] 2026-02-18T08:43:57.350 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_47.status 2026-02-18T08:43:57.352 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=25, tag_tid=2, entry_tid=1' 2026-02-18T08:43:57.353 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_47 2026-02-18T08:43:57.353 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:43:57.353 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:43:57.353 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_47 2026-02-18T08:43:57.353 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_47 '..*' 2026-02-18T08:43:57.353 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:43:57.354 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:43:57.354 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_47 2026-02-18T08:43:57.354 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:43:57.354 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_47.status 2026-02-18T08:43:57.354 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_47.status 2026-02-18T08:43:57.354 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:43:57.355 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_47.status 2026-02-18T08:43:57.355 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_47 2026-02-18T08:43:57.356 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_47.status 2026-02-18T08:43:57.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 6 2026-02-18T08:43:57.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 6 2026-02-18T08:43:57.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:43:57.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=25, tag_tid=2, entry_tid=1], [object_number=24, tag_tid=2, entry_tid=0], [object_number=27, tag_tid=1, entry_tid=26259], [object_number=26, tag_tid=1, entry_tid=26258]]], state=connected] 2026-02-18T08:43:57.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=24, tag_tid=1, entry_tid=26260], [object_number=27, tag_tid=1, entry_tid=26259], [object_number=26, tag_tid=1, entry_tid=26258], [object_number=25, tag_tid=1, entry_tid=26257]]], state=connected] 2026-02-18T08:43:57.395 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_47.status 2026-02-18T08:43:57.397 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=24, tag_tid=1, entry_tid=26260' 2026-02-18T08:43:57.397 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=25, tag_tid=2, entry_tid=1' -a 'object_number=25, tag_tid=2, entry_tid=1' = 'object_number=24, tag_tid=1, entry_tid=26260' 2026-02-18T08:43:57.397 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=24, tag_tid=1, entry_tid=26260' '!=' 'object_number=24, tag_tid=1, entry_tid=26260' 2026-02-18T08:43:57.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:43:57.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.4 2026-02-18T08:43:57.800 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_47 2026-02-18T08:43:57.800 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:43:57.800 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:43:57.800 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_47 2026-02-18T08:43:57.800 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:43:57.800 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_47 ']' 2026-02-18T08:43:57.801 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_47' 2026-02-18T08:43:57.801 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_47 2026-02-18T08:43:57.801 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:43:57.801 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:43:57.801 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:43:57.801 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:43:57.801 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:43:57.801 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:43:57.801 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:43:57.801 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:43:57.801 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:43:57.805 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:43:57.805 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_47 2026-02-18T08:43:57.805 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:43:57.805 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:43:57.805 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:43:57.805 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:43:57.805 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:43:57.805 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:43:57.806 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:43:57.806 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:43:57.806 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:43:57.806 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:43:57.806 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:43:57.806 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:43:57.806 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:43:57.806 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:43:57.806 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:43:57.807 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:43:57.807 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:43:57.807 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:43:57.807 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:43:57.807 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:43:57.807 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:43:57.807 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:43:57.807 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:43:57.807 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:43:57.808 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:43:57.808 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:43:57.808 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:43:57.808 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:43:57.808 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:43:57.808 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:43:57.814 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:43:57.815 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:43:57.815 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:43:57.815 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_47 2026-02-18T08:43:57.868 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_47'] 2026-02-18T08:43:57.868 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:43:57.868 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:43:57.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:43:57.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_47 2026-02-18T08:43:57.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:43:57.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:43:57.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:43:57.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:43:57.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:43:57.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:43:57.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:43:57.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:43:57.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:43:57.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:43:57.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:43:57.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:43:57.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:43:57.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:43:57.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:43:57.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:43:57.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:43:57.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:43:57.874 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:43:57.874 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:43:57.874 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:43:57.874 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:43:57.874 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:43:57.874 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:43:57.874 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:43:57.874 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:43:57.874 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:43:57.874 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:43:57.874 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:43:57.875 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:43:57.884 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:43:57.884 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:43:57.884 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:43:57.884 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_47 2026-02-18T08:43:57.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_47'] 2026-02-18T08:43:57.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:43:57.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:43:57.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:43:57.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_47 2026-02-18T08:43:57.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:43:57.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:43:57.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:43:57.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:43:57.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:43:57.946 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:43:57.946 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:43:57.946 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:43:57.946 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:43:57.946 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:43:57.946 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:43:57.946 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:43:57.946 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:43:57.946 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:43:57.946 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:43:57.947 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:43:57.947 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:43:57.947 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:43:57.947 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:43:57.947 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:43:57.947 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:43:57.947 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:43:57.947 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:43:57.947 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:43:57.948 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:43:57.948 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:43:57.948 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:43:57.948 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:43:57.948 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:43:57.948 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:43:57.955 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:43:57.956 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:43:57.956 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:43:57.956 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_47 2026-02-18T08:43:58.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_47'] 2026-02-18T08:43:58.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:43:58.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:43:58.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:43:58.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/image_47 2026-02-18T08:43:58.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:43:58.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:43:58.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:43:58.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:43:58.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:43:58.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:43:58.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:43:58.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:43:58.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:43:58.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:43:58.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:43:58.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:43:58.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:43:58.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:43:58.018 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:43:58.018 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:43:58.018 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:43:58.018 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:43:58.018 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:43:58.018 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:43:58.018 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:43:58.018 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:43:58.018 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:43:58.018 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:43:58.018 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:43:58.018 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:43:58.018 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:43:58.018 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:43:58.018 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:43:58.018 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:43:58.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:43:58.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:43:58.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:43:58.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/image_47 2026-02-18T08:43:58.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:43:58.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_47 2026-02-18T08:43:58.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:43:58.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:43:58.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_47 2026-02-18T08:43:58.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_47 '' 2026-02-18T08:43:58.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:43:58.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:43:58.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_47 2026-02-18T08:43:58.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:43:58.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_47.status 2026-02-18T08:43:58.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_47.status 2026-02-18T08:43:58.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:43:58.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_47.status 2026-02-18T08:43:58.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_47 2026-02-18T08:43:58.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_47.status 2026-02-18T08:43:58.129 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 6 2026-02-18T08:43:58.129 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 6 2026-02-18T08:43:58.129 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:43:58.129 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=25, tag_tid=2, entry_tid=1], [object_number=24, tag_tid=2, entry_tid=0], [object_number=27, tag_tid=1, entry_tid=26259], [object_number=26, tag_tid=1, entry_tid=26258]]], state=connected] 2026-02-18T08:43:58.129 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=24, tag_tid=1, entry_tid=26260], [object_number=27, tag_tid=1, entry_tid=26259], [object_number=26, tag_tid=1, entry_tid=26258], [object_number=25, tag_tid=1, entry_tid=26257]]], state=connected] 2026-02-18T08:43:58.132 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_47.status 2026-02-18T08:43:58.134 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=25, tag_tid=2, entry_tid=1' 2026-02-18T08:43:58.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_47 2026-02-18T08:43:58.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:43:58.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:43:58.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_47 2026-02-18T08:43:58.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_47 '..*' 2026-02-18T08:43:58.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:43:58.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:43:58.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_47 2026-02-18T08:43:58.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:43:58.136 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_47.status 2026-02-18T08:43:58.136 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_47.status 2026-02-18T08:43:58.136 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:43:58.137 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_47.status 2026-02-18T08:43:58.137 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_47 2026-02-18T08:43:58.137 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_47.status 2026-02-18T08:43:58.164 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 6 2026-02-18T08:43:58.164 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 6 2026-02-18T08:43:58.164 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:43:58.164 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=25, tag_tid=2, entry_tid=1], [object_number=24, tag_tid=2, entry_tid=0], [object_number=27, tag_tid=1, entry_tid=26259], [object_number=26, tag_tid=1, entry_tid=26258]]], state=connected] 2026-02-18T08:43:58.164 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=24, tag_tid=1, entry_tid=26260], [object_number=27, tag_tid=1, entry_tid=26259], [object_number=26, tag_tid=1, entry_tid=26258], [object_number=25, tag_tid=1, entry_tid=26257]]], state=connected] 2026-02-18T08:43:58.167 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_47.status 2026-02-18T08:43:58.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=24, tag_tid=1, entry_tid=26260' 2026-02-18T08:43:58.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=25, tag_tid=2, entry_tid=1' -a 'object_number=25, tag_tid=2, entry_tid=1' = 'object_number=24, tag_tid=1, entry_tid=26260' 2026-02-18T08:43:58.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=24, tag_tid=1, entry_tid=26260' '!=' 'object_number=24, tag_tid=1, entry_tid=26260' 2026-02-18T08:43:58.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:43:58.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.8 2026-02-18T08:43:58.971 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_47 2026-02-18T08:43:58.971 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:43:58.971 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:43:58.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_47 2026-02-18T08:43:58.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:43:58.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_47 ']' 2026-02-18T08:43:58.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_47' 2026-02-18T08:43:58.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_47 2026-02-18T08:43:58.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:43:58.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:43:58.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:43:58.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:43:58.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:43:58.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:43:58.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:43:58.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:43:58.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:43:58.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:43:58.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_47 2026-02-18T08:43:58.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:43:58.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:43:58.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:43:58.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:43:58.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:43:58.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:43:58.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:43:58.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:43:58.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:43:58.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:43:58.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:43:58.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:43:58.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:43:58.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:43:58.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:43:58.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:43:58.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:43:58.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:43:58.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:43:58.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:43:58.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:43:58.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:43:58.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:43:58.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:43:58.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:43:58.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:43:58.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:43:58.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:43:58.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:43:58.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:43:58.986 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:43:58.986 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:43:58.986 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:43:58.986 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_47 2026-02-18T08:43:59.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_47'] 2026-02-18T08:43:59.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:43:59.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:43:59.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:43:59.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_47 2026-02-18T08:43:59.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:43:59.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:43:59.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:43:59.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:43:59.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:43:59.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:43:59.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:43:59.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:43:59.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:43:59.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:43:59.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:43:59.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:43:59.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:43:59.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:43:59.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:43:59.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:43:59.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:43:59.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:43:59.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:43:59.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:43:59.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:43:59.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:43:59.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:43:59.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:43:59.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:43:59.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:43:59.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:43:59.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:43:59.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:43:59.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:43:59.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:43:59.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:43:59.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:43:59.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_47 2026-02-18T08:43:59.121 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_47'] 2026-02-18T08:43:59.121 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:43:59.121 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:43:59.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:43:59.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_47 2026-02-18T08:43:59.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:43:59.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:43:59.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:43:59.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:43:59.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:43:59.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:43:59.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:43:59.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:43:59.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:43:59.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:43:59.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:43:59.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:43:59.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:43:59.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:43:59.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:43:59.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:43:59.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:43:59.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:43:59.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:43:59.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:43:59.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:43:59.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:43:59.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:43:59.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:43:59.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:43:59.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:43:59.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:43:59.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:43:59.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:43:59.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:43:59.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:43:59.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:43:59.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:43:59.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_47 2026-02-18T08:43:59.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_47'] 2026-02-18T08:43:59.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:43:59.194 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:43:59.196 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:43:59.196 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/image_47 2026-02-18T08:43:59.197 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:43:59.197 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:43:59.197 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:43:59.197 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:43:59.197 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:43:59.197 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:43:59.197 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:43:59.197 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:43:59.198 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:43:59.198 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:43:59.198 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:43:59.198 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:43:59.198 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:43:59.198 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:43:59.198 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:43:59.198 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:43:59.198 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:43:59.198 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:43:59.199 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:43:59.199 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:43:59.199 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:43:59.199 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:43:59.199 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:43:59.199 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:43:59.199 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:43:59.199 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:43:59.199 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:43:59.199 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:43:59.200 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:43:59.200 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:43:59.206 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:43:59.206 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:43:59.206 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:43:59.206 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/image_47 2026-02-18T08:43:59.270 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:43:59.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_47 2026-02-18T08:43:59.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:43:59.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:43:59.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_47 2026-02-18T08:43:59.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_47 '' 2026-02-18T08:43:59.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:43:59.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:43:59.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_47 2026-02-18T08:43:59.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:43:59.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_47.status 2026-02-18T08:43:59.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_47.status 2026-02-18T08:43:59.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:43:59.273 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_47.status 2026-02-18T08:43:59.273 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_47 2026-02-18T08:43:59.273 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_47.status 2026-02-18T08:43:59.300 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 6 2026-02-18T08:43:59.300 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 6 2026-02-18T08:43:59.300 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:43:59.301 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=25, tag_tid=2, entry_tid=1], [object_number=24, tag_tid=2, entry_tid=0], [object_number=27, tag_tid=1, entry_tid=26259], [object_number=26, tag_tid=1, entry_tid=26258]]], state=connected] 2026-02-18T08:43:59.301 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=24, tag_tid=1, entry_tid=26260], [object_number=27, tag_tid=1, entry_tid=26259], [object_number=26, tag_tid=1, entry_tid=26258], [object_number=25, tag_tid=1, entry_tid=26257]]], state=connected] 2026-02-18T08:43:59.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_47.status 2026-02-18T08:43:59.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=25, tag_tid=2, entry_tid=1' 2026-02-18T08:43:59.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_47 2026-02-18T08:43:59.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:43:59.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:43:59.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_47 2026-02-18T08:43:59.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_47 '..*' 2026-02-18T08:43:59.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:43:59.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:43:59.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_47 2026-02-18T08:43:59.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:43:59.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_47.status 2026-02-18T08:43:59.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_47.status 2026-02-18T08:43:59.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:43:59.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_47.status 2026-02-18T08:43:59.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_47 2026-02-18T08:43:59.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_47.status 2026-02-18T08:43:59.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 6 2026-02-18T08:43:59.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 6 2026-02-18T08:43:59.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:43:59.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=25, tag_tid=2, entry_tid=1], [object_number=24, tag_tid=2, entry_tid=0], [object_number=27, tag_tid=1, entry_tid=26259], [object_number=26, tag_tid=1, entry_tid=26258]]], state=connected] 2026-02-18T08:43:59.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=24, tag_tid=1, entry_tid=26260], [object_number=27, tag_tid=1, entry_tid=26259], [object_number=26, tag_tid=1, entry_tid=26258], [object_number=25, tag_tid=1, entry_tid=26257]]], state=connected] 2026-02-18T08:43:59.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_47.status 2026-02-18T08:43:59.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=24, tag_tid=1, entry_tid=26260' 2026-02-18T08:43:59.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=25, tag_tid=2, entry_tid=1' -a 'object_number=25, tag_tid=2, entry_tid=1' = 'object_number=24, tag_tid=1, entry_tid=26260' 2026-02-18T08:43:59.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=24, tag_tid=1, entry_tid=26260' '!=' 'object_number=24, tag_tid=1, entry_tid=26260' 2026-02-18T08:43:59.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:43:59.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1.6 2026-02-18T08:44:00.950 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_47 2026-02-18T08:44:00.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:44:00.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:44:00.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_47 2026-02-18T08:44:00.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:44:00.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_47 ']' 2026-02-18T08:44:00.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_47' 2026-02-18T08:44:00.952 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_47 2026-02-18T08:44:00.952 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:44:00.952 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:44:00.952 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:44:00.952 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:44:00.952 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:44:00.952 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:44:00.952 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:44:00.953 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:44:00.953 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:44:00.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:44:00.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_47 2026-02-18T08:44:00.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:44:00.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:44:00.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:44:00.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:44:00.958 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:44:00.958 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:44:00.958 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:44:00.958 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:44:00.958 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:44:00.958 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:44:00.958 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:44:00.958 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:44:00.959 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:44:00.959 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:44:00.959 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:44:00.959 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:44:00.959 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:44:00.959 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:44:00.959 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:44:00.959 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:44:00.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:44:00.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:44:00.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:44:00.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:44:00.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:44:00.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:44:00.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:44:00.961 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:44:00.961 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:44:00.961 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:44:00.964 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:44:00.964 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:44:00.964 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:44:00.964 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_47 2026-02-18T08:44:01.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_47'] 2026-02-18T08:44:01.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:44:01.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:44:01.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:44:01.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_47 2026-02-18T08:44:01.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:44:01.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:44:01.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:44:01.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:44:01.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:44:01.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:44:01.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:44:01.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:44:01.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:44:01.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:44:01.027 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:44:01.027 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:44:01.027 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:44:01.027 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:44:01.027 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:44:01.027 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:44:01.027 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:44:01.027 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:44:01.027 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:44:01.027 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:44:01.027 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:44:01.027 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:44:01.027 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:44:01.027 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:44:01.027 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:44:01.027 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:44:01.027 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:44:01.027 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:44:01.027 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:44:01.027 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:44:01.038 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:44:01.038 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:44:01.038 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:44:01.038 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_47 2026-02-18T08:44:01.097 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_47'] 2026-02-18T08:44:01.097 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:44:01.097 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:44:01.100 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:44:01.100 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_47 2026-02-18T08:44:01.100 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:44:01.100 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:44:01.100 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:44:01.101 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:44:01.101 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:44:01.101 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:44:01.101 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:44:01.101 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:44:01.101 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:44:01.101 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:44:01.101 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:44:01.102 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:44:01.102 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:44:01.102 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:44:01.102 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:44:01.102 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:44:01.102 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:44:01.102 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:44:01.102 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:44:01.102 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:44:01.102 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:44:01.102 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:44:01.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:44:01.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:44:01.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:44:01.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:44:01.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:44:01.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:44:01.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:44:01.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:44:01.110 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:44:01.110 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:44:01.110 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:44:01.111 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_47 2026-02-18T08:44:01.174 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_47'] 2026-02-18T08:44:01.175 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:44:01.175 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:44:01.178 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:44:01.179 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/image_47 2026-02-18T08:44:01.179 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:44:01.179 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:44:01.179 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:44:01.179 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:44:01.179 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:44:01.179 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:44:01.179 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:44:01.180 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:44:01.180 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:44:01.180 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:44:01.180 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:44:01.180 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:44:01.180 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:44:01.180 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:44:01.180 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:44:01.180 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:44:01.180 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:44:01.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:44:01.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:44:01.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:44:01.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:44:01.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:44:01.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:44:01.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:44:01.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:44:01.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:44:01.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:44:01.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:44:01.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:44:01.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:44:01.188 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:44:01.189 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:44:01.189 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:44:01.189 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/image_47 2026-02-18T08:44:01.250 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:44:01.250 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_47 2026-02-18T08:44:01.250 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:44:01.250 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:44:01.251 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_47 2026-02-18T08:44:01.251 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_47 '' 2026-02-18T08:44:01.251 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:44:01.251 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:44:01.251 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_47 2026-02-18T08:44:01.251 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:44:01.251 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_47.status 2026-02-18T08:44:01.251 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_47.status 2026-02-18T08:44:01.251 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:44:01.252 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_47.status 2026-02-18T08:44:01.252 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_47 2026-02-18T08:44:01.252 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_47.status 2026-02-18T08:44:01.289 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 6 2026-02-18T08:44:01.289 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 6 2026-02-18T08:44:01.289 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:44:01.289 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=25, tag_tid=2, entry_tid=1], [object_number=24, tag_tid=2, entry_tid=0], [object_number=27, tag_tid=1, entry_tid=26259], [object_number=26, tag_tid=1, entry_tid=26258]]], state=connected] 2026-02-18T08:44:01.289 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=24, tag_tid=1, entry_tid=26260], [object_number=27, tag_tid=1, entry_tid=26259], [object_number=26, tag_tid=1, entry_tid=26258], [object_number=25, tag_tid=1, entry_tid=26257]]], state=connected] 2026-02-18T08:44:01.292 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_47.status 2026-02-18T08:44:01.294 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=25, tag_tid=2, entry_tid=1' 2026-02-18T08:44:01.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_47 2026-02-18T08:44:01.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:44:01.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:44:01.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_47 2026-02-18T08:44:01.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_47 '..*' 2026-02-18T08:44:01.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:44:01.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:44:01.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_47 2026-02-18T08:44:01.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:44:01.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_47.status 2026-02-18T08:44:01.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_47.status 2026-02-18T08:44:01.297 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:44:01.297 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_47.status 2026-02-18T08:44:01.297 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_47.status 2026-02-18T08:44:01.297 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_47 2026-02-18T08:44:01.324 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 6 2026-02-18T08:44:01.325 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 6 2026-02-18T08:44:01.325 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:44:01.325 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=25, tag_tid=2, entry_tid=1], [object_number=24, tag_tid=2, entry_tid=0], [object_number=27, tag_tid=1, entry_tid=26259], [object_number=26, tag_tid=1, entry_tid=26258]]], state=connected] 2026-02-18T08:44:01.325 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=24, tag_tid=1, entry_tid=26260], [object_number=27, tag_tid=1, entry_tid=26259], [object_number=26, tag_tid=1, entry_tid=26258], [object_number=25, tag_tid=1, entry_tid=26257]]], state=connected] 2026-02-18T08:44:01.327 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_47.status 2026-02-18T08:44:01.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=24, tag_tid=1, entry_tid=26260' 2026-02-18T08:44:01.329 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=25, tag_tid=2, entry_tid=1' -a 'object_number=25, tag_tid=2, entry_tid=1' = 'object_number=24, tag_tid=1, entry_tid=26260' 2026-02-18T08:44:01.329 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=24, tag_tid=1, entry_tid=26260' '!=' 'object_number=24, tag_tid=1, entry_tid=26260' 2026-02-18T08:44:01.329 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:44:01.329 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 2 2026-02-18T08:44:03.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_47 2026-02-18T08:44:03.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:44:03.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:44:03.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_47 2026-02-18T08:44:03.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:44:03.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_47 ']' 2026-02-18T08:44:03.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_47' 2026-02-18T08:44:03.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_47 2026-02-18T08:44:03.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:44:03.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:44:03.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:44:03.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:44:03.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:44:03.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:44:03.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:44:03.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:44:03.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:44:03.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:44:03.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_47 2026-02-18T08:44:03.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:44:03.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:44:03.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:44:03.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:44:03.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:44:03.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:44:03.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:44:03.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:44:03.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:44:03.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:44:03.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:44:03.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:44:03.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:44:03.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:44:03.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:44:03.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:44:03.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:44:03.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:44:03.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:44:03.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:44:03.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:44:03.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:44:03.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:44:03.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:44:03.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:44:03.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:44:03.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:44:03.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:44:03.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:44:03.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:44:03.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:44:03.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:44:03.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:44:03.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_47 2026-02-18T08:44:03.406 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_47'] 2026-02-18T08:44:03.406 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:44:03.407 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:44:03.409 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:44:03.409 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_47 2026-02-18T08:44:03.409 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:44:03.409 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:44:03.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:44:03.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:44:03.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:44:03.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:44:03.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:44:03.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:44:03.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:44:03.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:44:03.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:44:03.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:44:03.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:44:03.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:44:03.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:44:03.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:44:03.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:44:03.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:44:03.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:44:03.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:44:03.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:44:03.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:44:03.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:44:03.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:44:03.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:44:03.412 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:44:03.412 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:44:03.412 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:44:03.412 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:44:03.412 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:44:03.419 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:44:03.420 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:44:03.420 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:44:03.420 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_47 2026-02-18T08:44:03.477 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_47'] 2026-02-18T08:44:03.478 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:44:03.478 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:44:03.480 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:44:03.480 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_47 2026-02-18T08:44:03.480 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:44:03.481 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:44:03.481 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:44:03.481 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:44:03.481 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:44:03.481 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:44:03.481 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:44:03.481 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:44:03.481 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:44:03.481 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:44:03.481 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:44:03.481 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:44:03.481 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:44:03.482 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:44:03.482 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:44:03.482 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:44:03.482 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:44:03.482 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:44:03.482 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:44:03.482 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:44:03.482 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:44:03.482 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:44:03.482 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:44:03.482 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:44:03.483 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:44:03.483 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:44:03.483 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:44:03.483 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:44:03.483 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:44:03.483 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:44:03.491 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:44:03.491 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:44:03.491 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:44:03.491 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_47 2026-02-18T08:44:03.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_47'] 2026-02-18T08:44:03.551 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:44:03.551 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:44:03.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:44:03.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/image_47 2026-02-18T08:44:03.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:44:03.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:44:03.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:44:03.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:44:03.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:44:03.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:44:03.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:44:03.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:44:03.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:44:03.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:44:03.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:44:03.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:44:03.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:44:03.557 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:44:03.557 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:44:03.557 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:44:03.557 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:44:03.557 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:44:03.557 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:44:03.557 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:44:03.557 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:44:03.558 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:44:03.558 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:44:03.558 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:44:03.558 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:44:03.558 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:44:03.558 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:44:03.558 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:44:03.558 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:44:03.559 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:44:03.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:44:03.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:44:03.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:44:03.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/image_47 2026-02-18T08:44:03.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:44:03.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_47 2026-02-18T08:44:03.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:44:03.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:44:03.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_47 2026-02-18T08:44:03.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_47 '' 2026-02-18T08:44:03.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:44:03.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:44:03.635 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_47 2026-02-18T08:44:03.635 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:44:03.635 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_47.status 2026-02-18T08:44:03.636 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_47.status 2026-02-18T08:44:03.636 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:44:03.636 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_47.status 2026-02-18T08:44:03.636 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_47 2026-02-18T08:44:03.637 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_47.status 2026-02-18T08:44:03.664 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 6 2026-02-18T08:44:03.664 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 6 2026-02-18T08:44:03.664 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:44:03.664 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=25, tag_tid=2, entry_tid=1], [object_number=24, tag_tid=2, entry_tid=0], [object_number=27, tag_tid=1, entry_tid=26259], [object_number=26, tag_tid=1, entry_tid=26258]]], state=connected] 2026-02-18T08:44:03.664 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=25, tag_tid=2, entry_tid=1], [object_number=24, tag_tid=2, entry_tid=0], [object_number=27, tag_tid=1, entry_tid=26259], [object_number=26, tag_tid=1, entry_tid=26258]]], state=connected] 2026-02-18T08:44:03.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_47.status 2026-02-18T08:44:03.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=25, tag_tid=2, entry_tid=1' 2026-02-18T08:44:03.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_47 2026-02-18T08:44:03.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:44:03.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:44:03.670 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_47 2026-02-18T08:44:03.670 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_47 '..*' 2026-02-18T08:44:03.670 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:44:03.670 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:44:03.670 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_47 2026-02-18T08:44:03.670 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:44:03.670 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_47.status 2026-02-18T08:44:03.671 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_47.status 2026-02-18T08:44:03.671 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:44:03.671 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_47.status 2026-02-18T08:44:03.672 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_47 2026-02-18T08:44:03.672 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_47.status 2026-02-18T08:44:03.707 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 6 2026-02-18T08:44:03.707 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 6 2026-02-18T08:44:03.707 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:44:03.707 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=25, tag_tid=2, entry_tid=1], [object_number=24, tag_tid=2, entry_tid=0], [object_number=27, tag_tid=1, entry_tid=26259], [object_number=26, tag_tid=1, entry_tid=26258]]], state=connected] 2026-02-18T08:44:03.707 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=25, tag_tid=2, entry_tid=1], [object_number=24, tag_tid=2, entry_tid=0], [object_number=27, tag_tid=1, entry_tid=26259], [object_number=26, tag_tid=1, entry_tid=26258]]], state=connected] 2026-02-18T08:44:03.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_47.status 2026-02-18T08:44:03.712 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=25, tag_tid=2, entry_tid=1' 2026-02-18T08:44:03.712 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=25, tag_tid=2, entry_tid=1' -a 'object_number=25, tag_tid=2, entry_tid=1' = 'object_number=25, tag_tid=2, entry_tid=1' 2026-02-18T08:44:03.712 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:44:03.713 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_snap_present cluster1 mirror image_47 snap 2026-02-18T08:44:03.713 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:44:03.713 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:44:03.713 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_47 2026-02-18T08:44:03.713 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap 2026-02-18T08:44:03.713 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-18T08:44:03.713 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-18T08:44:03.713 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1 2026-02-18T08:44:04.713 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster1 info mirror/image_47@snap 2026-02-18T08:44:04.745 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:rbd image 'image_47': 2026-02-18T08:44:04.745 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: size 128 MiB in 32 objects 2026-02-18T08:44:04.745 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: order 22 (4 MiB objects) 2026-02-18T08:44:04.745 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: snapshot_count: 1 2026-02-18T08:44:04.746 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: id: 1229daf243a5 2026-02-18T08:44:04.746 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: block_name_prefix: rbd_data.1229daf243a5 2026-02-18T08:44:04.746 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: format: 2 2026-02-18T08:44:04.746 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: features: layering, exclusive-lock, journaling 2026-02-18T08:44:04.746 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: op_features: 2026-02-18T08:44:04.746 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: flags: 2026-02-18T08:44:04.746 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: create_timestamp: Wed Feb 18 08:33:30 2026 2026-02-18T08:44:04.746 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: access_timestamp: Wed Feb 18 08:33:30 2026 2026-02-18T08:44:04.746 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: modify_timestamp: Wed Feb 18 08:34:31 2026 2026-02-18T08:44:04.746 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: protected: False 2026-02-18T08:44:04.746 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: journal: 1229daf243a5 2026-02-18T08:44:04.746 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring state: enabled 2026-02-18T08:44:04.747 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring mode: journal 2026-02-18T08:44:04.747 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring global id: f2a5bc25-74a1-4066-b9da-9c7e03ba9a40 2026-02-18T08:44:04.747 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring primary: false 2026-02-18T08:44:04.748 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:44:04.749 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ compare_image_snaps mirror image_47 snap 2026-02-18T08:44:04.749 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:44:04.749 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_47 2026-02-18T08:44:04.749 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap 2026-02-18T08:44:04.749 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local ret=0 2026-02-18T08:44:04.749 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local rmt_export=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_47.export 2026-02-18T08:44:04.749 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loc_export=/tmp/tmp.vybh1m9yJF/cluster1-mirror-image_47.export 2026-02-18T08:44:04.749 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_47.export /tmp/tmp.vybh1m9yJF/cluster1-mirror-image_47.export 2026-02-18T08:44:04.749 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 -p mirror export image_47@snap /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_47.export 2026-02-18T08:44:04.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-18T08:44:04.914 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster1 -p mirror export image_47@snap /tmp/tmp.vybh1m9yJF/cluster1-mirror-image_47.export 2026-02-18T08:44:05.080 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-18T08:44:05.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmp /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_47.export /tmp/tmp.vybh1m9yJF/cluster1-mirror-image_47.export 2026-02-18T08:44:05.146 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_47.export /tmp/tmp.vybh1m9yJF/cluster1-mirror-image_47.export 2026-02-18T08:44:05.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:44:05.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T08:44:05.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_48 2026-02-18T08:44:05.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_snap cluster2 mirror image_48 snap 2026-02-18T08:44:05.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:44:05.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:44:05.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_48 2026-02-18T08:44:05.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap 2026-02-18T08:44:05.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 -p mirror snap create image_48@snap --debug-rbd=20 --debug-journaler=20 2026-02-18T08:44:05.670 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.0 with skip (0.8034304105202797) > weight (0.25) 2026-02-18T08:44:05.670 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.1 with skip (0.7079003495619156) > weight (0.25) 2026-02-18T08:44:05.670 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.2 with skip (0.47192712598795294) > weight (0.25) 2026-02-18T08:44:05.670 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.3 with skip (0.4942368187221322) > weight (0.25) 2026-02-18T08:44:05.670 INFO:tasks.rbd_mirror_thrash:waiting for 74.00258804802058 secs before thrashing 2026-02-18T08:44:07.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_replay_started cluster1 mirror image_48 2026-02-18T08:44:07.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:44:07.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:44:07.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_48 2026-02-18T08:44:07.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_replay_state cluster1 mirror image_48 started 2026-02-18T08:44:07.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:44:07.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:44:07.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_48 2026-02-18T08:44:07.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local state=started 2026-02-18T08:44:07.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-18T08:44:07.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:44:07.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1 2026-02-18T08:44:08.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test_image_replay_state cluster1 mirror image_48 started 2026-02-18T08:44:08.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:44:08.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:44:08.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_48 2026-02-18T08:44:08.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local test_state=started 2026-02-18T08:44:08.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local status_result 2026-02-18T08:44:08.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_state=stopped 2026-02-18T08:44:08.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_48 2026-02-18T08:44:08.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -i state 2026-02-18T08:44:08.982 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_instance=cluster1 2026-02-18T08:44:08.982 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-18T08:44:08.982 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:44:08.982 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance=cluster1 2026-02-18T08:44:08.982 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local loop_instance 2026-02-18T08:44:08.982 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:44:08.982 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sleep 0 2026-02-18T08:44:08.982 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:44:08.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ seq 0 3 2026-02-18T08:44:08.985 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:44:08.985 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_48 2026-02-18T08:44:08.985 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:0 2026-02-18T08:44:08.985 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-18T08:44:08.985 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:44:08.986 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:44:08.986 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:44:08.986 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-18T08:44:08.986 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:44:08.986 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:44:08.986 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:44:08.986 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:44:08.986 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-18T08:44:08.986 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:44:08.986 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:44:08.986 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:44:08.986 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:44:08.987 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:0 2026-02-18T08:44:08.987 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-18T08:44:08.987 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-18T08:44:08.987 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:44:08.987 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:0 2026-02-18T08:44:08.987 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T08:44:08.987 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-18T08:44:08.987 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:44:08.987 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:44:08.987 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-18T08:44:08.987 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-18T08:44:08.988 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-18T08:44:08.988 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=0 2026-02-18T08:44:08.988 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=0 2026-02-18T08:44:08.988 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:44:08.994 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:44:08.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:44:08.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:44:08.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_48 2026-02-18T08:44:09.055 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_48'] 2026-02-18T08:44:09.055 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status : get status for rbd mirror 2026-02-18T08:44:09.056 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:44:09.058 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:44:09.059 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_48 2026-02-18T08:44:09.059 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:1 2026-02-18T08:44:09.059 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-18T08:44:09.059 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:44:09.059 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:44:09.059 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:44:09.059 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-18T08:44:09.059 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:44:09.059 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:44:09.060 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:44:09.060 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:44:09.060 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-18T08:44:09.060 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:44:09.060 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:44:09.060 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:44:09.060 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:44:09.060 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:1 2026-02-18T08:44:09.060 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-18T08:44:09.060 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-18T08:44:09.060 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:44:09.060 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:1 2026-02-18T08:44:09.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T08:44:09.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-18T08:44:09.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:44:09.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:44:09.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-18T08:44:09.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-18T08:44:09.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-18T08:44:09.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=1 2026-02-18T08:44:09.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=1 2026-02-18T08:44:09.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:44:09.068 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:44:09.068 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:44:09.069 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:44:09.069 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_48 2026-02-18T08:44:09.107 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:44:09.102+0000 7fb27e7f3640 -1 reset not still connected to 0x55e170c6ec30 2026-02-18T08:44:09.107 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:44:09.102+0000 7fb27e7f3640 -1 reset not still connected to 0x55e188baf5f0 2026-02-18T08:44:09.108 INFO:tasks.ceph.cluster1.osd.2.trial034.stderr:2026-02-18T08:44:09.102+0000 7fb27e7f3640 -1 reset not still connected to 0x55e1b06460d0 2026-02-18T08:44:09.131 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_48'] 2026-02-18T08:44:09.131 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status : get status for rbd mirror 2026-02-18T08:44:09.131 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:44:09.134 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:44:09.134 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_48 2026-02-18T08:44:09.134 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:2 2026-02-18T08:44:09.134 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-18T08:44:09.134 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:44:09.134 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:44:09.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:44:09.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-18T08:44:09.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:44:09.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:44:09.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:44:09.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:44:09.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-18T08:44:09.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:44:09.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:44:09.136 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:44:09.136 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:44:09.136 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:2 2026-02-18T08:44:09.136 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-18T08:44:09.136 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-18T08:44:09.136 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:44:09.136 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:2 2026-02-18T08:44:09.136 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T08:44:09.136 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-18T08:44:09.136 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:44:09.136 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:44:09.137 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-18T08:44:09.137 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-18T08:44:09.137 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-18T08:44:09.137 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=2 2026-02-18T08:44:09.137 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=2 2026-02-18T08:44:09.137 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:44:09.146 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:44:09.146 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:44:09.146 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:44:09.146 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_48 2026-02-18T08:44:09.198 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_48'] 2026-02-18T08:44:09.198 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status : get status for rbd mirror 2026-02-18T08:44:09.198 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:44:09.200 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:44:09.200 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/image_48 2026-02-18T08:44:09.200 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:3 2026-02-18T08:44:09.200 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-18T08:44:09.200 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:44:09.200 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:44:09.200 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:44:09.200 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-18T08:44:09.200 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:44:09.200 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:44:09.200 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:44:09.200 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:44:09.200 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-18T08:44:09.200 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:44:09.200 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:44:09.200 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:44:09.200 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:44:09.200 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:3 2026-02-18T08:44:09.200 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-18T08:44:09.200 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-18T08:44:09.200 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:44:09.200 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:3 2026-02-18T08:44:09.200 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T08:44:09.200 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-18T08:44:09.201 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:44:09.201 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:44:09.201 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-18T08:44:09.201 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-18T08:44:09.201 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-18T08:44:09.201 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=3 2026-02-18T08:44:09.201 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=3 2026-02-18T08:44:09.201 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:44:09.210 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:44:09.210 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:44:09.211 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:44:09.211 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/image_48 2026-02-18T08:44:09.278 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ return 0 2026-02-18T08:44:09.278 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ status_result=' "state": "Replaying"' 2026-02-18T08:44:09.279 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ echo ' "state": "Replaying"' 2026-02-18T08:44:09.279 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ grep -i Replaying 2026-02-18T08:44:09.280 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: "state": "Replaying" 2026-02-18T08:44:09.281 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ current_state=started 2026-02-18T08:44:09.281 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test started = started 2026-02-18T08:44:09.281 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:44:09.281 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror image_48 2026-02-18T08:44:09.281 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_cluster=cluster1 2026-02-18T08:44:09.281 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:44:09.282 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_pool=mirror 2026-02-18T08:44:09.282 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local remote_pool=mirror 2026-02-18T08:44:09.282 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_48 2026-02-18T08:44:09.282 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' journal = journal ']' 2026-02-18T08:44:09.282 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror mirror image_48 2026-02-18T08:44:09.282 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_cluster=cluster1 2026-02-18T08:44:09.282 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:44:09.282 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_pool=mirror 2026-02-18T08:44:09.282 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local remote_pool=mirror 2026-02-18T08:44:09.282 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_48 2026-02-18T08:44:09.282 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-02-18T08:44:09.282 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-02-18T08:44:09.283 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:44:09.283 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:44:09.283 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:44:09.482 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_48 2026-02-18T08:44:09.482 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:44:09.482 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:44:09.482 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_48 2026-02-18T08:44:09.482 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:44:09.482 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_48 ']' 2026-02-18T08:44:09.482 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_48' 2026-02-18T08:44:09.483 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_48 2026-02-18T08:44:09.483 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:44:09.483 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:44:09.483 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:44:09.483 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:44:09.483 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:44:09.483 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:44:09.483 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:44:09.486 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:44:09.486 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:44:09.487 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:44:09.487 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_48 2026-02-18T08:44:09.487 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:44:09.487 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:44:09.487 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:44:09.487 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:44:09.487 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:44:09.487 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:44:09.487 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:44:09.487 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:44:09.487 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:44:09.488 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:44:09.488 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:44:09.488 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:44:09.488 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:44:09.488 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:44:09.488 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:44:09.488 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:44:09.488 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:44:09.488 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:44:09.488 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:44:09.488 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:44:09.488 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:44:09.489 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:44:09.489 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:44:09.489 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:44:09.489 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:44:09.489 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:44:09.489 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:44:09.489 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:44:09.489 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:44:09.489 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:44:09.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:44:09.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:44:09.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:44:09.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_48 2026-02-18T08:44:09.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_48'] 2026-02-18T08:44:09.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:44:09.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:44:09.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:44:09.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_48 2026-02-18T08:44:09.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:44:09.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:44:09.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:44:09.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:44:09.551 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:44:09.551 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:44:09.551 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:44:09.551 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:44:09.551 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:44:09.551 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:44:09.551 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:44:09.551 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:44:09.551 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:44:09.551 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:44:09.551 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:44:09.551 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:44:09.552 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:44:09.552 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:44:09.552 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:44:09.552 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:44:09.552 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:44:09.552 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:44:09.552 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:44:09.552 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:44:09.552 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:44:09.552 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:44:09.552 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:44:09.552 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:44:09.553 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:44:09.553 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:44:09.561 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:44:09.562 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:44:09.562 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:44:09.562 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_48 2026-02-18T08:44:09.613 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_48'] 2026-02-18T08:44:09.613 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:44:09.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:44:09.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:44:09.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_48 2026-02-18T08:44:09.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:44:09.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:44:09.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:44:09.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:44:09.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:44:09.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:44:09.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:44:09.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:44:09.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:44:09.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:44:09.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:44:09.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:44:09.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:44:09.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:44:09.618 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:44:09.618 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:44:09.618 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:44:09.618 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:44:09.618 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:44:09.618 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:44:09.618 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:44:09.618 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:44:09.618 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:44:09.618 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:44:09.618 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:44:09.618 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:44:09.619 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:44:09.619 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:44:09.619 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:44:09.619 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:44:09.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:44:09.628 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:44:09.628 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:44:09.628 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_48 2026-02-18T08:44:09.686 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_48'] 2026-02-18T08:44:09.686 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:44:09.686 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:44:09.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:44:09.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/image_48 2026-02-18T08:44:09.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:44:09.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:44:09.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:44:09.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:44:09.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:44:09.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:44:09.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:44:09.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:44:09.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:44:09.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:44:09.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:44:09.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:44:09.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:44:09.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:44:09.691 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:44:09.691 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:44:09.691 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:44:09.691 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:44:09.691 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:44:09.691 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:44:09.691 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:44:09.691 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:44:09.691 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:44:09.691 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:44:09.691 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:44:09.691 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:44:09.692 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:44:09.692 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:44:09.692 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:44:09.692 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:44:09.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:44:09.700 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:44:09.700 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:44:09.700 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/image_48 2026-02-18T08:44:09.766 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:44:09.768 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_48 2026-02-18T08:44:09.768 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:44:09.768 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:44:09.768 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_48 2026-02-18T08:44:09.768 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_48 '' 2026-02-18T08:44:09.768 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:44:09.768 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:44:09.768 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_48 2026-02-18T08:44:09.769 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:44:09.769 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_48.status 2026-02-18T08:44:09.769 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_48.status 2026-02-18T08:44:09.769 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:44:09.769 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_48.status 2026-02-18T08:44:09.770 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_48 2026-02-18T08:44:09.770 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_48.status 2026-02-18T08:44:09.796 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 8 2026-02-18T08:44:09.796 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 8 2026-02-18T08:44:09.796 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:44:09.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=33, tag_tid=2, entry_tid=1], [object_number=32, tag_tid=2, entry_tid=0], [object_number=35, tag_tid=1, entry_tid=36203], [object_number=34, tag_tid=1, entry_tid=36202]]], state=connected] 2026-02-18T08:44:09.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=32, tag_tid=1, entry_tid=36204], [object_number=35, tag_tid=1, entry_tid=36203], [object_number=34, tag_tid=1, entry_tid=36202], [object_number=33, tag_tid=1, entry_tid=36201]]], state=connected] 2026-02-18T08:44:09.799 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_48.status 2026-02-18T08:44:09.800 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=33, tag_tid=2, entry_tid=1' 2026-02-18T08:44:09.802 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_48 2026-02-18T08:44:09.802 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:44:09.802 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:44:09.802 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_48 2026-02-18T08:44:09.802 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_48 '..*' 2026-02-18T08:44:09.802 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:44:09.802 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:44:09.802 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_48 2026-02-18T08:44:09.802 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:44:09.803 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_48.status 2026-02-18T08:44:09.803 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_48.status 2026-02-18T08:44:09.803 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:44:09.803 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_48.status 2026-02-18T08:44:09.804 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_48 2026-02-18T08:44:09.804 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_48.status 2026-02-18T08:44:09.839 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 8 2026-02-18T08:44:09.839 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 8 2026-02-18T08:44:09.839 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:44:09.839 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=33, tag_tid=2, entry_tid=1], [object_number=32, tag_tid=2, entry_tid=0], [object_number=35, tag_tid=1, entry_tid=36203], [object_number=34, tag_tid=1, entry_tid=36202]]], state=connected] 2026-02-18T08:44:09.840 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=32, tag_tid=1, entry_tid=36204], [object_number=35, tag_tid=1, entry_tid=36203], [object_number=34, tag_tid=1, entry_tid=36202], [object_number=33, tag_tid=1, entry_tid=36201]]], state=connected] 2026-02-18T08:44:09.843 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_48.status 2026-02-18T08:44:09.845 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=32, tag_tid=1, entry_tid=36204' 2026-02-18T08:44:09.845 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=33, tag_tid=2, entry_tid=1' -a 'object_number=33, tag_tid=2, entry_tid=1' = 'object_number=32, tag_tid=1, entry_tid=36204' 2026-02-18T08:44:09.845 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=32, tag_tid=1, entry_tid=36204' '!=' '' 2026-02-18T08:44:09.845 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:44:09.845 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=32, tag_tid=1, entry_tid=36204' = '' 2026-02-18T08:44:09.845 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=32, tag_tid=1, entry_tid=36204' 2026-02-18T08:44:09.846 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=33, tag_tid=2, entry_tid=1' 2026-02-18T08:44:09.846 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:44:09.846 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:44:09.847 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=2 2026-02-18T08:44:09.848 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=32, tag_tid=1, entry_tid=36204' 2026-02-18T08:44:09.848 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:44:09.848 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:44:09.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=1 2026-02-18T08:44:09.850 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=33, tag_tid=2, entry_tid=1' 2026-02-18T08:44:09.850 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:44:09.850 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:44:09.851 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:44:09.852 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=32, tag_tid=1, entry_tid=36204' 2026-02-18T08:44:09.852 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:44:09.853 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:44:09.854 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=36204 2026-02-18T08:44:09.854 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 2 = 1 -a 1 -le 36204 2026-02-18T08:44:09.854 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:44:09.854 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:44:09.854 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:44:10.056 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_48 2026-02-18T08:44:10.056 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:44:10.056 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:44:10.056 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_48 2026-02-18T08:44:10.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:44:10.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_48 ']' 2026-02-18T08:44:10.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_48' 2026-02-18T08:44:10.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_48 2026-02-18T08:44:10.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:44:10.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:44:10.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:44:10.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:44:10.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:44:10.058 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:44:10.058 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:44:10.058 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:44:10.060 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:44:10.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:44:10.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_48 2026-02-18T08:44:10.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:44:10.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:44:10.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:44:10.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:44:10.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:44:10.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:44:10.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:44:10.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:44:10.062 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:44:10.062 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:44:10.062 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:44:10.062 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:44:10.062 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:44:10.062 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:44:10.062 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:44:10.062 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:44:10.062 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:44:10.062 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:44:10.062 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:44:10.062 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:44:10.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:44:10.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:44:10.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:44:10.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:44:10.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:44:10.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:44:10.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:44:10.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:44:10.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:44:10.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:44:10.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:44:10.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:44:10.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:44:10.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_48 2026-02-18T08:44:10.130 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_48'] 2026-02-18T08:44:10.130 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:44:10.130 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:44:10.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:44:10.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_48 2026-02-18T08:44:10.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:44:10.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:44:10.134 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:44:10.134 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:44:10.134 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:44:10.134 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:44:10.134 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:44:10.134 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:44:10.134 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:44:10.134 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:44:10.134 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:44:10.134 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:44:10.134 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:44:10.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:44:10.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:44:10.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:44:10.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:44:10.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:44:10.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:44:10.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:44:10.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:44:10.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:44:10.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:44:10.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:44:10.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:44:10.136 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:44:10.136 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:44:10.136 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:44:10.136 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:44:10.136 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:44:10.143 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:44:10.144 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:44:10.144 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:44:10.144 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_48 2026-02-18T08:44:10.196 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_48'] 2026-02-18T08:44:10.196 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:44:10.197 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:44:10.199 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:44:10.199 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_48 2026-02-18T08:44:10.199 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:44:10.199 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:44:10.199 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:44:10.200 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:44:10.200 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:44:10.200 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:44:10.200 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:44:10.200 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:44:10.200 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:44:10.200 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:44:10.200 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:44:10.200 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:44:10.200 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:44:10.200 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:44:10.200 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:44:10.201 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:44:10.201 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:44:10.201 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:44:10.201 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:44:10.201 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:44:10.201 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:44:10.201 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:44:10.201 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:44:10.201 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:44:10.201 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:44:10.201 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:44:10.201 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:44:10.202 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:44:10.202 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:44:10.202 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:44:10.209 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:44:10.210 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:44:10.210 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:44:10.210 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_48 2026-02-18T08:44:10.261 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_48'] 2026-02-18T08:44:10.261 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:44:10.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:44:10.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:44:10.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/image_48 2026-02-18T08:44:10.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:44:10.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:44:10.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:44:10.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:44:10.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:44:10.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:44:10.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:44:10.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:44:10.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:44:10.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:44:10.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:44:10.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:44:10.266 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:44:10.266 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:44:10.266 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:44:10.266 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:44:10.266 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:44:10.266 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:44:10.266 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:44:10.266 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:44:10.266 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:44:10.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:44:10.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:44:10.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:44:10.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:44:10.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:44:10.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:44:10.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:44:10.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:44:10.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:44:10.274 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:44:10.275 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:44:10.275 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:44:10.275 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/image_48 2026-02-18T08:44:10.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:44:10.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_48 2026-02-18T08:44:10.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:44:10.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:44:10.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_48 2026-02-18T08:44:10.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_48 '' 2026-02-18T08:44:10.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:44:10.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:44:10.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_48 2026-02-18T08:44:10.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:44:10.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_48.status 2026-02-18T08:44:10.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_48.status 2026-02-18T08:44:10.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:44:10.344 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_48.status 2026-02-18T08:44:10.344 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_48 2026-02-18T08:44:10.344 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_48.status 2026-02-18T08:44:10.380 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 8 2026-02-18T08:44:10.381 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 8 2026-02-18T08:44:10.381 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:44:10.381 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=33, tag_tid=2, entry_tid=1], [object_number=32, tag_tid=2, entry_tid=0], [object_number=35, tag_tid=1, entry_tid=36203], [object_number=34, tag_tid=1, entry_tid=36202]]], state=connected] 2026-02-18T08:44:10.381 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=32, tag_tid=1, entry_tid=36204], [object_number=35, tag_tid=1, entry_tid=36203], [object_number=34, tag_tid=1, entry_tid=36202], [object_number=33, tag_tid=1, entry_tid=36201]]], state=connected] 2026-02-18T08:44:10.384 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_48.status 2026-02-18T08:44:10.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=33, tag_tid=2, entry_tid=1' 2026-02-18T08:44:10.387 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_48 2026-02-18T08:44:10.387 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:44:10.387 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:44:10.388 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_48 2026-02-18T08:44:10.388 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_48 '..*' 2026-02-18T08:44:10.388 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:44:10.388 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:44:10.388 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_48 2026-02-18T08:44:10.388 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:44:10.388 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_48.status 2026-02-18T08:44:10.388 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_48.status 2026-02-18T08:44:10.388 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:44:10.389 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_48.status 2026-02-18T08:44:10.389 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_48 2026-02-18T08:44:10.389 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_48.status 2026-02-18T08:44:10.420 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 8 2026-02-18T08:44:10.421 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 8 2026-02-18T08:44:10.421 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:44:10.421 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=33, tag_tid=2, entry_tid=1], [object_number=32, tag_tid=2, entry_tid=0], [object_number=35, tag_tid=1, entry_tid=36203], [object_number=34, tag_tid=1, entry_tid=36202]]], state=connected] 2026-02-18T08:44:10.421 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=32, tag_tid=1, entry_tid=36204], [object_number=35, tag_tid=1, entry_tid=36203], [object_number=34, tag_tid=1, entry_tid=36202], [object_number=33, tag_tid=1, entry_tid=36201]]], state=connected] 2026-02-18T08:44:10.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_48.status 2026-02-18T08:44:10.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=32, tag_tid=1, entry_tid=36204' 2026-02-18T08:44:10.427 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=33, tag_tid=2, entry_tid=1' -a 'object_number=33, tag_tid=2, entry_tid=1' = 'object_number=32, tag_tid=1, entry_tid=36204' 2026-02-18T08:44:10.427 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=32, tag_tid=1, entry_tid=36204' '!=' 'object_number=32, tag_tid=1, entry_tid=36204' 2026-02-18T08:44:10.427 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:44:10.427 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.4 2026-02-18T08:44:10.829 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_48 2026-02-18T08:44:10.829 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:44:10.829 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:44:10.829 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_48 2026-02-18T08:44:10.829 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:44:10.830 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_48 ']' 2026-02-18T08:44:10.830 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_48' 2026-02-18T08:44:10.830 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_48 2026-02-18T08:44:10.830 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:44:10.830 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:44:10.830 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:44:10.830 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:44:10.830 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:44:10.830 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:44:10.830 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:44:10.831 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:44:10.831 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:44:10.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:44:10.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_48 2026-02-18T08:44:10.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:44:10.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:44:10.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:44:10.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:44:10.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:44:10.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:44:10.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:44:10.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:44:10.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:44:10.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:44:10.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:44:10.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:44:10.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:44:10.835 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:44:10.835 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:44:10.835 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:44:10.835 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:44:10.835 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:44:10.835 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:44:10.835 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:44:10.835 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:44:10.835 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:44:10.835 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:44:10.835 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:44:10.835 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:44:10.836 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:44:10.836 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:44:10.836 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:44:10.836 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:44:10.836 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:44:10.843 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:44:10.844 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:44:10.844 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:44:10.844 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_48 2026-02-18T08:44:10.896 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_48'] 2026-02-18T08:44:10.896 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:44:10.896 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:44:10.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:44:10.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_48 2026-02-18T08:44:10.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:44:10.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:44:10.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:44:10.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:44:10.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:44:10.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:44:10.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:44:10.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:44:10.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:44:10.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:44:10.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:44:10.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:44:10.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:44:10.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:44:10.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:44:10.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:44:10.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:44:10.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:44:10.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:44:10.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:44:10.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:44:10.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:44:10.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:44:10.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:44:10.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:44:10.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:44:10.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:44:10.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:44:10.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:44:10.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:44:10.910 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:44:10.911 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:44:10.911 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:44:10.911 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_48 2026-02-18T08:44:10.970 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_48'] 2026-02-18T08:44:10.970 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:44:10.970 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:44:10.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:44:10.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_48 2026-02-18T08:44:10.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:44:10.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:44:10.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:44:10.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:44:10.974 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:44:10.974 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:44:10.974 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:44:10.974 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:44:10.974 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:44:10.974 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:44:10.974 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:44:10.974 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:44:10.974 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:44:10.974 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:44:10.974 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:44:10.974 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:44:10.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:44:10.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:44:10.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:44:10.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:44:10.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:44:10.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:44:10.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:44:10.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:44:10.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:44:10.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:44:10.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:44:10.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:44:10.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:44:10.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:44:10.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:44:10.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:44:10.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:44:10.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_48 2026-02-18T08:44:11.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_48'] 2026-02-18T08:44:11.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:44:11.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:44:11.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:44:11.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/image_48 2026-02-18T08:44:11.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:44:11.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:44:11.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:44:11.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:44:11.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:44:11.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:44:11.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:44:11.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:44:11.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:44:11.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:44:11.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:44:11.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:44:11.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:44:11.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:44:11.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:44:11.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:44:11.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:44:11.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:44:11.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:44:11.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:44:11.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:44:11.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:44:11.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:44:11.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:44:11.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:44:11.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:44:11.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:44:11.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:44:11.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:44:11.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:44:11.058 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:44:11.059 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:44:11.059 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:44:11.059 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/image_48 2026-02-18T08:44:11.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:44:11.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_48 2026-02-18T08:44:11.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:44:11.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:44:11.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_48 2026-02-18T08:44:11.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_48 '' 2026-02-18T08:44:11.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:44:11.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:44:11.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_48 2026-02-18T08:44:11.118 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:44:11.118 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_48.status 2026-02-18T08:44:11.118 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_48.status 2026-02-18T08:44:11.118 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:44:11.118 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_48.status 2026-02-18T08:44:11.119 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_48 2026-02-18T08:44:11.119 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_48.status 2026-02-18T08:44:11.155 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 8 2026-02-18T08:44:11.155 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 8 2026-02-18T08:44:11.155 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:44:11.155 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=33, tag_tid=2, entry_tid=1], [object_number=32, tag_tid=2, entry_tid=0], [object_number=35, tag_tid=1, entry_tid=36203], [object_number=34, tag_tid=1, entry_tid=36202]]], state=connected] 2026-02-18T08:44:11.156 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=32, tag_tid=1, entry_tid=36204], [object_number=35, tag_tid=1, entry_tid=36203], [object_number=34, tag_tid=1, entry_tid=36202], [object_number=33, tag_tid=1, entry_tid=36201]]], state=connected] 2026-02-18T08:44:11.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_48.status 2026-02-18T08:44:11.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=33, tag_tid=2, entry_tid=1' 2026-02-18T08:44:11.162 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_48 2026-02-18T08:44:11.162 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:44:11.162 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:44:11.162 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_48 2026-02-18T08:44:11.162 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_48 '..*' 2026-02-18T08:44:11.163 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:44:11.163 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:44:11.163 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_48 2026-02-18T08:44:11.163 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:44:11.163 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_48.status 2026-02-18T08:44:11.163 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_48.status 2026-02-18T08:44:11.163 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:44:11.163 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_48.status 2026-02-18T08:44:11.164 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_48.status 2026-02-18T08:44:11.164 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_48 2026-02-18T08:44:11.190 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 8 2026-02-18T08:44:11.190 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 8 2026-02-18T08:44:11.190 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:44:11.190 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=33, tag_tid=2, entry_tid=1], [object_number=32, tag_tid=2, entry_tid=0], [object_number=35, tag_tid=1, entry_tid=36203], [object_number=34, tag_tid=1, entry_tid=36202]]], state=connected] 2026-02-18T08:44:11.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=32, tag_tid=1, entry_tid=36204], [object_number=35, tag_tid=1, entry_tid=36203], [object_number=34, tag_tid=1, entry_tid=36202], [object_number=33, tag_tid=1, entry_tid=36201]]], state=connected] 2026-02-18T08:44:11.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_48.status 2026-02-18T08:44:11.194 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=32, tag_tid=1, entry_tid=36204' 2026-02-18T08:44:11.194 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=33, tag_tid=2, entry_tid=1' -a 'object_number=33, tag_tid=2, entry_tid=1' = 'object_number=32, tag_tid=1, entry_tid=36204' 2026-02-18T08:44:11.195 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=32, tag_tid=1, entry_tid=36204' '!=' 'object_number=32, tag_tid=1, entry_tid=36204' 2026-02-18T08:44:11.195 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:44:11.195 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.8 2026-02-18T08:44:11.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_48 2026-02-18T08:44:11.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:44:11.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:44:11.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_48 2026-02-18T08:44:11.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:44:11.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_48 ']' 2026-02-18T08:44:11.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_48' 2026-02-18T08:44:11.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_48 2026-02-18T08:44:11.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:44:11.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:44:11.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:44:11.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:44:11.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:44:11.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:44:11.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:44:12.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:44:12.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:44:12.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:44:12.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_48 2026-02-18T08:44:12.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:44:12.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:44:12.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:44:12.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:44:12.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:44:12.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:44:12.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:44:12.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:44:12.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:44:12.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:44:12.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:44:12.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:44:12.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:44:12.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:44:12.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:44:12.003 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:44:12.003 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:44:12.003 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:44:12.003 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:44:12.003 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:44:12.003 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:44:12.003 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:44:12.003 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:44:12.003 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:44:12.003 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:44:12.003 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:44:12.004 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:44:12.004 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:44:12.004 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:44:12.004 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:44:12.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:44:12.012 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:44:12.012 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:44:12.012 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_48 2026-02-18T08:44:12.074 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_48'] 2026-02-18T08:44:12.074 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:44:12.074 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:44:12.076 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:44:12.076 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_48 2026-02-18T08:44:12.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:44:12.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:44:12.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:44:12.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:44:12.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:44:12.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:44:12.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:44:12.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:44:12.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:44:12.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:44:12.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:44:12.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:44:12.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:44:12.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:44:12.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:44:12.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:44:12.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:44:12.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:44:12.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:44:12.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:44:12.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:44:12.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:44:12.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:44:12.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:44:12.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:44:12.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:44:12.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:44:12.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:44:12.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:44:12.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:44:12.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:44:12.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:44:12.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:44:12.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_48 2026-02-18T08:44:12.146 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_48'] 2026-02-18T08:44:12.147 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:44:12.147 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:44:12.149 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:44:12.149 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_48 2026-02-18T08:44:12.150 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:44:12.150 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:44:12.150 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:44:12.150 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:44:12.150 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:44:12.150 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:44:12.150 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:44:12.150 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:44:12.150 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:44:12.150 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:44:12.150 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:44:12.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:44:12.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:44:12.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:44:12.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:44:12.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:44:12.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:44:12.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:44:12.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:44:12.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:44:12.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:44:12.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:44:12.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:44:12.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:44:12.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:44:12.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:44:12.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:44:12.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:44:12.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:44:12.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:44:12.161 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:44:12.162 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:44:12.162 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:44:12.162 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_48 2026-02-18T08:44:12.220 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_48'] 2026-02-18T08:44:12.224 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:44:12.225 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:44:12.227 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:44:12.227 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/image_48 2026-02-18T08:44:12.227 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:44:12.227 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:44:12.227 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:44:12.227 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T08:44:12.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-18T08:44:12.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:44:12.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:44:12.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:44:12.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-18T08:44:12.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T08:44:12.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:44:12.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:44:12.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-18T08:44:12.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:44:12.229 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-18T08:44:12.229 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-18T08:44:12.229 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:44:12.229 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:44:12.229 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T08:44:12.229 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-18T08:44:12.229 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:44:12.229 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:44:12.229 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:44:12.229 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-18T08:44:12.229 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T08:44:12.229 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:44:12.230 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:44:12.230 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-18T08:44:12.230 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-18T08:44:12.230 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T08:44:12.234 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:44:12.234 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:44:12.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T08:44:12.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/image_48 2026-02-18T08:44:12.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:44:12.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_48 2026-02-18T08:44:12.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:44:12.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:44:12.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_48 2026-02-18T08:44:12.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_48 '' 2026-02-18T08:44:12.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:44:12.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:44:12.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_48 2026-02-18T08:44:12.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:44:12.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_48.status 2026-02-18T08:44:12.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_48.status 2026-02-18T08:44:12.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:44:12.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_48.status 2026-02-18T08:44:12.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_48 2026-02-18T08:44:12.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_48.status 2026-02-18T08:44:12.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 8 2026-02-18T08:44:12.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 8 2026-02-18T08:44:12.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:44:12.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=33, tag_tid=2, entry_tid=1], [object_number=32, tag_tid=2, entry_tid=0], [object_number=35, tag_tid=1, entry_tid=36203], [object_number=34, tag_tid=1, entry_tid=36202]]], state=connected] 2026-02-18T08:44:12.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=33, tag_tid=2, entry_tid=1], [object_number=32, tag_tid=2, entry_tid=0], [object_number=35, tag_tid=1, entry_tid=36203], [object_number=34, tag_tid=1, entry_tid=36202]]], state=connected] 2026-02-18T08:44:12.348 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_48.status 2026-02-18T08:44:12.350 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=33, tag_tid=2, entry_tid=1' 2026-02-18T08:44:12.352 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_48 2026-02-18T08:44:12.352 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:44:12.352 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:44:12.352 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_48 2026-02-18T08:44:12.352 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_48 '..*' 2026-02-18T08:44:12.352 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:44:12.353 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:44:12.353 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_48 2026-02-18T08:44:12.353 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:44:12.353 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_48.status 2026-02-18T08:44:12.353 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_48.status 2026-02-18T08:44:12.353 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:44:12.355 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_48.status 2026-02-18T08:44:12.355 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_48 2026-02-18T08:44:12.355 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_48.status 2026-02-18T08:44:12.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 8 2026-02-18T08:44:12.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 8 2026-02-18T08:44:12.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:44:12.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=33, tag_tid=2, entry_tid=1], [object_number=32, tag_tid=2, entry_tid=0], [object_number=35, tag_tid=1, entry_tid=36203], [object_number=34, tag_tid=1, entry_tid=36202]]], state=connected] 2026-02-18T08:44:12.387 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=33, tag_tid=2, entry_tid=1], [object_number=32, tag_tid=2, entry_tid=0], [object_number=35, tag_tid=1, entry_tid=36203], [object_number=34, tag_tid=1, entry_tid=36202]]], state=connected] 2026-02-18T08:44:12.390 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_48.status 2026-02-18T08:44:12.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=33, tag_tid=2, entry_tid=1' 2026-02-18T08:44:12.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=33, tag_tid=2, entry_tid=1' -a 'object_number=33, tag_tid=2, entry_tid=1' = 'object_number=33, tag_tid=2, entry_tid=1' 2026-02-18T08:44:12.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:44:12.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_snap_present cluster1 mirror image_48 snap 2026-02-18T08:44:12.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:44:12.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:44:12.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_48 2026-02-18T08:44:12.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap 2026-02-18T08:44:12.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-18T08:44:12.393 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-18T08:44:12.393 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1 2026-02-18T08:44:13.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster1 info mirror/image_48@snap 2026-02-18T08:44:13.422 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:rbd image 'image_48': 2026-02-18T08:44:13.423 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: size 128 MiB in 32 objects 2026-02-18T08:44:13.423 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: order 22 (4 MiB objects) 2026-02-18T08:44:13.423 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: snapshot_count: 1 2026-02-18T08:44:13.423 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: id: 1229a5505e0b 2026-02-18T08:44:13.423 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: block_name_prefix: rbd_data.1229a5505e0b 2026-02-18T08:44:13.423 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: format: 2 2026-02-18T08:44:13.423 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: features: layering, exclusive-lock, journaling 2026-02-18T08:44:13.423 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: op_features: 2026-02-18T08:44:13.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: flags: 2026-02-18T08:44:13.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: create_timestamp: Wed Feb 18 08:33:37 2026 2026-02-18T08:44:13.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: access_timestamp: Wed Feb 18 08:33:37 2026 2026-02-18T08:44:13.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: modify_timestamp: Wed Feb 18 08:34:38 2026 2026-02-18T08:44:13.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: protected: False 2026-02-18T08:44:13.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: journal: 1229a5505e0b 2026-02-18T08:44:13.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring state: enabled 2026-02-18T08:44:13.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring mode: journal 2026-02-18T08:44:13.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring global id: d6c5d0ef-f921-4c2a-8cc3-35a6e7a0d803 2026-02-18T08:44:13.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring primary: false 2026-02-18T08:44:13.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:44:13.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ compare_image_snaps mirror image_48 snap 2026-02-18T08:44:13.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:44:13.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_48 2026-02-18T08:44:13.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap 2026-02-18T08:44:13.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local ret=0 2026-02-18T08:44:13.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local rmt_export=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_48.export 2026-02-18T08:44:13.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loc_export=/tmp/tmp.vybh1m9yJF/cluster1-mirror-image_48.export 2026-02-18T08:44:13.427 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_48.export /tmp/tmp.vybh1m9yJF/cluster1-mirror-image_48.export 2026-02-18T08:44:13.427 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 -p mirror export image_48@snap /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_48.export 2026-02-18T08:44:13.598 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-18T08:44:13.605 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster1 -p mirror export image_48@snap /tmp/tmp.vybh1m9yJF/cluster1-mirror-image_48.export 2026-02-18T08:44:13.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-18T08:44:13.824 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmp /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_48.export /tmp/tmp.vybh1m9yJF/cluster1-mirror-image_48.export 2026-02-18T08:44:13.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_48.export /tmp/tmp.vybh1m9yJF/cluster1-mirror-image_48.export 2026-02-18T08:44:13.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:44:13.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T08:44:13.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_49 2026-02-18T08:44:13.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_snap cluster2 mirror image_49 snap 2026-02-18T08:44:13.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:44:13.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:44:13.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_49 2026-02-18T08:44:13.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap 2026-02-18T08:44:13.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 -p mirror snap create image_49@snap --debug-rbd=20 --debug-journaler=20 2026-02-18T08:44:17.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_replay_started cluster1 mirror image_49 2026-02-18T08:44:17.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:44:17.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:44:17.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_49 2026-02-18T08:44:17.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_replay_state cluster1 mirror image_49 started 2026-02-18T08:44:17.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:44:17.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:44:17.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_49 2026-02-18T08:44:17.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local state=started 2026-02-18T08:44:17.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-18T08:44:17.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:44:17.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1 2026-02-18T08:44:18.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test_image_replay_state cluster1 mirror image_49 started 2026-02-18T08:44:18.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:44:18.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:44:18.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_49 2026-02-18T08:44:18.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local test_state=started 2026-02-18T08:44:18.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local status_result 2026-02-18T08:44:18.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_state=stopped 2026-02-18T08:44:18.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_49 2026-02-18T08:44:18.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -i state 2026-02-18T08:44:18.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_instance=cluster1 2026-02-18T08:44:18.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-18T08:44:18.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:44:18.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance=cluster1 2026-02-18T08:44:18.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local loop_instance 2026-02-18T08:44:18.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:44:18.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sleep 0 2026-02-18T08:44:19.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:44:19.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ seq 0 3 2026-02-18T08:44:19.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:44:19.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_49 2026-02-18T08:44:19.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:0 2026-02-18T08:44:19.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-18T08:44:19.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:44:19.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:44:19.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:44:19.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-18T08:44:19.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:44:19.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:44:19.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:44:19.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:44:19.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-18T08:44:19.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:44:19.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:44:19.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:44:19.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:44:19.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:0 2026-02-18T08:44:19.003 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-18T08:44:19.003 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-18T08:44:19.003 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:44:19.003 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:0 2026-02-18T08:44:19.003 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T08:44:19.003 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-18T08:44:19.003 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:44:19.003 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:44:19.003 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-18T08:44:19.003 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-18T08:44:19.003 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-18T08:44:19.003 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=0 2026-02-18T08:44:19.003 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=0 2026-02-18T08:44:19.004 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:44:19.010 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:44:19.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:44:19.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:44:19.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_49 2026-02-18T08:44:19.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_49'] 2026-02-18T08:44:19.074 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status : get status for rbd mirror 2026-02-18T08:44:19.074 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:44:19.076 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:44:19.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_49 2026-02-18T08:44:19.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:1 2026-02-18T08:44:19.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-18T08:44:19.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:44:19.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:44:19.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:44:19.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-18T08:44:19.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:44:19.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:44:19.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:44:19.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:44:19.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-18T08:44:19.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:44:19.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:44:19.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:44:19.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:44:19.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:1 2026-02-18T08:44:19.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-18T08:44:19.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-18T08:44:19.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:44:19.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:1 2026-02-18T08:44:19.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T08:44:19.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-18T08:44:19.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:44:19.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:44:19.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-18T08:44:19.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-18T08:44:19.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-18T08:44:19.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=1 2026-02-18T08:44:19.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=1 2026-02-18T08:44:19.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:44:19.086 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:44:19.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:44:19.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:44:19.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_49 2026-02-18T08:44:19.155 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ return 0 2026-02-18T08:44:19.155 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ status_result=' "state": "Replaying"' 2026-02-18T08:44:19.156 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ echo ' "state": "Replaying"' 2026-02-18T08:44:19.156 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ grep -i Replaying 2026-02-18T08:44:19.157 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: "state": "Replaying" 2026-02-18T08:44:19.158 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ current_state=started 2026-02-18T08:44:19.158 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test started = started 2026-02-18T08:44:19.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:44:19.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror image_49 2026-02-18T08:44:19.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_cluster=cluster1 2026-02-18T08:44:19.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:44:19.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_pool=mirror 2026-02-18T08:44:19.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local remote_pool=mirror 2026-02-18T08:44:19.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_49 2026-02-18T08:44:19.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' journal = journal ']' 2026-02-18T08:44:19.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror mirror image_49 2026-02-18T08:44:19.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_cluster=cluster1 2026-02-18T08:44:19.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:44:19.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_pool=mirror 2026-02-18T08:44:19.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local remote_pool=mirror 2026-02-18T08:44:19.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_49 2026-02-18T08:44:19.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-02-18T08:44:19.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-02-18T08:44:19.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:44:19.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:44:19.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:44:19.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_49 2026-02-18T08:44:19.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:44:19.360 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:44:19.360 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_49 2026-02-18T08:44:19.360 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:44:19.360 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_49 ']' 2026-02-18T08:44:19.360 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_49' 2026-02-18T08:44:19.360 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_49 2026-02-18T08:44:19.360 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:44:19.360 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:44:19.360 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:44:19.360 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:44:19.361 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:44:19.361 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:44:19.361 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:44:19.361 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:44:19.363 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:44:19.364 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:44:19.364 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_49 2026-02-18T08:44:19.364 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:44:19.364 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:44:19.364 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:44:19.364 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:44:19.364 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:44:19.364 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:44:19.364 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:44:19.364 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:44:19.365 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:44:19.365 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:44:19.365 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:44:19.365 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:44:19.365 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:44:19.365 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:44:19.365 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:44:19.365 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:44:19.365 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:44:19.365 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:44:19.365 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:44:19.365 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:44:19.366 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:44:19.366 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:44:19.366 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:44:19.366 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:44:19.366 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:44:19.366 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:44:19.366 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:44:19.366 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:44:19.366 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:44:19.366 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:44:19.374 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:44:19.374 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:44:19.375 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:44:19.375 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_49 2026-02-18T08:44:19.435 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_49'] 2026-02-18T08:44:19.435 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:44:19.435 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:44:19.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:44:19.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_49 2026-02-18T08:44:19.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:44:19.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:44:19.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:44:19.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:44:19.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:44:19.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:44:19.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:44:19.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:44:19.439 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:44:19.439 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:44:19.439 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:44:19.439 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:44:19.439 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:44:19.439 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:44:19.439 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:44:19.439 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:44:19.439 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:44:19.439 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:44:19.439 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:44:19.440 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:44:19.440 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:44:19.440 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:44:19.440 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:44:19.440 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:44:19.440 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:44:19.440 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:44:19.440 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:44:19.440 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:44:19.440 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:44:19.440 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:44:19.449 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:44:19.449 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:44:19.449 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:44:19.449 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_49 2026-02-18T08:44:19.514 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:44:19.514 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_49 2026-02-18T08:44:19.514 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:44:19.514 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:44:19.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_49 2026-02-18T08:44:19.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_49 '' 2026-02-18T08:44:19.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:44:19.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:44:19.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_49 2026-02-18T08:44:19.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:44:19.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_49.status 2026-02-18T08:44:19.516 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_49.status 2026-02-18T08:44:19.516 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:44:19.516 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_49.status 2026-02-18T08:44:19.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_49 2026-02-18T08:44:19.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_49.status 2026-02-18T08:44:19.553 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 10 2026-02-18T08:44:19.553 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 10 2026-02-18T08:44:19.553 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:44:19.553 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=41, tag_tid=2, entry_tid=1], [object_number=40, tag_tid=2, entry_tid=0], [object_number=43, tag_tid=1, entry_tid=43251], [object_number=42, tag_tid=1, entry_tid=43250]]], state=connected] 2026-02-18T08:44:19.553 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=43, tag_tid=1, entry_tid=43251], [object_number=42, tag_tid=1, entry_tid=43250], [object_number=41, tag_tid=1, entry_tid=43249], [object_number=40, tag_tid=1, entry_tid=43248]]], state=connected] 2026-02-18T08:44:19.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_49.status 2026-02-18T08:44:19.558 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=41, tag_tid=2, entry_tid=1' 2026-02-18T08:44:19.559 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_49 2026-02-18T08:44:19.559 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:44:19.559 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:44:19.559 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_49 2026-02-18T08:44:19.559 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_49 '..*' 2026-02-18T08:44:19.559 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:44:19.559 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:44:19.560 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_49 2026-02-18T08:44:19.560 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:44:19.560 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_49.status 2026-02-18T08:44:19.560 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_49.status 2026-02-18T08:44:19.560 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:44:19.561 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_49.status 2026-02-18T08:44:19.561 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_49 2026-02-18T08:44:19.561 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_49.status 2026-02-18T08:44:19.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 10 2026-02-18T08:44:19.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 10 2026-02-18T08:44:19.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:44:19.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=41, tag_tid=2, entry_tid=1], [object_number=40, tag_tid=2, entry_tid=0], [object_number=43, tag_tid=1, entry_tid=43251], [object_number=42, tag_tid=1, entry_tid=43250]]], state=connected] 2026-02-18T08:44:19.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=43, tag_tid=1, entry_tid=43251], [object_number=42, tag_tid=1, entry_tid=43250], [object_number=41, tag_tid=1, entry_tid=43249], [object_number=40, tag_tid=1, entry_tid=43248]]], state=connected] 2026-02-18T08:44:19.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_49.status 2026-02-18T08:44:19.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=43, tag_tid=1, entry_tid=43251' 2026-02-18T08:44:19.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=41, tag_tid=2, entry_tid=1' -a 'object_number=41, tag_tid=2, entry_tid=1' = 'object_number=43, tag_tid=1, entry_tid=43251' 2026-02-18T08:44:19.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=43, tag_tid=1, entry_tid=43251' '!=' '' 2026-02-18T08:44:19.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-18T08:44:19.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=43, tag_tid=1, entry_tid=43251' = '' 2026-02-18T08:44:19.594 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=43, tag_tid=1, entry_tid=43251' 2026-02-18T08:44:19.594 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=41, tag_tid=2, entry_tid=1' 2026-02-18T08:44:19.594 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:44:19.594 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:44:19.595 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=2 2026-02-18T08:44:19.596 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=43, tag_tid=1, entry_tid=43251' 2026-02-18T08:44:19.596 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-18T08:44:19.596 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:44:19.598 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=1 2026-02-18T08:44:19.598 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=41, tag_tid=2, entry_tid=1' 2026-02-18T08:44:19.598 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:44:19.599 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:44:19.600 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-18T08:44:19.601 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=43, tag_tid=1, entry_tid=43251' 2026-02-18T08:44:19.601 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-18T08:44:19.601 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-18T08:44:19.602 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=43251 2026-02-18T08:44:19.602 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 2 = 1 -a 1 -le 43251 2026-02-18T08:44:19.603 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:44:19.603 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:44:19.603 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:44:19.805 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_49 2026-02-18T08:44:19.805 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:44:19.805 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:44:19.805 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_49 2026-02-18T08:44:19.805 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:44:19.805 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_49 ']' 2026-02-18T08:44:19.805 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_49' 2026-02-18T08:44:19.805 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_49 2026-02-18T08:44:19.805 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:44:19.806 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:44:19.806 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:44:19.806 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:44:19.806 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:44:19.806 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:44:19.806 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:44:19.806 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:44:19.806 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:44:19.809 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:44:19.809 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_49 2026-02-18T08:44:19.809 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:44:19.809 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:44:19.809 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:44:19.809 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:44:19.809 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:44:19.810 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:44:19.810 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:44:19.810 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:44:19.810 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:44:19.810 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:44:19.810 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:44:19.810 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:44:19.810 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:44:19.810 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:44:19.810 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:44:19.810 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:44:19.810 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:44:19.810 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:44:19.811 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:44:19.811 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:44:19.811 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:44:19.811 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:44:19.811 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:44:19.811 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:44:19.811 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:44:19.811 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:44:19.811 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:44:19.811 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:44:19.811 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:44:19.811 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:44:19.819 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:44:19.819 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:44:19.819 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:44:19.819 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_49 2026-02-18T08:44:19.878 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_49'] 2026-02-18T08:44:19.879 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:44:19.879 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:44:19.881 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:44:19.881 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_49 2026-02-18T08:44:19.882 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:44:19.882 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:44:19.882 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:44:19.882 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:44:19.882 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:44:19.882 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:44:19.882 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:44:19.882 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:44:19.882 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:44:19.882 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:44:19.882 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:44:19.883 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:44:19.883 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:44:19.883 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:44:19.883 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:44:19.883 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:44:19.883 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:44:19.883 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:44:19.883 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:44:19.883 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:44:19.883 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:44:19.883 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:44:19.883 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:44:19.884 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:44:19.884 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:44:19.884 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:44:19.884 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:44:19.884 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:44:19.884 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:44:19.884 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:44:19.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:44:19.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:44:19.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:44:19.894 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_49 2026-02-18T08:44:19.956 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:44:19.956 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_49 2026-02-18T08:44:19.956 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:44:19.956 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:44:19.956 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_49 2026-02-18T08:44:19.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_49 '' 2026-02-18T08:44:19.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:44:19.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:44:19.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_49 2026-02-18T08:44:19.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:44:19.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_49.status 2026-02-18T08:44:19.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_49.status 2026-02-18T08:44:19.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:44:19.958 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_49.status 2026-02-18T08:44:19.958 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_49 2026-02-18T08:44:19.958 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_49.status 2026-02-18T08:44:19.985 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 10 2026-02-18T08:44:19.985 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 10 2026-02-18T08:44:19.985 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:44:19.985 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=41, tag_tid=2, entry_tid=1], [object_number=40, tag_tid=2, entry_tid=0], [object_number=43, tag_tid=1, entry_tid=43251], [object_number=42, tag_tid=1, entry_tid=43250]]], state=connected] 2026-02-18T08:44:19.985 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=43, tag_tid=1, entry_tid=43251], [object_number=42, tag_tid=1, entry_tid=43250], [object_number=41, tag_tid=1, entry_tid=43249], [object_number=40, tag_tid=1, entry_tid=43248]]], state=connected] 2026-02-18T08:44:19.987 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_49.status 2026-02-18T08:44:19.989 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=41, tag_tid=2, entry_tid=1' 2026-02-18T08:44:19.990 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_49 2026-02-18T08:44:19.990 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:44:19.990 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:44:19.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_49 2026-02-18T08:44:19.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_49 '..*' 2026-02-18T08:44:19.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:44:19.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:44:19.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_49 2026-02-18T08:44:19.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:44:19.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_49.status 2026-02-18T08:44:19.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_49.status 2026-02-18T08:44:19.992 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:44:19.992 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_49.status 2026-02-18T08:44:19.992 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_49 2026-02-18T08:44:19.992 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_49.status 2026-02-18T08:44:20.028 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 10 2026-02-18T08:44:20.028 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 10 2026-02-18T08:44:20.028 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:44:20.028 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=41, tag_tid=2, entry_tid=1], [object_number=40, tag_tid=2, entry_tid=0], [object_number=43, tag_tid=1, entry_tid=43251], [object_number=42, tag_tid=1, entry_tid=43250]]], state=connected] 2026-02-18T08:44:20.028 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=43, tag_tid=1, entry_tid=43251], [object_number=42, tag_tid=1, entry_tid=43250], [object_number=41, tag_tid=1, entry_tid=43249], [object_number=40, tag_tid=1, entry_tid=43248]]], state=connected] 2026-02-18T08:44:20.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_49.status 2026-02-18T08:44:20.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=43, tag_tid=1, entry_tid=43251' 2026-02-18T08:44:20.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=41, tag_tid=2, entry_tid=1' -a 'object_number=41, tag_tid=2, entry_tid=1' = 'object_number=43, tag_tid=1, entry_tid=43251' 2026-02-18T08:44:20.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=43, tag_tid=1, entry_tid=43251' '!=' 'object_number=43, tag_tid=1, entry_tid=43251' 2026-02-18T08:44:20.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:44:20.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.4 2026-02-18T08:44:20.436 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_49 2026-02-18T08:44:20.436 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:44:20.436 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:44:20.436 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_49 2026-02-18T08:44:20.436 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:44:20.436 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_49 ']' 2026-02-18T08:44:20.437 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_49' 2026-02-18T08:44:20.437 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_49 2026-02-18T08:44:20.437 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:44:20.437 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:44:20.437 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:44:20.437 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:44:20.437 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:44:20.437 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:44:20.437 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:44:20.437 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:44:20.437 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:44:20.440 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:44:20.440 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_49 2026-02-18T08:44:20.440 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:44:20.440 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:44:20.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:44:20.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:44:20.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:44:20.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:44:20.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:44:20.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:44:20.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:44:20.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:44:20.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:44:20.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:44:20.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:44:20.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:44:20.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:44:20.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:44:20.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:44:20.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:44:20.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:44:20.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:44:20.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:44:20.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:44:20.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:44:20.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:44:20.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:44:20.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:44:20.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:44:20.443 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:44:20.443 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:44:20.443 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:44:20.450 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:44:20.451 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:44:20.451 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:44:20.451 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_49 2026-02-18T08:44:20.510 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_49'] 2026-02-18T08:44:20.510 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:44:20.510 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:44:20.512 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:44:20.512 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_49 2026-02-18T08:44:20.512 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:44:20.512 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:44:20.512 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:44:20.512 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:44:20.512 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:44:20.512 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:44:20.512 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:44:20.512 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:44:20.512 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:44:20.512 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:44:20.512 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:44:20.512 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:44:20.512 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:44:20.512 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:44:20.513 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:44:20.513 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:44:20.513 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:44:20.513 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:44:20.513 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:44:20.513 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:44:20.513 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:44:20.513 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:44:20.513 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:44:20.513 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:44:20.513 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:44:20.513 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:44:20.513 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:44:20.513 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:44:20.513 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:44:20.513 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:44:20.525 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:44:20.526 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:44:20.526 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:44:20.526 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_49 2026-02-18T08:44:20.584 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:44:20.584 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_49 2026-02-18T08:44:20.584 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:44:20.584 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:44:20.584 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_49 2026-02-18T08:44:20.584 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_49 '' 2026-02-18T08:44:20.585 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:44:20.585 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:44:20.585 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_49 2026-02-18T08:44:20.585 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:44:20.585 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_49.status 2026-02-18T08:44:20.585 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_49.status 2026-02-18T08:44:20.585 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:44:20.586 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_49.status 2026-02-18T08:44:20.586 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_49 2026-02-18T08:44:20.586 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_49.status 2026-02-18T08:44:20.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 10 2026-02-18T08:44:20.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 10 2026-02-18T08:44:20.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:44:20.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=41, tag_tid=2, entry_tid=1], [object_number=40, tag_tid=2, entry_tid=0], [object_number=43, tag_tid=1, entry_tid=43251], [object_number=42, tag_tid=1, entry_tid=43250]]], state=connected] 2026-02-18T08:44:20.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=43, tag_tid=1, entry_tid=43251], [object_number=42, tag_tid=1, entry_tid=43250], [object_number=41, tag_tid=1, entry_tid=43249], [object_number=40, tag_tid=1, entry_tid=43248]]], state=connected] 2026-02-18T08:44:20.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_49.status 2026-02-18T08:44:20.618 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=41, tag_tid=2, entry_tid=1' 2026-02-18T08:44:20.619 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_49 2026-02-18T08:44:20.620 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:44:20.620 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:44:20.620 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_49 2026-02-18T08:44:20.620 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_49 '..*' 2026-02-18T08:44:20.620 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:44:20.620 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:44:20.620 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_49 2026-02-18T08:44:20.620 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:44:20.620 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_49.status 2026-02-18T08:44:20.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_49.status 2026-02-18T08:44:20.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:44:20.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_49.status 2026-02-18T08:44:20.622 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_49 2026-02-18T08:44:20.622 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_49.status 2026-02-18T08:44:20.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 10 2026-02-18T08:44:20.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 10 2026-02-18T08:44:20.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:44:20.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=41, tag_tid=2, entry_tid=1], [object_number=40, tag_tid=2, entry_tid=0], [object_number=43, tag_tid=1, entry_tid=43251], [object_number=42, tag_tid=1, entry_tid=43250]]], state=connected] 2026-02-18T08:44:20.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=43, tag_tid=1, entry_tid=43251], [object_number=42, tag_tid=1, entry_tid=43250], [object_number=41, tag_tid=1, entry_tid=43249], [object_number=40, tag_tid=1, entry_tid=43248]]], state=connected] 2026-02-18T08:44:20.659 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_49.status 2026-02-18T08:44:20.661 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=43, tag_tid=1, entry_tid=43251' 2026-02-18T08:44:20.661 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=41, tag_tid=2, entry_tid=1' -a 'object_number=41, tag_tid=2, entry_tid=1' = 'object_number=43, tag_tid=1, entry_tid=43251' 2026-02-18T08:44:20.661 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=43, tag_tid=1, entry_tid=43251' '!=' 'object_number=43, tag_tid=1, entry_tid=43251' 2026-02-18T08:44:20.661 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:44:20.661 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.8 2026-02-18T08:44:21.463 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_49 2026-02-18T08:44:21.463 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:44:21.463 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:44:21.463 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_49 2026-02-18T08:44:21.463 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:44:21.463 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_49 ']' 2026-02-18T08:44:21.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_49' 2026-02-18T08:44:21.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_49 2026-02-18T08:44:21.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:44:21.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:44:21.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:44:21.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:44:21.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:44:21.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:44:21.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:44:21.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:44:21.467 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:44:21.467 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:44:21.467 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_49 2026-02-18T08:44:21.467 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:44:21.467 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:44:21.468 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:44:21.468 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:44:21.468 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:44:21.468 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:44:21.468 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:44:21.468 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:44:21.468 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:44:21.468 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:44:21.468 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:44:21.468 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:44:21.468 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:44:21.468 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:44:21.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:44:21.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:44:21.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:44:21.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:44:21.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:44:21.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:44:21.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:44:21.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:44:21.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:44:21.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:44:21.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:44:21.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:44:21.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:44:21.470 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:44:21.470 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:44:21.470 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:44:21.478 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:44:21.478 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:44:21.479 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:44:21.479 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_49 2026-02-18T08:44:21.539 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_49'] 2026-02-18T08:44:21.539 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:44:21.539 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:44:21.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:44:21.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_49 2026-02-18T08:44:21.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:44:21.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:44:21.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:44:21.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:44:21.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:44:21.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:44:21.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:44:21.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:44:21.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:44:21.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:44:21.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:44:21.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:44:21.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:44:21.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:44:21.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:44:21.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:44:21.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:44:21.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:44:21.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:44:21.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:44:21.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:44:21.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:44:21.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:44:21.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:44:21.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:44:21.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:44:21.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:44:21.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:44:21.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:44:21.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:44:21.553 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:44:21.553 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:44:21.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:44:21.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_49 2026-02-18T08:44:21.610 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:44:21.612 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_49 2026-02-18T08:44:21.612 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:44:21.612 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:44:21.612 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_49 2026-02-18T08:44:21.612 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_49 '' 2026-02-18T08:44:21.612 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:44:21.612 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:44:21.612 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_49 2026-02-18T08:44:21.612 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:44:21.612 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_49.status 2026-02-18T08:44:21.613 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_49.status 2026-02-18T08:44:21.613 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:44:21.613 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_49.status 2026-02-18T08:44:21.613 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_49 2026-02-18T08:44:21.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_49.status 2026-02-18T08:44:21.641 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 10 2026-02-18T08:44:21.641 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 10 2026-02-18T08:44:21.641 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:44:21.641 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=41, tag_tid=2, entry_tid=1], [object_number=40, tag_tid=2, entry_tid=0], [object_number=43, tag_tid=1, entry_tid=43251], [object_number=42, tag_tid=1, entry_tid=43250]]], state=connected] 2026-02-18T08:44:21.641 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=43, tag_tid=1, entry_tid=43251], [object_number=42, tag_tid=1, entry_tid=43250], [object_number=41, tag_tid=1, entry_tid=43249], [object_number=40, tag_tid=1, entry_tid=43248]]], state=connected] 2026-02-18T08:44:21.643 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_49.status 2026-02-18T08:44:21.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=41, tag_tid=2, entry_tid=1' 2026-02-18T08:44:21.646 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_49 2026-02-18T08:44:21.646 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:44:21.646 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:44:21.647 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_49 2026-02-18T08:44:21.647 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_49 '..*' 2026-02-18T08:44:21.647 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:44:21.647 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:44:21.647 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_49 2026-02-18T08:44:21.647 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:44:21.647 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_49.status 2026-02-18T08:44:21.648 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_49.status 2026-02-18T08:44:21.648 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:44:21.648 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_49.status 2026-02-18T08:44:21.649 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_49.status 2026-02-18T08:44:21.649 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_49 2026-02-18T08:44:21.675 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 10 2026-02-18T08:44:21.675 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 10 2026-02-18T08:44:21.675 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:44:21.675 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=41, tag_tid=2, entry_tid=1], [object_number=40, tag_tid=2, entry_tid=0], [object_number=43, tag_tid=1, entry_tid=43251], [object_number=42, tag_tid=1, entry_tid=43250]]], state=connected] 2026-02-18T08:44:21.676 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=43, tag_tid=1, entry_tid=43251], [object_number=42, tag_tid=1, entry_tid=43250], [object_number=41, tag_tid=1, entry_tid=43249], [object_number=40, tag_tid=1, entry_tid=43248]]], state=connected] 2026-02-18T08:44:21.678 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_49.status 2026-02-18T08:44:21.679 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=43, tag_tid=1, entry_tid=43251' 2026-02-18T08:44:21.680 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=41, tag_tid=2, entry_tid=1' -a 'object_number=41, tag_tid=2, entry_tid=1' = 'object_number=43, tag_tid=1, entry_tid=43251' 2026-02-18T08:44:21.680 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=43, tag_tid=1, entry_tid=43251' '!=' 'object_number=43, tag_tid=1, entry_tid=43251' 2026-02-18T08:44:21.680 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:44:21.680 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1.6 2026-02-18T08:44:23.282 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_49 2026-02-18T08:44:23.282 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:44:23.282 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:44:23.282 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_49 2026-02-18T08:44:23.283 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:44:23.283 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_49 ']' 2026-02-18T08:44:23.283 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_49' 2026-02-18T08:44:23.283 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_49 2026-02-18T08:44:23.283 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:44:23.283 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:44:23.283 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:44:23.283 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:44:23.283 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:44:23.283 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:44:23.283 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:44:23.284 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:44:23.284 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:44:23.286 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:44:23.286 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_49 2026-02-18T08:44:23.286 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:44:23.286 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:44:23.287 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:44:23.287 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:44:23.287 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:44:23.287 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:44:23.287 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:44:23.287 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:44:23.287 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:44:23.287 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:44:23.287 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:44:23.287 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:44:23.287 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:44:23.287 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:44:23.288 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:44:23.288 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:44:23.288 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:44:23.288 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:44:23.288 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:44:23.288 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:44:23.288 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:44:23.288 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:44:23.288 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:44:23.288 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:44:23.288 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:44:23.288 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:44:23.288 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:44:23.289 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:44:23.289 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:44:23.289 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:44:23.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:44:23.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:44:23.297 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:44:23.297 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_49 2026-02-18T08:44:23.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_49'] 2026-02-18T08:44:23.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:44:23.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:44:23.362 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:44:23.362 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_49 2026-02-18T08:44:23.362 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:44:23.362 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:44:23.362 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:44:23.362 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:44:23.362 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:44:23.363 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:44:23.363 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:44:23.363 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:44:23.363 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:44:23.363 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:44:23.363 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:44:23.363 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:44:23.363 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:44:23.363 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:44:23.363 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:44:23.363 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:44:23.363 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:44:23.364 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:44:23.364 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:44:23.364 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:44:23.364 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:44:23.364 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:44:23.364 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:44:23.364 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:44:23.364 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:44:23.364 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:44:23.364 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:44:23.364 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:44:23.364 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:44:23.364 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:44:23.375 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:44:23.375 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:44:23.375 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:44:23.375 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_49 2026-02-18T08:44:23.443 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:44:23.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_49 2026-02-18T08:44:23.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:44:23.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:44:23.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_49 2026-02-18T08:44:23.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_49 '' 2026-02-18T08:44:23.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:44:23.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:44:23.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_49 2026-02-18T08:44:23.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:44:23.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_49.status 2026-02-18T08:44:23.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_49.status 2026-02-18T08:44:23.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:44:23.446 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_49.status 2026-02-18T08:44:23.446 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_49 2026-02-18T08:44:23.446 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_49.status 2026-02-18T08:44:23.474 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 10 2026-02-18T08:44:23.474 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 10 2026-02-18T08:44:23.474 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:44:23.474 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=41, tag_tid=2, entry_tid=1], [object_number=40, tag_tid=2, entry_tid=0], [object_number=43, tag_tid=1, entry_tid=43251], [object_number=42, tag_tid=1, entry_tid=43250]]], state=connected] 2026-02-18T08:44:23.474 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=41, tag_tid=2, entry_tid=1], [object_number=40, tag_tid=2, entry_tid=0], [object_number=43, tag_tid=1, entry_tid=43251], [object_number=42, tag_tid=1, entry_tid=43250]]], state=connected] 2026-02-18T08:44:23.476 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_49.status 2026-02-18T08:44:23.477 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=41, tag_tid=2, entry_tid=1' 2026-02-18T08:44:23.479 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_49 2026-02-18T08:44:23.479 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:44:23.479 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:44:23.479 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_49 2026-02-18T08:44:23.479 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_49 '..*' 2026-02-18T08:44:23.479 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:44:23.479 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:44:23.479 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_49 2026-02-18T08:44:23.479 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:44:23.479 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_49.status 2026-02-18T08:44:23.479 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_49.status 2026-02-18T08:44:23.480 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:44:23.480 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_49.status 2026-02-18T08:44:23.481 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_49 2026-02-18T08:44:23.481 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_49.status 2026-02-18T08:44:23.506 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 10 2026-02-18T08:44:23.506 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 10 2026-02-18T08:44:23.506 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:44:23.506 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=41, tag_tid=2, entry_tid=1], [object_number=40, tag_tid=2, entry_tid=0], [object_number=43, tag_tid=1, entry_tid=43251], [object_number=42, tag_tid=1, entry_tid=43250]]], state=connected] 2026-02-18T08:44:23.506 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=41, tag_tid=2, entry_tid=1], [object_number=40, tag_tid=2, entry_tid=0], [object_number=43, tag_tid=1, entry_tid=43251], [object_number=42, tag_tid=1, entry_tid=43250]]], state=connected] 2026-02-18T08:44:23.508 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_49.status 2026-02-18T08:44:23.511 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=41, tag_tid=2, entry_tid=1' 2026-02-18T08:44:23.511 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=41, tag_tid=2, entry_tid=1' -a 'object_number=41, tag_tid=2, entry_tid=1' = 'object_number=41, tag_tid=2, entry_tid=1' 2026-02-18T08:44:23.511 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:44:23.511 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_snap_present cluster1 mirror image_49 snap 2026-02-18T08:44:23.511 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:44:23.511 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:44:23.511 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_49 2026-02-18T08:44:23.511 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap 2026-02-18T08:44:23.512 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-18T08:44:23.512 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-18T08:44:23.512 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1 2026-02-18T08:44:24.512 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster1 info mirror/image_49@snap 2026-02-18T08:44:24.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:rbd image 'image_49': 2026-02-18T08:44:24.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: size 128 MiB in 32 objects 2026-02-18T08:44:24.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: order 22 (4 MiB objects) 2026-02-18T08:44:24.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: snapshot_count: 1 2026-02-18T08:44:24.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: id: 122997588807 2026-02-18T08:44:24.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: block_name_prefix: rbd_data.122997588807 2026-02-18T08:44:24.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: format: 2 2026-02-18T08:44:24.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: features: layering, exclusive-lock, journaling 2026-02-18T08:44:24.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: op_features: 2026-02-18T08:44:24.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: flags: 2026-02-18T08:44:24.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: create_timestamp: Wed Feb 18 08:33:47 2026 2026-02-18T08:44:24.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: access_timestamp: Wed Feb 18 08:33:47 2026 2026-02-18T08:44:24.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: modify_timestamp: Wed Feb 18 08:34:48 2026 2026-02-18T08:44:24.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: protected: False 2026-02-18T08:44:24.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: journal: 122997588807 2026-02-18T08:44:24.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring state: enabled 2026-02-18T08:44:24.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring mode: journal 2026-02-18T08:44:24.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring global id: 4ccddbab-d01f-4b3b-8788-c1087a2ce753 2026-02-18T08:44:24.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring primary: false 2026-02-18T08:44:24.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:44:24.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ compare_image_snaps mirror image_49 snap 2026-02-18T08:44:24.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:44:24.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_49 2026-02-18T08:44:24.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap 2026-02-18T08:44:24.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local ret=0 2026-02-18T08:44:24.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local rmt_export=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_49.export 2026-02-18T08:44:24.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loc_export=/tmp/tmp.vybh1m9yJF/cluster1-mirror-image_49.export 2026-02-18T08:44:24.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_49.export /tmp/tmp.vybh1m9yJF/cluster1-mirror-image_49.export 2026-02-18T08:44:24.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 -p mirror export image_49@snap /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_49.export 2026-02-18T08:44:24.684 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-18T08:44:24.694 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster1 -p mirror export image_49@snap /tmp/tmp.vybh1m9yJF/cluster1-mirror-image_49.export 2026-02-18T08:44:24.897 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-18T08:44:24.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmp /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_49.export /tmp/tmp.vybh1m9yJF/cluster1-mirror-image_49.export 2026-02-18T08:44:24.962 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_49.export /tmp/tmp.vybh1m9yJF/cluster1-mirror-image_49.export 2026-02-18T08:44:25.006 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:44:25.006 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T08:44:25.006 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_50 2026-02-18T08:44:25.006 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_snap cluster2 mirror image_50 snap 2026-02-18T08:44:25.006 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:44:25.006 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:44:25.006 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_50 2026-02-18T08:44:25.006 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap 2026-02-18T08:44:25.007 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 -p mirror snap create image_50@snap --debug-rbd=20 --debug-journaler=20 2026-02-18T08:44:28.004 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_replay_started cluster1 mirror image_50 2026-02-18T08:44:28.004 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:44:28.004 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:44:28.004 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_50 2026-02-18T08:44:28.005 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_replay_state cluster1 mirror image_50 started 2026-02-18T08:44:28.005 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:44:28.005 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:44:28.005 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_50 2026-02-18T08:44:28.005 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local state=started 2026-02-18T08:44:28.005 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-18T08:44:28.005 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:44:28.005 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1 2026-02-18T08:44:29.006 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test_image_replay_state cluster1 mirror image_50 started 2026-02-18T08:44:29.006 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:44:29.006 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:44:29.006 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_50 2026-02-18T08:44:29.006 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local test_state=started 2026-02-18T08:44:29.007 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local status_result 2026-02-18T08:44:29.007 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_state=stopped 2026-02-18T08:44:29.007 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_50 2026-02-18T08:44:29.007 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -i state 2026-02-18T08:44:29.007 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_instance=cluster1 2026-02-18T08:44:29.007 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-18T08:44:29.007 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:44:29.007 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance=cluster1 2026-02-18T08:44:29.007 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local loop_instance 2026-02-18T08:44:29.007 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:44:29.007 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sleep 0 2026-02-18T08:44:29.010 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:44:29.010 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ seq 0 3 2026-02-18T08:44:29.010 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:44:29.010 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_50 2026-02-18T08:44:29.010 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:0 2026-02-18T08:44:29.010 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-18T08:44:29.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:44:29.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:44:29.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:44:29.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-18T08:44:29.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:44:29.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:44:29.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:44:29.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:44:29.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-18T08:44:29.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:44:29.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:44:29.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:44:29.012 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:44:29.012 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:0 2026-02-18T08:44:29.012 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-18T08:44:29.012 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-18T08:44:29.012 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:44:29.012 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:0 2026-02-18T08:44:29.012 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T08:44:29.012 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-18T08:44:29.012 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:44:29.012 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:44:29.012 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-18T08:44:29.012 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-18T08:44:29.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-18T08:44:29.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=0 2026-02-18T08:44:29.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=0 2026-02-18T08:44:29.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:44:29.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:44:29.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:44:29.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:44:29.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_50 2026-02-18T08:44:29.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_50'] 2026-02-18T08:44:29.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status : get status for rbd mirror 2026-02-18T08:44:29.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:44:29.080 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:44:29.080 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_50 2026-02-18T08:44:29.080 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:1 2026-02-18T08:44:29.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-18T08:44:29.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:44:29.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:44:29.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:44:29.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-18T08:44:29.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:44:29.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:44:29.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:44:29.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:44:29.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-18T08:44:29.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:44:29.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:44:29.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:44:29.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:44:29.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:1 2026-02-18T08:44:29.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-18T08:44:29.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-18T08:44:29.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:44:29.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:1 2026-02-18T08:44:29.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T08:44:29.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-18T08:44:29.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:44:29.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:44:29.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-18T08:44:29.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-18T08:44:29.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-18T08:44:29.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=1 2026-02-18T08:44:29.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=1 2026-02-18T08:44:29.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:44:29.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:44:29.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:44:29.092 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:44:29.092 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_50 2026-02-18T08:44:29.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_50'] 2026-02-18T08:44:29.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status : get status for rbd mirror 2026-02-18T08:44:29.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:44:29.156 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:44:29.157 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_50 2026-02-18T08:44:29.157 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:2 2026-02-18T08:44:29.157 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-18T08:44:29.157 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:44:29.157 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:44:29.157 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:44:29.157 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-18T08:44:29.157 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:44:29.158 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:44:29.158 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:44:29.158 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:44:29.158 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-18T08:44:29.158 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:44:29.158 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:44:29.158 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:44:29.158 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:44:29.158 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:2 2026-02-18T08:44:29.158 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-18T08:44:29.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-18T08:44:29.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:44:29.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:2 2026-02-18T08:44:29.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T08:44:29.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-18T08:44:29.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-18T08:44:29.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:44:29.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-18T08:44:29.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-18T08:44:29.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-18T08:44:29.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=2 2026-02-18T08:44:29.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=2 2026-02-18T08:44:29.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:44:29.167 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:44:29.167 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:44:29.167 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:44:29.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_50 2026-02-18T08:44:29.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ return 0 2026-02-18T08:44:29.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ status_result=' "state": "Replaying"' 2026-02-18T08:44:29.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ echo ' "state": "Replaying"' 2026-02-18T08:44:29.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ grep -i Replaying 2026-02-18T08:44:29.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: "state": "Replaying" 2026-02-18T08:44:29.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ current_state=started 2026-02-18T08:44:29.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test started = started 2026-02-18T08:44:29.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:44:29.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror image_50 2026-02-18T08:44:29.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_cluster=cluster1 2026-02-18T08:44:29.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:44:29.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_pool=mirror 2026-02-18T08:44:29.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local remote_pool=mirror 2026-02-18T08:44:29.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_50 2026-02-18T08:44:29.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' journal = journal ']' 2026-02-18T08:44:29.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror mirror image_50 2026-02-18T08:44:29.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_cluster=cluster1 2026-02-18T08:44:29.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:44:29.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_pool=mirror 2026-02-18T08:44:29.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local remote_pool=mirror 2026-02-18T08:44:29.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_50 2026-02-18T08:44:29.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-02-18T08:44:29.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-02-18T08:44:29.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:44:29.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T08:44:29.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-18T08:44:29.440 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_50 2026-02-18T08:44:29.440 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:44:29.440 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:44:29.440 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_50 2026-02-18T08:44:29.440 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-18T08:44:29.440 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_50 ']' 2026-02-18T08:44:29.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_50' 2026-02-18T08:44:29.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_50 2026-02-18T08:44:29.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-18T08:44:29.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:44:29.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:44:29.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-18T08:44:29.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-18T08:44:29.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T08:44:29.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:44:29.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-18T08:44:29.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:44:29.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:44:29.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_50 2026-02-18T08:44:29.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:44:29.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:44:29.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:44:29.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T08:44:29.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-18T08:44:29.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:44:29.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:44:29.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:44:29.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:44:29.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T08:44:29.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:44:29.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:44:29.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:44:29.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:44:29.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-18T08:44:29.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-18T08:44:29.446 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:44:29.446 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:44:29.446 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T08:44:29.446 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-18T08:44:29.446 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:44:29.446 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:44:29.446 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:44:29.446 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:44:29.446 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T08:44:29.446 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:44:29.446 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:44:29.446 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-18T08:44:29.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-18T08:44:29.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T08:44:29.453 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:44:29.454 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:44:29.454 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T08:44:29.454 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_50 2026-02-18T08:44:29.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_50'] 2026-02-18T08:44:29.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:44:29.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:44:29.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:44:29.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_50 2026-02-18T08:44:29.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:44:29.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:44:29.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:44:29.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T08:44:29.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-18T08:44:29.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:44:29.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:44:29.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:44:29.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-18T08:44:29.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T08:44:29.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:44:29.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:44:29.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-18T08:44:29.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:44:29.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-18T08:44:29.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-18T08:44:29.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:44:29.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:44:29.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T08:44:29.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-18T08:44:29.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:44:29.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:44:29.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:44:29.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-18T08:44:29.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T08:44:29.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:44:29.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:44:29.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-18T08:44:29.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-18T08:44:29.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T08:44:29.529 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:44:29.530 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:44:29.530 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T08:44:29.530 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_50 2026-02-18T08:44:29.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_50'] 2026-02-18T08:44:29.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-18T08:44:29.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-18T08:44:29.586 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:44:29.586 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_50 2026-02-18T08:44:29.586 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:44:29.586 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:44:29.586 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:44:29.586 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T08:44:29.586 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-18T08:44:29.586 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:44:29.586 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:44:29.587 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:44:29.587 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-18T08:44:29.587 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T08:44:29.587 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:44:29.587 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:44:29.587 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-18T08:44:29.587 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:44:29.587 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-18T08:44:29.587 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-18T08:44:29.587 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:44:29.587 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-18T08:44:29.587 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T08:44:29.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-18T08:44:29.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-18T08:44:29.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-18T08:44:29.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:44:29.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-18T08:44:29.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T08:44:29.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-18T08:44:29.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-18T08:44:29.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-18T08:44:29.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-18T08:44:29.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T08:44:29.597 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:44:29.597 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:44:29.598 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T08:44:29.598 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_50 2026-02-18T08:44:29.660 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:44:29.661 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_50 2026-02-18T08:44:29.661 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:44:29.661 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:44:29.661 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_50 2026-02-18T08:44:29.661 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_50 '' 2026-02-18T08:44:29.661 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:44:29.662 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:44:29.662 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_50 2026-02-18T08:44:29.662 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-18T08:44:29.662 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_50.status 2026-02-18T08:44:29.662 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_50.status 2026-02-18T08:44:29.662 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:44:29.662 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_50.status 2026-02-18T08:44:29.663 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_50 2026-02-18T08:44:29.663 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_50.status 2026-02-18T08:44:29.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 5 2026-02-18T08:44:29.691 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 5 2026-02-18T08:44:29.691 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:44:29.691 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=21, tag_tid=2, entry_tid=1], [object_number=20, tag_tid=2, entry_tid=0], [object_number=23, tag_tid=1, entry_tid=22091], [object_number=22, tag_tid=1, entry_tid=22090]]], state=connected] 2026-02-18T08:44:29.691 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=21, tag_tid=2, entry_tid=1], [object_number=20, tag_tid=2, entry_tid=0], [object_number=23, tag_tid=1, entry_tid=22091], [object_number=22, tag_tid=1, entry_tid=22090]]], state=connected] 2026-02-18T08:44:29.693 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_50.status 2026-02-18T08:44:29.695 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=21, tag_tid=2, entry_tid=1' 2026-02-18T08:44:29.696 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_50 2026-02-18T08:44:29.696 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:44:29.696 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:44:29.696 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_50 2026-02-18T08:44:29.697 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_50 '..*' 2026-02-18T08:44:29.697 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:44:29.697 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:44:29.697 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_50 2026-02-18T08:44:29.697 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-18T08:44:29.697 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_50.status 2026-02-18T08:44:29.697 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_50.status 2026-02-18T08:44:29.697 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-18T08:44:29.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_50.status 2026-02-18T08:44:29.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_50 2026-02-18T08:44:29.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_50.status 2026-02-18T08:44:29.725 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 5 2026-02-18T08:44:29.725 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 5 2026-02-18T08:44:29.725 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-18T08:44:29.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=21, tag_tid=2, entry_tid=1], [object_number=20, tag_tid=2, entry_tid=0], [object_number=23, tag_tid=1, entry_tid=22091], [object_number=22, tag_tid=1, entry_tid=22090]]], state=connected] 2026-02-18T08:44:29.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=590ab87c-e426-4a76-8b8a-636aa4a7d086, commit_position=[positions=[[object_number=21, tag_tid=2, entry_tid=1], [object_number=20, tag_tid=2, entry_tid=0], [object_number=23, tag_tid=1, entry_tid=22091], [object_number=22, tag_tid=1, entry_tid=22090]]], state=connected] 2026-02-18T08:44:29.728 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_50.status 2026-02-18T08:44:29.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=21, tag_tid=2, entry_tid=1' 2026-02-18T08:44:29.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=21, tag_tid=2, entry_tid=1' -a 'object_number=21, tag_tid=2, entry_tid=1' = 'object_number=21, tag_tid=2, entry_tid=1' 2026-02-18T08:44:29.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:44:29.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_snap_present cluster1 mirror image_50 snap 2026-02-18T08:44:29.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:44:29.731 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:44:29.731 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_50 2026-02-18T08:44:29.731 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap 2026-02-18T08:44:29.731 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-18T08:44:29.731 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-18T08:44:29.731 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1 2026-02-18T08:44:30.731 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster1 info mirror/image_50@snap 2026-02-18T08:44:30.760 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:rbd image 'image_50': 2026-02-18T08:44:30.760 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: size 128 MiB in 32 objects 2026-02-18T08:44:30.761 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: order 22 (4 MiB objects) 2026-02-18T08:44:30.761 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: snapshot_count: 1 2026-02-18T08:44:30.761 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: id: 11f9f1ac6ba8 2026-02-18T08:44:30.761 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: block_name_prefix: rbd_data.11f9f1ac6ba8 2026-02-18T08:44:30.761 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: format: 2 2026-02-18T08:44:30.761 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: features: layering, exclusive-lock, journaling 2026-02-18T08:44:30.761 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: op_features: 2026-02-18T08:44:30.761 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: flags: 2026-02-18T08:44:30.761 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: create_timestamp: Wed Feb 18 08:34:00 2026 2026-02-18T08:44:30.761 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: access_timestamp: Wed Feb 18 08:34:00 2026 2026-02-18T08:44:30.761 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: modify_timestamp: Wed Feb 18 08:34:00 2026 2026-02-18T08:44:30.762 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: protected: False 2026-02-18T08:44:30.762 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: journal: 11f9f1ac6ba8 2026-02-18T08:44:30.762 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring state: enabled 2026-02-18T08:44:30.762 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring mode: journal 2026-02-18T08:44:30.762 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring global id: fd348964-3f19-49f8-832d-1e739ea53fe0 2026-02-18T08:44:30.762 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring primary: false 2026-02-18T08:44:30.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:44:30.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ compare_image_snaps mirror image_50 snap 2026-02-18T08:44:30.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:44:30.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_50 2026-02-18T08:44:30.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap 2026-02-18T08:44:30.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local ret=0 2026-02-18T08:44:30.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local rmt_export=/tmp/tmp.vybh1m9yJF/cluster2-mirror-image_50.export 2026-02-18T08:44:30.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loc_export=/tmp/tmp.vybh1m9yJF/cluster1-mirror-image_50.export 2026-02-18T08:44:30.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_50.export /tmp/tmp.vybh1m9yJF/cluster1-mirror-image_50.export 2026-02-18T08:44:30.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 -p mirror export image_50@snap /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_50.export 2026-02-18T08:44:30.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-18T08:44:30.909 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster1 -p mirror export image_50@snap /tmp/tmp.vybh1m9yJF/cluster1-mirror-image_50.export 2026-02-18T08:44:31.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-18T08:44:31.092 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmp /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_50.export /tmp/tmp.vybh1m9yJF/cluster1-mirror-image_50.export 2026-02-18T08:44:31.150 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.vybh1m9yJF/cluster2-mirror-image_50.export /tmp/tmp.vybh1m9yJF/cluster1-mirror-image_50.export 2026-02-18T08:44:31.194 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:44:31.195 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ testlog 'TEST: delete many images' 2026-02-18T08:44:31.195 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ tee -a /tmp/tmp.vybh1m9yJF/rbd-mirror.test.log 2026-02-18T08:44:31.195 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ date '+%F %T' 2026-02-18T08:44:31.196 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ echo 2026-02-18 08:44:31 TEST: delete many images 2026-02-18T08:44:31.197 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:2026-02-18 08:44:31 TEST: delete many images 2026-02-18T08:44:31.197 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 1 50 2026-02-18T08:44:31.198 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T08:44:31.198 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_1 2026-02-18T08:44:31.199 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_snapshot cluster2 mirror image_1 snap 2026-02-18T08:44:31.199 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:44:31.199 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:44:31.199 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_1 2026-02-18T08:44:31.199 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap=snap 2026-02-18T08:44:31.199 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 snap rm mirror/image_1@snap 2026-02-18T08:44:31.999 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Removing snap: 100% complete...done. 2026-02-18T08:44:32.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_image_retry cluster2 mirror image_1 2026-02-18T08:44:32.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:44:32.010 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:44:32.010 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_1 2026-02-18T08:44:32.010 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-18T08:44:32.010 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:44:32.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_image cluster2 mirror image_1 2026-02-18T08:44:32.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:44:32.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:44:32.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_1 2026-02-18T08:44:32.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_1 2026-02-18T08:44:32.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster=cluster2 rm mirror/image_1 2026-02-18T08:44:32.175 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-18T08:44:32.180 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:44:32.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T08:44:32.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_2 2026-02-18T08:44:32.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_snapshot cluster2 mirror image_2 snap 2026-02-18T08:44:32.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:44:32.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:44:32.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_2 2026-02-18T08:44:32.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap=snap 2026-02-18T08:44:32.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 snap rm mirror/image_2@snap 2026-02-18T08:44:33.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Removing snap: 100% complete...done. 2026-02-18T08:44:33.012 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_image_retry cluster2 mirror image_2 2026-02-18T08:44:33.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:44:33.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:44:33.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_2 2026-02-18T08:44:33.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-18T08:44:33.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:44:33.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_image cluster2 mirror image_2 2026-02-18T08:44:33.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:44:33.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:44:33.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_2 2026-02-18T08:44:33.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_2 2026-02-18T08:44:33.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster=cluster2 rm mirror/image_2 2026-02-18T08:44:33.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-18T08:44:33.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:44:33.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T08:44:33.187 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_3 2026-02-18T08:44:33.187 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_snapshot cluster2 mirror image_3 snap 2026-02-18T08:44:33.187 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:44:33.187 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:44:33.187 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_3 2026-02-18T08:44:33.187 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap=snap 2026-02-18T08:44:33.187 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 snap rm mirror/image_3@snap 2026-02-18T08:44:34.003 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Removing snap: 100% complete...done. 2026-02-18T08:44:34.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_image_retry cluster2 mirror image_3 2026-02-18T08:44:34.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:44:34.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:44:34.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_3 2026-02-18T08:44:34.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-18T08:44:34.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:44:34.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_image cluster2 mirror image_3 2026-02-18T08:44:34.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:44:34.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:44:34.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_3 2026-02-18T08:44:34.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_3 2026-02-18T08:44:34.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster=cluster2 rm mirror/image_3 2026-02-18T08:44:34.389 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-18T08:44:34.394 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:44:34.394 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T08:44:34.394 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_4 2026-02-18T08:44:34.394 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_snapshot cluster2 mirror image_4 snap 2026-02-18T08:44:34.394 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:44:34.394 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:44:34.394 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_4 2026-02-18T08:44:34.395 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap=snap 2026-02-18T08:44:34.395 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 snap rm mirror/image_4@snap 2026-02-18T08:44:35.004 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Removing snap: 100% complete...done. 2026-02-18T08:44:35.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_image_retry cluster2 mirror image_4 2026-02-18T08:44:35.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:44:35.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:44:35.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_4 2026-02-18T08:44:35.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-18T08:44:35.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:44:35.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_image cluster2 mirror image_4 2026-02-18T08:44:35.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:44:35.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:44:35.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_4 2026-02-18T08:44:35.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_4 2026-02-18T08:44:35.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster=cluster2 rm mirror/image_4 2026-02-18T08:44:35.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-18T08:44:35.189 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:44:35.189 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T08:44:35.189 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_5 2026-02-18T08:44:35.189 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_snapshot cluster2 mirror image_5 snap 2026-02-18T08:44:35.189 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:44:35.189 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:44:35.189 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_5 2026-02-18T08:44:35.190 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap=snap 2026-02-18T08:44:35.190 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 snap rm mirror/image_5@snap 2026-02-18T08:44:36.007 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Removing snap: 100% complete...done. 2026-02-18T08:44:36.020 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_image_retry cluster2 mirror image_5 2026-02-18T08:44:36.020 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:44:36.020 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:44:36.020 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_5 2026-02-18T08:44:36.021 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-18T08:44:36.021 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:44:36.021 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_image cluster2 mirror image_5 2026-02-18T08:44:36.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:44:36.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:44:36.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_5 2026-02-18T08:44:36.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_5 2026-02-18T08:44:36.055 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster=cluster2 rm mirror/image_5 2026-02-18T08:44:36.180 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-18T08:44:36.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:44:36.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T08:44:36.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_6 2026-02-18T08:44:36.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_snapshot cluster2 mirror image_6 snap 2026-02-18T08:44:36.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:44:36.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:44:36.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_6 2026-02-18T08:44:36.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap=snap 2026-02-18T08:44:36.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 snap rm mirror/image_6@snap 2026-02-18T08:44:37.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Removing snap: 100% complete...done. 2026-02-18T08:44:37.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_image_retry cluster2 mirror image_6 2026-02-18T08:44:37.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:44:37.027 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:44:37.027 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_6 2026-02-18T08:44:37.027 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-18T08:44:37.027 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:44:37.028 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_image cluster2 mirror image_6 2026-02-18T08:44:37.028 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:44:37.028 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:44:37.028 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_6 2026-02-18T08:44:37.028 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_6 2026-02-18T08:44:37.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster=cluster2 rm mirror/image_6 2026-02-18T08:44:37.195 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-18T08:44:37.200 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:44:37.200 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T08:44:37.201 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_7 2026-02-18T08:44:37.201 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_snapshot cluster2 mirror image_7 snap 2026-02-18T08:44:37.201 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:44:37.201 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:44:37.201 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_7 2026-02-18T08:44:37.201 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap=snap 2026-02-18T08:44:37.201 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 snap rm mirror/image_7@snap 2026-02-18T08:44:38.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Removing snap: 100% complete...done. 2026-02-18T08:44:38.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_image_retry cluster2 mirror image_7 2026-02-18T08:44:38.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:44:38.027 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:44:38.027 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_7 2026-02-18T08:44:38.027 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-18T08:44:38.027 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:44:38.028 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_image cluster2 mirror image_7 2026-02-18T08:44:38.028 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:44:38.028 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:44:38.028 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_7 2026-02-18T08:44:38.028 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_7 2026-02-18T08:44:38.059 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster=cluster2 rm mirror/image_7 2026-02-18T08:44:38.189 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-18T08:44:38.194 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:44:38.195 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T08:44:38.195 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_8 2026-02-18T08:44:38.195 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_snapshot cluster2 mirror image_8 snap 2026-02-18T08:44:38.195 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:44:38.195 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:44:38.195 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_8 2026-02-18T08:44:38.195 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap=snap 2026-02-18T08:44:38.195 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 snap rm mirror/image_8@snap 2026-02-18T08:44:39.020 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Removing snap: 100% complete...done. 2026-02-18T08:44:39.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_image_retry cluster2 mirror image_8 2026-02-18T08:44:39.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:44:39.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:44:39.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_8 2026-02-18T08:44:39.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-18T08:44:39.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:44:39.035 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_image cluster2 mirror image_8 2026-02-18T08:44:39.035 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:44:39.035 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:44:39.035 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_8 2026-02-18T08:44:39.035 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_8 2026-02-18T08:44:39.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster=cluster2 rm mirror/image_8 2026-02-18T08:44:39.201 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-18T08:44:39.206 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:44:39.206 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T08:44:39.206 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_9 2026-02-18T08:44:39.207 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_snapshot cluster2 mirror image_9 snap 2026-02-18T08:44:39.207 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:44:39.207 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:44:39.207 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_9 2026-02-18T08:44:39.207 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap=snap 2026-02-18T08:44:39.207 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 snap rm mirror/image_9@snap 2026-02-18T08:44:40.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Removing snap: 100% complete...done. 2026-02-18T08:44:40.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_image_retry cluster2 mirror image_9 2026-02-18T08:44:40.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:44:40.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:44:40.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_9 2026-02-18T08:44:40.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-18T08:44:40.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:44:40.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_image cluster2 mirror image_9 2026-02-18T08:44:40.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:44:40.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:44:40.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_9 2026-02-18T08:44:40.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_9 2026-02-18T08:44:40.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster=cluster2 rm mirror/image_9 2026-02-18T08:44:40.209 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-18T08:44:40.214 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:44:40.214 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T08:44:40.214 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_10 2026-02-18T08:44:40.214 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_snapshot cluster2 mirror image_10 snap 2026-02-18T08:44:40.214 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:44:40.214 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:44:40.214 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_10 2026-02-18T08:44:40.214 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap=snap 2026-02-18T08:44:40.215 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 snap rm mirror/image_10@snap 2026-02-18T08:44:41.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Removing snap: 100% complete...done. 2026-02-18T08:44:41.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_image_retry cluster2 mirror image_10 2026-02-18T08:44:41.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:44:41.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:44:41.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_10 2026-02-18T08:44:41.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-18T08:44:41.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:44:41.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_image cluster2 mirror image_10 2026-02-18T08:44:41.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:44:41.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:44:41.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_10 2026-02-18T08:44:41.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_10 2026-02-18T08:44:41.075 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster=cluster2 rm mirror/image_10 2026-02-18T08:44:41.204 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-18T08:44:41.210 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:44:41.210 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T08:44:41.210 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_11 2026-02-18T08:44:41.210 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_snapshot cluster2 mirror image_11 snap 2026-02-18T08:44:41.210 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:44:41.210 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:44:41.210 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_11 2026-02-18T08:44:41.210 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap=snap 2026-02-18T08:44:41.211 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 snap rm mirror/image_11@snap 2026-02-18T08:44:42.036 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Removing snap: 100% complete...done. 2026-02-18T08:44:42.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_image_retry cluster2 mirror image_11 2026-02-18T08:44:42.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:44:42.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:44:42.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_11 2026-02-18T08:44:42.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-18T08:44:42.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:44:42.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_image cluster2 mirror image_11 2026-02-18T08:44:42.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:44:42.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:44:42.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_11 2026-02-18T08:44:42.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_11 2026-02-18T08:44:42.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster=cluster2 rm mirror/image_11 2026-02-18T08:44:42.416 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-18T08:44:42.421 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:44:42.421 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T08:44:42.421 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_12 2026-02-18T08:44:42.421 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_snapshot cluster2 mirror image_12 snap 2026-02-18T08:44:42.422 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:44:42.422 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:44:42.422 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_12 2026-02-18T08:44:42.422 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap=snap 2026-02-18T08:44:42.422 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 snap rm mirror/image_12@snap 2026-02-18T08:44:43.041 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Removing snap: 100% complete...done. 2026-02-18T08:44:43.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_image_retry cluster2 mirror image_12 2026-02-18T08:44:43.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:44:43.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:44:43.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_12 2026-02-18T08:44:43.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-18T08:44:43.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:44:43.055 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_image cluster2 mirror image_12 2026-02-18T08:44:43.055 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:44:43.055 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:44:43.055 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_12 2026-02-18T08:44:43.055 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_12 2026-02-18T08:44:43.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster=cluster2 rm mirror/image_12 2026-02-18T08:44:43.218 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-18T08:44:43.223 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:44:43.223 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T08:44:43.224 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_13 2026-02-18T08:44:43.224 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_snapshot cluster2 mirror image_13 snap 2026-02-18T08:44:43.224 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:44:43.224 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:44:43.224 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_13 2026-02-18T08:44:43.224 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap=snap 2026-02-18T08:44:43.224 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 snap rm mirror/image_13@snap 2026-02-18T08:44:44.039 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Removing snap: 100% complete...done. 2026-02-18T08:44:44.056 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_image_retry cluster2 mirror image_13 2026-02-18T08:44:44.056 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:44:44.056 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:44:44.056 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_13 2026-02-18T08:44:44.056 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-18T08:44:44.056 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:44:44.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_image cluster2 mirror image_13 2026-02-18T08:44:44.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:44:44.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:44:44.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_13 2026-02-18T08:44:44.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_13 2026-02-18T08:44:44.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster=cluster2 rm mirror/image_13 2026-02-18T08:44:44.218 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-18T08:44:44.223 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:44:44.223 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T08:44:44.223 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_14 2026-02-18T08:44:44.223 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_snapshot cluster2 mirror image_14 snap 2026-02-18T08:44:44.223 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:44:44.223 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:44:44.223 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_14 2026-02-18T08:44:44.223 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap=snap 2026-02-18T08:44:44.224 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 snap rm mirror/image_14@snap 2026-02-18T08:44:45.039 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Removing snap: 100% complete...done. 2026-02-18T08:44:45.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_image_retry cluster2 mirror image_14 2026-02-18T08:44:45.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:44:45.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:44:45.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_14 2026-02-18T08:44:45.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-18T08:44:45.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:44:45.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_image cluster2 mirror image_14 2026-02-18T08:44:45.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:44:45.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:44:45.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_14 2026-02-18T08:44:45.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_14 2026-02-18T08:44:45.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster=cluster2 rm mirror/image_14 2026-02-18T08:44:45.198 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-18T08:44:45.203 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:44:45.203 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T08:44:45.203 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_15 2026-02-18T08:44:45.203 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_snapshot cluster2 mirror image_15 snap 2026-02-18T08:44:45.203 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:44:45.203 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:44:45.203 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_15 2026-02-18T08:44:45.204 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap=snap 2026-02-18T08:44:45.204 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 snap rm mirror/image_15@snap 2026-02-18T08:44:46.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Removing snap: 100% complete...done. 2026-02-18T08:44:46.056 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_image_retry cluster2 mirror image_15 2026-02-18T08:44:46.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:44:46.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:44:46.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_15 2026-02-18T08:44:46.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-18T08:44:46.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:44:46.058 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_image cluster2 mirror image_15 2026-02-18T08:44:46.058 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:44:46.058 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:44:46.058 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_15 2026-02-18T08:44:46.058 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_15 2026-02-18T08:44:46.092 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster=cluster2 rm mirror/image_15 2026-02-18T08:44:46.217 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-18T08:44:46.222 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:44:46.222 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T08:44:46.222 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_16 2026-02-18T08:44:46.222 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_snapshot cluster2 mirror image_16 snap 2026-02-18T08:44:46.222 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:44:46.222 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:44:46.222 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_16 2026-02-18T08:44:46.223 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap=snap 2026-02-18T08:44:46.223 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 snap rm mirror/image_16@snap 2026-02-18T08:44:47.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Removing snap: 100% complete...done. 2026-02-18T08:44:47.062 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_image_retry cluster2 mirror image_16 2026-02-18T08:44:47.062 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:44:47.062 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:44:47.062 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_16 2026-02-18T08:44:47.062 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-18T08:44:47.062 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:44:47.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_image cluster2 mirror image_16 2026-02-18T08:44:47.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:44:47.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:44:47.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_16 2026-02-18T08:44:47.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_16 2026-02-18T08:44:47.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster=cluster2 rm mirror/image_16 2026-02-18T08:44:47.231 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-18T08:44:47.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:44:47.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T08:44:47.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_17 2026-02-18T08:44:47.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_snapshot cluster2 mirror image_17 snap 2026-02-18T08:44:47.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:44:47.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:44:47.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_17 2026-02-18T08:44:47.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap=snap 2026-02-18T08:44:47.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 snap rm mirror/image_17@snap 2026-02-18T08:44:48.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Removing snap: 100% complete...done. 2026-02-18T08:44:48.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_image_retry cluster2 mirror image_17 2026-02-18T08:44:48.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:44:48.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:44:48.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_17 2026-02-18T08:44:48.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-18T08:44:48.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:44:48.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_image cluster2 mirror image_17 2026-02-18T08:44:48.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:44:48.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:44:48.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_17 2026-02-18T08:44:48.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_17 2026-02-18T08:44:48.096 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster=cluster2 rm mirror/image_17 2026-02-18T08:44:48.223 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-18T08:44:48.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:44:48.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T08:44:48.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_18 2026-02-18T08:44:48.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_snapshot cluster2 mirror image_18 snap 2026-02-18T08:44:48.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:44:48.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:44:48.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_18 2026-02-18T08:44:48.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap=snap 2026-02-18T08:44:48.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 snap rm mirror/image_18@snap 2026-02-18T08:44:49.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Removing snap: 100% complete...done. 2026-02-18T08:44:49.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_image_retry cluster2 mirror image_18 2026-02-18T08:44:49.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:44:49.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:44:49.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_18 2026-02-18T08:44:49.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-18T08:44:49.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:44:49.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_image cluster2 mirror image_18 2026-02-18T08:44:49.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:44:49.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:44:49.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_18 2026-02-18T08:44:49.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_18 2026-02-18T08:44:49.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster=cluster2 rm mirror/image_18 2026-02-18T08:44:49.221 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-18T08:44:49.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:44:49.227 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T08:44:49.227 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_19 2026-02-18T08:44:49.227 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_snapshot cluster2 mirror image_19 snap 2026-02-18T08:44:49.227 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:44:49.227 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:44:49.227 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_19 2026-02-18T08:44:49.227 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap=snap 2026-02-18T08:44:49.227 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 snap rm mirror/image_19@snap 2026-02-18T08:44:50.056 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Removing snap: 100% complete...done. 2026-02-18T08:44:50.068 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_image_retry cluster2 mirror image_19 2026-02-18T08:44:50.068 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:44:50.068 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:44:50.068 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_19 2026-02-18T08:44:50.069 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-18T08:44:50.069 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:44:50.069 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_image cluster2 mirror image_19 2026-02-18T08:44:50.069 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:44:50.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:44:50.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_19 2026-02-18T08:44:50.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_19 2026-02-18T08:44:50.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster=cluster2 rm mirror/image_19 2026-02-18T08:44:50.223 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-18T08:44:50.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:44:50.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T08:44:50.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_20 2026-02-18T08:44:50.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_snapshot cluster2 mirror image_20 snap 2026-02-18T08:44:50.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:44:50.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:44:50.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_20 2026-02-18T08:44:50.229 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap=snap 2026-02-18T08:44:50.229 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 snap rm mirror/image_20@snap 2026-02-18T08:44:51.059 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Removing snap: 100% complete...done. 2026-02-18T08:44:51.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_image_retry cluster2 mirror image_20 2026-02-18T08:44:51.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:44:51.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:44:51.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_20 2026-02-18T08:44:51.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-18T08:44:51.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:44:51.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_image cluster2 mirror image_20 2026-02-18T08:44:51.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:44:51.074 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:44:51.074 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_20 2026-02-18T08:44:51.074 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_20 2026-02-18T08:44:51.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster=cluster2 rm mirror/image_20 2026-02-18T08:44:51.230 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-18T08:44:51.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:44:51.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T08:44:51.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_21 2026-02-18T08:44:51.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_snapshot cluster2 mirror image_21 snap 2026-02-18T08:44:51.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:44:51.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:44:51.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_21 2026-02-18T08:44:51.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap=snap 2026-02-18T08:44:51.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 snap rm mirror/image_21@snap 2026-02-18T08:44:52.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Removing snap: 100% complete...done. 2026-02-18T08:44:52.074 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_image_retry cluster2 mirror image_21 2026-02-18T08:44:52.075 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:44:52.075 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:44:52.075 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_21 2026-02-18T08:44:52.075 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-18T08:44:52.075 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:44:52.076 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_image cluster2 mirror image_21 2026-02-18T08:44:52.076 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:44:52.076 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:44:52.076 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_21 2026-02-18T08:44:52.076 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_21 2026-02-18T08:44:52.110 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster=cluster2 rm mirror/image_21 2026-02-18T08:44:52.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-18T08:44:52.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:44:52.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T08:44:52.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_22 2026-02-18T08:44:52.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_snapshot cluster2 mirror image_22 snap 2026-02-18T08:44:52.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:44:52.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:44:52.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_22 2026-02-18T08:44:52.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap=snap 2026-02-18T08:44:52.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 snap rm mirror/image_22@snap 2026-02-18T08:44:53.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Removing snap: 100% complete...done. 2026-02-18T08:44:53.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_image_retry cluster2 mirror image_22 2026-02-18T08:44:53.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:44:53.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:44:53.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_22 2026-02-18T08:44:53.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-18T08:44:53.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:44:53.080 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_image cluster2 mirror image_22 2026-02-18T08:44:53.080 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:44:53.080 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:44:53.080 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_22 2026-02-18T08:44:53.080 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_22 2026-02-18T08:44:53.114 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster=cluster2 rm mirror/image_22 2026-02-18T08:44:53.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-18T08:44:53.245 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:44:53.245 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T08:44:53.245 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_23 2026-02-18T08:44:53.245 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_snapshot cluster2 mirror image_23 snap 2026-02-18T08:44:53.246 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:44:53.246 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:44:53.246 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_23 2026-02-18T08:44:53.246 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap=snap 2026-02-18T08:44:53.246 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 snap rm mirror/image_23@snap 2026-02-18T08:44:54.037 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Removing snap: 100% complete...done. 2026-02-18T08:44:54.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_image_retry cluster2 mirror image_23 2026-02-18T08:44:54.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:44:54.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:44:54.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_23 2026-02-18T08:44:54.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-18T08:44:54.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:44:54.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_image cluster2 mirror image_23 2026-02-18T08:44:54.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:44:54.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:44:54.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_23 2026-02-18T08:44:54.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_23 2026-02-18T08:44:54.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster=cluster2 rm mirror/image_23 2026-02-18T08:44:54.388 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-18T08:44:54.393 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:44:54.393 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T08:44:54.394 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_24 2026-02-18T08:44:54.394 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_snapshot cluster2 mirror image_24 snap 2026-02-18T08:44:54.394 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:44:54.394 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:44:54.394 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_24 2026-02-18T08:44:54.394 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap=snap 2026-02-18T08:44:54.394 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 snap rm mirror/image_24@snap 2026-02-18T08:44:55.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Removing snap: 100% complete...done. 2026-02-18T08:44:55.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_image_retry cluster2 mirror image_24 2026-02-18T08:44:55.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:44:55.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:44:55.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_24 2026-02-18T08:44:55.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-18T08:44:55.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:44:55.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_image cluster2 mirror image_24 2026-02-18T08:44:55.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:44:55.080 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:44:55.080 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_24 2026-02-18T08:44:55.080 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_24 2026-02-18T08:44:55.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster=cluster2 rm mirror/image_24 2026-02-18T08:44:55.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-18T08:44:55.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:44:55.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T08:44:55.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_25 2026-02-18T08:44:55.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_snapshot cluster2 mirror image_25 snap 2026-02-18T08:44:55.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:44:55.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:44:55.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_25 2026-02-18T08:44:55.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap=snap 2026-02-18T08:44:55.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 snap rm mirror/image_25@snap 2026-02-18T08:44:55.349 INFO:tasks.ceph.cluster1.osd.1.trial034.stderr:2026-02-18T08:44:55.345+0000 7f20d7266640 -1 reset not still connected to 0x55640afd7ba0 2026-02-18T08:44:55.349 INFO:tasks.ceph.cluster1.osd.1.trial034.stderr:2026-02-18T08:44:55.345+0000 7f20d7266640 -1 reset not still connected to 0x55640bbf1e10 2026-02-18T08:44:55.351 INFO:tasks.ceph.cluster1.osd.1.trial034.stderr:2026-02-18T08:44:55.345+0000 7f20d7266640 -1 reset not still connected to 0x55640bcd1ad0 2026-02-18T08:44:55.351 INFO:tasks.ceph.cluster1.osd.1.trial034.stderr:2026-02-18T08:44:55.345+0000 7f20d7266640 -1 reset not still connected to 0x55640ccb9380 2026-02-18T08:44:55.351 INFO:tasks.ceph.cluster1.osd.1.trial034.stderr:2026-02-18T08:44:55.345+0000 7f20d7266640 -1 reset not still connected to 0x55642f6a01a0 2026-02-18T08:44:55.351 INFO:tasks.ceph.cluster1.osd.1.trial034.stderr:2026-02-18T08:44:55.345+0000 7f20d7266640 -1 reset not still connected to 0x55642f6a0a90 2026-02-18T08:44:56.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Removing snap: 100% complete...done. 2026-02-18T08:44:56.080 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_image_retry cluster2 mirror image_25 2026-02-18T08:44:56.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:44:56.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:44:56.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_25 2026-02-18T08:44:56.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-18T08:44:56.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:44:56.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_image cluster2 mirror image_25 2026-02-18T08:44:56.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:44:56.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:44:56.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_25 2026-02-18T08:44:56.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_25 2026-02-18T08:44:56.119 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster=cluster2 rm mirror/image_25 2026-02-18T08:44:56.238 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-18T08:44:56.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:44:56.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T08:44:56.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_26 2026-02-18T08:44:56.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_snapshot cluster2 mirror image_26 snap 2026-02-18T08:44:56.244 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:44:56.244 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:44:56.244 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_26 2026-02-18T08:44:56.244 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap=snap 2026-02-18T08:44:56.244 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 snap rm mirror/image_26@snap 2026-02-18T08:44:57.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Removing snap: 100% complete...done. 2026-02-18T08:44:57.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_image_retry cluster2 mirror image_26 2026-02-18T08:44:57.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:44:57.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:44:57.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_26 2026-02-18T08:44:57.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-18T08:44:57.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:44:57.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_image cluster2 mirror image_26 2026-02-18T08:44:57.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:44:57.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:44:57.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_26 2026-02-18T08:44:57.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_26 2026-02-18T08:44:57.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster=cluster2 rm mirror/image_26 2026-02-18T08:44:57.252 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-18T08:44:57.257 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:44:57.257 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T08:44:57.257 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_27 2026-02-18T08:44:57.257 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_snapshot cluster2 mirror image_27 snap 2026-02-18T08:44:57.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:44:57.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:44:57.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_27 2026-02-18T08:44:57.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap=snap 2026-02-18T08:44:57.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 snap rm mirror/image_27@snap 2026-02-18T08:44:58.076 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Removing snap: 100% complete...done. 2026-02-18T08:44:58.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_image_retry cluster2 mirror image_27 2026-02-18T08:44:58.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:44:58.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:44:58.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_27 2026-02-18T08:44:58.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-18T08:44:58.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:44:58.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_image cluster2 mirror image_27 2026-02-18T08:44:58.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:44:58.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:44:58.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_27 2026-02-18T08:44:58.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_27 2026-02-18T08:44:58.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster=cluster2 rm mirror/image_27 2026-02-18T08:44:58.254 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-18T08:44:58.260 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:44:58.260 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T08:44:58.260 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_28 2026-02-18T08:44:58.260 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_snapshot cluster2 mirror image_28 snap 2026-02-18T08:44:58.261 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:44:58.261 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:44:58.261 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_28 2026-02-18T08:44:58.261 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap=snap 2026-02-18T08:44:58.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 snap rm mirror/image_28@snap 2026-02-18T08:44:59.037 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Removing snap: 100% complete...done. 2026-02-18T08:44:59.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_image_retry cluster2 mirror image_28 2026-02-18T08:44:59.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:44:59.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:44:59.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_28 2026-02-18T08:44:59.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-18T08:44:59.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:44:59.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_image cluster2 mirror image_28 2026-02-18T08:44:59.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:44:59.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:44:59.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_28 2026-02-18T08:44:59.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_28 2026-02-18T08:44:59.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster=cluster2 rm mirror/image_28 2026-02-18T08:44:59.203 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-18T08:44:59.208 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:44:59.209 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T08:44:59.209 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_29 2026-02-18T08:44:59.209 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_snapshot cluster2 mirror image_29 snap 2026-02-18T08:44:59.209 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:44:59.209 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:44:59.209 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_29 2026-02-18T08:44:59.209 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap=snap 2026-02-18T08:44:59.209 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 snap rm mirror/image_29@snap 2026-02-18T08:45:00.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Removing snap: 100% complete...done. 2026-02-18T08:45:00.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_image_retry cluster2 mirror image_29 2026-02-18T08:45:00.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:45:00.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:45:00.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_29 2026-02-18T08:45:00.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-18T08:45:00.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:45:00.092 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_image cluster2 mirror image_29 2026-02-18T08:45:00.092 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:45:00.092 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:45:00.092 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_29 2026-02-18T08:45:00.092 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_29 2026-02-18T08:45:00.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster=cluster2 rm mirror/image_29 2026-02-18T08:45:00.252 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-18T08:45:00.257 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:45:00.257 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T08:45:00.257 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_30 2026-02-18T08:45:00.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_snapshot cluster2 mirror image_30 snap 2026-02-18T08:45:00.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:45:00.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:45:00.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_30 2026-02-18T08:45:00.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap=snap 2026-02-18T08:45:00.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 snap rm mirror/image_30@snap 2026-02-18T08:45:01.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Removing snap: 100% complete...done. 2026-02-18T08:45:01.095 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_image_retry cluster2 mirror image_30 2026-02-18T08:45:01.096 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:45:01.096 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:45:01.096 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_30 2026-02-18T08:45:01.096 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-18T08:45:01.096 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:45:01.097 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_image cluster2 mirror image_30 2026-02-18T08:45:01.097 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:45:01.097 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:45:01.097 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_30 2026-02-18T08:45:01.097 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_30 2026-02-18T08:45:01.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster=cluster2 rm mirror/image_30 2026-02-18T08:45:01.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-18T08:45:01.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:45:01.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T08:45:01.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_31 2026-02-18T08:45:01.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_snapshot cluster2 mirror image_31 snap 2026-02-18T08:45:01.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:45:01.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:45:01.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_31 2026-02-18T08:45:01.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap=snap 2026-02-18T08:45:01.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 snap rm mirror/image_31@snap 2026-02-18T08:45:02.092 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Removing snap: 100% complete...done. 2026-02-18T08:45:02.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_image_retry cluster2 mirror image_31 2026-02-18T08:45:02.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:45:02.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:45:02.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_31 2026-02-18T08:45:02.105 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-18T08:45:02.105 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:45:02.105 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_image cluster2 mirror image_31 2026-02-18T08:45:02.106 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:45:02.106 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:45:02.106 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_31 2026-02-18T08:45:02.106 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_31 2026-02-18T08:45:02.138 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster=cluster2 rm mirror/image_31 2026-02-18T08:45:02.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-18T08:45:02.269 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:45:02.269 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T08:45:02.270 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_32 2026-02-18T08:45:02.270 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_snapshot cluster2 mirror image_32 snap 2026-02-18T08:45:02.270 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:45:02.270 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:45:02.270 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_32 2026-02-18T08:45:02.270 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap=snap 2026-02-18T08:45:02.270 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 snap rm mirror/image_32@snap 2026-02-18T08:45:03.092 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Removing snap: 100% complete...done. 2026-02-18T08:45:03.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_image_retry cluster2 mirror image_32 2026-02-18T08:45:03.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:45:03.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:45:03.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_32 2026-02-18T08:45:03.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-18T08:45:03.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:45:03.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_image cluster2 mirror image_32 2026-02-18T08:45:03.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:45:03.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:45:03.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_32 2026-02-18T08:45:03.105 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_32 2026-02-18T08:45:03.137 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster=cluster2 rm mirror/image_32 2026-02-18T08:45:03.257 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-18T08:45:03.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:45:03.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T08:45:03.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_33 2026-02-18T08:45:03.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_snapshot cluster2 mirror image_33 snap 2026-02-18T08:45:03.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:45:03.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:45:03.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_33 2026-02-18T08:45:03.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap=snap 2026-02-18T08:45:03.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 snap rm mirror/image_33@snap 2026-02-18T08:45:04.040 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Removing snap: 100% complete...done. 2026-02-18T08:45:04.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_image_retry cluster2 mirror image_33 2026-02-18T08:45:04.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:45:04.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:45:04.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_33 2026-02-18T08:45:04.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-18T08:45:04.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:45:04.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_image cluster2 mirror image_33 2026-02-18T08:45:04.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:45:04.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:45:04.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_33 2026-02-18T08:45:04.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_33 2026-02-18T08:45:04.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster=cluster2 rm mirror/image_33 2026-02-18T08:45:04.419 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-18T08:45:04.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:45:04.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T08:45:04.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_34 2026-02-18T08:45:04.425 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_snapshot cluster2 mirror image_34 snap 2026-02-18T08:45:04.425 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:45:04.425 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:45:04.425 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_34 2026-02-18T08:45:04.425 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap=snap 2026-02-18T08:45:04.425 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 snap rm mirror/image_34@snap 2026-02-18T08:45:05.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Removing snap: 100% complete...done. 2026-02-18T08:45:05.101 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_image_retry cluster2 mirror image_34 2026-02-18T08:45:05.101 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:45:05.102 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:45:05.102 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_34 2026-02-18T08:45:05.102 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-18T08:45:05.102 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:45:05.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_image cluster2 mirror image_34 2026-02-18T08:45:05.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:45:05.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:45:05.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_34 2026-02-18T08:45:05.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_34 2026-02-18T08:45:05.139 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster=cluster2 rm mirror/image_34 2026-02-18T08:45:05.270 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-18T08:45:05.275 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:45:05.275 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T08:45:05.275 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_35 2026-02-18T08:45:05.275 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_snapshot cluster2 mirror image_35 snap 2026-02-18T08:45:05.275 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:45:05.276 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:45:05.276 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_35 2026-02-18T08:45:05.276 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap=snap 2026-02-18T08:45:05.276 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 snap rm mirror/image_35@snap 2026-02-18T08:45:06.094 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Removing snap: 100% complete...done. 2026-02-18T08:45:06.106 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_image_retry cluster2 mirror image_35 2026-02-18T08:45:06.106 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:45:06.106 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:45:06.106 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_35 2026-02-18T08:45:06.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-18T08:45:06.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:45:06.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_image cluster2 mirror image_35 2026-02-18T08:45:06.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:45:06.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:45:06.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_35 2026-02-18T08:45:06.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_35 2026-02-18T08:45:06.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster=cluster2 rm mirror/image_35 2026-02-18T08:45:06.261 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-18T08:45:06.266 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:45:06.266 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T08:45:06.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_36 2026-02-18T08:45:06.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_snapshot cluster2 mirror image_36 snap 2026-02-18T08:45:06.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:45:06.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:45:06.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_36 2026-02-18T08:45:06.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap=snap 2026-02-18T08:45:06.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 snap rm mirror/image_36@snap 2026-02-18T08:45:07.100 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Removing snap: 100% complete...done. 2026-02-18T08:45:07.111 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_image_retry cluster2 mirror image_36 2026-02-18T08:45:07.111 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:45:07.111 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:45:07.111 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_36 2026-02-18T08:45:07.111 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-18T08:45:07.112 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:45:07.112 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_image cluster2 mirror image_36 2026-02-18T08:45:07.113 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:45:07.113 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:45:07.113 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_36 2026-02-18T08:45:07.113 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_36 2026-02-18T08:45:07.139 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster=cluster2 rm mirror/image_36 2026-02-18T08:45:07.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-18T08:45:07.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:45:07.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T08:45:07.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_37 2026-02-18T08:45:07.273 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_snapshot cluster2 mirror image_37 snap 2026-02-18T08:45:07.273 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:45:07.273 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:45:07.273 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_37 2026-02-18T08:45:07.273 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap=snap 2026-02-18T08:45:07.273 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 snap rm mirror/image_37@snap 2026-02-18T08:45:08.101 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Removing snap: 100% complete...done. 2026-02-18T08:45:08.113 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_image_retry cluster2 mirror image_37 2026-02-18T08:45:08.113 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:45:08.113 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:45:08.113 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_37 2026-02-18T08:45:08.113 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-18T08:45:08.114 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:45:08.114 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_image cluster2 mirror image_37 2026-02-18T08:45:08.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:45:08.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:45:08.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_37 2026-02-18T08:45:08.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_37 2026-02-18T08:45:08.150 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster=cluster2 rm mirror/image_37 2026-02-18T08:45:08.274 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-18T08:45:08.280 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:45:08.280 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T08:45:08.280 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_38 2026-02-18T08:45:08.280 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_snapshot cluster2 mirror image_38 snap 2026-02-18T08:45:08.280 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:45:08.280 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:45:08.280 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_38 2026-02-18T08:45:08.281 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap=snap 2026-02-18T08:45:08.281 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 snap rm mirror/image_38@snap 2026-02-18T08:45:09.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Removing snap: 100% complete...done. 2026-02-18T08:45:09.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_image_retry cluster2 mirror image_38 2026-02-18T08:45:09.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:45:09.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:45:09.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_38 2026-02-18T08:45:09.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-18T08:45:09.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:45:09.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_image cluster2 mirror image_38 2026-02-18T08:45:09.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:45:09.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:45:09.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_38 2026-02-18T08:45:09.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_38 2026-02-18T08:45:09.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster=cluster2 rm mirror/image_38 2026-02-18T08:45:09.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-18T08:45:09.222 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:45:09.222 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T08:45:09.222 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_39 2026-02-18T08:45:09.222 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_snapshot cluster2 mirror image_39 snap 2026-02-18T08:45:09.222 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:45:09.222 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:45:09.222 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_39 2026-02-18T08:45:09.222 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap=snap 2026-02-18T08:45:09.223 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 snap rm mirror/image_39@snap 2026-02-18T08:45:10.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Removing snap: 100% complete...done. 2026-02-18T08:45:10.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_image_retry cluster2 mirror image_39 2026-02-18T08:45:10.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:45:10.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:45:10.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_39 2026-02-18T08:45:10.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-18T08:45:10.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:45:10.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_image cluster2 mirror image_39 2026-02-18T08:45:10.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:45:10.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:45:10.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_39 2026-02-18T08:45:10.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_39 2026-02-18T08:45:10.150 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster=cluster2 rm mirror/image_39 2026-02-18T08:45:10.279 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-18T08:45:10.284 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:45:10.284 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T08:45:10.284 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_40 2026-02-18T08:45:10.284 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_snapshot cluster2 mirror image_40 snap 2026-02-18T08:45:10.284 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:45:10.284 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:45:10.285 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_40 2026-02-18T08:45:10.285 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap=snap 2026-02-18T08:45:10.285 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 snap rm mirror/image_40@snap 2026-02-18T08:45:11.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Removing snap: 100% complete...done. 2026-02-18T08:45:11.118 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_image_retry cluster2 mirror image_40 2026-02-18T08:45:11.119 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:45:11.119 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:45:11.119 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_40 2026-02-18T08:45:11.119 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-18T08:45:11.119 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:45:11.120 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_image cluster2 mirror image_40 2026-02-18T08:45:11.120 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:45:11.120 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:45:11.120 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_40 2026-02-18T08:45:11.120 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_40 2026-02-18T08:45:11.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster=cluster2 rm mirror/image_40 2026-02-18T08:45:11.283 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-18T08:45:11.288 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:45:11.288 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T08:45:11.288 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_41 2026-02-18T08:45:11.289 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_snapshot cluster2 mirror image_41 snap 2026-02-18T08:45:11.289 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:45:11.289 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:45:11.289 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_41 2026-02-18T08:45:11.289 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap=snap 2026-02-18T08:45:11.289 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 snap rm mirror/image_41@snap 2026-02-18T08:45:12.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Removing snap: 100% complete...done. 2026-02-18T08:45:12.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_image_retry cluster2 mirror image_41 2026-02-18T08:45:12.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:45:12.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:45:12.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_41 2026-02-18T08:45:12.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-18T08:45:12.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:45:12.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_image cluster2 mirror image_41 2026-02-18T08:45:12.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:45:12.129 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:45:12.129 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_41 2026-02-18T08:45:12.129 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_41 2026-02-18T08:45:12.158 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster=cluster2 rm mirror/image_41 2026-02-18T08:45:12.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-18T08:45:12.300 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:45:12.300 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T08:45:12.300 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_42 2026-02-18T08:45:12.300 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_snapshot cluster2 mirror image_42 snap 2026-02-18T08:45:12.300 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:45:12.301 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:45:12.301 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_42 2026-02-18T08:45:12.301 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap=snap 2026-02-18T08:45:12.301 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 snap rm mirror/image_42@snap 2026-02-18T08:45:13.114 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Removing snap: 100% complete...done. 2026-02-18T08:45:13.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_image_retry cluster2 mirror image_42 2026-02-18T08:45:13.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:45:13.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:45:13.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_42 2026-02-18T08:45:13.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-18T08:45:13.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:45:13.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_image cluster2 mirror image_42 2026-02-18T08:45:13.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:45:13.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:45:13.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_42 2026-02-18T08:45:13.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_42 2026-02-18T08:45:13.162 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster=cluster2 rm mirror/image_42 2026-02-18T08:45:13.287 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-18T08:45:13.293 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:45:13.293 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T08:45:13.293 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_43 2026-02-18T08:45:13.293 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_snapshot cluster2 mirror image_43 snap 2026-02-18T08:45:13.293 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:45:13.293 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:45:13.293 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_43 2026-02-18T08:45:13.294 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap=snap 2026-02-18T08:45:13.294 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 snap rm mirror/image_43@snap 2026-02-18T08:45:14.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Removing snap: 100% complete...done. 2026-02-18T08:45:14.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_image_retry cluster2 mirror image_43 2026-02-18T08:45:14.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:45:14.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:45:14.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_43 2026-02-18T08:45:14.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-18T08:45:14.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:45:14.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_image cluster2 mirror image_43 2026-02-18T08:45:14.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:45:14.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:45:14.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_43 2026-02-18T08:45:14.055 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_43 2026-02-18T08:45:14.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster=cluster2 rm mirror/image_43 2026-02-18T08:45:14.218 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-18T08:45:14.223 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:45:14.223 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T08:45:14.223 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_44 2026-02-18T08:45:14.224 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_snapshot cluster2 mirror image_44 snap 2026-02-18T08:45:14.224 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:45:14.224 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:45:14.224 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_44 2026-02-18T08:45:14.224 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap=snap 2026-02-18T08:45:14.224 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 snap rm mirror/image_44@snap 2026-02-18T08:45:15.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Removing snap: 100% complete...done. 2026-02-18T08:45:15.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_image_retry cluster2 mirror image_44 2026-02-18T08:45:15.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:45:15.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:45:15.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_44 2026-02-18T08:45:15.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-18T08:45:15.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:45:15.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_image cluster2 mirror image_44 2026-02-18T08:45:15.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:45:15.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:45:15.129 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_44 2026-02-18T08:45:15.129 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_44 2026-02-18T08:45:15.156 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster=cluster2 rm mirror/image_44 2026-02-18T08:45:15.289 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-18T08:45:15.294 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:45:15.294 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T08:45:15.294 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_45 2026-02-18T08:45:15.294 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_snapshot cluster2 mirror image_45 snap 2026-02-18T08:45:15.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:45:15.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:45:15.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_45 2026-02-18T08:45:15.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap=snap 2026-02-18T08:45:15.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 snap rm mirror/image_45@snap 2026-02-18T08:45:16.121 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Removing snap: 100% complete...done. 2026-02-18T08:45:16.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_image_retry cluster2 mirror image_45 2026-02-18T08:45:16.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:45:16.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:45:16.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_45 2026-02-18T08:45:16.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-18T08:45:16.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:45:16.134 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_image cluster2 mirror image_45 2026-02-18T08:45:16.134 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:45:16.134 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:45:16.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_45 2026-02-18T08:45:16.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_45 2026-02-18T08:45:16.169 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster=cluster2 rm mirror/image_45 2026-02-18T08:45:16.286 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-18T08:45:16.291 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:45:16.291 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T08:45:16.291 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_46 2026-02-18T08:45:16.291 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_snapshot cluster2 mirror image_46 snap 2026-02-18T08:45:16.292 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:45:16.292 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:45:16.292 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_46 2026-02-18T08:45:16.292 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap=snap 2026-02-18T08:45:16.292 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 snap rm mirror/image_46@snap 2026-02-18T08:45:17.129 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Removing snap: 100% complete...done. 2026-02-18T08:45:17.140 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_image_retry cluster2 mirror image_46 2026-02-18T08:45:17.140 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:45:17.140 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:45:17.140 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_46 2026-02-18T08:45:17.140 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-18T08:45:17.140 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:45:17.141 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_image cluster2 mirror image_46 2026-02-18T08:45:17.141 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:45:17.141 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:45:17.141 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_46 2026-02-18T08:45:17.142 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_46 2026-02-18T08:45:17.174 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster=cluster2 rm mirror/image_46 2026-02-18T08:45:17.301 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-18T08:45:17.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:45:17.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T08:45:17.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_47 2026-02-18T08:45:17.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_snapshot cluster2 mirror image_47 snap 2026-02-18T08:45:17.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:45:17.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:45:17.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_47 2026-02-18T08:45:17.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap=snap 2026-02-18T08:45:17.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 snap rm mirror/image_47@snap 2026-02-18T08:45:18.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Removing snap: 100% complete...done. 2026-02-18T08:45:18.139 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_image_retry cluster2 mirror image_47 2026-02-18T08:45:18.139 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:45:18.139 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:45:18.139 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_47 2026-02-18T08:45:18.139 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-18T08:45:18.139 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:45:18.140 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_image cluster2 mirror image_47 2026-02-18T08:45:18.140 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:45:18.140 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:45:18.141 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_47 2026-02-18T08:45:18.141 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_47 2026-02-18T08:45:18.169 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster=cluster2 rm mirror/image_47 2026-02-18T08:45:18.286 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-18T08:45:18.291 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:45:18.291 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T08:45:18.291 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_48 2026-02-18T08:45:18.291 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_snapshot cluster2 mirror image_48 snap 2026-02-18T08:45:18.291 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:45:18.291 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:45:18.292 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_48 2026-02-18T08:45:18.292 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap=snap 2026-02-18T08:45:18.292 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 snap rm mirror/image_48@snap 2026-02-18T08:45:19.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Removing snap: 100% complete...done. 2026-02-18T08:45:19.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_image_retry cluster2 mirror image_48 2026-02-18T08:45:19.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:45:19.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:45:19.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_48 2026-02-18T08:45:19.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-18T08:45:19.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:45:19.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_image cluster2 mirror image_48 2026-02-18T08:45:19.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:45:19.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:45:19.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_48 2026-02-18T08:45:19.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_48 2026-02-18T08:45:19.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster=cluster2 rm mirror/image_48 2026-02-18T08:45:19.202 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-18T08:45:19.207 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:45:19.207 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T08:45:19.207 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_49 2026-02-18T08:45:19.207 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_snapshot cluster2 mirror image_49 snap 2026-02-18T08:45:19.207 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:45:19.208 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:45:19.208 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_49 2026-02-18T08:45:19.208 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap=snap 2026-02-18T08:45:19.208 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 snap rm mirror/image_49@snap 2026-02-18T08:45:19.674 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.0 with skip (0.39593801980689813) > weight (0.25) 2026-02-18T08:45:19.674 INFO:tasks.rbd_mirror_thrash:kill cluster2.client.mirror.1 2026-02-18T08:45:19.674 INFO:tasks.rbd_mirror.cluster2.client.mirror.1:Sent signal 15 2026-02-18T08:45:19.675 INFO:tasks.rbd_mirror_thrash:waiting for 0.09614747124494172 secs before reviving daemons 2026-02-18T08:45:19.675 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.trial163.stderr:2026-02-18T08:45:19.667+0000 7fe4b7056640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.1 (PID: 90853) UID: 1000 2026-02-18T08:45:19.772 INFO:tasks.rbd_mirror_thrash:waiting for cluster2.client.mirror.1 2026-02-18T08:45:19.772 INFO:teuthology.orchestra.run:waiting for 600 2026-02-18T08:45:20.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Removing snap: 100% complete...done. 2026-02-18T08:45:20.145 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_image_retry cluster2 mirror image_49 2026-02-18T08:45:20.146 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:45:20.146 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:45:20.146 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_49 2026-02-18T08:45:20.146 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-18T08:45:20.146 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:45:20.147 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_image cluster2 mirror image_49 2026-02-18T08:45:20.147 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:45:20.147 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:45:20.147 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_49 2026-02-18T08:45:20.147 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_49 2026-02-18T08:45:20.177 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster=cluster2 rm mirror/image_49 2026-02-18T08:45:20.294 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-18T08:45:20.299 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:45:20.299 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T08:45:20.299 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_50 2026-02-18T08:45:20.299 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_snapshot cluster2 mirror image_50 snap 2026-02-18T08:45:20.299 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:45:20.299 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:45:20.299 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_50 2026-02-18T08:45:20.299 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap=snap 2026-02-18T08:45:20.299 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 snap rm mirror/image_50@snap 2026-02-18T08:45:21.132 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Removing snap: 100% complete...done. 2026-02-18T08:45:21.143 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_image_retry cluster2 mirror image_50 2026-02-18T08:45:21.144 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:45:21.144 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:45:21.144 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_50 2026-02-18T08:45:21.144 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-18T08:45:21.144 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:45:21.145 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_image cluster2 mirror image_50 2026-02-18T08:45:21.145 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:45:21.145 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:45:21.145 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_50 2026-02-18T08:45:21.145 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_50 2026-02-18T08:45:21.178 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster=cluster2 rm mirror/image_50 2026-02-18T08:45:21.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-18T08:45:21.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:45:21.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ testlog 'TEST: image deletions should propagate' 2026-02-18T08:45:21.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ tee -a /tmp/tmp.vybh1m9yJF/rbd-mirror.test.log 2026-02-18T08:45:21.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ date '+%F %T' 2026-02-18T08:45:21.310 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ echo 2026-02-18 08:45:21 TEST: image deletions should propagate 2026-02-18T08:45:21.310 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:2026-02-18 08:45:21 TEST: image deletions should propagate 2026-02-18T08:45:21.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_pool_images cluster1 mirror 0 2026-02-18T08:45:21.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:45:21.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:45:21.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image_count=0 2026-02-18T08:45:21.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-18T08:45:21.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local count 2026-02-18T08:45:21.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local last_count=0 2026-02-18T08:45:21.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-18T08:45:21.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 1 40 2026-02-18T08:45:21.312 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in `seq 1 40` 2026-02-18T08:45:21.312 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 1 -ne 1 2026-02-18T08:45:21.312 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster1 -p mirror mirror pool status 2026-02-18T08:45:21.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep 'images: ' 2026-02-18T08:45:21.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut '-d ' -f 2 2026-02-18T08:45:21.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd: failed to query services: (13) Permission denied 2026-02-18T08:45:21.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ count=0 2026-02-18T08:45:21.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 0 = 0 2026-02-18T08:45:21.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:45:21.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_pool_healthy cluster1 mirror 0 2026-02-18T08:45:21.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:45:21.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:45:21.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-18T08:45:21.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local state 2026-02-18T08:45:21.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 1 40 2026-02-18T08:45:21.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in `seq 1 40` 2026-02-18T08:45:21.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 1 -ne 1 2026-02-18T08:45:21.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster1 -p mirror mirror pool status 2026-02-18T08:45:21.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep 'image health:' 2026-02-18T08:45:21.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut '-d ' -f 3 2026-02-18T08:45:21.361 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd: failed to query services: (13) Permission denied 2026-02-18T08:45:21.364 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ state=OK 2026-02-18T08:45:21.364 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test OK = ERROR 2026-02-18T08:45:21.364 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test OK = OK 2026-02-18T08:45:21.364 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:45:21.365 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 1 50 2026-02-18T08:45:21.366 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T08:45:21.366 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_1 2026-02-18T08:45:21.366 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_present cluster1 mirror image_1 deleted 2026-02-18T08:45:21.367 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:45:21.367 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:45:21.367 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_1 2026-02-18T08:45:21.367 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local state=deleted 2026-02-18T08:45:21.367 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image_id= 2026-02-18T08:45:21.367 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-18T08:45:21.367 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n '' 2026-02-18T08:45:21.367 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_image_id cluster1 mirror image_1 2026-02-18T08:45:21.367 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:45:21.367 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:45:21.367 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_1 2026-02-18T08:45:21.367 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster=cluster1 info mirror/image_1 2026-02-18T08:45:21.368 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-18T08:45:21.387 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd: error opening image image_1: (2) No such file or directory 2026-02-18T08:45:21.390 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image_id= 2026-02-18T08:45:21.390 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-18T08:45:21.390 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.1 2026-02-18T08:45:21.492 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test_image_present cluster1 mirror image_1 deleted '' 2026-02-18T08:45:21.492 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:45:21.492 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:45:21.492 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_1 2026-02-18T08:45:21.492 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local test_state=deleted 2026-02-18T08:45:21.493 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image_id= 2026-02-18T08:45:21.493 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_state=deleted 2026-02-18T08:45:21.493 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_image_id 2026-02-18T08:45:21.493 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_image_id cluster1 mirror image_1 2026-02-18T08:45:21.493 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:45:21.493 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:45:21.493 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_1 2026-02-18T08:45:21.493 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster=cluster1 info mirror/image_1 2026-02-18T08:45:21.493 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-18T08:45:21.512 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd: error opening image image_1: (2) No such file or directory 2026-02-18T08:45:21.516 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ current_image_id= 2026-02-18T08:45:21.516 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n '' 2026-02-18T08:45:21.516 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test deleted = deleted 2026-02-18T08:45:21.516 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:45:21.516 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T08:45:21.516 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_2 2026-02-18T08:45:21.516 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_present cluster1 mirror image_2 deleted 2026-02-18T08:45:21.516 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:45:21.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:45:21.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_2 2026-02-18T08:45:21.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local state=deleted 2026-02-18T08:45:21.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image_id= 2026-02-18T08:45:21.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-18T08:45:21.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n '' 2026-02-18T08:45:21.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_image_id cluster1 mirror image_2 2026-02-18T08:45:21.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:45:21.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:45:21.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_2 2026-02-18T08:45:21.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster=cluster1 info mirror/image_2 2026-02-18T08:45:21.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-18T08:45:21.535 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd: error opening image image_2: (2) No such file or directory 2026-02-18T08:45:21.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image_id= 2026-02-18T08:45:21.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-18T08:45:21.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.1 2026-02-18T08:45:21.640 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test_image_present cluster1 mirror image_2 deleted '' 2026-02-18T08:45:21.641 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:45:21.641 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:45:21.641 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_2 2026-02-18T08:45:21.641 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local test_state=deleted 2026-02-18T08:45:21.641 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image_id= 2026-02-18T08:45:21.641 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_state=deleted 2026-02-18T08:45:21.641 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_image_id 2026-02-18T08:45:21.641 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_image_id cluster1 mirror image_2 2026-02-18T08:45:21.641 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:45:21.642 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:45:21.642 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_2 2026-02-18T08:45:21.642 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster=cluster1 info mirror/image_2 2026-02-18T08:45:21.642 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-18T08:45:21.660 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd: error opening image image_2: (2) No such file or directory 2026-02-18T08:45:21.664 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ current_image_id= 2026-02-18T08:45:21.664 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n '' 2026-02-18T08:45:21.664 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test deleted = deleted 2026-02-18T08:45:21.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:45:21.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T08:45:21.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_3 2026-02-18T08:45:21.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_present cluster1 mirror image_3 deleted 2026-02-18T08:45:21.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:45:21.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:45:21.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_3 2026-02-18T08:45:21.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local state=deleted 2026-02-18T08:45:21.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image_id= 2026-02-18T08:45:21.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-18T08:45:21.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n '' 2026-02-18T08:45:21.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_image_id cluster1 mirror image_3 2026-02-18T08:45:21.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:45:21.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:45:21.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_3 2026-02-18T08:45:21.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster=cluster1 info mirror/image_3 2026-02-18T08:45:21.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-18T08:45:21.683 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd: error opening image image_3: (2) No such file or directory 2026-02-18T08:45:21.686 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image_id= 2026-02-18T08:45:21.686 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-18T08:45:21.687 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.1 2026-02-18T08:45:21.788 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test_image_present cluster1 mirror image_3 deleted '' 2026-02-18T08:45:21.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:45:21.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:45:21.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_3 2026-02-18T08:45:21.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local test_state=deleted 2026-02-18T08:45:21.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image_id= 2026-02-18T08:45:21.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_state=deleted 2026-02-18T08:45:21.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_image_id 2026-02-18T08:45:21.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_image_id cluster1 mirror image_3 2026-02-18T08:45:21.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:45:21.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:45:21.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_3 2026-02-18T08:45:21.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster=cluster1 info mirror/image_3 2026-02-18T08:45:21.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-18T08:45:21.810 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd: error opening image image_3: (2) No such file or directory 2026-02-18T08:45:21.814 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ current_image_id= 2026-02-18T08:45:21.815 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n '' 2026-02-18T08:45:21.815 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test deleted = deleted 2026-02-18T08:45:21.815 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:45:21.815 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T08:45:21.815 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_4 2026-02-18T08:45:21.815 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_present cluster1 mirror image_4 deleted 2026-02-18T08:45:21.815 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:45:21.815 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:45:21.815 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_4 2026-02-18T08:45:21.815 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local state=deleted 2026-02-18T08:45:21.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image_id= 2026-02-18T08:45:21.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-18T08:45:21.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n '' 2026-02-18T08:45:21.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_image_id cluster1 mirror image_4 2026-02-18T08:45:21.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:45:21.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:45:21.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_4 2026-02-18T08:45:21.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster=cluster1 info mirror/image_4 2026-02-18T08:45:21.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-18T08:45:21.835 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd: error opening image image_4: (2) No such file or directory 2026-02-18T08:45:21.838 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image_id= 2026-02-18T08:45:21.838 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-18T08:45:21.838 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.1 2026-02-18T08:45:21.861 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.0 with skip (0.5401464354356784) > weight (0.25) 2026-02-18T08:45:21.861 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.1 with skip (0.5833861779331784) > weight (0.25) 2026-02-18T08:45:21.861 INFO:tasks.rbd_mirror_thrash:kill cluster1.client.mirror.2 2026-02-18T08:45:21.862 INFO:tasks.rbd_mirror.cluster1.client.mirror.2:Sent signal 15 2026-02-18T08:45:21.862 INFO:tasks.rbd_mirror_thrash:waiting for 6.420621781824501 secs before reviving daemons 2026-02-18T08:45:21.862 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.trial163.stderr:2026-02-18T08:45:21.855+0000 7f97fac49640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.2 (PID: 92598) UID: 1000 2026-02-18T08:45:21.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test_image_present cluster1 mirror image_4 deleted '' 2026-02-18T08:45:21.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:45:21.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:45:21.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_4 2026-02-18T08:45:21.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local test_state=deleted 2026-02-18T08:45:21.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image_id= 2026-02-18T08:45:21.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_state=deleted 2026-02-18T08:45:21.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_image_id 2026-02-18T08:45:21.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_image_id cluster1 mirror image_4 2026-02-18T08:45:21.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:45:21.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:45:21.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_4 2026-02-18T08:45:21.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster=cluster1 info mirror/image_4 2026-02-18T08:45:21.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-18T08:45:21.961 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd: error opening image image_4: (2) No such file or directory 2026-02-18T08:45:21.965 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ current_image_id= 2026-02-18T08:45:21.966 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n '' 2026-02-18T08:45:21.966 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test deleted = deleted 2026-02-18T08:45:21.966 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:45:21.966 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T08:45:21.966 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_5 2026-02-18T08:45:21.966 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_present cluster1 mirror image_5 deleted 2026-02-18T08:45:21.966 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:45:21.966 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:45:21.966 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_5 2026-02-18T08:45:21.966 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local state=deleted 2026-02-18T08:45:21.966 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image_id= 2026-02-18T08:45:21.967 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-18T08:45:21.967 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n '' 2026-02-18T08:45:21.967 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_image_id cluster1 mirror image_5 2026-02-18T08:45:21.967 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:45:21.967 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:45:21.967 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_5 2026-02-18T08:45:21.967 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-18T08:45:21.967 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster=cluster1 info mirror/image_5 2026-02-18T08:45:21.986 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd: error opening image image_5: (2) No such file or directory 2026-02-18T08:45:21.989 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image_id= 2026-02-18T08:45:21.989 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-18T08:45:21.989 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.1 2026-02-18T08:45:22.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test_image_present cluster1 mirror image_5 deleted '' 2026-02-18T08:45:22.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:45:22.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:45:22.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_5 2026-02-18T08:45:22.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local test_state=deleted 2026-02-18T08:45:22.092 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image_id= 2026-02-18T08:45:22.092 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_state=deleted 2026-02-18T08:45:22.092 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_image_id 2026-02-18T08:45:22.092 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_image_id cluster1 mirror image_5 2026-02-18T08:45:22.092 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:45:22.092 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:45:22.092 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_5 2026-02-18T08:45:22.092 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster=cluster1 info mirror/image_5 2026-02-18T08:45:22.092 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-18T08:45:22.111 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd: error opening image image_5: (2) No such file or directory 2026-02-18T08:45:22.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ current_image_id= 2026-02-18T08:45:22.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n '' 2026-02-18T08:45:22.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test deleted = deleted 2026-02-18T08:45:22.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:45:22.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T08:45:22.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_6 2026-02-18T08:45:22.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_present cluster1 mirror image_6 deleted 2026-02-18T08:45:22.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:45:22.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:45:22.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_6 2026-02-18T08:45:22.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local state=deleted 2026-02-18T08:45:22.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image_id= 2026-02-18T08:45:22.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-18T08:45:22.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n '' 2026-02-18T08:45:22.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_image_id cluster1 mirror image_6 2026-02-18T08:45:22.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:45:22.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:45:22.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_6 2026-02-18T08:45:22.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster=cluster1 info mirror/image_6 2026-02-18T08:45:22.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-18T08:45:22.137 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd: error opening image image_6: (2) No such file or directory 2026-02-18T08:45:22.140 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image_id= 2026-02-18T08:45:22.140 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-18T08:45:22.140 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.1 2026-02-18T08:45:22.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test_image_present cluster1 mirror image_6 deleted '' 2026-02-18T08:45:22.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:45:22.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:45:22.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_6 2026-02-18T08:45:22.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local test_state=deleted 2026-02-18T08:45:22.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image_id= 2026-02-18T08:45:22.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_state=deleted 2026-02-18T08:45:22.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_image_id 2026-02-18T08:45:22.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_image_id cluster1 mirror image_6 2026-02-18T08:45:22.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:45:22.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:45:22.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_6 2026-02-18T08:45:22.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster=cluster1 info mirror/image_6 2026-02-18T08:45:22.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-18T08:45:22.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd: error opening image image_6: (2) No such file or directory 2026-02-18T08:45:22.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ current_image_id= 2026-02-18T08:45:22.269 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n '' 2026-02-18T08:45:22.269 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test deleted = deleted 2026-02-18T08:45:22.269 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:45:22.269 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T08:45:22.269 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_7 2026-02-18T08:45:22.269 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_present cluster1 mirror image_7 deleted 2026-02-18T08:45:22.269 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:45:22.269 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:45:22.269 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_7 2026-02-18T08:45:22.269 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local state=deleted 2026-02-18T08:45:22.269 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image_id= 2026-02-18T08:45:22.270 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-18T08:45:22.270 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n '' 2026-02-18T08:45:22.270 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_image_id cluster1 mirror image_7 2026-02-18T08:45:22.270 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:45:22.270 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:45:22.270 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_7 2026-02-18T08:45:22.270 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-18T08:45:22.270 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster=cluster1 info mirror/image_7 2026-02-18T08:45:22.294 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd: error opening image image_7: (2) No such file or directory 2026-02-18T08:45:22.297 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image_id= 2026-02-18T08:45:22.297 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-18T08:45:22.297 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.1 2026-02-18T08:45:22.399 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test_image_present cluster1 mirror image_7 deleted '' 2026-02-18T08:45:22.399 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:45:22.399 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:45:22.399 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_7 2026-02-18T08:45:22.399 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local test_state=deleted 2026-02-18T08:45:22.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image_id= 2026-02-18T08:45:22.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_state=deleted 2026-02-18T08:45:22.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_image_id 2026-02-18T08:45:22.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_image_id cluster1 mirror image_7 2026-02-18T08:45:22.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:45:22.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:45:22.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_7 2026-02-18T08:45:22.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster=cluster1 info mirror/image_7 2026-02-18T08:45:22.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-18T08:45:22.419 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd: error opening image image_7: (2) No such file or directory 2026-02-18T08:45:22.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ current_image_id= 2026-02-18T08:45:22.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n '' 2026-02-18T08:45:22.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test deleted = deleted 2026-02-18T08:45:22.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:45:22.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T08:45:22.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_8 2026-02-18T08:45:22.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_present cluster1 mirror image_8 deleted 2026-02-18T08:45:22.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:45:22.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:45:22.425 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_8 2026-02-18T08:45:22.425 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local state=deleted 2026-02-18T08:45:22.425 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image_id= 2026-02-18T08:45:22.425 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-18T08:45:22.425 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n '' 2026-02-18T08:45:22.425 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_image_id cluster1 mirror image_8 2026-02-18T08:45:22.425 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:45:22.425 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:45:22.425 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_8 2026-02-18T08:45:22.425 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster=cluster1 info mirror/image_8 2026-02-18T08:45:22.425 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-18T08:45:22.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd: error opening image image_8: (2) No such file or directory 2026-02-18T08:45:22.448 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image_id= 2026-02-18T08:45:22.448 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-18T08:45:22.448 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.1 2026-02-18T08:45:22.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test_image_present cluster1 mirror image_8 deleted '' 2026-02-18T08:45:22.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:45:22.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:45:22.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_8 2026-02-18T08:45:22.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local test_state=deleted 2026-02-18T08:45:22.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image_id= 2026-02-18T08:45:22.551 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_state=deleted 2026-02-18T08:45:22.551 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_image_id 2026-02-18T08:45:22.551 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_image_id cluster1 mirror image_8 2026-02-18T08:45:22.551 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:45:22.551 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:45:22.551 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_8 2026-02-18T08:45:22.551 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster=cluster1 info mirror/image_8 2026-02-18T08:45:22.551 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-18T08:45:22.571 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd: error opening image image_8: (2) No such file or directory 2026-02-18T08:45:22.576 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ current_image_id= 2026-02-18T08:45:22.576 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n '' 2026-02-18T08:45:22.576 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test deleted = deleted 2026-02-18T08:45:22.576 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:45:22.576 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T08:45:22.576 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_9 2026-02-18T08:45:22.576 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_present cluster1 mirror image_9 deleted 2026-02-18T08:45:22.577 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:45:22.577 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:45:22.577 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_9 2026-02-18T08:45:22.577 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local state=deleted 2026-02-18T08:45:22.577 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image_id= 2026-02-18T08:45:22.577 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-18T08:45:22.577 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n '' 2026-02-18T08:45:22.577 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_image_id cluster1 mirror image_9 2026-02-18T08:45:22.577 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:45:22.577 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:45:22.577 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_9 2026-02-18T08:45:22.578 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster=cluster1 info mirror/image_9 2026-02-18T08:45:22.578 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-18T08:45:22.596 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd: error opening image image_9: (2) No such file or directory 2026-02-18T08:45:22.599 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image_id= 2026-02-18T08:45:22.599 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-18T08:45:22.600 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.1 2026-02-18T08:45:22.701 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test_image_present cluster1 mirror image_9 deleted '' 2026-02-18T08:45:22.701 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:45:22.702 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:45:22.702 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_9 2026-02-18T08:45:22.702 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local test_state=deleted 2026-02-18T08:45:22.702 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image_id= 2026-02-18T08:45:22.702 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_state=deleted 2026-02-18T08:45:22.702 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_image_id 2026-02-18T08:45:22.702 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_image_id cluster1 mirror image_9 2026-02-18T08:45:22.702 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:45:22.702 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:45:22.702 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_9 2026-02-18T08:45:22.702 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster=cluster1 info mirror/image_9 2026-02-18T08:45:22.703 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-18T08:45:22.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd: error opening image image_9: (2) No such file or directory 2026-02-18T08:45:22.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ current_image_id= 2026-02-18T08:45:22.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n '' 2026-02-18T08:45:22.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test deleted = deleted 2026-02-18T08:45:22.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:45:22.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T08:45:22.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_10 2026-02-18T08:45:22.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_present cluster1 mirror image_10 deleted 2026-02-18T08:45:22.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:45:22.728 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:45:22.728 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_10 2026-02-18T08:45:22.728 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local state=deleted 2026-02-18T08:45:22.728 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image_id= 2026-02-18T08:45:22.728 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-18T08:45:22.728 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n '' 2026-02-18T08:45:22.728 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_image_id cluster1 mirror image_10 2026-02-18T08:45:22.728 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:45:22.728 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:45:22.728 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_10 2026-02-18T08:45:22.728 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster=cluster1 info mirror/image_10 2026-02-18T08:45:22.728 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-18T08:45:22.747 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd: error opening image image_10: (2) No such file or directory 2026-02-18T08:45:22.750 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image_id= 2026-02-18T08:45:22.750 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-18T08:45:22.750 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.1 2026-02-18T08:45:22.852 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test_image_present cluster1 mirror image_10 deleted '' 2026-02-18T08:45:22.852 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:45:22.852 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:45:22.853 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_10 2026-02-18T08:45:22.853 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local test_state=deleted 2026-02-18T08:45:22.853 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image_id= 2026-02-18T08:45:22.853 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_state=deleted 2026-02-18T08:45:22.853 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_image_id 2026-02-18T08:45:22.853 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_image_id cluster1 mirror image_10 2026-02-18T08:45:22.853 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:45:22.853 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:45:22.853 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_10 2026-02-18T08:45:22.854 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster=cluster1 info mirror/image_10 2026-02-18T08:45:22.854 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-18T08:45:22.875 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd: error opening image image_10: (2) No such file or directory 2026-02-18T08:45:22.879 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ current_image_id= 2026-02-18T08:45:22.879 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n '' 2026-02-18T08:45:22.879 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test deleted = deleted 2026-02-18T08:45:22.879 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:45:22.879 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T08:45:22.879 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_11 2026-02-18T08:45:22.880 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_present cluster1 mirror image_11 deleted 2026-02-18T08:45:22.880 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:45:22.880 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:45:22.880 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_11 2026-02-18T08:45:22.880 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local state=deleted 2026-02-18T08:45:22.880 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image_id= 2026-02-18T08:45:22.880 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-18T08:45:22.880 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n '' 2026-02-18T08:45:22.880 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_image_id cluster1 mirror image_11 2026-02-18T08:45:22.880 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:45:22.880 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:45:22.880 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_11 2026-02-18T08:45:22.881 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster=cluster1 info mirror/image_11 2026-02-18T08:45:22.881 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-18T08:45:22.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd: error opening image image_11: (2) No such file or directory 2026-02-18T08:45:22.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image_id= 2026-02-18T08:45:22.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-18T08:45:22.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.1 2026-02-18T08:45:23.004 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test_image_present cluster1 mirror image_11 deleted '' 2026-02-18T08:45:23.004 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:45:23.005 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:45:23.005 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_11 2026-02-18T08:45:23.005 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local test_state=deleted 2026-02-18T08:45:23.005 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image_id= 2026-02-18T08:45:23.005 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_state=deleted 2026-02-18T08:45:23.005 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_image_id 2026-02-18T08:45:23.005 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_image_id cluster1 mirror image_11 2026-02-18T08:45:23.005 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:45:23.005 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:45:23.005 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_11 2026-02-18T08:45:23.006 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster=cluster1 info mirror/image_11 2026-02-18T08:45:23.006 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-18T08:45:23.028 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd: error opening image image_11: (2) No such file or directory 2026-02-18T08:45:23.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ current_image_id= 2026-02-18T08:45:23.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n '' 2026-02-18T08:45:23.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test deleted = deleted 2026-02-18T08:45:23.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:45:23.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T08:45:23.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_12 2026-02-18T08:45:23.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_present cluster1 mirror image_12 deleted 2026-02-18T08:45:23.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:45:23.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:45:23.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_12 2026-02-18T08:45:23.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local state=deleted 2026-02-18T08:45:23.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image_id= 2026-02-18T08:45:23.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-18T08:45:23.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n '' 2026-02-18T08:45:23.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_image_id cluster1 mirror image_12 2026-02-18T08:45:23.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:45:23.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:45:23.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_12 2026-02-18T08:45:23.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster=cluster1 info mirror/image_12 2026-02-18T08:45:23.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-18T08:45:23.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd: error opening image image_12: (2) No such file or directory 2026-02-18T08:45:23.055 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image_id= 2026-02-18T08:45:23.055 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-18T08:45:23.055 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.1 2026-02-18T08:45:23.157 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test_image_present cluster1 mirror image_12 deleted '' 2026-02-18T08:45:23.157 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:45:23.158 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:45:23.158 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_12 2026-02-18T08:45:23.158 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local test_state=deleted 2026-02-18T08:45:23.158 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image_id= 2026-02-18T08:45:23.158 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_state=deleted 2026-02-18T08:45:23.158 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_image_id 2026-02-18T08:45:23.158 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_image_id cluster1 mirror image_12 2026-02-18T08:45:23.158 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:45:23.158 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:45:23.158 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_12 2026-02-18T08:45:23.158 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster=cluster1 info mirror/image_12 2026-02-18T08:45:23.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-18T08:45:23.178 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd: error opening image image_12: (2) No such file or directory 2026-02-18T08:45:23.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ current_image_id= 2026-02-18T08:45:23.183 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n '' 2026-02-18T08:45:23.183 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test deleted = deleted 2026-02-18T08:45:23.183 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:45:23.183 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T08:45:23.183 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_13 2026-02-18T08:45:23.183 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_present cluster1 mirror image_13 deleted 2026-02-18T08:45:23.183 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:45:23.183 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:45:23.183 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_13 2026-02-18T08:45:23.183 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local state=deleted 2026-02-18T08:45:23.183 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image_id= 2026-02-18T08:45:23.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-18T08:45:23.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n '' 2026-02-18T08:45:23.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_image_id cluster1 mirror image_13 2026-02-18T08:45:23.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:45:23.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:45:23.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_13 2026-02-18T08:45:23.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster=cluster1 info mirror/image_13 2026-02-18T08:45:23.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-18T08:45:23.202 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd: error opening image image_13: (2) No such file or directory 2026-02-18T08:45:23.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image_id= 2026-02-18T08:45:23.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-18T08:45:23.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.1 2026-02-18T08:45:23.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test_image_present cluster1 mirror image_13 deleted '' 2026-02-18T08:45:23.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:45:23.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:45:23.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_13 2026-02-18T08:45:23.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local test_state=deleted 2026-02-18T08:45:23.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image_id= 2026-02-18T08:45:23.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_state=deleted 2026-02-18T08:45:23.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_image_id 2026-02-18T08:45:23.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_image_id cluster1 mirror image_13 2026-02-18T08:45:23.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:45:23.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:45:23.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_13 2026-02-18T08:45:23.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster=cluster1 info mirror/image_13 2026-02-18T08:45:23.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-18T08:45:23.329 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd: error opening image image_13: (2) No such file or directory 2026-02-18T08:45:23.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ current_image_id= 2026-02-18T08:45:23.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n '' 2026-02-18T08:45:23.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test deleted = deleted 2026-02-18T08:45:23.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:45:23.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T08:45:23.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_14 2026-02-18T08:45:23.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_present cluster1 mirror image_14 deleted 2026-02-18T08:45:23.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:45:23.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:45:23.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_14 2026-02-18T08:45:23.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local state=deleted 2026-02-18T08:45:23.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image_id= 2026-02-18T08:45:23.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-18T08:45:23.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n '' 2026-02-18T08:45:23.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_image_id cluster1 mirror image_14 2026-02-18T08:45:23.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:45:23.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:45:23.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_14 2026-02-18T08:45:23.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster=cluster1 info mirror/image_14 2026-02-18T08:45:23.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-18T08:45:23.356 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd: error opening image image_14: (2) No such file or directory 2026-02-18T08:45:23.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image_id= 2026-02-18T08:45:23.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-18T08:45:23.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.1 2026-02-18T08:45:23.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test_image_present cluster1 mirror image_14 deleted '' 2026-02-18T08:45:23.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:45:23.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:45:23.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_14 2026-02-18T08:45:23.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local test_state=deleted 2026-02-18T08:45:23.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image_id= 2026-02-18T08:45:23.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_state=deleted 2026-02-18T08:45:23.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_image_id 2026-02-18T08:45:23.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_image_id cluster1 mirror image_14 2026-02-18T08:45:23.462 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:45:23.462 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:45:23.462 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_14 2026-02-18T08:45:23.462 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster=cluster1 info mirror/image_14 2026-02-18T08:45:23.462 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-18T08:45:23.482 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd: error opening image image_14: (2) No such file or directory 2026-02-18T08:45:23.486 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ current_image_id= 2026-02-18T08:45:23.486 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n '' 2026-02-18T08:45:23.486 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test deleted = deleted 2026-02-18T08:45:23.486 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:45:23.486 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T08:45:23.487 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_15 2026-02-18T08:45:23.487 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_present cluster1 mirror image_15 deleted 2026-02-18T08:45:23.487 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:45:23.487 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:45:23.487 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_15 2026-02-18T08:45:23.487 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local state=deleted 2026-02-18T08:45:23.487 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image_id= 2026-02-18T08:45:23.487 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-18T08:45:23.487 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n '' 2026-02-18T08:45:23.487 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_image_id cluster1 mirror image_15 2026-02-18T08:45:23.487 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:45:23.487 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:45:23.488 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_15 2026-02-18T08:45:23.488 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster=cluster1 info mirror/image_15 2026-02-18T08:45:23.488 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-18T08:45:23.506 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd: error opening image image_15: (2) No such file or directory 2026-02-18T08:45:23.509 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image_id= 2026-02-18T08:45:23.510 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-18T08:45:23.510 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.1 2026-02-18T08:45:23.611 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test_image_present cluster1 mirror image_15 deleted '' 2026-02-18T08:45:23.612 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:45:23.612 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:45:23.612 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_15 2026-02-18T08:45:23.612 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local test_state=deleted 2026-02-18T08:45:23.612 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image_id= 2026-02-18T08:45:23.612 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_state=deleted 2026-02-18T08:45:23.612 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_image_id 2026-02-18T08:45:23.612 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_image_id cluster1 mirror image_15 2026-02-18T08:45:23.612 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:45:23.613 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:45:23.613 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_15 2026-02-18T08:45:23.613 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster=cluster1 info mirror/image_15 2026-02-18T08:45:23.613 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-18T08:45:23.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd: error opening image image_15: (2) No such file or directory 2026-02-18T08:45:23.636 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ current_image_id= 2026-02-18T08:45:23.636 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n '' 2026-02-18T08:45:23.637 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test deleted = deleted 2026-02-18T08:45:23.637 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:45:23.637 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T08:45:23.637 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_16 2026-02-18T08:45:23.637 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_present cluster1 mirror image_16 deleted 2026-02-18T08:45:23.637 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:45:23.637 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:45:23.637 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_16 2026-02-18T08:45:23.637 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local state=deleted 2026-02-18T08:45:23.637 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image_id= 2026-02-18T08:45:23.637 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-18T08:45:23.637 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n '' 2026-02-18T08:45:23.638 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_image_id cluster1 mirror image_16 2026-02-18T08:45:23.638 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:45:23.638 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:45:23.638 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_16 2026-02-18T08:45:23.638 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster=cluster1 info mirror/image_16 2026-02-18T08:45:23.638 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-18T08:45:23.658 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd: error opening image image_16: (2) No such file or directory 2026-02-18T08:45:23.660 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image_id= 2026-02-18T08:45:23.660 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-18T08:45:23.661 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.1 2026-02-18T08:45:23.763 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test_image_present cluster1 mirror image_16 deleted '' 2026-02-18T08:45:23.763 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:45:23.763 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:45:23.763 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_16 2026-02-18T08:45:23.763 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local test_state=deleted 2026-02-18T08:45:23.763 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image_id= 2026-02-18T08:45:23.763 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_state=deleted 2026-02-18T08:45:23.763 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_image_id 2026-02-18T08:45:23.763 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_image_id cluster1 mirror image_16 2026-02-18T08:45:23.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:45:23.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:45:23.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_16 2026-02-18T08:45:23.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster=cluster1 info mirror/image_16 2026-02-18T08:45:23.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-18T08:45:23.784 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd: error opening image image_16: (2) No such file or directory 2026-02-18T08:45:23.788 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ current_image_id= 2026-02-18T08:45:23.788 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n '' 2026-02-18T08:45:23.788 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test deleted = deleted 2026-02-18T08:45:23.788 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:45:23.788 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T08:45:23.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_17 2026-02-18T08:45:23.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_present cluster1 mirror image_17 deleted 2026-02-18T08:45:23.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:45:23.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:45:23.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_17 2026-02-18T08:45:23.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local state=deleted 2026-02-18T08:45:23.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image_id= 2026-02-18T08:45:23.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-18T08:45:23.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n '' 2026-02-18T08:45:23.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_image_id cluster1 mirror image_17 2026-02-18T08:45:23.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:45:23.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:45:23.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_17 2026-02-18T08:45:23.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster=cluster1 info mirror/image_17 2026-02-18T08:45:23.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-18T08:45:23.807 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd: error opening image image_17: (2) No such file or directory 2026-02-18T08:45:23.810 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image_id= 2026-02-18T08:45:23.810 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-18T08:45:23.810 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.1 2026-02-18T08:45:23.912 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test_image_present cluster1 mirror image_17 deleted '' 2026-02-18T08:45:23.912 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:45:23.913 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:45:23.913 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_17 2026-02-18T08:45:23.913 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local test_state=deleted 2026-02-18T08:45:23.913 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image_id= 2026-02-18T08:45:23.913 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_state=deleted 2026-02-18T08:45:23.913 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_image_id 2026-02-18T08:45:23.913 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_image_id cluster1 mirror image_17 2026-02-18T08:45:23.913 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:45:23.913 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:45:23.913 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_17 2026-02-18T08:45:23.913 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster=cluster1 info mirror/image_17 2026-02-18T08:45:23.914 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-18T08:45:23.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd: error opening image image_17: (2) No such file or directory 2026-02-18T08:45:23.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ current_image_id= 2026-02-18T08:45:23.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n '' 2026-02-18T08:45:23.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test deleted = deleted 2026-02-18T08:45:23.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:45:23.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T08:45:23.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_18 2026-02-18T08:45:23.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_present cluster1 mirror image_18 deleted 2026-02-18T08:45:23.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:45:23.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:45:23.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_18 2026-02-18T08:45:23.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local state=deleted 2026-02-18T08:45:23.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image_id= 2026-02-18T08:45:23.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-18T08:45:23.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n '' 2026-02-18T08:45:23.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_image_id cluster1 mirror image_18 2026-02-18T08:45:23.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:45:23.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:45:23.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_18 2026-02-18T08:45:23.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster=cluster1 info mirror/image_18 2026-02-18T08:45:23.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-18T08:45:23.958 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd: error opening image image_18: (2) No such file or directory 2026-02-18T08:45:23.961 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image_id= 2026-02-18T08:45:23.961 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-18T08:45:23.961 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.1 2026-02-18T08:45:24.062 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test_image_present cluster1 mirror image_18 deleted '' 2026-02-18T08:45:24.062 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:45:24.062 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:45:24.062 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_18 2026-02-18T08:45:24.062 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local test_state=deleted 2026-02-18T08:45:24.062 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image_id= 2026-02-18T08:45:24.062 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_state=deleted 2026-02-18T08:45:24.062 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_image_id 2026-02-18T08:45:24.062 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_image_id cluster1 mirror image_18 2026-02-18T08:45:24.062 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:45:24.062 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:45:24.062 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_18 2026-02-18T08:45:24.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster=cluster1 info mirror/image_18 2026-02-18T08:45:24.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-18T08:45:24.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd: error opening image image_18: (2) No such file or directory 2026-02-18T08:45:24.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ current_image_id= 2026-02-18T08:45:24.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n '' 2026-02-18T08:45:24.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test deleted = deleted 2026-02-18T08:45:24.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:45:24.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T08:45:24.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_19 2026-02-18T08:45:24.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_present cluster1 mirror image_19 deleted 2026-02-18T08:45:24.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:45:24.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:45:24.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_19 2026-02-18T08:45:24.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local state=deleted 2026-02-18T08:45:24.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image_id= 2026-02-18T08:45:24.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-18T08:45:24.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n '' 2026-02-18T08:45:24.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_image_id cluster1 mirror image_19 2026-02-18T08:45:24.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:45:24.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:45:24.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_19 2026-02-18T08:45:24.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster=cluster1 info mirror/image_19 2026-02-18T08:45:24.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-18T08:45:24.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd: error opening image image_19: (2) No such file or directory 2026-02-18T08:45:24.111 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image_id= 2026-02-18T08:45:24.111 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-18T08:45:24.111 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.1 2026-02-18T08:45:24.213 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test_image_present cluster1 mirror image_19 deleted '' 2026-02-18T08:45:24.213 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:45:24.213 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:45:24.214 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_19 2026-02-18T08:45:24.214 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local test_state=deleted 2026-02-18T08:45:24.214 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image_id= 2026-02-18T08:45:24.214 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_state=deleted 2026-02-18T08:45:24.214 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_image_id 2026-02-18T08:45:24.214 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_image_id cluster1 mirror image_19 2026-02-18T08:45:24.214 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:45:24.214 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:45:24.214 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_19 2026-02-18T08:45:24.215 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster=cluster1 info mirror/image_19 2026-02-18T08:45:24.215 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-18T08:45:24.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd: error opening image image_19: (2) No such file or directory 2026-02-18T08:45:24.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ current_image_id= 2026-02-18T08:45:24.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n '' 2026-02-18T08:45:24.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test deleted = deleted 2026-02-18T08:45:24.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:45:24.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T08:45:24.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_20 2026-02-18T08:45:24.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_present cluster1 mirror image_20 deleted 2026-02-18T08:45:24.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:45:24.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:45:24.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_20 2026-02-18T08:45:24.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local state=deleted 2026-02-18T08:45:24.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image_id= 2026-02-18T08:45:24.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-18T08:45:24.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n '' 2026-02-18T08:45:24.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_image_id cluster1 mirror image_20 2026-02-18T08:45:24.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:45:24.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:45:24.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_20 2026-02-18T08:45:24.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster=cluster1 info mirror/image_20 2026-02-18T08:45:24.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-18T08:45:24.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd: error opening image image_20: (2) No such file or directory 2026-02-18T08:45:24.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image_id= 2026-02-18T08:45:24.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-18T08:45:24.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.1 2026-02-18T08:45:24.366 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test_image_present cluster1 mirror image_20 deleted '' 2026-02-18T08:45:24.366 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:45:24.367 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:45:24.367 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_20 2026-02-18T08:45:24.367 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local test_state=deleted 2026-02-18T08:45:24.367 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image_id= 2026-02-18T08:45:24.367 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_state=deleted 2026-02-18T08:45:24.367 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_image_id 2026-02-18T08:45:24.367 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_image_id cluster1 mirror image_20 2026-02-18T08:45:24.367 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:45:24.367 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:45:24.367 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_20 2026-02-18T08:45:24.368 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster=cluster1 info mirror/image_20 2026-02-18T08:45:24.368 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-18T08:45:24.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd: error opening image image_20: (2) No such file or directory 2026-02-18T08:45:24.390 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ current_image_id= 2026-02-18T08:45:24.390 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n '' 2026-02-18T08:45:24.390 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test deleted = deleted 2026-02-18T08:45:24.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:45:24.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T08:45:24.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_21 2026-02-18T08:45:24.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_present cluster1 mirror image_21 deleted 2026-02-18T08:45:24.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:45:24.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:45:24.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_21 2026-02-18T08:45:24.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local state=deleted 2026-02-18T08:45:24.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image_id= 2026-02-18T08:45:24.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-18T08:45:24.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n '' 2026-02-18T08:45:24.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_image_id cluster1 mirror image_21 2026-02-18T08:45:24.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:45:24.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:45:24.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_21 2026-02-18T08:45:24.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster=cluster1 info mirror/image_21 2026-02-18T08:45:24.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-18T08:45:24.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd: error opening image image_21: (2) No such file or directory 2026-02-18T08:45:24.414 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image_id= 2026-02-18T08:45:24.414 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-18T08:45:24.414 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.1 2026-02-18T08:45:24.516 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test_image_present cluster1 mirror image_21 deleted '' 2026-02-18T08:45:24.516 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:45:24.516 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:45:24.516 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_21 2026-02-18T08:45:24.516 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local test_state=deleted 2026-02-18T08:45:24.516 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image_id= 2026-02-18T08:45:24.516 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_state=deleted 2026-02-18T08:45:24.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_image_id 2026-02-18T08:45:24.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_image_id cluster1 mirror image_21 2026-02-18T08:45:24.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:45:24.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:45:24.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_21 2026-02-18T08:45:24.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster=cluster1 info mirror/image_21 2026-02-18T08:45:24.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-18T08:45:24.536 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd: error opening image image_21: (2) No such file or directory 2026-02-18T08:45:24.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ current_image_id= 2026-02-18T08:45:24.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n '' 2026-02-18T08:45:24.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test deleted = deleted 2026-02-18T08:45:24.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:45:24.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T08:45:24.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_22 2026-02-18T08:45:24.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_present cluster1 mirror image_22 deleted 2026-02-18T08:45:24.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:45:24.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:45:24.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_22 2026-02-18T08:45:24.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local state=deleted 2026-02-18T08:45:24.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image_id= 2026-02-18T08:45:24.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-18T08:45:24.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n '' 2026-02-18T08:45:24.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_image_id cluster1 mirror image_22 2026-02-18T08:45:24.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:45:24.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:45:24.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_22 2026-02-18T08:45:24.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster=cluster1 info mirror/image_22 2026-02-18T08:45:24.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-18T08:45:24.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd: error opening image image_22: (2) No such file or directory 2026-02-18T08:45:24.567 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image_id= 2026-02-18T08:45:24.567 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-18T08:45:24.567 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.1 2026-02-18T08:45:24.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test_image_present cluster1 mirror image_22 deleted '' 2026-02-18T08:45:24.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:45:24.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:45:24.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_22 2026-02-18T08:45:24.670 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local test_state=deleted 2026-02-18T08:45:24.670 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image_id= 2026-02-18T08:45:24.670 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_state=deleted 2026-02-18T08:45:24.670 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_image_id 2026-02-18T08:45:24.670 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_image_id cluster1 mirror image_22 2026-02-18T08:45:24.670 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:45:24.670 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:45:24.670 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_22 2026-02-18T08:45:24.671 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster=cluster1 info mirror/image_22 2026-02-18T08:45:24.671 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-18T08:45:24.693 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd: error opening image image_22: (2) No such file or directory 2026-02-18T08:45:24.697 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ current_image_id= 2026-02-18T08:45:24.697 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n '' 2026-02-18T08:45:24.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test deleted = deleted 2026-02-18T08:45:24.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:45:24.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T08:45:24.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_23 2026-02-18T08:45:24.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_present cluster1 mirror image_23 deleted 2026-02-18T08:45:24.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:45:24.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:45:24.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_23 2026-02-18T08:45:24.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local state=deleted 2026-02-18T08:45:24.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image_id= 2026-02-18T08:45:24.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-18T08:45:24.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n '' 2026-02-18T08:45:24.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_image_id cluster1 mirror image_23 2026-02-18T08:45:24.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:45:24.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:45:24.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_23 2026-02-18T08:45:24.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster=cluster1 info mirror/image_23 2026-02-18T08:45:24.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-18T08:45:24.718 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd: error opening image image_23: (2) No such file or directory 2026-02-18T08:45:24.721 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image_id= 2026-02-18T08:45:24.721 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-18T08:45:24.721 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.1 2026-02-18T08:45:24.823 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test_image_present cluster1 mirror image_23 deleted '' 2026-02-18T08:45:24.823 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:45:24.823 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:45:24.823 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_23 2026-02-18T08:45:24.823 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local test_state=deleted 2026-02-18T08:45:24.823 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image_id= 2026-02-18T08:45:24.824 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_state=deleted 2026-02-18T08:45:24.824 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_image_id 2026-02-18T08:45:24.824 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_image_id cluster1 mirror image_23 2026-02-18T08:45:24.824 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:45:24.824 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:45:24.824 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_23 2026-02-18T08:45:24.824 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster=cluster1 info mirror/image_23 2026-02-18T08:45:24.824 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-18T08:45:24.844 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd: error opening image image_23: (2) No such file or directory 2026-02-18T08:45:24.848 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ current_image_id= 2026-02-18T08:45:24.848 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n '' 2026-02-18T08:45:24.848 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test deleted = deleted 2026-02-18T08:45:24.848 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:45:24.848 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T08:45:24.848 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_24 2026-02-18T08:45:24.848 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_present cluster1 mirror image_24 deleted 2026-02-18T08:45:24.848 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:45:24.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:45:24.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_24 2026-02-18T08:45:24.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local state=deleted 2026-02-18T08:45:24.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image_id= 2026-02-18T08:45:24.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-18T08:45:24.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n '' 2026-02-18T08:45:24.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_image_id cluster1 mirror image_24 2026-02-18T08:45:24.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:45:24.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:45:24.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_24 2026-02-18T08:45:24.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster=cluster1 info mirror/image_24 2026-02-18T08:45:24.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-18T08:45:24.867 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd: error opening image image_24: (2) No such file or directory 2026-02-18T08:45:24.870 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image_id= 2026-02-18T08:45:24.870 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-18T08:45:24.870 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.1 2026-02-18T08:45:24.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test_image_present cluster1 mirror image_24 deleted '' 2026-02-18T08:45:24.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:45:24.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:45:24.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_24 2026-02-18T08:45:24.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local test_state=deleted 2026-02-18T08:45:24.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image_id= 2026-02-18T08:45:24.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_state=deleted 2026-02-18T08:45:24.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_image_id 2026-02-18T08:45:24.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_image_id cluster1 mirror image_24 2026-02-18T08:45:24.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:45:24.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:45:24.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_24 2026-02-18T08:45:24.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster=cluster1 info mirror/image_24 2026-02-18T08:45:24.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-18T08:45:24.992 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd: error opening image image_24: (2) No such file or directory 2026-02-18T08:45:24.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ current_image_id= 2026-02-18T08:45:24.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n '' 2026-02-18T08:45:24.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test deleted = deleted 2026-02-18T08:45:24.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:45:24.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T08:45:24.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_25 2026-02-18T08:45:24.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_present cluster1 mirror image_25 deleted 2026-02-18T08:45:24.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:45:24.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:45:24.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_25 2026-02-18T08:45:24.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local state=deleted 2026-02-18T08:45:24.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image_id= 2026-02-18T08:45:24.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-18T08:45:24.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n '' 2026-02-18T08:45:24.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_image_id cluster1 mirror image_25 2026-02-18T08:45:24.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:45:24.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:45:24.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_25 2026-02-18T08:45:24.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster=cluster1 info mirror/image_25 2026-02-18T08:45:24.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-18T08:45:25.018 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd: error opening image image_25: (2) No such file or directory 2026-02-18T08:45:25.021 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image_id= 2026-02-18T08:45:25.021 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-18T08:45:25.021 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.1 2026-02-18T08:45:25.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test_image_present cluster1 mirror image_25 deleted '' 2026-02-18T08:45:25.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:45:25.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:45:25.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_25 2026-02-18T08:45:25.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local test_state=deleted 2026-02-18T08:45:25.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image_id= 2026-02-18T08:45:25.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_state=deleted 2026-02-18T08:45:25.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_image_id 2026-02-18T08:45:25.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_image_id cluster1 mirror image_25 2026-02-18T08:45:25.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:45:25.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:45:25.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_25 2026-02-18T08:45:25.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster=cluster1 info mirror/image_25 2026-02-18T08:45:25.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-18T08:45:25.144 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd: error opening image image_25: (2) No such file or directory 2026-02-18T08:45:25.148 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ current_image_id= 2026-02-18T08:45:25.148 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n '' 2026-02-18T08:45:25.148 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test deleted = deleted 2026-02-18T08:45:25.148 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:45:25.148 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T08:45:25.148 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_26 2026-02-18T08:45:25.148 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_present cluster1 mirror image_26 deleted 2026-02-18T08:45:25.148 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:45:25.148 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:45:25.148 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_26 2026-02-18T08:45:25.148 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local state=deleted 2026-02-18T08:45:25.149 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image_id= 2026-02-18T08:45:25.149 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-18T08:45:25.149 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n '' 2026-02-18T08:45:25.149 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_image_id cluster1 mirror image_26 2026-02-18T08:45:25.149 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:45:25.149 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:45:25.149 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_26 2026-02-18T08:45:25.149 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster=cluster1 info mirror/image_26 2026-02-18T08:45:25.149 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-18T08:45:25.167 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd: error opening image image_26: (2) No such file or directory 2026-02-18T08:45:25.170 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image_id= 2026-02-18T08:45:25.170 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-18T08:45:25.170 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.1 2026-02-18T08:45:25.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test_image_present cluster1 mirror image_26 deleted '' 2026-02-18T08:45:25.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:45:25.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:45:25.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_26 2026-02-18T08:45:25.273 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local test_state=deleted 2026-02-18T08:45:25.273 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image_id= 2026-02-18T08:45:25.273 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_state=deleted 2026-02-18T08:45:25.273 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_image_id 2026-02-18T08:45:25.273 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_image_id cluster1 mirror image_26 2026-02-18T08:45:25.273 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:45:25.273 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:45:25.273 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_26 2026-02-18T08:45:25.273 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster=cluster1 info mirror/image_26 2026-02-18T08:45:25.273 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-18T08:45:25.292 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd: error opening image image_26: (2) No such file or directory 2026-02-18T08:45:25.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ current_image_id= 2026-02-18T08:45:25.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n '' 2026-02-18T08:45:25.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test deleted = deleted 2026-02-18T08:45:25.297 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:45:25.297 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T08:45:25.297 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_27 2026-02-18T08:45:25.297 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_present cluster1 mirror image_27 deleted 2026-02-18T08:45:25.297 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:45:25.297 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:45:25.297 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_27 2026-02-18T08:45:25.297 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local state=deleted 2026-02-18T08:45:25.297 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image_id= 2026-02-18T08:45:25.297 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-18T08:45:25.297 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n '' 2026-02-18T08:45:25.298 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_image_id cluster1 mirror image_27 2026-02-18T08:45:25.298 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:45:25.298 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:45:25.298 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_27 2026-02-18T08:45:25.298 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster=cluster1 info mirror/image_27 2026-02-18T08:45:25.298 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-18T08:45:25.318 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd: error opening image image_27: (2) No such file or directory 2026-02-18T08:45:25.320 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image_id= 2026-02-18T08:45:25.321 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-18T08:45:25.321 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.1 2026-02-18T08:45:25.423 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test_image_present cluster1 mirror image_27 deleted '' 2026-02-18T08:45:25.423 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:45:25.423 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:45:25.423 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_27 2026-02-18T08:45:25.423 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local test_state=deleted 2026-02-18T08:45:25.423 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image_id= 2026-02-18T08:45:25.423 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_state=deleted 2026-02-18T08:45:25.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_image_id 2026-02-18T08:45:25.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_image_id cluster1 mirror image_27 2026-02-18T08:45:25.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:45:25.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:45:25.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_27 2026-02-18T08:45:25.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster=cluster1 info mirror/image_27 2026-02-18T08:45:25.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-18T08:45:25.443 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd: error opening image image_27: (2) No such file or directory 2026-02-18T08:45:25.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ current_image_id= 2026-02-18T08:45:25.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n '' 2026-02-18T08:45:25.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test deleted = deleted 2026-02-18T08:45:25.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:45:25.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T08:45:25.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_28 2026-02-18T08:45:25.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_present cluster1 mirror image_28 deleted 2026-02-18T08:45:25.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:45:25.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:45:25.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_28 2026-02-18T08:45:25.448 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local state=deleted 2026-02-18T08:45:25.448 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image_id= 2026-02-18T08:45:25.448 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-18T08:45:25.448 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n '' 2026-02-18T08:45:25.448 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_image_id cluster1 mirror image_28 2026-02-18T08:45:25.448 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:45:25.448 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:45:25.448 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_28 2026-02-18T08:45:25.448 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster=cluster1 info mirror/image_28 2026-02-18T08:45:25.448 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-18T08:45:25.467 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd: error opening image image_28: (2) No such file or directory 2026-02-18T08:45:25.470 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image_id= 2026-02-18T08:45:25.470 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-18T08:45:25.470 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.1 2026-02-18T08:45:25.572 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test_image_present cluster1 mirror image_28 deleted '' 2026-02-18T08:45:25.572 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:45:25.572 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:45:25.572 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_28 2026-02-18T08:45:25.572 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local test_state=deleted 2026-02-18T08:45:25.572 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image_id= 2026-02-18T08:45:25.572 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_state=deleted 2026-02-18T08:45:25.573 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_image_id 2026-02-18T08:45:25.573 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_image_id cluster1 mirror image_28 2026-02-18T08:45:25.573 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:45:25.573 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:45:25.573 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_28 2026-02-18T08:45:25.573 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster=cluster1 info mirror/image_28 2026-02-18T08:45:25.574 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-18T08:45:25.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd: error opening image image_28: (2) No such file or directory 2026-02-18T08:45:25.597 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ current_image_id= 2026-02-18T08:45:25.597 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n '' 2026-02-18T08:45:25.597 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test deleted = deleted 2026-02-18T08:45:25.597 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:45:25.598 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T08:45:25.598 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_29 2026-02-18T08:45:25.598 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_present cluster1 mirror image_29 deleted 2026-02-18T08:45:25.598 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:45:25.598 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:45:25.598 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_29 2026-02-18T08:45:25.598 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local state=deleted 2026-02-18T08:45:25.598 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image_id= 2026-02-18T08:45:25.598 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-18T08:45:25.598 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n '' 2026-02-18T08:45:25.598 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_image_id cluster1 mirror image_29 2026-02-18T08:45:25.598 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:45:25.599 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:45:25.599 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_29 2026-02-18T08:45:25.599 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster=cluster1 info mirror/image_29 2026-02-18T08:45:25.599 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-18T08:45:25.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd: error opening image image_29: (2) No such file or directory 2026-02-18T08:45:25.620 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image_id= 2026-02-18T08:45:25.620 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-18T08:45:25.620 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.1 2026-02-18T08:45:25.722 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test_image_present cluster1 mirror image_29 deleted '' 2026-02-18T08:45:25.722 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:45:25.722 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:45:25.722 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_29 2026-02-18T08:45:25.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local test_state=deleted 2026-02-18T08:45:25.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image_id= 2026-02-18T08:45:25.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_state=deleted 2026-02-18T08:45:25.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_image_id 2026-02-18T08:45:25.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_image_id cluster1 mirror image_29 2026-02-18T08:45:25.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:45:25.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:45:25.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_29 2026-02-18T08:45:25.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster=cluster1 info mirror/image_29 2026-02-18T08:45:25.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-18T08:45:25.742 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd: error opening image image_29: (2) No such file or directory 2026-02-18T08:45:25.746 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ current_image_id= 2026-02-18T08:45:25.747 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n '' 2026-02-18T08:45:25.747 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test deleted = deleted 2026-02-18T08:45:25.747 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:45:25.747 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T08:45:25.747 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_30 2026-02-18T08:45:25.747 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_present cluster1 mirror image_30 deleted 2026-02-18T08:45:25.747 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:45:25.747 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:45:25.747 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_30 2026-02-18T08:45:25.747 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local state=deleted 2026-02-18T08:45:25.747 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image_id= 2026-02-18T08:45:25.748 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-18T08:45:25.748 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n '' 2026-02-18T08:45:25.748 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_image_id cluster1 mirror image_30 2026-02-18T08:45:25.748 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:45:25.748 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:45:25.748 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_30 2026-02-18T08:45:25.748 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster=cluster1 info mirror/image_30 2026-02-18T08:45:25.748 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-18T08:45:25.767 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd: error opening image image_30: (2) No such file or directory 2026-02-18T08:45:25.770 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image_id= 2026-02-18T08:45:25.770 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-18T08:45:25.770 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.1 2026-02-18T08:45:25.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test_image_present cluster1 mirror image_30 deleted '' 2026-02-18T08:45:25.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:45:25.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:45:25.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_30 2026-02-18T08:45:25.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local test_state=deleted 2026-02-18T08:45:25.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image_id= 2026-02-18T08:45:25.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_state=deleted 2026-02-18T08:45:25.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_image_id 2026-02-18T08:45:25.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_image_id cluster1 mirror image_30 2026-02-18T08:45:25.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:45:25.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:45:25.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_30 2026-02-18T08:45:25.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster=cluster1 info mirror/image_30 2026-02-18T08:45:25.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-18T08:45:25.873 INFO:tasks.rbd_mirror_thrash:reviving cluster2.client.mirror.1 2026-02-18T08:45:25.873 INFO:tasks.rbd_mirror.cluster2.client.mirror.1:Restarting daemon 2026-02-18T08:45:25.874 DEBUG:teuthology.orchestra.run.trial163:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.1 2026-02-18T08:45:25.875 INFO:tasks.rbd_mirror.cluster2.client.mirror.1:Started 2026-02-18T08:45:25.876 INFO:tasks.rbd_mirror_thrash:waiting for 109.42505011148883 secs before thrashing 2026-02-18T08:45:25.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd: error opening image image_30: (2) No such file or directory 2026-02-18T08:45:25.896 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ current_image_id= 2026-02-18T08:45:25.896 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n '' 2026-02-18T08:45:25.896 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test deleted = deleted 2026-02-18T08:45:25.896 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:45:25.896 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T08:45:25.896 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_31 2026-02-18T08:45:25.896 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_present cluster1 mirror image_31 deleted 2026-02-18T08:45:25.897 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:45:25.897 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:45:25.897 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_31 2026-02-18T08:45:25.897 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local state=deleted 2026-02-18T08:45:25.897 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image_id= 2026-02-18T08:45:25.897 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-18T08:45:25.897 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n '' 2026-02-18T08:45:25.897 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_image_id cluster1 mirror image_31 2026-02-18T08:45:25.897 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:45:25.897 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:45:25.897 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_31 2026-02-18T08:45:25.897 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster=cluster1 info mirror/image_31 2026-02-18T08:45:25.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-18T08:45:25.917 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd: error opening image image_31: (2) No such file or directory 2026-02-18T08:45:25.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image_id= 2026-02-18T08:45:25.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-18T08:45:25.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.1 2026-02-18T08:45:26.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test_image_present cluster1 mirror image_31 deleted '' 2026-02-18T08:45:26.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:45:26.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:45:26.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_31 2026-02-18T08:45:26.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local test_state=deleted 2026-02-18T08:45:26.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image_id= 2026-02-18T08:45:26.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_state=deleted 2026-02-18T08:45:26.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_image_id 2026-02-18T08:45:26.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_image_id cluster1 mirror image_31 2026-02-18T08:45:26.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:45:26.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:45:26.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_31 2026-02-18T08:45:26.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster=cluster1 info mirror/image_31 2026-02-18T08:45:26.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-18T08:45:26.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd: error opening image image_31: (2) No such file or directory 2026-02-18T08:45:26.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ current_image_id= 2026-02-18T08:45:26.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n '' 2026-02-18T08:45:26.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test deleted = deleted 2026-02-18T08:45:26.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:45:26.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T08:45:26.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_32 2026-02-18T08:45:26.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_present cluster1 mirror image_32 deleted 2026-02-18T08:45:26.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:45:26.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:45:26.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_32 2026-02-18T08:45:26.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local state=deleted 2026-02-18T08:45:26.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image_id= 2026-02-18T08:45:26.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-18T08:45:26.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n '' 2026-02-18T08:45:26.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_image_id cluster1 mirror image_32 2026-02-18T08:45:26.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:45:26.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:45:26.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_32 2026-02-18T08:45:26.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-18T08:45:26.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster=cluster1 info mirror/image_32 2026-02-18T08:45:26.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd: error opening image image_32: (2) No such file or directory 2026-02-18T08:45:26.074 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image_id= 2026-02-18T08:45:26.074 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-18T08:45:26.074 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.1 2026-02-18T08:45:26.176 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test_image_present cluster1 mirror image_32 deleted '' 2026-02-18T08:45:26.176 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:45:26.176 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:45:26.176 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_32 2026-02-18T08:45:26.176 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local test_state=deleted 2026-02-18T08:45:26.176 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image_id= 2026-02-18T08:45:26.176 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_state=deleted 2026-02-18T08:45:26.176 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_image_id 2026-02-18T08:45:26.176 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_image_id cluster1 mirror image_32 2026-02-18T08:45:26.176 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:45:26.177 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:45:26.177 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_32 2026-02-18T08:45:26.177 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster=cluster1 info mirror/image_32 2026-02-18T08:45:26.177 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-18T08:45:26.201 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd: error opening image image_32: (2) No such file or directory 2026-02-18T08:45:26.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ current_image_id= 2026-02-18T08:45:26.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n '' 2026-02-18T08:45:26.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test deleted = deleted 2026-02-18T08:45:26.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:45:26.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T08:45:26.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_33 2026-02-18T08:45:26.206 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_present cluster1 mirror image_33 deleted 2026-02-18T08:45:26.206 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:45:26.206 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:45:26.206 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_33 2026-02-18T08:45:26.206 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local state=deleted 2026-02-18T08:45:26.206 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image_id= 2026-02-18T08:45:26.206 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-18T08:45:26.206 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n '' 2026-02-18T08:45:26.206 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_image_id cluster1 mirror image_33 2026-02-18T08:45:26.206 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:45:26.206 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:45:26.207 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_33 2026-02-18T08:45:26.207 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster=cluster1 info mirror/image_33 2026-02-18T08:45:26.207 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-18T08:45:26.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd: error opening image image_33: (2) No such file or directory 2026-02-18T08:45:26.229 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image_id= 2026-02-18T08:45:26.229 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-18T08:45:26.229 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.1 2026-02-18T08:45:26.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test_image_present cluster1 mirror image_33 deleted '' 2026-02-18T08:45:26.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:45:26.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:45:26.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_33 2026-02-18T08:45:26.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local test_state=deleted 2026-02-18T08:45:26.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image_id= 2026-02-18T08:45:26.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_state=deleted 2026-02-18T08:45:26.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_image_id 2026-02-18T08:45:26.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_image_id cluster1 mirror image_33 2026-02-18T08:45:26.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:45:26.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:45:26.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_33 2026-02-18T08:45:26.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster=cluster1 info mirror/image_33 2026-02-18T08:45:26.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-18T08:45:26.354 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd: error opening image image_33: (2) No such file or directory 2026-02-18T08:45:26.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ current_image_id= 2026-02-18T08:45:26.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n '' 2026-02-18T08:45:26.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test deleted = deleted 2026-02-18T08:45:26.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:45:26.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T08:45:26.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_34 2026-02-18T08:45:26.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_present cluster1 mirror image_34 deleted 2026-02-18T08:45:26.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:45:26.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:45:26.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_34 2026-02-18T08:45:26.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local state=deleted 2026-02-18T08:45:26.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image_id= 2026-02-18T08:45:26.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-18T08:45:26.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n '' 2026-02-18T08:45:26.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_image_id cluster1 mirror image_34 2026-02-18T08:45:26.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:45:26.360 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:45:26.360 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_34 2026-02-18T08:45:26.360 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster=cluster1 info mirror/image_34 2026-02-18T08:45:26.360 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-18T08:45:26.380 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd: error opening image image_34: (2) No such file or directory 2026-02-18T08:45:26.383 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image_id= 2026-02-18T08:45:26.383 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-18T08:45:26.383 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.1 2026-02-18T08:45:26.485 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test_image_present cluster1 mirror image_34 deleted '' 2026-02-18T08:45:26.485 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:45:26.485 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:45:26.485 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_34 2026-02-18T08:45:26.485 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local test_state=deleted 2026-02-18T08:45:26.485 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image_id= 2026-02-18T08:45:26.485 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_state=deleted 2026-02-18T08:45:26.485 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_image_id 2026-02-18T08:45:26.486 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_image_id cluster1 mirror image_34 2026-02-18T08:45:26.486 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:45:26.486 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:45:26.486 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_34 2026-02-18T08:45:26.486 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster=cluster1 info mirror/image_34 2026-02-18T08:45:26.486 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-18T08:45:26.507 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd: error opening image image_34: (2) No such file or directory 2026-02-18T08:45:26.511 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ current_image_id= 2026-02-18T08:45:26.511 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n '' 2026-02-18T08:45:26.512 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test deleted = deleted 2026-02-18T08:45:26.512 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:45:26.512 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T08:45:26.512 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_35 2026-02-18T08:45:26.512 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_present cluster1 mirror image_35 deleted 2026-02-18T08:45:26.512 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:45:26.512 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:45:26.512 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_35 2026-02-18T08:45:26.512 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local state=deleted 2026-02-18T08:45:26.512 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image_id= 2026-02-18T08:45:26.512 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-18T08:45:26.513 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n '' 2026-02-18T08:45:26.513 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_image_id cluster1 mirror image_35 2026-02-18T08:45:26.513 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:45:26.513 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:45:26.513 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_35 2026-02-18T08:45:26.513 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster=cluster1 info mirror/image_35 2026-02-18T08:45:26.513 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-18T08:45:26.532 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd: error opening image image_35: (2) No such file or directory 2026-02-18T08:45:26.535 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image_id= 2026-02-18T08:45:26.535 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-18T08:45:26.535 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.1 2026-02-18T08:45:26.637 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test_image_present cluster1 mirror image_35 deleted '' 2026-02-18T08:45:26.637 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:45:26.637 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:45:26.637 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_35 2026-02-18T08:45:26.637 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local test_state=deleted 2026-02-18T08:45:26.637 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image_id= 2026-02-18T08:45:26.638 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_state=deleted 2026-02-18T08:45:26.638 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_image_id 2026-02-18T08:45:26.638 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_image_id cluster1 mirror image_35 2026-02-18T08:45:26.638 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:45:26.638 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:45:26.638 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_35 2026-02-18T08:45:26.638 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster=cluster1 info mirror/image_35 2026-02-18T08:45:26.638 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-18T08:45:26.660 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd: error opening image image_35: (2) No such file or directory 2026-02-18T08:45:26.664 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ current_image_id= 2026-02-18T08:45:26.664 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n '' 2026-02-18T08:45:26.664 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test deleted = deleted 2026-02-18T08:45:26.664 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:45:26.664 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T08:45:26.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_36 2026-02-18T08:45:26.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_present cluster1 mirror image_36 deleted 2026-02-18T08:45:26.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:45:26.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:45:26.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_36 2026-02-18T08:45:26.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local state=deleted 2026-02-18T08:45:26.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image_id= 2026-02-18T08:45:26.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-18T08:45:26.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n '' 2026-02-18T08:45:26.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_image_id cluster1 mirror image_36 2026-02-18T08:45:26.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:45:26.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:45:26.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_36 2026-02-18T08:45:26.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster=cluster1 info mirror/image_36 2026-02-18T08:45:26.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-18T08:45:26.684 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd: error opening image image_36: (2) No such file or directory 2026-02-18T08:45:26.687 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image_id= 2026-02-18T08:45:26.688 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-18T08:45:26.688 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.1 2026-02-18T08:45:26.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test_image_present cluster1 mirror image_36 deleted '' 2026-02-18T08:45:26.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:45:26.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:45:26.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_36 2026-02-18T08:45:26.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local test_state=deleted 2026-02-18T08:45:26.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image_id= 2026-02-18T08:45:26.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_state=deleted 2026-02-18T08:45:26.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_image_id 2026-02-18T08:45:26.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_image_id cluster1 mirror image_36 2026-02-18T08:45:26.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:45:26.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:45:26.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_36 2026-02-18T08:45:26.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster=cluster1 info mirror/image_36 2026-02-18T08:45:26.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-18T08:45:26.811 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd: error opening image image_36: (2) No such file or directory 2026-02-18T08:45:26.815 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ current_image_id= 2026-02-18T08:45:26.815 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n '' 2026-02-18T08:45:26.815 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test deleted = deleted 2026-02-18T08:45:26.815 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:45:26.815 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T08:45:26.815 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_37 2026-02-18T08:45:26.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_present cluster1 mirror image_37 deleted 2026-02-18T08:45:26.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:45:26.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:45:26.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_37 2026-02-18T08:45:26.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local state=deleted 2026-02-18T08:45:26.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image_id= 2026-02-18T08:45:26.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-18T08:45:26.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n '' 2026-02-18T08:45:26.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_image_id cluster1 mirror image_37 2026-02-18T08:45:26.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:45:26.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:45:26.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_37 2026-02-18T08:45:26.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster=cluster1 info mirror/image_37 2026-02-18T08:45:26.817 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-18T08:45:26.835 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd: error opening image image_37: (2) No such file or directory 2026-02-18T08:45:26.838 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image_id= 2026-02-18T08:45:26.838 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-18T08:45:26.838 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.1 2026-02-18T08:45:26.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test_image_present cluster1 mirror image_37 deleted '' 2026-02-18T08:45:26.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:45:26.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:45:26.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_37 2026-02-18T08:45:26.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local test_state=deleted 2026-02-18T08:45:26.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image_id= 2026-02-18T08:45:26.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_state=deleted 2026-02-18T08:45:26.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_image_id 2026-02-18T08:45:26.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_image_id cluster1 mirror image_37 2026-02-18T08:45:26.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:45:26.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:45:26.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_37 2026-02-18T08:45:26.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster=cluster1 info mirror/image_37 2026-02-18T08:45:26.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-18T08:45:26.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd: error opening image image_37: (2) No such file or directory 2026-02-18T08:45:26.964 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ current_image_id= 2026-02-18T08:45:26.964 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n '' 2026-02-18T08:45:26.964 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test deleted = deleted 2026-02-18T08:45:26.965 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:45:26.965 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T08:45:26.965 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_38 2026-02-18T08:45:26.965 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_present cluster1 mirror image_38 deleted 2026-02-18T08:45:26.965 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:45:26.965 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:45:26.965 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_38 2026-02-18T08:45:26.965 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local state=deleted 2026-02-18T08:45:26.965 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image_id= 2026-02-18T08:45:26.965 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-18T08:45:26.965 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n '' 2026-02-18T08:45:26.965 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_image_id cluster1 mirror image_38 2026-02-18T08:45:26.966 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:45:26.966 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:45:26.966 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_38 2026-02-18T08:45:26.966 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster=cluster1 info mirror/image_38 2026-02-18T08:45:26.966 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-18T08:45:26.985 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd: error opening image image_38: (2) No such file or directory 2026-02-18T08:45:26.988 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image_id= 2026-02-18T08:45:26.988 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-18T08:45:26.988 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.1 2026-02-18T08:45:27.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test_image_present cluster1 mirror image_38 deleted '' 2026-02-18T08:45:27.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:45:27.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:45:27.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_38 2026-02-18T08:45:27.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local test_state=deleted 2026-02-18T08:45:27.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image_id= 2026-02-18T08:45:27.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_state=deleted 2026-02-18T08:45:27.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_image_id 2026-02-18T08:45:27.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_image_id cluster1 mirror image_38 2026-02-18T08:45:27.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:45:27.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:45:27.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_38 2026-02-18T08:45:27.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster=cluster1 info mirror/image_38 2026-02-18T08:45:27.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-18T08:45:27.110 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd: error opening image image_38: (2) No such file or directory 2026-02-18T08:45:27.114 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ current_image_id= 2026-02-18T08:45:27.114 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n '' 2026-02-18T08:45:27.114 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test deleted = deleted 2026-02-18T08:45:27.114 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:45:27.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T08:45:27.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_39 2026-02-18T08:45:27.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_present cluster1 mirror image_39 deleted 2026-02-18T08:45:27.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:45:27.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:45:27.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_39 2026-02-18T08:45:27.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local state=deleted 2026-02-18T08:45:27.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image_id= 2026-02-18T08:45:27.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-18T08:45:27.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n '' 2026-02-18T08:45:27.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_image_id cluster1 mirror image_39 2026-02-18T08:45:27.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:45:27.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:45:27.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_39 2026-02-18T08:45:27.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster=cluster1 info mirror/image_39 2026-02-18T08:45:27.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-18T08:45:27.134 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd: error opening image image_39: (2) No such file or directory 2026-02-18T08:45:27.137 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image_id= 2026-02-18T08:45:27.137 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-18T08:45:27.138 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.1 2026-02-18T08:45:27.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test_image_present cluster1 mirror image_39 deleted '' 2026-02-18T08:45:27.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:45:27.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:45:27.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_39 2026-02-18T08:45:27.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local test_state=deleted 2026-02-18T08:45:27.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image_id= 2026-02-18T08:45:27.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_state=deleted 2026-02-18T08:45:27.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_image_id 2026-02-18T08:45:27.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_image_id cluster1 mirror image_39 2026-02-18T08:45:27.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:45:27.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:45:27.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_39 2026-02-18T08:45:27.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster=cluster1 info mirror/image_39 2026-02-18T08:45:27.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-18T08:45:27.260 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd: error opening image image_39: (2) No such file or directory 2026-02-18T08:45:27.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ current_image_id= 2026-02-18T08:45:27.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n '' 2026-02-18T08:45:27.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test deleted = deleted 2026-02-18T08:45:27.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:45:27.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T08:45:27.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_40 2026-02-18T08:45:27.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_present cluster1 mirror image_40 deleted 2026-02-18T08:45:27.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:45:27.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:45:27.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_40 2026-02-18T08:45:27.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local state=deleted 2026-02-18T08:45:27.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image_id= 2026-02-18T08:45:27.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-18T08:45:27.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n '' 2026-02-18T08:45:27.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_image_id cluster1 mirror image_40 2026-02-18T08:45:27.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:45:27.266 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:45:27.266 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_40 2026-02-18T08:45:27.266 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster=cluster1 info mirror/image_40 2026-02-18T08:45:27.266 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-18T08:45:27.286 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd: error opening image image_40: (2) No such file or directory 2026-02-18T08:45:27.288 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image_id= 2026-02-18T08:45:27.289 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-18T08:45:27.289 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.1 2026-02-18T08:45:27.390 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test_image_present cluster1 mirror image_40 deleted '' 2026-02-18T08:45:27.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:45:27.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:45:27.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_40 2026-02-18T08:45:27.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local test_state=deleted 2026-02-18T08:45:27.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image_id= 2026-02-18T08:45:27.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_state=deleted 2026-02-18T08:45:27.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_image_id 2026-02-18T08:45:27.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_image_id cluster1 mirror image_40 2026-02-18T08:45:27.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:45:27.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:45:27.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_40 2026-02-18T08:45:27.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster=cluster1 info mirror/image_40 2026-02-18T08:45:27.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-18T08:45:27.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd: error opening image image_40: (2) No such file or directory 2026-02-18T08:45:27.415 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ current_image_id= 2026-02-18T08:45:27.415 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n '' 2026-02-18T08:45:27.415 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test deleted = deleted 2026-02-18T08:45:27.415 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:45:27.415 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T08:45:27.415 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_41 2026-02-18T08:45:27.415 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_present cluster1 mirror image_41 deleted 2026-02-18T08:45:27.415 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:45:27.416 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:45:27.416 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_41 2026-02-18T08:45:27.416 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local state=deleted 2026-02-18T08:45:27.416 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image_id= 2026-02-18T08:45:27.416 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-18T08:45:27.416 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n '' 2026-02-18T08:45:27.416 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_image_id cluster1 mirror image_41 2026-02-18T08:45:27.416 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:45:27.416 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:45:27.416 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_41 2026-02-18T08:45:27.416 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster=cluster1 info mirror/image_41 2026-02-18T08:45:27.416 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-18T08:45:27.435 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd: error opening image image_41: (2) No such file or directory 2026-02-18T08:45:27.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image_id= 2026-02-18T08:45:27.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-18T08:45:27.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.1 2026-02-18T08:45:27.540 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test_image_present cluster1 mirror image_41 deleted '' 2026-02-18T08:45:27.540 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:45:27.540 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:45:27.540 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_41 2026-02-18T08:45:27.540 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local test_state=deleted 2026-02-18T08:45:27.540 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image_id= 2026-02-18T08:45:27.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_state=deleted 2026-02-18T08:45:27.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_image_id 2026-02-18T08:45:27.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_image_id cluster1 mirror image_41 2026-02-18T08:45:27.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:45:27.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:45:27.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_41 2026-02-18T08:45:27.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster=cluster1 info mirror/image_41 2026-02-18T08:45:27.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-18T08:45:27.561 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd: error opening image image_41: (2) No such file or directory 2026-02-18T08:45:27.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ current_image_id= 2026-02-18T08:45:27.565 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n '' 2026-02-18T08:45:27.565 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test deleted = deleted 2026-02-18T08:45:27.565 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:45:27.565 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T08:45:27.565 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_42 2026-02-18T08:45:27.565 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_present cluster1 mirror image_42 deleted 2026-02-18T08:45:27.565 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:45:27.565 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:45:27.565 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_42 2026-02-18T08:45:27.565 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local state=deleted 2026-02-18T08:45:27.565 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image_id= 2026-02-18T08:45:27.565 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-18T08:45:27.566 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n '' 2026-02-18T08:45:27.566 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_image_id cluster1 mirror image_42 2026-02-18T08:45:27.566 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:45:27.566 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:45:27.566 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_42 2026-02-18T08:45:27.566 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster=cluster1 info mirror/image_42 2026-02-18T08:45:27.566 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-18T08:45:27.585 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd: error opening image image_42: (2) No such file or directory 2026-02-18T08:45:27.587 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image_id= 2026-02-18T08:45:27.587 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-18T08:45:27.587 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.1 2026-02-18T08:45:27.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test_image_present cluster1 mirror image_42 deleted '' 2026-02-18T08:45:27.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:45:27.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:45:27.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_42 2026-02-18T08:45:27.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local test_state=deleted 2026-02-18T08:45:27.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image_id= 2026-02-18T08:45:27.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_state=deleted 2026-02-18T08:45:27.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_image_id 2026-02-18T08:45:27.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_image_id cluster1 mirror image_42 2026-02-18T08:45:27.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:45:27.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:45:27.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_42 2026-02-18T08:45:27.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster=cluster1 info mirror/image_42 2026-02-18T08:45:27.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-18T08:45:27.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd: error opening image image_42: (2) No such file or directory 2026-02-18T08:45:27.714 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ current_image_id= 2026-02-18T08:45:27.714 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n '' 2026-02-18T08:45:27.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test deleted = deleted 2026-02-18T08:45:27.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:45:27.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T08:45:27.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_43 2026-02-18T08:45:27.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_present cluster1 mirror image_43 deleted 2026-02-18T08:45:27.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:45:27.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:45:27.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_43 2026-02-18T08:45:27.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local state=deleted 2026-02-18T08:45:27.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image_id= 2026-02-18T08:45:27.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-18T08:45:27.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n '' 2026-02-18T08:45:27.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_image_id cluster1 mirror image_43 2026-02-18T08:45:27.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:45:27.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:45:27.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_43 2026-02-18T08:45:27.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster=cluster1 info mirror/image_43 2026-02-18T08:45:27.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-18T08:45:27.735 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd: error opening image image_43: (2) No such file or directory 2026-02-18T08:45:27.737 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image_id= 2026-02-18T08:45:27.738 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-18T08:45:27.738 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.1 2026-02-18T08:45:27.839 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test_image_present cluster1 mirror image_43 deleted '' 2026-02-18T08:45:27.840 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:45:27.840 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:45:27.840 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_43 2026-02-18T08:45:27.840 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local test_state=deleted 2026-02-18T08:45:27.840 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image_id= 2026-02-18T08:45:27.840 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_state=deleted 2026-02-18T08:45:27.840 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_image_id 2026-02-18T08:45:27.840 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_image_id cluster1 mirror image_43 2026-02-18T08:45:27.840 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:45:27.840 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:45:27.840 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_43 2026-02-18T08:45:27.840 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster=cluster1 info mirror/image_43 2026-02-18T08:45:27.841 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-18T08:45:27.860 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd: error opening image image_43: (2) No such file or directory 2026-02-18T08:45:27.864 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ current_image_id= 2026-02-18T08:45:27.864 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n '' 2026-02-18T08:45:27.865 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test deleted = deleted 2026-02-18T08:45:27.865 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:45:27.865 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T08:45:27.865 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_44 2026-02-18T08:45:27.865 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_present cluster1 mirror image_44 deleted 2026-02-18T08:45:27.865 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:45:27.865 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:45:27.865 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_44 2026-02-18T08:45:27.865 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local state=deleted 2026-02-18T08:45:27.865 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image_id= 2026-02-18T08:45:27.865 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-18T08:45:27.865 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n '' 2026-02-18T08:45:27.866 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_image_id cluster1 mirror image_44 2026-02-18T08:45:27.866 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:45:27.866 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:45:27.866 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_44 2026-02-18T08:45:27.866 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster=cluster1 info mirror/image_44 2026-02-18T08:45:27.866 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-18T08:45:27.885 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd: error opening image image_44: (2) No such file or directory 2026-02-18T08:45:27.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image_id= 2026-02-18T08:45:27.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-18T08:45:27.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.1 2026-02-18T08:45:27.990 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test_image_present cluster1 mirror image_44 deleted '' 2026-02-18T08:45:27.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:45:27.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:45:27.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_44 2026-02-18T08:45:27.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local test_state=deleted 2026-02-18T08:45:27.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image_id= 2026-02-18T08:45:27.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_state=deleted 2026-02-18T08:45:27.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_image_id 2026-02-18T08:45:27.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_image_id cluster1 mirror image_44 2026-02-18T08:45:27.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:45:27.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:45:27.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_44 2026-02-18T08:45:27.992 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster=cluster1 info mirror/image_44 2026-02-18T08:45:27.992 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-18T08:45:28.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd: error opening image image_44: (2) No such file or directory 2026-02-18T08:45:28.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ current_image_id= 2026-02-18T08:45:28.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n '' 2026-02-18T08:45:28.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test deleted = deleted 2026-02-18T08:45:28.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:45:28.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T08:45:28.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_45 2026-02-18T08:45:28.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_present cluster1 mirror image_45 deleted 2026-02-18T08:45:28.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:45:28.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:45:28.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_45 2026-02-18T08:45:28.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local state=deleted 2026-02-18T08:45:28.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image_id= 2026-02-18T08:45:28.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-18T08:45:28.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n '' 2026-02-18T08:45:28.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_image_id cluster1 mirror image_45 2026-02-18T08:45:28.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:45:28.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:45:28.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_45 2026-02-18T08:45:28.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster=cluster1 info mirror/image_45 2026-02-18T08:45:28.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-18T08:45:28.035 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd: error opening image image_45: (2) No such file or directory 2026-02-18T08:45:28.037 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image_id= 2026-02-18T08:45:28.038 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-18T08:45:28.038 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.1 2026-02-18T08:45:28.139 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test_image_present cluster1 mirror image_45 deleted '' 2026-02-18T08:45:28.139 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:45:28.139 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:45:28.140 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_45 2026-02-18T08:45:28.140 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local test_state=deleted 2026-02-18T08:45:28.140 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image_id= 2026-02-18T08:45:28.140 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_state=deleted 2026-02-18T08:45:28.140 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_image_id 2026-02-18T08:45:28.140 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_image_id cluster1 mirror image_45 2026-02-18T08:45:28.140 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:45:28.140 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:45:28.140 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_45 2026-02-18T08:45:28.140 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster=cluster1 info mirror/image_45 2026-02-18T08:45:28.140 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-18T08:45:28.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd: error opening image image_45: (2) No such file or directory 2026-02-18T08:45:28.164 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ current_image_id= 2026-02-18T08:45:28.164 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n '' 2026-02-18T08:45:28.164 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test deleted = deleted 2026-02-18T08:45:28.164 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:45:28.164 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T08:45:28.164 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_46 2026-02-18T08:45:28.165 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_present cluster1 mirror image_46 deleted 2026-02-18T08:45:28.165 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:45:28.165 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:45:28.165 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_46 2026-02-18T08:45:28.165 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local state=deleted 2026-02-18T08:45:28.165 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image_id= 2026-02-18T08:45:28.165 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-18T08:45:28.165 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n '' 2026-02-18T08:45:28.165 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_image_id cluster1 mirror image_46 2026-02-18T08:45:28.165 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:45:28.165 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:45:28.165 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_46 2026-02-18T08:45:28.166 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster=cluster1 info mirror/image_46 2026-02-18T08:45:28.166 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-18T08:45:28.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd: error opening image image_46: (2) No such file or directory 2026-02-18T08:45:28.187 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image_id= 2026-02-18T08:45:28.187 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-18T08:45:28.187 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.1 2026-02-18T08:45:28.284 INFO:tasks.rbd_mirror_thrash:waiting for cluster1.client.mirror.2 2026-02-18T08:45:28.284 INFO:teuthology.orchestra.run:waiting for 600 2026-02-18T08:45:28.284 INFO:tasks.rbd_mirror_thrash:reviving cluster1.client.mirror.2 2026-02-18T08:45:28.284 INFO:tasks.rbd_mirror.cluster1.client.mirror.2:Restarting daemon 2026-02-18T08:45:28.285 DEBUG:teuthology.orchestra.run.trial163:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.2 2026-02-18T08:45:28.286 INFO:tasks.rbd_mirror.cluster1.client.mirror.2:Started 2026-02-18T08:45:28.287 INFO:tasks.rbd_mirror_thrash:waiting for 111.16642330574513 secs before thrashing 2026-02-18T08:45:28.289 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test_image_present cluster1 mirror image_46 deleted '' 2026-02-18T08:45:28.289 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:45:28.289 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:45:28.289 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_46 2026-02-18T08:45:28.289 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local test_state=deleted 2026-02-18T08:45:28.289 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image_id= 2026-02-18T08:45:28.290 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_state=deleted 2026-02-18T08:45:28.290 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_image_id 2026-02-18T08:45:28.290 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_image_id cluster1 mirror image_46 2026-02-18T08:45:28.290 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:45:28.290 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:45:28.290 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_46 2026-02-18T08:45:28.290 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster=cluster1 info mirror/image_46 2026-02-18T08:45:28.290 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-18T08:45:28.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd: error opening image image_46: (2) No such file or directory 2026-02-18T08:45:28.318 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ current_image_id= 2026-02-18T08:45:28.318 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n '' 2026-02-18T08:45:28.318 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test deleted = deleted 2026-02-18T08:45:28.318 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:45:28.318 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T08:45:28.318 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_47 2026-02-18T08:45:28.318 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_present cluster1 mirror image_47 deleted 2026-02-18T08:45:28.318 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:45:28.318 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:45:28.318 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_47 2026-02-18T08:45:28.319 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local state=deleted 2026-02-18T08:45:28.319 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image_id= 2026-02-18T08:45:28.319 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-18T08:45:28.319 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n '' 2026-02-18T08:45:28.319 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_image_id cluster1 mirror image_47 2026-02-18T08:45:28.319 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:45:28.319 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:45:28.319 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_47 2026-02-18T08:45:28.319 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster=cluster1 info mirror/image_47 2026-02-18T08:45:28.319 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-18T08:45:28.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd: error opening image image_47: (2) No such file or directory 2026-02-18T08:45:28.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image_id= 2026-02-18T08:45:28.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-18T08:45:28.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.1 2026-02-18T08:45:28.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test_image_present cluster1 mirror image_47 deleted '' 2026-02-18T08:45:28.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:45:28.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:45:28.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_47 2026-02-18T08:45:28.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local test_state=deleted 2026-02-18T08:45:28.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image_id= 2026-02-18T08:45:28.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_state=deleted 2026-02-18T08:45:28.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_image_id 2026-02-18T08:45:28.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_image_id cluster1 mirror image_47 2026-02-18T08:45:28.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:45:28.446 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:45:28.446 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_47 2026-02-18T08:45:28.446 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster=cluster1 info mirror/image_47 2026-02-18T08:45:28.446 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-18T08:45:28.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd: error opening image image_47: (2) No such file or directory 2026-02-18T08:45:28.470 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ current_image_id= 2026-02-18T08:45:28.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n '' 2026-02-18T08:45:28.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test deleted = deleted 2026-02-18T08:45:28.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:45:28.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T08:45:28.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_48 2026-02-18T08:45:28.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_present cluster1 mirror image_48 deleted 2026-02-18T08:45:28.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:45:28.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:45:28.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_48 2026-02-18T08:45:28.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local state=deleted 2026-02-18T08:45:28.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image_id= 2026-02-18T08:45:28.472 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-18T08:45:28.472 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n '' 2026-02-18T08:45:28.472 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_image_id cluster1 mirror image_48 2026-02-18T08:45:28.472 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:45:28.472 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:45:28.472 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_48 2026-02-18T08:45:28.472 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster=cluster1 info mirror/image_48 2026-02-18T08:45:28.472 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-18T08:45:28.492 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd: error opening image image_48: (2) No such file or directory 2026-02-18T08:45:28.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image_id= 2026-02-18T08:45:28.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-18T08:45:28.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.1 2026-02-18T08:45:28.597 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test_image_present cluster1 mirror image_48 deleted '' 2026-02-18T08:45:28.597 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:45:28.597 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:45:28.597 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_48 2026-02-18T08:45:28.597 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local test_state=deleted 2026-02-18T08:45:28.597 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image_id= 2026-02-18T08:45:28.597 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_state=deleted 2026-02-18T08:45:28.598 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_image_id 2026-02-18T08:45:28.598 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_image_id cluster1 mirror image_48 2026-02-18T08:45:28.598 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:45:28.598 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:45:28.598 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_48 2026-02-18T08:45:28.598 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster=cluster1 info mirror/image_48 2026-02-18T08:45:28.598 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-18T08:45:28.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd: error opening image image_48: (2) No such file or directory 2026-02-18T08:45:28.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ current_image_id= 2026-02-18T08:45:28.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n '' 2026-02-18T08:45:28.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test deleted = deleted 2026-02-18T08:45:28.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:45:28.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T08:45:28.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_49 2026-02-18T08:45:28.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_present cluster1 mirror image_49 deleted 2026-02-18T08:45:28.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:45:28.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:45:28.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_49 2026-02-18T08:45:28.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local state=deleted 2026-02-18T08:45:28.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image_id= 2026-02-18T08:45:28.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-18T08:45:28.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n '' 2026-02-18T08:45:28.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_image_id cluster1 mirror image_49 2026-02-18T08:45:28.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:45:28.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:45:28.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_49 2026-02-18T08:45:28.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster=cluster1 info mirror/image_49 2026-02-18T08:45:28.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-18T08:45:28.646 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd: error opening image image_49: (2) No such file or directory 2026-02-18T08:45:28.649 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image_id= 2026-02-18T08:45:28.649 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-18T08:45:28.649 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.1 2026-02-18T08:45:28.751 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test_image_present cluster1 mirror image_49 deleted '' 2026-02-18T08:45:28.751 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:45:28.751 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:45:28.751 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_49 2026-02-18T08:45:28.751 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local test_state=deleted 2026-02-18T08:45:28.751 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image_id= 2026-02-18T08:45:28.752 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_state=deleted 2026-02-18T08:45:28.752 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_image_id 2026-02-18T08:45:28.752 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_image_id cluster1 mirror image_49 2026-02-18T08:45:28.752 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:45:28.752 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:45:28.752 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_49 2026-02-18T08:45:28.752 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster=cluster1 info mirror/image_49 2026-02-18T08:45:28.752 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-18T08:45:28.770 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd: error opening image image_49: (2) No such file or directory 2026-02-18T08:45:28.774 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ current_image_id= 2026-02-18T08:45:28.774 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n '' 2026-02-18T08:45:28.774 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test deleted = deleted 2026-02-18T08:45:28.774 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:45:28.774 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T08:45:28.774 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_50 2026-02-18T08:45:28.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_present cluster1 mirror image_50 deleted 2026-02-18T08:45:28.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:45:28.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:45:28.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_50 2026-02-18T08:45:28.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local state=deleted 2026-02-18T08:45:28.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image_id= 2026-02-18T08:45:28.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-18T08:45:28.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n '' 2026-02-18T08:45:28.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_image_id cluster1 mirror image_50 2026-02-18T08:45:28.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:45:28.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:45:28.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_50 2026-02-18T08:45:28.776 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster=cluster1 info mirror/image_50 2026-02-18T08:45:28.776 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-18T08:45:28.794 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd: error opening image image_50: (2) No such file or directory 2026-02-18T08:45:28.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image_id= 2026-02-18T08:45:28.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-18T08:45:28.798 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.1 2026-02-18T08:45:28.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test_image_present cluster1 mirror image_50 deleted '' 2026-02-18T08:45:28.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:45:28.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:45:28.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_50 2026-02-18T08:45:28.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local test_state=deleted 2026-02-18T08:45:28.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image_id= 2026-02-18T08:45:28.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_state=deleted 2026-02-18T08:45:28.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_image_id 2026-02-18T08:45:28.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_image_id cluster1 mirror image_50 2026-02-18T08:45:28.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:45:28.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:45:28.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_50 2026-02-18T08:45:28.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster=cluster1 info mirror/image_50 2026-02-18T08:45:28.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-18T08:45:28.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd: error opening image image_50: (2) No such file or directory 2026-02-18T08:45:28.923 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ current_image_id= 2026-02-18T08:45:28.923 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n '' 2026-02-18T08:45:28.924 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test deleted = deleted 2026-02-18T08:45:28.924 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:45:28.924 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ testlog 'TEST: delete images during bootstrap' 2026-02-18T08:45:28.924 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ tee -a /tmp/tmp.vybh1m9yJF/rbd-mirror.test.log 2026-02-18T08:45:28.924 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ date '+%F %T' 2026-02-18T08:45:28.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ echo 2026-02-18 08:45:28 TEST: delete images during bootstrap 2026-02-18T08:45:28.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:2026-02-18 08:45:28 TEST: delete images during bootstrap 2026-02-18T08:45:28.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_pool_mirror_mode cluster1 mirror image 2026-02-18T08:45:28.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:45:28.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:45:28.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local mode=image 2026-02-18T08:45:28.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster=cluster1 mirror pool enable mirror image 2026-02-18T08:45:28.954 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:note: changing mirroring mode from pool to image 2026-02-18T08:45:28.961 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_pool_mirror_mode cluster2 mirror image 2026-02-18T08:45:28.962 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:45:28.962 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:45:28.962 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local mode=image 2026-02-18T08:45:28.962 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster=cluster2 mirror pool enable mirror image 2026-02-18T08:45:28.986 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:note: changing mirroring mode from pool to image 2026-02-18T08:45:28.994 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ start_mirror cluster1 2026-02-18T08:45:28.994 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:45:28.994 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-18T08:45:28.994 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1 cluster instance 2026-02-18T08:45:28.994 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1 2026-02-18T08:45:28.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-18T08:45:28.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-18T08:45:28.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-18T08:45:28.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=cluster1 2026-02-18T08:45:28.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 = cluster1 ']' 2026-02-18T08:45:28.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-18T08:45:28.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-18T08:45:28.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-18T08:45:28.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-18T08:45:28.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-18T08:45:28.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 1 2026-02-18T08:45:28.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 2026-02-18T08:45:28.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=test 2026-02-18T08:45:28.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 1 10 2026-02-18T08:45:28.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 10` 2026-02-18T08:45:28.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_1 2026-02-18T08:45:28.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_image cluster2 mirror image_1 512M 2026-02-18T08:45:28.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:45:28.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:45:28.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:45:28.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:45:28.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_1 2026-02-18T08:45:28.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:45:28.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local size=128 2026-02-18T08:45:28.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n 512M ']' 2026-02-18T08:45:28.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ size=512M 2026-02-18T08:45:28.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:45:28.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 create --size 512M --image-feature layering,exclusive-lock,journaling mirror/image_1 2026-02-18T08:45:29.035 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ enable_mirror cluster2 mirror image_1 2026-02-18T08:45:29.036 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:45:29.036 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:45:29.036 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_1 2026-02-18T08:45:29.036 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local mode=journal 2026-02-18T08:45:29.036 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_1 journal 2026-02-18T08:45:29.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:Mirroring enabled 2026-02-18T08:45:29.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster=cluster2 info mirror/image_1 2026-02-18T08:45:29.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:rbd image 'image_1': 2026-02-18T08:45:29.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: size 512 MiB in 128 objects 2026-02-18T08:45:29.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: order 22 (4 MiB objects) 2026-02-18T08:45:29.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: snapshot_count: 0 2026-02-18T08:45:29.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: id: 39724347b20b 2026-02-18T08:45:29.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: block_name_prefix: rbd_data.39724347b20b 2026-02-18T08:45:29.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: format: 2 2026-02-18T08:45:29.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: features: layering, exclusive-lock, journaling 2026-02-18T08:45:29.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: op_features: 2026-02-18T08:45:29.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: flags: 2026-02-18T08:45:29.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: create_timestamp: Wed Feb 18 08:45:29 2026 2026-02-18T08:45:29.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: access_timestamp: Wed Feb 18 08:45:29 2026 2026-02-18T08:45:29.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: modify_timestamp: Wed Feb 18 08:45:29 2026 2026-02-18T08:45:29.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: journal: 39724347b20b 2026-02-18T08:45:29.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring state: enabled 2026-02-18T08:45:29.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring mode: journal 2026-02-18T08:45:29.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring global id: d9c38237-37b1-4fed-a01c-73fd83039f9a 2026-02-18T08:45:29.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring primary: true 2026-02-18T08:45:29.111 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ stress_write_image cluster2 mirror image_1 2026-02-18T08:45:29.111 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:45:29.112 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:45:29.112 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_1 2026-02-18T08:45:29.112 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-18T08:45:29.114 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local duration=13 2026-02-18T08:45:29.114 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set +e 2026-02-18T08:45:29.114 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ timeout 13s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_1 --debug-rbd=20 --debug-journaler=20 2026-02-18T08:45:29.817 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:100 IOs, 35870720 bytes 2026-02-18T08:45:30.476 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:200 IOs, 83576832 bytes 2026-02-18T08:45:30.801 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:300 IOs, 124899328 bytes 2026-02-18T08:45:31.738 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:400 IOs, 198477824 bytes 2026-02-18T08:45:32.364 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:500 IOs, 215374848 bytes 2026-02-18T08:45:33.201 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:600 IOs, 288244736 bytes 2026-02-18T08:45:34.163 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:700 IOs, 343590912 bytes 2026-02-18T08:45:34.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:800 IOs, 380123136 bytes 2026-02-18T08:45:35.171 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:900 IOs, 417745920 bytes 2026-02-18T08:45:36.134 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1000 IOs, 435260416 bytes 2026-02-18T08:45:37.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1100 IOs, 498660352 bytes 2026-02-18T08:45:37.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1200 IOs, 549275648 bytes 2026-02-18T08:45:38.577 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1300 IOs, 593478656 bytes 2026-02-18T08:45:40.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1400 IOs, 728284160 bytes 2026-02-18T08:45:41.793 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1500 IOs, 801116160 bytes 2026-02-18T08:45:42.122 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ error_code=124 2026-02-18T08:45:42.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set -e 2026-02-18T08:45:42.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 124 -eq 124 ']' 2026-02-18T08:45:42.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:45:42.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_present cluster1 mirror image_1 present 2026-02-18T08:45:42.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:45:42.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:45:42.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_1 2026-02-18T08:45:42.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local state=present 2026-02-18T08:45:42.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image_id= 2026-02-18T08:45:42.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-18T08:45:42.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n '' 2026-02-18T08:45:42.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_image_id cluster1 mirror image_1 2026-02-18T08:45:42.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:45:42.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:45:42.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_1 2026-02-18T08:45:42.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster=cluster1 info mirror/image_1 2026-02-18T08:45:42.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-18T08:45:42.155 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image_id=120d733b3831 2026-02-18T08:45:42.155 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-18T08:45:42.155 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.1 2026-02-18T08:45:42.257 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test_image_present cluster1 mirror image_1 present 120d733b3831 2026-02-18T08:45:42.257 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:45:42.257 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:45:42.257 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_1 2026-02-18T08:45:42.257 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local test_state=present 2026-02-18T08:45:42.257 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image_id=120d733b3831 2026-02-18T08:45:42.257 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_state=deleted 2026-02-18T08:45:42.257 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_image_id 2026-02-18T08:45:42.257 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_image_id cluster1 mirror image_1 2026-02-18T08:45:42.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:45:42.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:45:42.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_1 2026-02-18T08:45:42.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster=cluster1 info mirror/image_1 2026-02-18T08:45:42.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-18T08:45:42.290 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ current_image_id=120d733b3831 2026-02-18T08:45:42.290 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 120d733b3831 2026-02-18T08:45:42.290 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -z 120d733b3831 -o 120d733b3831 = 120d733b3831 2026-02-18T08:45:42.290 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ current_state=present 2026-02-18T08:45:42.290 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test present = present 2026-02-18T08:45:42.290 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:45:42.290 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ disable_mirror cluster2 mirror image_1 2026-02-18T08:45:42.290 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:45:42.290 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:45:42.290 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_1 2026-02-18T08:45:42.290 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster=cluster2 mirror image disable mirror/image_1 2026-02-18T08:46:13.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:Mirroring disabled 2026-02-18T08:46:13.260 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_present cluster1 mirror image_1 deleted 2026-02-18T08:46:13.260 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:46:13.260 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:46:13.260 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_1 2026-02-18T08:46:13.261 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local state=deleted 2026-02-18T08:46:13.261 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image_id= 2026-02-18T08:46:13.261 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-18T08:46:13.261 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n '' 2026-02-18T08:46:13.261 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_image_id cluster1 mirror image_1 2026-02-18T08:46:13.261 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:46:13.261 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:46:13.261 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_1 2026-02-18T08:46:13.261 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster=cluster1 info mirror/image_1 2026-02-18T08:46:13.261 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-18T08:46:13.261 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd: error opening image image_1: (2) No such file or directory 2026-02-18T08:46:13.261 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image_id= 2026-02-18T08:46:13.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-18T08:46:13.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.1 2026-02-18T08:46:13.323 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test_image_present cluster1 mirror image_1 deleted '' 2026-02-18T08:46:13.323 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:46:13.323 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:46:13.323 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_1 2026-02-18T08:46:13.323 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local test_state=deleted 2026-02-18T08:46:13.323 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image_id= 2026-02-18T08:46:13.323 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_state=deleted 2026-02-18T08:46:13.323 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_image_id 2026-02-18T08:46:13.323 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_image_id cluster1 mirror image_1 2026-02-18T08:46:13.323 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:46:13.324 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:46:13.324 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_1 2026-02-18T08:46:13.324 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster=cluster1 info mirror/image_1 2026-02-18T08:46:13.324 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-18T08:46:13.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd: error opening image image_1: (2) No such file or directory 2026-02-18T08:46:13.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ current_image_id= 2026-02-18T08:46:13.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n '' 2026-02-18T08:46:13.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test deleted = deleted 2026-02-18T08:46:13.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:46:13.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ purge_snapshots cluster2 mirror image_1 2026-02-18T08:46:13.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:46:13.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:46:13.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_1 2026-02-18T08:46:13.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 snap purge mirror/image_1 2026-02-18T08:46:13.380 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_image_retry cluster2 mirror image_1 2026-02-18T08:46:13.380 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:46:13.380 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:46:13.381 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_1 2026-02-18T08:46:13.381 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-18T08:46:13.381 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:46:13.382 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_image cluster2 mirror image_1 2026-02-18T08:46:13.382 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:46:13.382 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:46:13.382 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_1 2026-02-18T08:46:13.382 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_1 2026-02-18T08:46:13.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster=cluster2 rm mirror/image_1 2026-02-18T08:46:13.540 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Removing image: 1% complete... Removing image: 2% complete... Removing image: 3% complete... Removing image: 4% complete... Removing image: 5% complete... Removing image: 6% complete... Removing image: 7% complete... Removing image: 8% complete... Removing image: 9% complete... Removing image: 10% complete... Removing image: 11% complete... Removing image: 12% complete... Removing image: 13% complete... Removing image: 14% complete... Removing image: 15% complete... Removing image: 16% complete... Removing image: 17% complete... Removing image: 18% complete... Removing image: 19% complete... Removing image: 20% complete... Removing image: 21% complete... Removing image: 22% complete... Removing image: 23% complete... Removing image: 24% complete... Removing image: 25% complete... Removing image: 26% complete... Removing image: 27% complete... Removing image: 28% complete... Removing image: 29% complete... Removing image: 30% complete... Removing image: 31% complete... Removing image: 32% complete... Removing image: 33% complete... Removing image: 34% complete... Removing image: 35% complete... Removing image: 36% complete... Removing image: 37% complete... Removing image: 38% complete... Removing image: 39% complete... Removing image: 40% complete... Removing image: 41% complete... Removing image: 42% complete... Removing image: 43% complete... Removing image: 44% complete... Removing image: 45% complete... Removing image: 46% complete... Removing image: 47% complete... Removing image: 48% complete... Removing image: 49% complete... Removing image: 50% complete... Removing image: 51% complete... Removing image: 52% complete... Removing image: 53% complete... Removing image: 54% complete... Removing image: 55% complete... Removing image: 56% complete... Removing image: 57% complete... Removing image: 58% complete... Removing image: 59% complete... Removing image: 60% complete... Removing image: 61% complete... Removing image: 62% complete... Removing image: 63% complete... Removing image: 64% complete... Removing image: 65% complete... Removing image: 66% complete... Removing image: 67% complete... Removing image: 68% complete... Removing image: 69% complete... Removing image: 70% complete... Removing image: 71% complete... Removing image: 72% complete... Removing image: 73% complete... Removing image: 74% complete... Removing image: 75% complete... Removing image: 76% complete... Removing image: 77% complete... Removing image: 78% complete... Removing image: 79% complete... Removing image: 80% complete... Removing image: 81% complete... Removing image: 82% complete... Removing image: 83% complete... Removing image: 84% complete... Removing image: 85% complete... Removing image: 86% complete... Removing image: 87% complete... Removing image: 88% complete... Removing image: 89% complete... Removing image: 90% complete... Removing image: 91% complete... Removing image: 92% complete... Removing image: 93% complete... Removing image: 94% complete... Removing image: 95% complete... Removing image: 96% complete... Removing image: 97% complete... Removing image: 98% complete... Removing image: 99% complete... Removing image: 100% complete...done. 2026-02-18T08:46:13.545 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:46:13.545 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 10` 2026-02-18T08:46:13.545 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_2 2026-02-18T08:46:13.545 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_image cluster2 mirror image_2 512M 2026-02-18T08:46:13.545 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:46:13.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:46:13.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:46:13.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:46:13.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_2 2026-02-18T08:46:13.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:46:13.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local size=128 2026-02-18T08:46:13.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n 512M ']' 2026-02-18T08:46:13.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ size=512M 2026-02-18T08:46:13.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:46:13.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 create --size 512M --image-feature layering,exclusive-lock,journaling mirror/image_2 2026-02-18T08:46:13.582 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ enable_mirror cluster2 mirror image_2 2026-02-18T08:46:13.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:46:13.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:46:13.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_2 2026-02-18T08:46:13.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local mode=journal 2026-02-18T08:46:13.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_2 journal 2026-02-18T08:46:13.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:Mirroring enabled 2026-02-18T08:46:13.619 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster=cluster2 info mirror/image_2 2026-02-18T08:46:13.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:rbd image 'image_2': 2026-02-18T08:46:13.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: size 512 MiB in 128 objects 2026-02-18T08:46:13.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: order 22 (4 MiB objects) 2026-02-18T08:46:13.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: snapshot_count: 0 2026-02-18T08:46:13.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: id: 398a6155f08f 2026-02-18T08:46:13.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: block_name_prefix: rbd_data.398a6155f08f 2026-02-18T08:46:13.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: format: 2 2026-02-18T08:46:13.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: features: layering, exclusive-lock, journaling 2026-02-18T08:46:13.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: op_features: 2026-02-18T08:46:13.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: flags: 2026-02-18T08:46:13.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: create_timestamp: Wed Feb 18 08:46:13 2026 2026-02-18T08:46:13.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: access_timestamp: Wed Feb 18 08:46:13 2026 2026-02-18T08:46:13.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: modify_timestamp: Wed Feb 18 08:46:13 2026 2026-02-18T08:46:13.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: journal: 398a6155f08f 2026-02-18T08:46:13.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring state: enabled 2026-02-18T08:46:13.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring mode: journal 2026-02-18T08:46:13.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring global id: 21988282-3570-4c62-9563-c69e6acbbedd 2026-02-18T08:46:13.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring primary: true 2026-02-18T08:46:13.654 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ stress_write_image cluster2 mirror image_2 2026-02-18T08:46:13.654 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:46:13.654 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:46:13.654 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_2 2026-02-18T08:46:13.654 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-18T08:46:13.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local duration=10 2026-02-18T08:46:13.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set +e 2026-02-18T08:46:13.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ timeout 10s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_2 --debug-rbd=20 --debug-journaler=20 2026-02-18T08:46:14.578 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:100 IOs, 41131008 bytes 2026-02-18T08:46:14.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:200 IOs, 81072128 bytes 2026-02-18T08:46:15.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:300 IOs, 90335232 bytes 2026-02-18T08:46:15.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:400 IOs, 112297984 bytes 2026-02-18T08:46:16.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:500 IOs, 172347392 bytes 2026-02-18T08:46:17.865 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:600 IOs, 253765632 bytes 2026-02-18T08:46:18.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:700 IOs, 293356544 bytes 2026-02-18T08:46:19.131 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:800 IOs, 315979776 bytes 2026-02-18T08:46:20.121 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:900 IOs, 365096960 bytes 2026-02-18T08:46:20.930 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1000 IOs, 412753920 bytes 2026-02-18T08:46:21.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1100 IOs, 432132096 bytes 2026-02-18T08:46:21.425 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1200 IOs, 446218240 bytes 2026-02-18T08:46:22.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1300 IOs, 475623424 bytes 2026-02-18T08:46:22.796 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1400 IOs, 511397888 bytes 2026-02-18T08:46:23.211 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1500 IOs, 530757632 bytes 2026-02-18T08:46:23.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1600 IOs, 549114880 bytes 2026-02-18T08:46:23.667 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ error_code=124 2026-02-18T08:46:23.667 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set -e 2026-02-18T08:46:23.667 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 124 -eq 124 ']' 2026-02-18T08:46:23.667 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:46:23.667 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_present cluster1 mirror image_2 present 2026-02-18T08:46:23.667 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:46:23.667 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:46:23.667 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_2 2026-02-18T08:46:23.667 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local state=present 2026-02-18T08:46:23.667 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image_id= 2026-02-18T08:46:23.667 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-18T08:46:23.667 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n '' 2026-02-18T08:46:23.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_image_id cluster1 mirror image_2 2026-02-18T08:46:23.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:46:23.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:46:23.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_2 2026-02-18T08:46:23.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster=cluster1 info mirror/image_2 2026-02-18T08:46:23.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-18T08:46:23.702 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image_id=120d1405b326 2026-02-18T08:46:23.702 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-18T08:46:23.702 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.1 2026-02-18T08:46:23.805 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test_image_present cluster1 mirror image_2 present 120d1405b326 2026-02-18T08:46:23.805 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:46:23.805 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:46:23.805 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_2 2026-02-18T08:46:23.805 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local test_state=present 2026-02-18T08:46:23.805 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image_id=120d1405b326 2026-02-18T08:46:23.805 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_state=deleted 2026-02-18T08:46:23.805 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_image_id 2026-02-18T08:46:23.805 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_image_id cluster1 mirror image_2 2026-02-18T08:46:23.806 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:46:23.806 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:46:23.806 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_2 2026-02-18T08:46:23.806 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster=cluster1 info mirror/image_2 2026-02-18T08:46:23.806 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-18T08:46:23.837 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ current_image_id=120d1405b326 2026-02-18T08:46:23.837 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 120d1405b326 2026-02-18T08:46:23.837 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -z 120d1405b326 -o 120d1405b326 = 120d1405b326 2026-02-18T08:46:23.837 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ current_state=present 2026-02-18T08:46:23.837 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test present = present 2026-02-18T08:46:23.837 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:46:23.838 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ disable_mirror cluster2 mirror image_2 2026-02-18T08:46:23.838 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:46:23.838 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:46:23.838 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_2 2026-02-18T08:46:23.838 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster=cluster2 mirror image disable mirror/image_2 2026-02-18T08:46:54.249 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:Mirroring disabled 2026-02-18T08:46:54.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_present cluster1 mirror image_2 deleted 2026-02-18T08:46:54.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:46:54.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:46:54.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_2 2026-02-18T08:46:54.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local state=deleted 2026-02-18T08:46:54.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image_id= 2026-02-18T08:46:54.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-18T08:46:54.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n '' 2026-02-18T08:46:54.259 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_image_id cluster1 mirror image_2 2026-02-18T08:46:54.259 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:46:54.259 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:46:54.259 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_2 2026-02-18T08:46:54.259 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster=cluster1 info mirror/image_2 2026-02-18T08:46:54.259 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-18T08:46:54.282 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd: error opening image image_2: (2) No such file or directory 2026-02-18T08:46:54.285 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image_id= 2026-02-18T08:46:54.285 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-18T08:46:54.285 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.1 2026-02-18T08:46:54.387 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test_image_present cluster1 mirror image_2 deleted '' 2026-02-18T08:46:54.387 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:46:54.387 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:46:54.387 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_2 2026-02-18T08:46:54.387 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local test_state=deleted 2026-02-18T08:46:54.387 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image_id= 2026-02-18T08:46:54.387 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_state=deleted 2026-02-18T08:46:54.387 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_image_id 2026-02-18T08:46:54.388 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_image_id cluster1 mirror image_2 2026-02-18T08:46:54.388 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:46:54.388 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:46:54.388 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_2 2026-02-18T08:46:54.388 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster=cluster1 info mirror/image_2 2026-02-18T08:46:54.388 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-18T08:46:54.408 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd: error opening image image_2: (2) No such file or directory 2026-02-18T08:46:54.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ current_image_id= 2026-02-18T08:46:54.412 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n '' 2026-02-18T08:46:54.412 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test deleted = deleted 2026-02-18T08:46:54.412 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:46:54.412 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ purge_snapshots cluster2 mirror image_2 2026-02-18T08:46:54.412 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:46:54.412 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:46:54.412 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_2 2026-02-18T08:46:54.412 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 snap purge mirror/image_2 2026-02-18T08:46:54.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_image_retry cluster2 mirror image_2 2026-02-18T08:46:54.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:46:54.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:46:54.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_2 2026-02-18T08:46:54.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-18T08:46:54.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:46:54.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_image cluster2 mirror image_2 2026-02-18T08:46:54.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:46:54.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:46:54.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_2 2026-02-18T08:46:54.446 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_2 2026-02-18T08:46:54.475 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster=cluster2 rm mirror/image_2 2026-02-18T08:46:54.612 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Removing image: 1% complete... Removing image: 2% complete... Removing image: 3% complete... Removing image: 4% complete... Removing image: 5% complete... Removing image: 6% complete... Removing image: 7% complete... Removing image: 8% complete... Removing image: 9% complete... Removing image: 10% complete... Removing image: 11% complete... Removing image: 12% complete... Removing image: 13% complete... Removing image: 14% complete... Removing image: 15% complete... Removing image: 16% complete... Removing image: 17% complete... Removing image: 18% complete... Removing image: 19% complete... Removing image: 20% complete... Removing image: 21% complete... Removing image: 22% complete... Removing image: 23% complete... Removing image: 24% complete... Removing image: 25% complete... Removing image: 26% complete... Removing image: 27% complete... Removing image: 28% complete... Removing image: 29% complete... Removing image: 30% complete... Removing image: 31% complete... Removing image: 32% complete... Removing image: 33% complete... Removing image: 34% complete... Removing image: 35% complete... Removing image: 36% complete... Removing image: 37% complete... Removing image: 38% complete... Removing image: 39% complete... Removing image: 40% complete... Removing image: 41% complete... Removing image: 42% complete... Removing image: 43% complete... Removing image: 44% complete... Removing image: 45% complete... Removing image: 46% complete... Removing image: 47% complete... Removing image: 48% complete... Removing image: 49% complete... Removing image: 50% complete... Removing image: 51% complete... Removing image: 52% complete... Removing image: 53% complete... Removing image: 54% complete... Removing image: 55% complete... Removing image: 56% complete... Removing image: 57% complete... Removing image: 58% complete... Removing image: 59% complete... Removing image: 60% complete... Removing image: 61% complete... Removing image: 62% complete... Removing image: 63% complete... Removing image: 64% complete... Removing image: 65% complete... Removing image: 66% complete... Removing image: 67% complete... Removing image: 68% complete... Removing image: 69% complete... Removing image: 70% complete... Removing image: 71% complete... Removing image: 72% complete... Removing image: 73% complete... Removing image: 74% complete... Removing image: 75% complete... Removing image: 76% complete... Removing image: 77% complete... Removing image: 78% complete... Removing image: 79% complete... Removing image: 80% complete... Removing image: 81% complete... Removing image: 82% complete... Removing image: 83% complete... Removing image: 84% complete... Removing image: 85% complete... Removing image: 86% complete... Removing image: 87% complete... Removing image: 88% complete... Removing image: 89% complete... Removing image: 90% complete... Removing image: 91% complete... Removing image: 92% complete... Removing image: 93% complete... Removing image: 94% complete... Removing image: 95% complete... Removing image: 96% complete... Removing image: 97% complete... Removing image: 98% complete... Removing image: 99% complete... Removing image: 100% complete...done. 2026-02-18T08:46:54.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:46:54.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 10` 2026-02-18T08:46:54.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_3 2026-02-18T08:46:54.618 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_image cluster2 mirror image_3 512M 2026-02-18T08:46:54.618 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:46:54.618 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:46:54.618 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:46:54.618 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:46:54.618 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_3 2026-02-18T08:46:54.618 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:46:54.618 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local size=128 2026-02-18T08:46:54.618 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n 512M ']' 2026-02-18T08:46:54.618 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ size=512M 2026-02-18T08:46:54.618 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:46:54.618 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 create --size 512M --image-feature layering,exclusive-lock,journaling mirror/image_3 2026-02-18T08:46:54.655 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ enable_mirror cluster2 mirror image_3 2026-02-18T08:46:54.655 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:46:54.655 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:46:54.655 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_3 2026-02-18T08:46:54.655 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local mode=journal 2026-02-18T08:46:54.655 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_3 journal 2026-02-18T08:46:54.685 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:Mirroring enabled 2026-02-18T08:46:54.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster=cluster2 info mirror/image_3 2026-02-18T08:46:54.718 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:rbd image 'image_3': 2026-02-18T08:46:54.718 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: size 512 MiB in 128 objects 2026-02-18T08:46:54.719 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: order 22 (4 MiB objects) 2026-02-18T08:46:54.719 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: snapshot_count: 0 2026-02-18T08:46:54.719 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: id: 39a2d8b6135d 2026-02-18T08:46:54.719 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: block_name_prefix: rbd_data.39a2d8b6135d 2026-02-18T08:46:54.719 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: format: 2 2026-02-18T08:46:54.719 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: features: layering, exclusive-lock, journaling 2026-02-18T08:46:54.719 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: op_features: 2026-02-18T08:46:54.719 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: flags: 2026-02-18T08:46:54.719 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: create_timestamp: Wed Feb 18 08:46:54 2026 2026-02-18T08:46:54.719 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: access_timestamp: Wed Feb 18 08:46:54 2026 2026-02-18T08:46:54.719 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: modify_timestamp: Wed Feb 18 08:46:54 2026 2026-02-18T08:46:54.720 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: journal: 39a2d8b6135d 2026-02-18T08:46:54.720 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring state: enabled 2026-02-18T08:46:54.720 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring mode: journal 2026-02-18T08:46:54.720 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring global id: cbc4f2a9-76c6-40c2-8d7b-51d75e785629 2026-02-18T08:46:54.720 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring primary: true 2026-02-18T08:46:54.722 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ stress_write_image cluster2 mirror image_3 2026-02-18T08:46:54.722 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:46:54.722 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:46:54.722 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_3 2026-02-18T08:46:54.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-18T08:46:54.724 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local duration=8 2026-02-18T08:46:54.725 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set +e 2026-02-18T08:46:54.725 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ timeout 8s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_3 --debug-rbd=20 --debug-journaler=20 2026-02-18T08:46:55.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:100 IOs, 41278464 bytes 2026-02-18T08:46:55.923 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:200 IOs, 77191168 bytes 2026-02-18T08:46:56.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:300 IOs, 80458752 bytes 2026-02-18T08:46:56.695 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:400 IOs, 123109376 bytes 2026-02-18T08:46:57.924 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:500 IOs, 195375104 bytes 2026-02-18T08:46:58.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:600 IOs, 264070144 bytes 2026-02-18T08:46:59.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:700 IOs, 296083456 bytes 2026-02-18T08:46:59.950 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:800 IOs, 326032384 bytes 2026-02-18T08:47:00.200 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:900 IOs, 342866944 bytes 2026-02-18T08:47:01.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1000 IOs, 406468608 bytes 2026-02-18T08:47:02.735 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ error_code=124 2026-02-18T08:47:02.735 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set -e 2026-02-18T08:47:02.736 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 124 -eq 124 ']' 2026-02-18T08:47:02.736 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:47:02.736 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_present cluster1 mirror image_3 present 2026-02-18T08:47:02.736 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:47:02.736 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:47:02.736 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_3 2026-02-18T08:47:02.736 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local state=present 2026-02-18T08:47:02.736 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image_id= 2026-02-18T08:47:02.736 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-18T08:47:02.737 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n '' 2026-02-18T08:47:02.737 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_image_id cluster1 mirror image_3 2026-02-18T08:47:02.737 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:47:02.737 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:47:02.737 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_3 2026-02-18T08:47:02.737 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster=cluster1 info mirror/image_3 2026-02-18T08:47:02.737 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-18T08:47:02.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image_id=120d82f7e8ac 2026-02-18T08:47:02.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-18T08:47:02.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.1 2026-02-18T08:47:02.867 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test_image_present cluster1 mirror image_3 present 120d82f7e8ac 2026-02-18T08:47:02.868 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:47:02.868 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:47:02.868 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_3 2026-02-18T08:47:02.868 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local test_state=present 2026-02-18T08:47:02.868 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image_id=120d82f7e8ac 2026-02-18T08:47:02.868 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_state=deleted 2026-02-18T08:47:02.868 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_image_id 2026-02-18T08:47:02.868 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_image_id cluster1 mirror image_3 2026-02-18T08:47:02.868 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:47:02.868 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:47:02.869 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_3 2026-02-18T08:47:02.869 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster=cluster1 info mirror/image_3 2026-02-18T08:47:02.869 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-18T08:47:02.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ current_image_id=120d82f7e8ac 2026-02-18T08:47:02.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 120d82f7e8ac 2026-02-18T08:47:02.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -z 120d82f7e8ac -o 120d82f7e8ac = 120d82f7e8ac 2026-02-18T08:47:02.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ current_state=present 2026-02-18T08:47:02.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test present = present 2026-02-18T08:47:02.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:47:02.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ disable_mirror cluster2 mirror image_3 2026-02-18T08:47:02.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:47:02.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:47:02.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_3 2026-02-18T08:47:02.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster=cluster2 mirror image disable mirror/image_3 2026-02-18T08:47:15.301 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.0 with skip (0.4682608239981332) > weight (0.25) 2026-02-18T08:47:15.301 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.1 with skip (0.3666736409938328) > weight (0.25) 2026-02-18T08:47:15.301 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.2 with skip (0.3267710839460489) > weight (0.25) 2026-02-18T08:47:15.301 INFO:tasks.rbd_mirror_thrash:kill cluster2.client.mirror.3 2026-02-18T08:47:15.302 INFO:tasks.rbd_mirror.cluster2.client.mirror.3:Sent signal 15 2026-02-18T08:47:15.302 INFO:tasks.rbd_mirror_thrash:waiting for 8.03684159138125 secs before reviving daemons 2026-02-18T08:47:15.302 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.trial163.stderr:2026-02-18T08:47:15.293+0000 7f7977312640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.3 (PID: 89782) UID: 1000 2026-02-18T08:47:19.454 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.0 with skip (0.6860881980332866) > weight (0.25) 2026-02-18T08:47:19.454 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.1 with skip (0.4561058963527673) > weight (0.25) 2026-02-18T08:47:19.454 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.2 with skip (0.573222279948492) > weight (0.25) 2026-02-18T08:47:19.454 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.3 with skip (0.7646182706118138) > weight (0.25) 2026-02-18T08:47:19.455 INFO:tasks.rbd_mirror_thrash:waiting for 77.38922286022537 secs before thrashing 2026-02-18T08:47:23.340 INFO:tasks.rbd_mirror_thrash:waiting for cluster2.client.mirror.3 2026-02-18T08:47:23.340 INFO:teuthology.orchestra.run:waiting for 600 2026-02-18T08:47:23.340 INFO:tasks.rbd_mirror_thrash:reviving cluster2.client.mirror.3 2026-02-18T08:47:23.340 INFO:tasks.rbd_mirror.cluster2.client.mirror.3:Restarting daemon 2026-02-18T08:47:23.341 DEBUG:teuthology.orchestra.run.trial163:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.3 2026-02-18T08:47:23.343 INFO:tasks.rbd_mirror.cluster2.client.mirror.3:Started 2026-02-18T08:47:23.343 INFO:tasks.rbd_mirror_thrash:waiting for 83.33998868706648 secs before thrashing 2026-02-18T08:47:33.246 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:Mirroring disabled 2026-02-18T08:47:33.256 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_present cluster1 mirror image_3 deleted 2026-02-18T08:47:33.256 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:47:33.256 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:47:33.256 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_3 2026-02-18T08:47:33.256 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local state=deleted 2026-02-18T08:47:33.256 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image_id= 2026-02-18T08:47:33.256 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-18T08:47:33.256 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n '' 2026-02-18T08:47:33.257 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_image_id cluster1 mirror image_3 2026-02-18T08:47:33.257 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:47:33.257 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:47:33.257 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_3 2026-02-18T08:47:33.257 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster=cluster1 info mirror/image_3 2026-02-18T08:47:33.257 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-18T08:47:33.279 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd: error opening image image_3: (2) No such file or directory 2026-02-18T08:47:33.282 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image_id= 2026-02-18T08:47:33.282 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-18T08:47:33.282 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.1 2026-02-18T08:47:33.384 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test_image_present cluster1 mirror image_3 deleted '' 2026-02-18T08:47:33.384 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:47:33.384 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:47:33.384 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_3 2026-02-18T08:47:33.385 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local test_state=deleted 2026-02-18T08:47:33.385 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image_id= 2026-02-18T08:47:33.385 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_state=deleted 2026-02-18T08:47:33.385 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_image_id 2026-02-18T08:47:33.385 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_image_id cluster1 mirror image_3 2026-02-18T08:47:33.385 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:47:33.385 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:47:33.385 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_3 2026-02-18T08:47:33.385 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster=cluster1 info mirror/image_3 2026-02-18T08:47:33.385 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-18T08:47:33.406 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd: error opening image image_3: (2) No such file or directory 2026-02-18T08:47:33.409 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ current_image_id= 2026-02-18T08:47:33.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n '' 2026-02-18T08:47:33.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test deleted = deleted 2026-02-18T08:47:33.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:47:33.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ purge_snapshots cluster2 mirror image_3 2026-02-18T08:47:33.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:47:33.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:47:33.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_3 2026-02-18T08:47:33.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 snap purge mirror/image_3 2026-02-18T08:47:33.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_image_retry cluster2 mirror image_3 2026-02-18T08:47:33.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:47:33.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:47:33.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_3 2026-02-18T08:47:33.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-18T08:47:33.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:47:33.443 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_image cluster2 mirror image_3 2026-02-18T08:47:33.443 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:47:33.443 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:47:33.443 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_3 2026-02-18T08:47:33.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_3 2026-02-18T08:47:33.481 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster=cluster2 rm mirror/image_3 2026-02-18T08:47:33.613 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Removing image: 1% complete... Removing image: 2% complete... Removing image: 3% complete... Removing image: 4% complete... Removing image: 5% complete... Removing image: 6% complete... Removing image: 7% complete... Removing image: 8% complete... Removing image: 9% complete... Removing image: 10% complete... Removing image: 11% complete... Removing image: 12% complete... Removing image: 13% complete... Removing image: 14% complete... Removing image: 15% complete... Removing image: 16% complete... Removing image: 17% complete... Removing image: 18% complete... Removing image: 19% complete... Removing image: 20% complete... Removing image: 21% complete... Removing image: 22% complete... Removing image: 23% complete... Removing image: 24% complete... Removing image: 25% complete... Removing image: 26% complete... Removing image: 27% complete... Removing image: 28% complete... Removing image: 29% complete... Removing image: 30% complete... Removing image: 31% complete... Removing image: 32% complete... Removing image: 33% complete... Removing image: 34% complete... Removing image: 35% complete... Removing image: 36% complete... Removing image: 37% complete... Removing image: 38% complete... Removing image: 39% complete... Removing image: 40% complete... Removing image: 41% complete... Removing image: 42% complete... Removing image: 43% complete... Removing image: 44% complete... Removing image: 45% complete... Removing image: 46% complete... Removing image: 47% complete... Removing image: 48% complete... Removing image: 49% complete... Removing image: 50% complete... Removing image: 51% complete... Removing image: 52% complete... Removing image: 53% complete... Removing image: 54% complete... Removing image: 55% complete... Removing image: 56% complete... Removing image: 57% complete... Removing image: 58% complete... Removing image: 59% complete... Removing image: 60% complete... Removing image: 61% complete... Removing image: 62% complete... Removing image: 63% complete... Removing image: 64% complete... Removing image: 65% complete... Removing image: 66% complete... Removing image: 67% complete... Removing image: 68% complete... Removing image: 69% complete... Removing image: 70% complete... Removing image: 71% complete... Removing image: 72% complete... Removing image: 73% complete... Removing image: 74% complete... Removing image: 75% complete... Removing image: 76% complete... Removing image: 77% complete... Removing image: 78% complete... Removing image: 79% complete... Removing image: 80% complete... Removing image: 81% complete... Removing image: 82% complete... Removing image: 83% complete... Removing image: 84% complete... Removing image: 85% complete... Removing image: 86% complete... Removing image: 87% complete... Removing image: 88% complete... Removing image: 89% complete... Removing image: 90% complete... Removing image: 91% complete... Removing image: 92% complete... Removing image: 93% complete... Removing image: 94% complete... Removing image: 95% complete... Removing image: 96% complete... Removing image: 97% complete... Removing image: 98% complete... Removing image: 99% complete... Removing image: 100% complete...done. 2026-02-18T08:47:33.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:47:33.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 10` 2026-02-18T08:47:33.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_4 2026-02-18T08:47:33.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_image cluster2 mirror image_4 512M 2026-02-18T08:47:33.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:47:33.618 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:47:33.618 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:47:33.618 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:47:33.618 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_4 2026-02-18T08:47:33.618 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:47:33.618 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local size=128 2026-02-18T08:47:33.618 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n 512M ']' 2026-02-18T08:47:33.618 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ size=512M 2026-02-18T08:47:33.618 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:47:33.618 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 create --size 512M --image-feature layering,exclusive-lock,journaling mirror/image_4 2026-02-18T08:47:33.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ enable_mirror cluster2 mirror image_4 2026-02-18T08:47:33.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:47:33.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:47:33.653 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_4 2026-02-18T08:47:33.653 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local mode=journal 2026-02-18T08:47:33.653 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_4 journal 2026-02-18T08:47:33.682 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:Mirroring enabled 2026-02-18T08:47:33.688 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster=cluster2 info mirror/image_4 2026-02-18T08:47:33.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:rbd image 'image_4': 2026-02-18T08:47:33.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: size 512 MiB in 128 objects 2026-02-18T08:47:33.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: order 22 (4 MiB objects) 2026-02-18T08:47:33.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: snapshot_count: 0 2026-02-18T08:47:33.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: id: 39c38217fdaf 2026-02-18T08:47:33.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: block_name_prefix: rbd_data.39c38217fdaf 2026-02-18T08:47:33.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: format: 2 2026-02-18T08:47:33.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: features: layering, exclusive-lock, journaling 2026-02-18T08:47:33.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: op_features: 2026-02-18T08:47:33.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: flags: 2026-02-18T08:47:33.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: create_timestamp: Wed Feb 18 08:47:33 2026 2026-02-18T08:47:33.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: access_timestamp: Wed Feb 18 08:47:33 2026 2026-02-18T08:47:33.717 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: modify_timestamp: Wed Feb 18 08:47:33 2026 2026-02-18T08:47:33.717 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: journal: 39c38217fdaf 2026-02-18T08:47:33.717 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring state: enabled 2026-02-18T08:47:33.717 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring mode: journal 2026-02-18T08:47:33.717 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring global id: 5daf8659-e268-41a8-9b01-eaf4410b8e1d 2026-02-18T08:47:33.717 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring primary: true 2026-02-18T08:47:33.718 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ stress_write_image cluster2 mirror image_4 2026-02-18T08:47:33.718 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:47:33.719 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:47:33.719 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_4 2026-02-18T08:47:33.719 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-18T08:47:33.720 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local duration=6 2026-02-18T08:47:33.721 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set +e 2026-02-18T08:47:33.721 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ timeout 6s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_4 --debug-rbd=20 --debug-journaler=20 2026-02-18T08:47:34.500 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:100 IOs, 62114816 bytes 2026-02-18T08:47:35.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:200 IOs, 105094144 bytes 2026-02-18T08:47:36.145 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:300 IOs, 140875776 bytes 2026-02-18T08:47:37.929 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:400 IOs, 192464896 bytes 2026-02-18T08:47:38.413 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:500 IOs, 237411328 bytes 2026-02-18T08:47:39.283 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:600 IOs, 308911104 bytes 2026-02-18T08:47:39.732 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ error_code=124 2026-02-18T08:47:39.733 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set -e 2026-02-18T08:47:39.733 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 124 -eq 124 ']' 2026-02-18T08:47:39.733 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:47:39.733 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_present cluster1 mirror image_4 present 2026-02-18T08:47:39.733 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:47:39.733 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:47:39.733 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_4 2026-02-18T08:47:39.733 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local state=present 2026-02-18T08:47:39.734 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image_id= 2026-02-18T08:47:39.734 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-18T08:47:39.734 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n '' 2026-02-18T08:47:39.734 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_image_id cluster1 mirror image_4 2026-02-18T08:47:39.734 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:47:39.734 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:47:39.734 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_4 2026-02-18T08:47:39.734 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster=cluster1 info mirror/image_4 2026-02-18T08:47:39.734 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-18T08:47:39.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image_id=120d7d7ae020 2026-02-18T08:47:39.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-18T08:47:39.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.1 2026-02-18T08:47:39.867 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test_image_present cluster1 mirror image_4 present 120d7d7ae020 2026-02-18T08:47:39.867 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:47:39.867 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:47:39.867 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_4 2026-02-18T08:47:39.867 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local test_state=present 2026-02-18T08:47:39.867 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image_id=120d7d7ae020 2026-02-18T08:47:39.867 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_state=deleted 2026-02-18T08:47:39.867 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_image_id 2026-02-18T08:47:39.867 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_image_id cluster1 mirror image_4 2026-02-18T08:47:39.868 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:47:39.868 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:47:39.868 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_4 2026-02-18T08:47:39.868 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster=cluster1 info mirror/image_4 2026-02-18T08:47:39.868 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-18T08:47:39.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ current_image_id=120d7d7ae020 2026-02-18T08:47:39.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 120d7d7ae020 2026-02-18T08:47:39.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -z 120d7d7ae020 -o 120d7d7ae020 = 120d7d7ae020 2026-02-18T08:47:39.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ current_state=present 2026-02-18T08:47:39.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test present = present 2026-02-18T08:47:39.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:47:39.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ disable_mirror cluster2 mirror image_4 2026-02-18T08:47:39.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:47:39.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:47:39.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_4 2026-02-18T08:47:39.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster=cluster2 mirror image disable mirror/image_4 2026-02-18T08:48:10.156 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:Mirroring disabled 2026-02-18T08:48:10.164 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_present cluster1 mirror image_4 deleted 2026-02-18T08:48:10.164 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:48:10.164 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:48:10.164 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_4 2026-02-18T08:48:10.165 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local state=deleted 2026-02-18T08:48:10.165 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image_id= 2026-02-18T08:48:10.165 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-18T08:48:10.165 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n '' 2026-02-18T08:48:10.166 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_image_id cluster1 mirror image_4 2026-02-18T08:48:10.166 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:48:10.166 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:48:10.166 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_4 2026-02-18T08:48:10.166 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster=cluster1 info mirror/image_4 2026-02-18T08:48:10.166 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-18T08:48:10.189 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd: error opening image image_4: (2) No such file or directory 2026-02-18T08:48:10.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image_id= 2026-02-18T08:48:10.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-18T08:48:10.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.1 2026-02-18T08:48:10.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test_image_present cluster1 mirror image_4 deleted '' 2026-02-18T08:48:10.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:48:10.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:48:10.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_4 2026-02-18T08:48:10.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local test_state=deleted 2026-02-18T08:48:10.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image_id= 2026-02-18T08:48:10.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_state=deleted 2026-02-18T08:48:10.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_image_id 2026-02-18T08:48:10.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_image_id cluster1 mirror image_4 2026-02-18T08:48:10.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:48:10.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:48:10.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_4 2026-02-18T08:48:10.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster=cluster1 info mirror/image_4 2026-02-18T08:48:10.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-18T08:48:10.317 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd: error opening image image_4: (2) No such file or directory 2026-02-18T08:48:10.321 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ current_image_id= 2026-02-18T08:48:10.321 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n '' 2026-02-18T08:48:10.321 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test deleted = deleted 2026-02-18T08:48:10.321 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:48:10.321 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ purge_snapshots cluster2 mirror image_4 2026-02-18T08:48:10.322 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:48:10.322 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:48:10.322 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_4 2026-02-18T08:48:10.322 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 snap purge mirror/image_4 2026-02-18T08:48:10.353 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_image_retry cluster2 mirror image_4 2026-02-18T08:48:10.353 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:48:10.353 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:48:10.353 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_4 2026-02-18T08:48:10.353 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-18T08:48:10.353 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:48:10.354 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_image cluster2 mirror image_4 2026-02-18T08:48:10.354 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:48:10.354 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:48:10.355 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_4 2026-02-18T08:48:10.355 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_4 2026-02-18T08:48:10.384 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster=cluster2 rm mirror/image_4 2026-02-18T08:48:10.516 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Removing image: 1% complete... Removing image: 2% complete... Removing image: 3% complete... Removing image: 4% complete... Removing image: 5% complete... Removing image: 6% complete... Removing image: 7% complete... Removing image: 8% complete... Removing image: 9% complete... Removing image: 10% complete... Removing image: 11% complete... Removing image: 12% complete... Removing image: 13% complete... Removing image: 14% complete... Removing image: 15% complete... Removing image: 16% complete... Removing image: 17% complete... Removing image: 18% complete... Removing image: 19% complete... Removing image: 20% complete... Removing image: 21% complete... Removing image: 22% complete... Removing image: 23% complete... Removing image: 24% complete... Removing image: 25% complete... Removing image: 26% complete... Removing image: 27% complete... Removing image: 28% complete... Removing image: 29% complete... Removing image: 30% complete... Removing image: 31% complete... Removing image: 32% complete... Removing image: 33% complete... Removing image: 34% complete... Removing image: 35% complete... Removing image: 36% complete... Removing image: 37% complete... Removing image: 38% complete... Removing image: 39% complete... Removing image: 40% complete... Removing image: 41% complete... Removing image: 42% complete... Removing image: 43% complete... Removing image: 44% complete... Removing image: 45% complete... Removing image: 46% complete... Removing image: 47% complete... Removing image: 48% complete... Removing image: 49% complete... Removing image: 50% complete... Removing image: 51% complete... Removing image: 52% complete... Removing image: 53% complete... Removing image: 54% complete... Removing image: 55% complete... Removing image: 56% complete... Removing image: 57% complete... Removing image: 58% complete... Removing image: 59% complete... Removing image: 60% complete... Removing image: 61% complete... Removing image: 62% complete... Removing image: 63% complete... Removing image: 64% complete... Removing image: 65% complete... Removing image: 66% complete... Removing image: 67% complete... Removing image: 68% complete... Removing image: 69% complete... Removing image: 70% complete... Removing image: 71% complete... Removing image: 72% complete... Removing image: 73% complete... Removing image: 74% complete... Removing image: 75% complete... Removing image: 76% complete... Removing image: 77% complete... Removing image: 78% complete... Removing image: 79% complete... Removing image: 80% complete... Removing image: 81% complete... Removing image: 82% complete... Removing image: 83% complete... Removing image: 84% complete... Removing image: 85% complete... Removing image: 86% complete... Removing image: 87% complete... Removing image: 88% complete... Removing image: 89% complete... Removing image: 90% complete... Removing image: 91% complete... Removing image: 92% complete... Removing image: 93% complete... Removing image: 94% complete... Removing image: 95% complete... Removing image: 96% complete... Removing image: 97% complete... Removing image: 98% complete... Removing image: 99% complete... Removing image: 100% complete...done. 2026-02-18T08:48:10.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:48:10.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 10` 2026-02-18T08:48:10.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_5 2026-02-18T08:48:10.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_image cluster2 mirror image_5 512M 2026-02-18T08:48:10.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:48:10.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:48:10.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:48:10.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:48:10.523 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_5 2026-02-18T08:48:10.523 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:48:10.523 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local size=128 2026-02-18T08:48:10.523 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n 512M ']' 2026-02-18T08:48:10.523 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ size=512M 2026-02-18T08:48:10.523 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:48:10.523 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 create --size 512M --image-feature layering,exclusive-lock,journaling mirror/image_5 2026-02-18T08:48:10.558 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ enable_mirror cluster2 mirror image_5 2026-02-18T08:48:10.558 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:48:10.559 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:48:10.559 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_5 2026-02-18T08:48:10.559 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local mode=journal 2026-02-18T08:48:10.559 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_5 journal 2026-02-18T08:48:10.594 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:Mirroring enabled 2026-02-18T08:48:10.599 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster=cluster2 info mirror/image_5 2026-02-18T08:48:10.623 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:rbd image 'image_5': 2026-02-18T08:48:10.623 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: size 512 MiB in 128 objects 2026-02-18T08:48:10.623 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: order 22 (4 MiB objects) 2026-02-18T08:48:10.623 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: snapshot_count: 0 2026-02-18T08:48:10.623 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: id: 39db65fb633d 2026-02-18T08:48:10.623 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: block_name_prefix: rbd_data.39db65fb633d 2026-02-18T08:48:10.623 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: format: 2 2026-02-18T08:48:10.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: features: layering, exclusive-lock, journaling 2026-02-18T08:48:10.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: op_features: 2026-02-18T08:48:10.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: flags: 2026-02-18T08:48:10.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: create_timestamp: Wed Feb 18 08:48:10 2026 2026-02-18T08:48:10.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: access_timestamp: Wed Feb 18 08:48:10 2026 2026-02-18T08:48:10.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: modify_timestamp: Wed Feb 18 08:48:10 2026 2026-02-18T08:48:10.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: journal: 39db65fb633d 2026-02-18T08:48:10.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring state: enabled 2026-02-18T08:48:10.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring mode: journal 2026-02-18T08:48:10.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring global id: afdb7ed5-31d5-486a-87f1-d31ecabe8d64 2026-02-18T08:48:10.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring primary: true 2026-02-18T08:48:10.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ stress_write_image cluster2 mirror image_5 2026-02-18T08:48:10.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:48:10.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:48:10.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_5 2026-02-18T08:48:10.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-18T08:48:10.628 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local duration=13 2026-02-18T08:48:10.629 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set +e 2026-02-18T08:48:10.629 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ timeout 13s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_5 --debug-rbd=20 --debug-journaler=20 2026-02-18T08:48:10.817 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:100 IOs, 5309440 bytes 2026-02-18T08:48:11.909 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:200 IOs, 96111616 bytes 2026-02-18T08:48:13.434 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:300 IOs, 144919552 bytes 2026-02-18T08:48:14.594 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:400 IOs, 212335616 bytes 2026-02-18T08:48:15.278 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:500 IOs, 245372928 bytes 2026-02-18T08:48:16.738 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:600 IOs, 312637440 bytes 2026-02-18T08:48:16.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:700 IOs, 313265152 bytes 2026-02-18T08:48:18.203 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:800 IOs, 342306816 bytes 2026-02-18T08:48:19.280 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:900 IOs, 405807104 bytes 2026-02-18T08:48:19.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1000 IOs, 432850944 bytes 2026-02-18T08:48:20.283 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1100 IOs, 474343424 bytes 2026-02-18T08:48:20.802 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1200 IOs, 524409856 bytes 2026-02-18T08:48:22.300 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1300 IOs, 574126080 bytes 2026-02-18T08:48:22.825 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1400 IOs, 632393728 bytes 2026-02-18T08:48:23.641 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ error_code=124 2026-02-18T08:48:23.641 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set -e 2026-02-18T08:48:23.641 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 124 -eq 124 ']' 2026-02-18T08:48:23.641 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:48:23.641 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_present cluster1 mirror image_5 present 2026-02-18T08:48:23.641 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:48:23.642 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:48:23.642 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_5 2026-02-18T08:48:23.642 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local state=present 2026-02-18T08:48:23.642 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image_id= 2026-02-18T08:48:23.642 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-18T08:48:23.642 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n '' 2026-02-18T08:48:23.642 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_image_id cluster1 mirror image_5 2026-02-18T08:48:23.642 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:48:23.642 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:48:23.642 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_5 2026-02-18T08:48:23.642 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster=cluster1 info mirror/image_5 2026-02-18T08:48:23.642 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-18T08:48:23.671 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image_id=120da9fd94ea 2026-02-18T08:48:23.671 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-18T08:48:23.671 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.1 2026-02-18T08:48:23.773 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test_image_present cluster1 mirror image_5 present 120da9fd94ea 2026-02-18T08:48:23.773 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:48:23.773 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:48:23.773 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_5 2026-02-18T08:48:23.773 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local test_state=present 2026-02-18T08:48:23.773 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image_id=120da9fd94ea 2026-02-18T08:48:23.773 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_state=deleted 2026-02-18T08:48:23.773 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_image_id 2026-02-18T08:48:23.774 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_image_id cluster1 mirror image_5 2026-02-18T08:48:23.774 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:48:23.774 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:48:23.774 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_5 2026-02-18T08:48:23.774 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster=cluster1 info mirror/image_5 2026-02-18T08:48:23.774 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-18T08:48:23.804 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ current_image_id=120da9fd94ea 2026-02-18T08:48:23.804 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 120da9fd94ea 2026-02-18T08:48:23.805 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -z 120da9fd94ea -o 120da9fd94ea = 120da9fd94ea 2026-02-18T08:48:23.805 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ current_state=present 2026-02-18T08:48:23.805 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test present = present 2026-02-18T08:48:23.805 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:48:23.805 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ disable_mirror cluster2 mirror image_5 2026-02-18T08:48:23.805 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:48:23.805 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:48:23.805 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_5 2026-02-18T08:48:23.805 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster=cluster2 mirror image disable mirror/image_5 2026-02-18T08:48:36.844 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.0 with skip (0.7049573905089083) > weight (0.25) 2026-02-18T08:48:36.844 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.1 with skip (0.8408274430420796) > weight (0.25) 2026-02-18T08:48:36.844 INFO:tasks.rbd_mirror_thrash:kill cluster1.client.mirror.2 2026-02-18T08:48:36.845 INFO:tasks.rbd_mirror.cluster1.client.mirror.2:Sent signal 15 2026-02-18T08:48:36.845 INFO:tasks.rbd_mirror_thrash:waiting for 7.992344348521383 secs before reviving daemons 2026-02-18T08:48:36.846 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.trial163.stderr:2026-02-18T08:48:36.836+0000 7fc8fbff0640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.2 (PID: 131252) UID: 1000 2026-02-18T08:48:44.838 INFO:tasks.rbd_mirror_thrash:waiting for cluster1.client.mirror.2 2026-02-18T08:48:44.838 INFO:teuthology.orchestra.run:waiting for 600 2026-02-18T08:48:44.838 INFO:tasks.rbd_mirror_thrash:reviving cluster1.client.mirror.2 2026-02-18T08:48:44.838 INFO:tasks.rbd_mirror.cluster1.client.mirror.2:Restarting daemon 2026-02-18T08:48:44.839 DEBUG:teuthology.orchestra.run.trial163:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.2 2026-02-18T08:48:44.841 INFO:tasks.rbd_mirror.cluster1.client.mirror.2:Started 2026-02-18T08:48:44.841 INFO:tasks.rbd_mirror_thrash:waiting for 112.5540324828217 secs before thrashing 2026-02-18T08:48:46.683 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.0 with skip (0.8988033745007711) > weight (0.25) 2026-02-18T08:48:46.684 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.1 with skip (0.7531178638305588) > weight (0.25) 2026-02-18T08:48:46.684 INFO:tasks.rbd_mirror_thrash:kill cluster2.client.mirror.2 2026-02-18T08:48:46.684 INFO:tasks.rbd_mirror.cluster2.client.mirror.2:Sent signal 15 2026-02-18T08:48:46.684 INFO:tasks.rbd_mirror_thrash:waiting for 4.062652085730804 secs before reviving daemons 2026-02-18T08:48:46.685 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial163.stderr:2026-02-18T08:48:46.676+0000 7f0cf75c5640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.2 (PID: 109400) UID: 1000 2026-02-18T08:48:50.747 INFO:tasks.rbd_mirror_thrash:waiting for cluster2.client.mirror.2 2026-02-18T08:48:50.748 INFO:teuthology.orchestra.run:waiting for 600 2026-02-18T08:48:50.748 INFO:tasks.rbd_mirror_thrash:reviving cluster2.client.mirror.2 2026-02-18T08:48:50.748 INFO:tasks.rbd_mirror.cluster2.client.mirror.2:Restarting daemon 2026-02-18T08:48:50.748 DEBUG:teuthology.orchestra.run.trial163:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.2 2026-02-18T08:48:50.751 INFO:tasks.rbd_mirror.cluster2.client.mirror.2:Started 2026-02-18T08:48:50.751 INFO:tasks.rbd_mirror_thrash:waiting for 64.10609726186006 secs before thrashing 2026-02-18T08:48:54.253 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:Mirroring disabled 2026-02-18T08:48:54.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_present cluster1 mirror image_5 deleted 2026-02-18T08:48:54.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:48:54.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:48:54.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_5 2026-02-18T08:48:54.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local state=deleted 2026-02-18T08:48:54.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image_id= 2026-02-18T08:48:54.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-18T08:48:54.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n '' 2026-02-18T08:48:54.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_image_id cluster1 mirror image_5 2026-02-18T08:48:54.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:48:54.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:48:54.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_5 2026-02-18T08:48:54.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster=cluster1 info mirror/image_5 2026-02-18T08:48:54.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-18T08:48:54.286 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd: error opening image image_5: (2) No such file or directory 2026-02-18T08:48:54.289 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image_id= 2026-02-18T08:48:54.290 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-18T08:48:54.290 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.1 2026-02-18T08:48:54.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test_image_present cluster1 mirror image_5 deleted '' 2026-02-18T08:48:54.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:48:54.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:48:54.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_5 2026-02-18T08:48:54.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local test_state=deleted 2026-02-18T08:48:54.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image_id= 2026-02-18T08:48:54.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_state=deleted 2026-02-18T08:48:54.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_image_id 2026-02-18T08:48:54.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_image_id cluster1 mirror image_5 2026-02-18T08:48:54.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:48:54.393 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:48:54.393 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_5 2026-02-18T08:48:54.393 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster=cluster1 info mirror/image_5 2026-02-18T08:48:54.393 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-18T08:48:54.413 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd: error opening image image_5: (2) No such file or directory 2026-02-18T08:48:54.416 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ current_image_id= 2026-02-18T08:48:54.416 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n '' 2026-02-18T08:48:54.417 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test deleted = deleted 2026-02-18T08:48:54.417 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:48:54.417 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ purge_snapshots cluster2 mirror image_5 2026-02-18T08:48:54.417 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:48:54.417 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:48:54.417 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_5 2026-02-18T08:48:54.417 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 snap purge mirror/image_5 2026-02-18T08:48:54.449 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_image_retry cluster2 mirror image_5 2026-02-18T08:48:54.449 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:48:54.449 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:48:54.449 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_5 2026-02-18T08:48:54.449 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-18T08:48:54.450 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:48:54.450 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_image cluster2 mirror image_5 2026-02-18T08:48:54.450 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:48:54.451 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:48:54.451 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_5 2026-02-18T08:48:54.451 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_5 2026-02-18T08:48:54.481 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster=cluster2 rm mirror/image_5 2026-02-18T08:48:54.619 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Removing image: 1% complete... Removing image: 2% complete... Removing image: 3% complete... Removing image: 4% complete... Removing image: 5% complete... Removing image: 6% complete... Removing image: 7% complete... Removing image: 8% complete... Removing image: 9% complete... Removing image: 10% complete... Removing image: 11% complete... Removing image: 12% complete... Removing image: 13% complete... Removing image: 14% complete... Removing image: 15% complete... Removing image: 16% complete... Removing image: 17% complete... Removing image: 18% complete... Removing image: 19% complete... Removing image: 20% complete... Removing image: 21% complete... Removing image: 22% complete... Removing image: 23% complete... Removing image: 24% complete... Removing image: 25% complete... Removing image: 26% complete... Removing image: 27% complete... Removing image: 28% complete... Removing image: 29% complete... Removing image: 30% complete... Removing image: 31% complete... Removing image: 32% complete... Removing image: 33% complete... Removing image: 34% complete... Removing image: 35% complete... Removing image: 36% complete... Removing image: 37% complete... Removing image: 38% complete... Removing image: 39% complete... Removing image: 40% complete... Removing image: 41% complete... Removing image: 42% complete... Removing image: 43% complete... Removing image: 44% complete... Removing image: 45% complete... Removing image: 46% complete... Removing image: 47% complete... Removing image: 48% complete... Removing image: 49% complete... Removing image: 50% complete... Removing image: 51% complete... Removing image: 52% complete... Removing image: 53% complete... Removing image: 54% complete... Removing image: 55% complete... Removing image: 56% complete... Removing image: 57% complete... Removing image: 58% complete... Removing image: 59% complete... Removing image: 60% complete... Removing image: 61% complete... Removing image: 62% complete... Removing image: 63% complete... Removing image: 64% complete... Removing image: 65% complete... Removing image: 66% complete... Removing image: 67% complete... Removing image: 68% complete... Removing image: 69% complete... Removing image: 70% complete... Removing image: 71% complete... Removing image: 72% complete... Removing image: 73% complete... Removing image: 74% complete... Removing image: 75% complete... Removing image: 76% complete... Removing image: 77% complete... Removing image: 78% complete... Removing image: 79% complete... Removing image: 80% complete... Removing image: 81% complete... Removing image: 82% complete... Removing image: 83% complete... Removing image: 84% complete... Removing image: 85% complete... Removing image: 86% complete... Removing image: 87% complete... Removing image: 88% complete... Removing image: 89% complete... Removing image: 90% complete... Removing image: 91% complete... Removing image: 92% complete... Removing image: 93% complete... Removing image: 94% complete... Removing image: 95% complete... Removing image: 96% complete... Removing image: 97% complete... Removing image: 98% complete... Removing image: 99% complete... Removing image: 100% complete...done. 2026-02-18T08:48:54.623 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:48:54.623 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 10` 2026-02-18T08:48:54.623 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_6 2026-02-18T08:48:54.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_image cluster2 mirror image_6 512M 2026-02-18T08:48:54.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:48:54.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:48:54.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:48:54.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:48:54.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_6 2026-02-18T08:48:54.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:48:54.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local size=128 2026-02-18T08:48:54.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n 512M ']' 2026-02-18T08:48:54.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ size=512M 2026-02-18T08:48:54.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:48:54.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 create --size 512M --image-feature layering,exclusive-lock,journaling mirror/image_6 2026-02-18T08:48:54.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ enable_mirror cluster2 mirror image_6 2026-02-18T08:48:54.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:48:54.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:48:54.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_6 2026-02-18T08:48:54.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local mode=journal 2026-02-18T08:48:54.658 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_6 journal 2026-02-18T08:48:54.685 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:Mirroring enabled 2026-02-18T08:48:54.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster=cluster2 info mirror/image_6 2026-02-18T08:48:54.713 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:rbd image 'image_6': 2026-02-18T08:48:54.713 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: size 512 MiB in 128 objects 2026-02-18T08:48:54.713 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: order 22 (4 MiB objects) 2026-02-18T08:48:54.713 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: snapshot_count: 0 2026-02-18T08:48:54.714 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: id: 3a0228777556 2026-02-18T08:48:54.714 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: block_name_prefix: rbd_data.3a0228777556 2026-02-18T08:48:54.714 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: format: 2 2026-02-18T08:48:54.714 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: features: layering, exclusive-lock, journaling 2026-02-18T08:48:54.714 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: op_features: 2026-02-18T08:48:54.714 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: flags: 2026-02-18T08:48:54.714 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: create_timestamp: Wed Feb 18 08:48:54 2026 2026-02-18T08:48:54.714 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: access_timestamp: Wed Feb 18 08:48:54 2026 2026-02-18T08:48:54.714 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: modify_timestamp: Wed Feb 18 08:48:54 2026 2026-02-18T08:48:54.714 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: journal: 3a0228777556 2026-02-18T08:48:54.714 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring state: enabled 2026-02-18T08:48:54.714 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring mode: journal 2026-02-18T08:48:54.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring global id: a9445ac5-4e60-4647-9e70-d0ea6fa6cf0b 2026-02-18T08:48:54.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring primary: true 2026-02-18T08:48:54.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ stress_write_image cluster2 mirror image_6 2026-02-18T08:48:54.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:48:54.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:48:54.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_6 2026-02-18T08:48:54.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-18T08:48:54.717 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local duration=8 2026-02-18T08:48:54.717 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set +e 2026-02-18T08:48:54.718 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ timeout 8s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_6 --debug-rbd=20 --debug-journaler=20 2026-02-18T08:48:55.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:100 IOs, 57909248 bytes 2026-02-18T08:48:55.696 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:200 IOs, 58953728 bytes 2026-02-18T08:48:55.868 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:300 IOs, 94682112 bytes 2026-02-18T08:48:56.291 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:400 IOs, 124818432 bytes 2026-02-18T08:48:58.195 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:500 IOs, 174040064 bytes 2026-02-18T08:48:59.914 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:600 IOs, 264372224 bytes 2026-02-18T08:48:59.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:700 IOs, 281693184 bytes 2026-02-18T08:49:01.038 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:800 IOs, 367329280 bytes 2026-02-18T08:49:01.766 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:900 IOs, 408162304 bytes 2026-02-18T08:49:02.100 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1000 IOs, 433936384 bytes 2026-02-18T08:49:02.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ error_code=124 2026-02-18T08:49:02.728 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set -e 2026-02-18T08:49:02.728 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 124 -eq 124 ']' 2026-02-18T08:49:02.728 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:49:02.728 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_present cluster1 mirror image_6 present 2026-02-18T08:49:02.728 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:49:02.728 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:49:02.728 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_6 2026-02-18T08:49:02.728 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local state=present 2026-02-18T08:49:02.729 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image_id= 2026-02-18T08:49:02.729 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-18T08:49:02.729 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n '' 2026-02-18T08:49:02.729 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_image_id cluster1 mirror image_6 2026-02-18T08:49:02.729 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:49:02.729 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:49:02.729 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_6 2026-02-18T08:49:02.729 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster=cluster1 info mirror/image_6 2026-02-18T08:49:02.729 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-18T08:49:02.759 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image_id=120d761c913a 2026-02-18T08:49:02.759 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-18T08:49:02.759 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.1 2026-02-18T08:49:02.861 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test_image_present cluster1 mirror image_6 present 120d761c913a 2026-02-18T08:49:02.861 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:49:02.861 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:49:02.861 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_6 2026-02-18T08:49:02.861 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local test_state=present 2026-02-18T08:49:02.861 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image_id=120d761c913a 2026-02-18T08:49:02.861 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_state=deleted 2026-02-18T08:49:02.861 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_image_id 2026-02-18T08:49:02.861 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_image_id cluster1 mirror image_6 2026-02-18T08:49:02.861 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:49:02.861 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:49:02.861 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_6 2026-02-18T08:49:02.861 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster=cluster1 info mirror/image_6 2026-02-18T08:49:02.862 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-18T08:49:02.895 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ current_image_id=120d761c913a 2026-02-18T08:49:02.895 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 120d761c913a 2026-02-18T08:49:02.895 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -z 120d761c913a -o 120d761c913a = 120d761c913a 2026-02-18T08:49:02.895 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ current_state=present 2026-02-18T08:49:02.895 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test present = present 2026-02-18T08:49:02.896 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:49:02.896 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ disable_mirror cluster2 mirror image_6 2026-02-18T08:49:02.896 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:49:02.896 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:49:02.896 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_6 2026-02-18T08:49:02.896 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster=cluster2 mirror image disable mirror/image_6 2026-02-18T08:49:02.987 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.trial163.stderr:2026-02-18T08:49:02.979+0000 7f3a17ef2640 -1 rbd::mirror::ImageMap: 0x55ac22a65320 handle_peer_ack_remove: failed to remove global_image_id=a9445ac5-4e60-4647-9e70-d0ea6fa6cf0b 2026-02-18T08:49:32.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:Mirroring disabled 2026-02-18T08:49:32.433 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_present cluster1 mirror image_6 deleted 2026-02-18T08:49:32.433 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:49:32.433 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:49:32.433 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_6 2026-02-18T08:49:32.433 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local state=deleted 2026-02-18T08:49:32.434 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image_id= 2026-02-18T08:49:32.434 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-18T08:49:32.434 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n '' 2026-02-18T08:49:32.434 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_image_id cluster1 mirror image_6 2026-02-18T08:49:32.434 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:49:32.434 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:49:32.434 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_6 2026-02-18T08:49:32.434 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster=cluster1 info mirror/image_6 2026-02-18T08:49:32.434 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-18T08:49:32.458 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd: error opening image image_6: (2) No such file or directory 2026-02-18T08:49:32.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image_id= 2026-02-18T08:49:32.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-18T08:49:32.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.1 2026-02-18T08:49:32.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test_image_present cluster1 mirror image_6 deleted '' 2026-02-18T08:49:32.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:49:32.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:49:32.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_6 2026-02-18T08:49:32.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local test_state=deleted 2026-02-18T08:49:32.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image_id= 2026-02-18T08:49:32.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_state=deleted 2026-02-18T08:49:32.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_image_id 2026-02-18T08:49:32.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_image_id cluster1 mirror image_6 2026-02-18T08:49:32.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:49:32.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:49:32.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_6 2026-02-18T08:49:32.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster=cluster1 info mirror/image_6 2026-02-18T08:49:32.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-18T08:49:32.585 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd: error opening image image_6: (2) No such file or directory 2026-02-18T08:49:32.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ current_image_id= 2026-02-18T08:49:32.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n '' 2026-02-18T08:49:32.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test deleted = deleted 2026-02-18T08:49:32.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:49:32.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ purge_snapshots cluster2 mirror image_6 2026-02-18T08:49:32.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:49:32.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:49:32.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_6 2026-02-18T08:49:32.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 snap purge mirror/image_6 2026-02-18T08:49:32.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_image_retry cluster2 mirror image_6 2026-02-18T08:49:32.622 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:49:32.622 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:49:32.622 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_6 2026-02-18T08:49:32.622 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-18T08:49:32.622 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:49:32.623 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_image cluster2 mirror image_6 2026-02-18T08:49:32.623 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:49:32.623 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:49:32.623 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_6 2026-02-18T08:49:32.623 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_6 2026-02-18T08:49:32.655 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster=cluster2 rm mirror/image_6 2026-02-18T08:49:32.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Removing image: 1% complete... Removing image: 2% complete... Removing image: 3% complete... Removing image: 4% complete... Removing image: 5% complete... Removing image: 6% complete... Removing image: 7% complete... Removing image: 8% complete... Removing image: 9% complete... Removing image: 10% complete... Removing image: 11% complete... Removing image: 12% complete... Removing image: 13% complete... Removing image: 14% complete... Removing image: 15% complete... Removing image: 16% complete... Removing image: 17% complete... Removing image: 18% complete... Removing image: 19% complete... Removing image: 20% complete... Removing image: 21% complete... Removing image: 22% complete... Removing image: 23% complete... Removing image: 24% complete... Removing image: 25% complete... Removing image: 26% complete... Removing image: 27% complete... Removing image: 28% complete... Removing image: 29% complete... Removing image: 30% complete... Removing image: 31% complete... Removing image: 32% complete... Removing image: 33% complete... Removing image: 34% complete... Removing image: 35% complete... Removing image: 36% complete... Removing image: 37% complete... Removing image: 38% complete... Removing image: 39% complete... Removing image: 40% complete... Removing image: 41% complete... Removing image: 42% complete... Removing image: 43% complete... Removing image: 44% complete... Removing image: 45% complete... Removing image: 46% complete... Removing image: 47% complete... Removing image: 48% complete... Removing image: 49% complete... Removing image: 50% complete... Removing image: 51% complete... Removing image: 52% complete... Removing image: 53% complete... Removing image: 54% complete... Removing image: 55% complete... Removing image: 56% complete... Removing image: 57% complete... Removing image: 58% complete... Removing image: 59% complete... Removing image: 60% complete... Removing image: 61% complete... Removing image: 62% complete... Removing image: 63% complete... Removing image: 64% complete... Removing image: 65% complete... Removing image: 66% complete... Removing image: 67% complete... Removing image: 68% complete... Removing image: 69% complete... Removing image: 70% complete... Removing image: 71% complete... Removing image: 72% complete... Removing image: 73% complete... Removing image: 74% complete... Removing image: 75% complete... Removing image: 76% complete... Removing image: 77% complete... Removing image: 78% complete... Removing image: 79% complete... Removing image: 80% complete... Removing image: 81% complete... Removing image: 82% complete... Removing image: 83% complete... Removing image: 84% complete... Removing image: 85% complete... Removing image: 86% complete... Removing image: 87% complete... Removing image: 88% complete... Removing image: 89% complete... Removing image: 90% complete... Removing image: 91% complete... Removing image: 92% complete... Removing image: 93% complete... Removing image: 94% complete... Removing image: 95% complete... Removing image: 96% complete... Removing image: 97% complete... Removing image: 98% complete... Removing image: 99% complete... Removing image: 100% complete...done. 2026-02-18T08:49:32.796 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:49:32.796 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 10` 2026-02-18T08:49:32.796 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_7 2026-02-18T08:49:32.796 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_image cluster2 mirror image_7 512M 2026-02-18T08:49:32.796 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:49:32.796 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:49:32.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:49:32.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:49:32.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_7 2026-02-18T08:49:32.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:49:32.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local size=128 2026-02-18T08:49:32.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n 512M ']' 2026-02-18T08:49:32.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ size=512M 2026-02-18T08:49:32.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:49:32.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 create --size 512M --image-feature layering,exclusive-lock,journaling mirror/image_7 2026-02-18T08:49:32.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ enable_mirror cluster2 mirror image_7 2026-02-18T08:49:32.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:49:32.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:49:32.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_7 2026-02-18T08:49:32.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local mode=journal 2026-02-18T08:49:32.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_7 journal 2026-02-18T08:49:32.865 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:Mirroring enabled 2026-02-18T08:49:32.870 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster=cluster2 info mirror/image_7 2026-02-18T08:49:32.894 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:rbd image 'image_7': 2026-02-18T08:49:32.894 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: size 512 MiB in 128 objects 2026-02-18T08:49:32.894 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: order 22 (4 MiB objects) 2026-02-18T08:49:32.894 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: snapshot_count: 0 2026-02-18T08:49:32.895 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: id: 3a1a264e3cc0 2026-02-18T08:49:32.895 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: block_name_prefix: rbd_data.3a1a264e3cc0 2026-02-18T08:49:32.895 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: format: 2 2026-02-18T08:49:32.895 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: features: layering, exclusive-lock, journaling 2026-02-18T08:49:32.895 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: op_features: 2026-02-18T08:49:32.895 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: flags: 2026-02-18T08:49:32.895 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: create_timestamp: Wed Feb 18 08:49:32 2026 2026-02-18T08:49:32.895 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: access_timestamp: Wed Feb 18 08:49:32 2026 2026-02-18T08:49:32.895 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: modify_timestamp: Wed Feb 18 08:49:32 2026 2026-02-18T08:49:32.895 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: journal: 3a1a264e3cc0 2026-02-18T08:49:32.895 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring state: enabled 2026-02-18T08:49:32.895 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring mode: journal 2026-02-18T08:49:32.896 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring global id: 1b7133df-0095-4b33-8ed6-554f963e1119 2026-02-18T08:49:32.896 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring primary: true 2026-02-18T08:49:32.897 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ stress_write_image cluster2 mirror image_7 2026-02-18T08:49:32.897 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:49:32.897 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:49:32.897 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_7 2026-02-18T08:49:32.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-18T08:49:32.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local duration=6 2026-02-18T08:49:32.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set +e 2026-02-18T08:49:32.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ timeout 6s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_7 --debug-rbd=20 --debug-journaler=20 2026-02-18T08:49:33.695 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:100 IOs, 29828096 bytes 2026-02-18T08:49:34.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:200 IOs, 51391488 bytes 2026-02-18T08:49:35.130 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:300 IOs, 99573760 bytes 2026-02-18T08:49:35.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:400 IOs, 121431040 bytes 2026-02-18T08:49:37.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:500 IOs, 211497984 bytes 2026-02-18T08:49:37.706 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:600 IOs, 261068800 bytes 2026-02-18T08:49:38.714 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:700 IOs, 318775296 bytes 2026-02-18T08:49:38.909 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ error_code=124 2026-02-18T08:49:38.909 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set -e 2026-02-18T08:49:38.909 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 124 -eq 124 ']' 2026-02-18T08:49:38.909 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:49:38.909 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_present cluster1 mirror image_7 present 2026-02-18T08:49:38.909 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:49:38.909 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:49:38.909 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_7 2026-02-18T08:49:38.909 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local state=present 2026-02-18T08:49:38.910 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image_id= 2026-02-18T08:49:38.910 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-18T08:49:38.910 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n '' 2026-02-18T08:49:38.910 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_image_id cluster1 mirror image_7 2026-02-18T08:49:38.910 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:49:38.910 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:49:38.910 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_7 2026-02-18T08:49:38.910 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster=cluster1 info mirror/image_7 2026-02-18T08:49:38.910 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-18T08:49:38.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image_id=120d1444c20c 2026-02-18T08:49:38.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-18T08:49:38.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.1 2026-02-18T08:49:39.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test_image_present cluster1 mirror image_7 present 120d1444c20c 2026-02-18T08:49:39.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:49:39.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:49:39.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_7 2026-02-18T08:49:39.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local test_state=present 2026-02-18T08:49:39.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image_id=120d1444c20c 2026-02-18T08:49:39.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_state=deleted 2026-02-18T08:49:39.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_image_id 2026-02-18T08:49:39.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_image_id cluster1 mirror image_7 2026-02-18T08:49:39.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:49:39.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:49:39.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_7 2026-02-18T08:49:39.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster=cluster1 info mirror/image_7 2026-02-18T08:49:39.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-18T08:49:39.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ current_image_id=120d1444c20c 2026-02-18T08:49:39.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 120d1444c20c 2026-02-18T08:49:39.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -z 120d1444c20c -o 120d1444c20c = 120d1444c20c 2026-02-18T08:49:39.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ current_state=present 2026-02-18T08:49:39.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test present = present 2026-02-18T08:49:39.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:49:39.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ disable_mirror cluster2 mirror image_7 2026-02-18T08:49:39.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:49:39.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:49:39.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_7 2026-02-18T08:49:39.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster=cluster2 mirror image disable mirror/image_7 2026-02-18T08:49:54.858 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.0 with skip (0.5423560672848194) > weight (0.25) 2026-02-18T08:49:54.858 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.1 with skip (0.7372780592604877) > weight (0.25) 2026-02-18T08:49:54.858 INFO:tasks.rbd_mirror_thrash:kill cluster2.client.mirror.2 2026-02-18T08:49:54.859 INFO:tasks.rbd_mirror.cluster2.client.mirror.2:Sent signal 15 2026-02-18T08:49:54.859 INFO:tasks.rbd_mirror_thrash:waiting for 9.727948899869173 secs before reviving daemons 2026-02-18T08:49:54.860 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial163.stderr:2026-02-18T08:49:54.854+0000 7f571f76a640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.2 (PID: 133096) UID: 1000 2026-02-18T08:50:04.588 INFO:tasks.rbd_mirror_thrash:waiting for cluster2.client.mirror.2 2026-02-18T08:50:04.588 INFO:teuthology.orchestra.run:waiting for 600 2026-02-18T08:50:04.588 INFO:tasks.rbd_mirror_thrash:reviving cluster2.client.mirror.2 2026-02-18T08:50:04.588 INFO:tasks.rbd_mirror.cluster2.client.mirror.2:Restarting daemon 2026-02-18T08:50:04.588 DEBUG:teuthology.orchestra.run.trial163:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.2 2026-02-18T08:50:04.590 INFO:tasks.rbd_mirror.cluster2.client.mirror.2:Started 2026-02-18T08:50:04.590 INFO:tasks.rbd_mirror_thrash:waiting for 78.23816970335963 secs before thrashing 2026-02-18T08:50:08.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:Mirroring disabled 2026-02-18T08:50:08.225 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_present cluster1 mirror image_7 deleted 2026-02-18T08:50:08.225 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:50:08.225 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:50:08.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_7 2026-02-18T08:50:08.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local state=deleted 2026-02-18T08:50:08.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image_id= 2026-02-18T08:50:08.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-18T08:50:08.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n '' 2026-02-18T08:50:08.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_image_id cluster1 mirror image_7 2026-02-18T08:50:08.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:50:08.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:50:08.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_7 2026-02-18T08:50:08.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster=cluster1 info mirror/image_7 2026-02-18T08:50:08.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-18T08:50:08.249 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd: error opening image image_7: (2) No such file or directory 2026-02-18T08:50:08.252 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image_id= 2026-02-18T08:50:08.252 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-18T08:50:08.252 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.1 2026-02-18T08:50:08.354 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test_image_present cluster1 mirror image_7 deleted '' 2026-02-18T08:50:08.354 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:50:08.354 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:50:08.354 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_7 2026-02-18T08:50:08.354 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local test_state=deleted 2026-02-18T08:50:08.354 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image_id= 2026-02-18T08:50:08.355 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_state=deleted 2026-02-18T08:50:08.355 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_image_id 2026-02-18T08:50:08.355 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_image_id cluster1 mirror image_7 2026-02-18T08:50:08.355 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:50:08.355 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:50:08.355 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_7 2026-02-18T08:50:08.355 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster=cluster1 info mirror/image_7 2026-02-18T08:50:08.355 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-18T08:50:08.374 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd: error opening image image_7: (2) No such file or directory 2026-02-18T08:50:08.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ current_image_id= 2026-02-18T08:50:08.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n '' 2026-02-18T08:50:08.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test deleted = deleted 2026-02-18T08:50:08.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:50:08.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ purge_snapshots cluster2 mirror image_7 2026-02-18T08:50:08.379 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:50:08.379 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:50:08.379 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_7 2026-02-18T08:50:08.379 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 snap purge mirror/image_7 2026-02-18T08:50:08.409 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_image_retry cluster2 mirror image_7 2026-02-18T08:50:08.409 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:50:08.409 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:50:08.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_7 2026-02-18T08:50:08.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-18T08:50:08.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:50:08.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_image cluster2 mirror image_7 2026-02-18T08:50:08.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:50:08.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:50:08.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_7 2026-02-18T08:50:08.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_7 2026-02-18T08:50:08.440 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster=cluster2 rm mirror/image_7 2026-02-18T08:50:08.562 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Removing image: 1% complete... Removing image: 2% complete... Removing image: 3% complete... Removing image: 4% complete... Removing image: 5% complete... Removing image: 6% complete... Removing image: 7% complete... Removing image: 8% complete... Removing image: 9% complete... Removing image: 10% complete... Removing image: 11% complete... Removing image: 12% complete... Removing image: 13% complete... Removing image: 14% complete... Removing image: 15% complete... Removing image: 16% complete... Removing image: 17% complete... Removing image: 18% complete... Removing image: 19% complete... Removing image: 20% complete... Removing image: 21% complete... Removing image: 22% complete... Removing image: 23% complete... Removing image: 24% complete... Removing image: 25% complete... Removing image: 26% complete... Removing image: 27% complete... Removing image: 28% complete... Removing image: 29% complete... Removing image: 30% complete... Removing image: 31% complete... Removing image: 32% complete... Removing image: 33% complete... Removing image: 34% complete... Removing image: 35% complete... Removing image: 36% complete... Removing image: 37% complete... Removing image: 38% complete... Removing image: 39% complete... Removing image: 40% complete... Removing image: 41% complete... Removing image: 42% complete... Removing image: 43% complete... Removing image: 44% complete... Removing image: 45% complete... Removing image: 46% complete... Removing image: 47% complete... Removing image: 48% complete... Removing image: 49% complete... Removing image: 50% complete... Removing image: 51% complete... Removing image: 52% complete... Removing image: 53% complete... Removing image: 54% complete... Removing image: 55% complete... Removing image: 56% complete... Removing image: 57% complete... Removing image: 58% complete... Removing image: 59% complete... Removing image: 60% complete... Removing image: 61% complete... Removing image: 62% complete... Removing image: 63% complete... Removing image: 64% complete... Removing image: 65% complete... Removing image: 66% complete... Removing image: 67% complete... Removing image: 68% complete... Removing image: 69% complete... Removing image: 70% complete... Removing image: 71% complete... Removing image: 72% complete... Removing image: 73% complete... Removing image: 74% complete... Removing image: 75% complete... Removing image: 76% complete... Removing image: 77% complete... Removing image: 78% complete... Removing image: 79% complete... Removing image: 80% complete... Removing image: 81% complete... Removing image: 82% complete... Removing image: 83% complete... Removing image: 84% complete... Removing image: 85% complete... Removing image: 86% complete... Removing image: 87% complete... Removing image: 88% complete... Removing image: 89% complete... Removing image: 90% complete... Removing image: 91% complete... Removing image: 92% complete... Removing image: 93% complete... Removing image: 94% complete... Removing image: 95% complete... Removing image: 96% complete... Removing image: 97% complete... Removing image: 98% complete... Removing image: 99% complete... Removing image: 100% complete...done. 2026-02-18T08:50:08.567 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:50:08.567 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 10` 2026-02-18T08:50:08.567 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_8 2026-02-18T08:50:08.567 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_image cluster2 mirror image_8 512M 2026-02-18T08:50:08.567 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:50:08.568 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:50:08.568 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:50:08.568 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:50:08.568 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_8 2026-02-18T08:50:08.568 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:50:08.568 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local size=128 2026-02-18T08:50:08.568 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n 512M ']' 2026-02-18T08:50:08.568 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ size=512M 2026-02-18T08:50:08.568 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:50:08.568 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 create --size 512M --image-feature layering,exclusive-lock,journaling mirror/image_8 2026-02-18T08:50:08.604 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ enable_mirror cluster2 mirror image_8 2026-02-18T08:50:08.604 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:50:08.604 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:50:08.604 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_8 2026-02-18T08:50:08.604 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local mode=journal 2026-02-18T08:50:08.604 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_8 journal 2026-02-18T08:50:08.636 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:Mirroring enabled 2026-02-18T08:50:08.641 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster=cluster2 info mirror/image_8 2026-02-18T08:50:08.664 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:rbd image 'image_8': 2026-02-18T08:50:08.664 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: size 512 MiB in 128 objects 2026-02-18T08:50:08.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: order 22 (4 MiB objects) 2026-02-18T08:50:08.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: snapshot_count: 0 2026-02-18T08:50:08.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: id: 3a3c1c4b21c9 2026-02-18T08:50:08.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: block_name_prefix: rbd_data.3a3c1c4b21c9 2026-02-18T08:50:08.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: format: 2 2026-02-18T08:50:08.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: features: layering, exclusive-lock, journaling 2026-02-18T08:50:08.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: op_features: 2026-02-18T08:50:08.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: flags: 2026-02-18T08:50:08.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: create_timestamp: Wed Feb 18 08:50:08 2026 2026-02-18T08:50:08.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: access_timestamp: Wed Feb 18 08:50:08 2026 2026-02-18T08:50:08.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: modify_timestamp: Wed Feb 18 08:50:08 2026 2026-02-18T08:50:08.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: journal: 3a3c1c4b21c9 2026-02-18T08:50:08.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring state: enabled 2026-02-18T08:50:08.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring mode: journal 2026-02-18T08:50:08.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring global id: 9809b3b7-9356-42a3-8310-59e8a3d727cb 2026-02-18T08:50:08.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring primary: true 2026-02-18T08:50:08.667 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ stress_write_image cluster2 mirror image_8 2026-02-18T08:50:08.667 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:50:08.667 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:50:08.667 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_8 2026-02-18T08:50:08.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-18T08:50:08.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local duration=14 2026-02-18T08:50:08.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set +e 2026-02-18T08:50:08.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ timeout 14s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_8 --debug-rbd=20 --debug-journaler=20 2026-02-18T08:50:09.961 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:100 IOs, 70792192 bytes 2026-02-18T08:50:11.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:200 IOs, 136998912 bytes 2026-02-18T08:50:12.324 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:300 IOs, 182425600 bytes 2026-02-18T08:50:13.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:400 IOs, 211910656 bytes 2026-02-18T08:50:13.607 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:500 IOs, 230814720 bytes 2026-02-18T08:50:13.952 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:600 IOs, 248839168 bytes 2026-02-18T08:50:14.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:700 IOs, 313506816 bytes 2026-02-18T08:50:15.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:800 IOs, 385564672 bytes 2026-02-18T08:50:17.038 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:900 IOs, 494282752 bytes 2026-02-18T08:50:17.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1000 IOs, 503334912 bytes 2026-02-18T08:50:18.102 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1100 IOs, 533265408 bytes 2026-02-18T08:50:18.664 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1200 IOs, 556881920 bytes 2026-02-18T08:50:19.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1300 IOs, 602442752 bytes 2026-02-18T08:50:19.986 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1400 IOs, 619703296 bytes 2026-02-18T08:50:21.164 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1500 IOs, 698870784 bytes 2026-02-18T08:50:21.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1600 IOs, 716425216 bytes 2026-02-18T08:50:22.148 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1700 IOs, 754201600 bytes 2026-02-18T08:50:22.532 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1800 IOs, 776611840 bytes 2026-02-18T08:50:22.678 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ error_code=124 2026-02-18T08:50:22.678 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set -e 2026-02-18T08:50:22.678 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 124 -eq 124 ']' 2026-02-18T08:50:22.678 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:50:22.678 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_present cluster1 mirror image_8 present 2026-02-18T08:50:22.678 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:50:22.678 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:50:22.678 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_8 2026-02-18T08:50:22.678 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local state=present 2026-02-18T08:50:22.678 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image_id= 2026-02-18T08:50:22.678 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-18T08:50:22.678 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n '' 2026-02-18T08:50:22.679 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_image_id cluster1 mirror image_8 2026-02-18T08:50:22.679 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:50:22.679 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:50:22.679 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_8 2026-02-18T08:50:22.679 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster=cluster1 info mirror/image_8 2026-02-18T08:50:22.679 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-18T08:50:22.712 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image_id=120db0a395b 2026-02-18T08:50:22.712 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-18T08:50:22.712 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.1 2026-02-18T08:50:22.814 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test_image_present cluster1 mirror image_8 present 120db0a395b 2026-02-18T08:50:22.814 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:50:22.814 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:50:22.814 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_8 2026-02-18T08:50:22.814 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local test_state=present 2026-02-18T08:50:22.814 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image_id=120db0a395b 2026-02-18T08:50:22.814 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_state=deleted 2026-02-18T08:50:22.814 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_image_id 2026-02-18T08:50:22.815 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_image_id cluster1 mirror image_8 2026-02-18T08:50:22.815 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:50:22.815 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:50:22.815 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_8 2026-02-18T08:50:22.815 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster=cluster1 info mirror/image_8 2026-02-18T08:50:22.815 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-18T08:50:22.846 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ current_image_id=120db0a395b 2026-02-18T08:50:22.846 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 120db0a395b 2026-02-18T08:50:22.846 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -z 120db0a395b -o 120db0a395b = 120db0a395b 2026-02-18T08:50:22.846 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ current_state=present 2026-02-18T08:50:22.846 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test present = present 2026-02-18T08:50:22.846 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:50:22.846 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ disable_mirror cluster2 mirror image_8 2026-02-18T08:50:22.846 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:50:22.846 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:50:22.846 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_8 2026-02-18T08:50:22.847 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster=cluster2 mirror image disable mirror/image_8 2026-02-18T08:50:37.396 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.0 with skip (0.7080110687588591) > weight (0.25) 2026-02-18T08:50:37.396 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.1 with skip (0.3835289655415671) > weight (0.25) 2026-02-18T08:50:37.396 INFO:tasks.rbd_mirror_thrash:kill cluster1.client.mirror.2 2026-02-18T08:50:37.397 INFO:tasks.rbd_mirror.cluster1.client.mirror.2:Sent signal 15 2026-02-18T08:50:37.397 INFO:tasks.rbd_mirror_thrash:waiting for 6.143482454618231 secs before reviving daemons 2026-02-18T08:50:37.398 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.trial163.stderr:2026-02-18T08:50:37.390+0000 7f1012484640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.2 (PID: 133005) UID: 1000 2026-02-18T08:50:43.541 INFO:tasks.rbd_mirror_thrash:waiting for cluster1.client.mirror.2 2026-02-18T08:50:43.541 INFO:teuthology.orchestra.run:waiting for 600 2026-02-18T08:50:43.541 INFO:tasks.rbd_mirror_thrash:reviving cluster1.client.mirror.2 2026-02-18T08:50:43.542 INFO:tasks.rbd_mirror.cluster1.client.mirror.2:Restarting daemon 2026-02-18T08:50:43.542 DEBUG:teuthology.orchestra.run.trial163:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.2 2026-02-18T08:50:43.544 INFO:tasks.rbd_mirror.cluster1.client.mirror.2:Started 2026-02-18T08:50:43.544 INFO:tasks.rbd_mirror_thrash:waiting for 71.55687476338808 secs before thrashing 2026-02-18T08:50:53.298 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:Mirroring disabled 2026-02-18T08:50:53.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_present cluster1 mirror image_8 deleted 2026-02-18T08:50:53.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:50:53.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:50:53.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_8 2026-02-18T08:50:53.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local state=deleted 2026-02-18T08:50:53.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image_id= 2026-02-18T08:50:53.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-18T08:50:53.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n '' 2026-02-18T08:50:53.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_image_id cluster1 mirror image_8 2026-02-18T08:50:53.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:50:53.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:50:53.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_8 2026-02-18T08:50:53.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster=cluster1 info mirror/image_8 2026-02-18T08:50:53.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-18T08:50:53.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd: error opening image image_8: (2) No such file or directory 2026-02-18T08:50:53.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image_id= 2026-02-18T08:50:53.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-18T08:50:53.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.1 2026-02-18T08:50:53.435 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test_image_present cluster1 mirror image_8 deleted '' 2026-02-18T08:50:53.435 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:50:53.436 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:50:53.436 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_8 2026-02-18T08:50:53.436 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local test_state=deleted 2026-02-18T08:50:53.436 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image_id= 2026-02-18T08:50:53.436 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_state=deleted 2026-02-18T08:50:53.436 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_image_id 2026-02-18T08:50:53.436 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_image_id cluster1 mirror image_8 2026-02-18T08:50:53.436 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:50:53.436 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:50:53.436 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_8 2026-02-18T08:50:53.437 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster=cluster1 info mirror/image_8 2026-02-18T08:50:53.437 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-18T08:50:53.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd: error opening image image_8: (2) No such file or directory 2026-02-18T08:50:53.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ current_image_id= 2026-02-18T08:50:53.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n '' 2026-02-18T08:50:53.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test deleted = deleted 2026-02-18T08:50:53.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:50:53.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ purge_snapshots cluster2 mirror image_8 2026-02-18T08:50:53.462 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:50:53.462 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:50:53.462 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_8 2026-02-18T08:50:53.462 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 snap purge mirror/image_8 2026-02-18T08:50:53.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_image_retry cluster2 mirror image_8 2026-02-18T08:50:53.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:50:53.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:50:53.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_8 2026-02-18T08:50:53.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-18T08:50:53.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:50:53.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_image cluster2 mirror image_8 2026-02-18T08:50:53.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:50:53.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:50:53.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_8 2026-02-18T08:50:53.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_8 2026-02-18T08:50:53.528 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster=cluster2 rm mirror/image_8 2026-02-18T08:50:53.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Removing image: 1% complete... Removing image: 2% complete... Removing image: 3% complete... Removing image: 4% complete... Removing image: 5% complete... Removing image: 6% complete... Removing image: 7% complete... Removing image: 8% complete... Removing image: 9% complete... Removing image: 10% complete... Removing image: 11% complete... Removing image: 12% complete... Removing image: 13% complete... Removing image: 14% complete... Removing image: 15% complete... Removing image: 16% complete... Removing image: 17% complete... Removing image: 18% complete... Removing image: 19% complete... Removing image: 20% complete... Removing image: 21% complete... Removing image: 22% complete... Removing image: 23% complete... Removing image: 24% complete... Removing image: 25% complete... Removing image: 26% complete... Removing image: 27% complete... Removing image: 28% complete... Removing image: 29% complete... Removing image: 30% complete... Removing image: 31% complete... Removing image: 32% complete... Removing image: 33% complete... Removing image: 34% complete... Removing image: 35% complete... Removing image: 36% complete... Removing image: 37% complete... Removing image: 38% complete... Removing image: 39% complete... Removing image: 40% complete... Removing image: 41% complete... Removing image: 42% complete... Removing image: 43% complete... Removing image: 44% complete... Removing image: 45% complete... Removing image: 46% complete... Removing image: 47% complete... Removing image: 48% complete... Removing image: 49% complete... Removing image: 50% complete... Removing image: 51% complete... Removing image: 52% complete... Removing image: 53% complete... Removing image: 54% complete... Removing image: 55% complete... Removing image: 56% complete... Removing image: 57% complete... Removing image: 58% complete... Removing image: 59% complete... Removing image: 60% complete... Removing image: 61% complete... Removing image: 62% complete... Removing image: 63% complete... Removing image: 64% complete... Removing image: 65% complete... Removing image: 66% complete... Removing image: 67% complete... Removing image: 68% complete... Removing image: 69% complete... Removing image: 70% complete... Removing image: 71% complete... Removing image: 72% complete... Removing image: 73% complete... Removing image: 74% complete... Removing image: 75% complete... Removing image: 76% complete... Removing image: 77% complete... Removing image: 78% complete... Removing image: 79% complete... Removing image: 80% complete... Removing image: 81% complete... Removing image: 82% complete... Removing image: 83% complete... Removing image: 84% complete... Removing image: 85% complete... Removing image: 86% complete... Removing image: 87% complete... Removing image: 88% complete... Removing image: 89% complete... Removing image: 90% complete... Removing image: 91% complete... Removing image: 92% complete... Removing image: 93% complete... Removing image: 94% complete... Removing image: 95% complete... Removing image: 96% complete... Removing image: 97% complete... Removing image: 98% complete... Removing image: 99% complete... Removing image: 100% complete...done. 2026-02-18T08:50:53.662 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:50:53.662 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 10` 2026-02-18T08:50:53.662 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_9 2026-02-18T08:50:53.662 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_image cluster2 mirror image_9 512M 2026-02-18T08:50:53.662 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:50:53.662 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:50:53.662 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:50:53.662 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:50:53.662 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_9 2026-02-18T08:50:53.663 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:50:53.663 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local size=128 2026-02-18T08:50:53.663 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n 512M ']' 2026-02-18T08:50:53.663 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ size=512M 2026-02-18T08:50:53.663 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:50:53.663 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 create --size 512M --image-feature layering,exclusive-lock,journaling mirror/image_9 2026-02-18T08:50:53.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ enable_mirror cluster2 mirror image_9 2026-02-18T08:50:53.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:50:53.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:50:53.700 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_9 2026-02-18T08:50:53.700 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local mode=journal 2026-02-18T08:50:53.700 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_9 journal 2026-02-18T08:50:53.729 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:Mirroring enabled 2026-02-18T08:50:53.734 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster=cluster2 info mirror/image_9 2026-02-18T08:50:53.761 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:rbd image 'image_9': 2026-02-18T08:50:53.761 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: size 512 MiB in 128 objects 2026-02-18T08:50:53.761 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: order 22 (4 MiB objects) 2026-02-18T08:50:53.761 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: snapshot_count: 0 2026-02-18T08:50:53.761 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: id: 3a5aacc6a956 2026-02-18T08:50:53.761 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: block_name_prefix: rbd_data.3a5aacc6a956 2026-02-18T08:50:53.762 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: format: 2 2026-02-18T08:50:53.762 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: features: layering, exclusive-lock, journaling 2026-02-18T08:50:53.762 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: op_features: 2026-02-18T08:50:53.762 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: flags: 2026-02-18T08:50:53.762 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: create_timestamp: Wed Feb 18 08:50:53 2026 2026-02-18T08:50:53.762 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: access_timestamp: Wed Feb 18 08:50:53 2026 2026-02-18T08:50:53.762 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: modify_timestamp: Wed Feb 18 08:50:53 2026 2026-02-18T08:50:53.762 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: journal: 3a5aacc6a956 2026-02-18T08:50:53.762 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring state: enabled 2026-02-18T08:50:53.762 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring mode: journal 2026-02-18T08:50:53.762 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring global id: c39cb7c3-f05d-494d-9989-446e072efaa7 2026-02-18T08:50:53.762 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring primary: true 2026-02-18T08:50:53.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ stress_write_image cluster2 mirror image_9 2026-02-18T08:50:53.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:50:53.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:50:53.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_9 2026-02-18T08:50:53.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-18T08:50:53.766 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local duration=11 2026-02-18T08:50:53.766 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set +e 2026-02-18T08:50:53.766 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ timeout 11s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_9 --debug-rbd=20 --debug-journaler=20 2026-02-18T08:50:55.010 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:100 IOs, 101083136 bytes 2026-02-18T08:50:56.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:200 IOs, 116502528 bytes 2026-02-18T08:50:56.560 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:300 IOs, 155303936 bytes 2026-02-18T08:50:57.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:400 IOs, 223053824 bytes 2026-02-18T08:50:57.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:500 IOs, 223915008 bytes 2026-02-18T08:50:58.251 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:600 IOs, 261814272 bytes 2026-02-18T08:50:58.527 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:700 IOs, 278692864 bytes 2026-02-18T08:50:58.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:800 IOs, 279315456 bytes 2026-02-18T08:50:59.139 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:900 IOs, 309437440 bytes 2026-02-18T08:50:59.326 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1000 IOs, 315314176 bytes 2026-02-18T08:50:59.786 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1100 IOs, 349438976 bytes 2026-02-18T08:51:00.252 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1200 IOs, 372544512 bytes 2026-02-18T08:51:00.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1300 IOs, 400654336 bytes 2026-02-18T08:51:02.688 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1400 IOs, 480452608 bytes 2026-02-18T08:51:03.112 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1500 IOs, 520219648 bytes 2026-02-18T08:51:03.732 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1600 IOs, 543631360 bytes 2026-02-18T08:51:04.780 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ error_code=124 2026-02-18T08:51:04.780 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set -e 2026-02-18T08:51:04.780 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 124 -eq 124 ']' 2026-02-18T08:51:04.780 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:51:04.780 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_present cluster1 mirror image_9 present 2026-02-18T08:51:04.781 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:51:04.781 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:51:04.781 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_9 2026-02-18T08:51:04.781 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local state=present 2026-02-18T08:51:04.781 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image_id= 2026-02-18T08:51:04.781 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-18T08:51:04.781 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n '' 2026-02-18T08:51:04.781 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_image_id cluster1 mirror image_9 2026-02-18T08:51:04.781 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:51:04.781 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:51:04.781 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_9 2026-02-18T08:51:04.781 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster=cluster1 info mirror/image_9 2026-02-18T08:51:04.782 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-18T08:51:04.813 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image_id=120dbb81aba9 2026-02-18T08:51:04.813 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-18T08:51:04.813 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.1 2026-02-18T08:51:04.915 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test_image_present cluster1 mirror image_9 present 120dbb81aba9 2026-02-18T08:51:04.915 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:51:04.915 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:51:04.915 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_9 2026-02-18T08:51:04.915 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local test_state=present 2026-02-18T08:51:04.915 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image_id=120dbb81aba9 2026-02-18T08:51:04.916 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_state=deleted 2026-02-18T08:51:04.916 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_image_id 2026-02-18T08:51:04.916 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_image_id cluster1 mirror image_9 2026-02-18T08:51:04.916 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:51:04.916 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:51:04.916 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_9 2026-02-18T08:51:04.916 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster=cluster1 info mirror/image_9 2026-02-18T08:51:04.916 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-18T08:51:04.947 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ current_image_id=120dbb81aba9 2026-02-18T08:51:04.947 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 120dbb81aba9 2026-02-18T08:51:04.948 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -z 120dbb81aba9 -o 120dbb81aba9 = 120dbb81aba9 2026-02-18T08:51:04.948 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ current_state=present 2026-02-18T08:51:04.948 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test present = present 2026-02-18T08:51:04.948 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:51:04.948 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ disable_mirror cluster2 mirror image_9 2026-02-18T08:51:04.948 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:51:04.948 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:51:04.948 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_9 2026-02-18T08:51:04.948 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster=cluster2 mirror image disable mirror/image_9 2026-02-18T08:51:22.829 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.0 with skip (0.27151528747523035) > weight (0.25) 2026-02-18T08:51:22.830 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.1 with skip (0.4357438918834833) > weight (0.25) 2026-02-18T08:51:22.830 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.2 with skip (0.9275440035071671) > weight (0.25) 2026-02-18T08:51:22.830 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.3 with skip (0.8068489777939765) > weight (0.25) 2026-02-18T08:51:22.830 INFO:tasks.rbd_mirror_thrash:waiting for 99.48005787887072 secs before thrashing 2026-02-18T08:51:34.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:Mirroring disabled 2026-02-18T08:51:34.280 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_present cluster1 mirror image_9 deleted 2026-02-18T08:51:34.280 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:51:34.280 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:51:34.280 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_9 2026-02-18T08:51:34.280 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local state=deleted 2026-02-18T08:51:34.280 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image_id= 2026-02-18T08:51:34.280 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-18T08:51:34.280 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n '' 2026-02-18T08:51:34.280 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_image_id cluster1 mirror image_9 2026-02-18T08:51:34.280 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:51:34.281 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:51:34.281 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_9 2026-02-18T08:51:34.281 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster=cluster1 info mirror/image_9 2026-02-18T08:51:34.281 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-18T08:51:34.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd: error opening image image_9: (2) No such file or directory 2026-02-18T08:51:34.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image_id= 2026-02-18T08:51:34.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-18T08:51:34.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.1 2026-02-18T08:51:34.409 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test_image_present cluster1 mirror image_9 deleted '' 2026-02-18T08:51:34.409 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:51:34.409 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:51:34.409 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_9 2026-02-18T08:51:34.409 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local test_state=deleted 2026-02-18T08:51:34.409 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image_id= 2026-02-18T08:51:34.409 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_state=deleted 2026-02-18T08:51:34.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_image_id 2026-02-18T08:51:34.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_image_id cluster1 mirror image_9 2026-02-18T08:51:34.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:51:34.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:51:34.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_9 2026-02-18T08:51:34.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster=cluster1 info mirror/image_9 2026-02-18T08:51:34.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-18T08:51:34.432 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd: error opening image image_9: (2) No such file or directory 2026-02-18T08:51:34.435 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ current_image_id= 2026-02-18T08:51:34.435 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n '' 2026-02-18T08:51:34.435 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test deleted = deleted 2026-02-18T08:51:34.435 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:51:34.435 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ purge_snapshots cluster2 mirror image_9 2026-02-18T08:51:34.435 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:51:34.435 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:51:34.435 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_9 2026-02-18T08:51:34.435 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 snap purge mirror/image_9 2026-02-18T08:51:34.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_image_retry cluster2 mirror image_9 2026-02-18T08:51:34.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:51:34.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:51:34.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_9 2026-02-18T08:51:34.467 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-18T08:51:34.467 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:51:34.467 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_image cluster2 mirror image_9 2026-02-18T08:51:34.467 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:51:34.467 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:51:34.467 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_9 2026-02-18T08:51:34.467 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_9 2026-02-18T08:51:34.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster=cluster2 rm mirror/image_9 2026-02-18T08:51:34.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Removing image: 1% complete... Removing image: 2% complete... Removing image: 3% complete... Removing image: 4% complete... Removing image: 5% complete... Removing image: 6% complete... Removing image: 7% complete... Removing image: 8% complete... Removing image: 9% complete... Removing image: 10% complete... Removing image: 11% complete... Removing image: 12% complete... Removing image: 13% complete... Removing image: 14% complete... Removing image: 15% complete... Removing image: 16% complete... Removing image: 17% complete... Removing image: 18% complete... Removing image: 19% complete... Removing image: 20% complete... Removing image: 21% complete... Removing image: 22% complete... Removing image: 23% complete... Removing image: 24% complete... Removing image: 25% complete... Removing image: 26% complete... Removing image: 27% complete... Removing image: 28% complete... Removing image: 29% complete... Removing image: 30% complete... Removing image: 31% complete... Removing image: 32% complete... Removing image: 33% complete... Removing image: 34% complete... Removing image: 35% complete... Removing image: 36% complete... Removing image: 37% complete... Removing image: 38% complete... Removing image: 39% complete... Removing image: 40% complete... Removing image: 41% complete... Removing image: 42% complete... Removing image: 43% complete... Removing image: 44% complete... Removing image: 45% complete... Removing image: 46% complete... Removing image: 47% complete... Removing image: 48% complete... Removing image: 49% complete... Removing image: 50% complete... Removing image: 51% complete... Removing image: 52% complete... Removing image: 53% complete... Removing image: 54% complete... Removing image: 55% complete... Removing image: 56% complete... Removing image: 57% complete... Removing image: 58% complete... Removing image: 59% complete... Removing image: 60% complete... Removing image: 61% complete... Removing image: 62% complete... Removing image: 63% complete... Removing image: 64% complete... Removing image: 65% complete... Removing image: 66% complete... Removing image: 67% complete... Removing image: 68% complete... Removing image: 69% complete... Removing image: 70% complete... Removing image: 71% complete... Removing image: 72% complete... Removing image: 73% complete... Removing image: 74% complete... Removing image: 75% complete... Removing image: 76% complete... Removing image: 77% complete... Removing image: 78% complete... Removing image: 79% complete... Removing image: 80% complete... Removing image: 81% complete... Removing image: 82% complete... Removing image: 83% complete... Removing image: 84% complete... Removing image: 85% complete... Removing image: 86% complete... Removing image: 87% complete... Removing image: 88% complete... Removing image: 89% complete... Removing image: 90% complete... Removing image: 91% complete... Removing image: 92% complete... Removing image: 93% complete... Removing image: 94% complete... Removing image: 95% complete... Removing image: 96% complete... Removing image: 97% complete... Removing image: 98% complete... Removing image: 99% complete... Removing image: 100% complete...done. 2026-02-18T08:51:34.639 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:51:34.639 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 10` 2026-02-18T08:51:34.639 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_10 2026-02-18T08:51:34.639 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_image cluster2 mirror image_10 512M 2026-02-18T08:51:34.639 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:51:34.639 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:51:34.639 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:51:34.639 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:51:34.639 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_10 2026-02-18T08:51:34.639 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:51:34.639 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local size=128 2026-02-18T08:51:34.639 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n 512M ']' 2026-02-18T08:51:34.639 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ size=512M 2026-02-18T08:51:34.639 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-18T08:51:34.639 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 create --size 512M --image-feature layering,exclusive-lock,journaling mirror/image_10 2026-02-18T08:51:34.678 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ enable_mirror cluster2 mirror image_10 2026-02-18T08:51:34.678 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:51:34.678 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:51:34.678 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_10 2026-02-18T08:51:34.678 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local mode=journal 2026-02-18T08:51:34.678 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_10 journal 2026-02-18T08:51:34.707 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:Mirroring enabled 2026-02-18T08:51:34.711 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster=cluster2 info mirror/image_10 2026-02-18T08:51:34.735 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:rbd image 'image_10': 2026-02-18T08:51:34.735 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: size 512 MiB in 128 objects 2026-02-18T08:51:34.735 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: order 22 (4 MiB objects) 2026-02-18T08:51:34.735 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: snapshot_count: 0 2026-02-18T08:51:34.735 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: id: 3a721c91b314 2026-02-18T08:51:34.735 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: block_name_prefix: rbd_data.3a721c91b314 2026-02-18T08:51:34.735 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: format: 2 2026-02-18T08:51:34.735 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: features: layering, exclusive-lock, journaling 2026-02-18T08:51:34.735 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: op_features: 2026-02-18T08:51:34.735 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: flags: 2026-02-18T08:51:34.735 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: create_timestamp: Wed Feb 18 08:51:34 2026 2026-02-18T08:51:34.735 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: access_timestamp: Wed Feb 18 08:51:34 2026 2026-02-18T08:51:34.736 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: modify_timestamp: Wed Feb 18 08:51:34 2026 2026-02-18T08:51:34.736 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: journal: 3a721c91b314 2026-02-18T08:51:34.736 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring state: enabled 2026-02-18T08:51:34.736 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring mode: journal 2026-02-18T08:51:34.736 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring global id: af7f9fd1-b324-47f4-9629-8e5252dc3db9 2026-02-18T08:51:34.736 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring primary: true 2026-02-18T08:51:34.739 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ stress_write_image cluster2 mirror image_10 2026-02-18T08:51:34.739 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:51:34.739 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:51:34.739 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_10 2026-02-18T08:51:34.739 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-18T08:51:34.741 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local duration=5 2026-02-18T08:51:34.741 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set +e 2026-02-18T08:51:34.741 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ timeout 5s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_10 --debug-rbd=20 --debug-journaler=20 2026-02-18T08:51:34.856 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:100 IOs, 19579904 bytes 2026-02-18T08:51:35.930 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:200 IOs, 96109568 bytes 2026-02-18T08:51:36.754 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:300 IOs, 147635200 bytes 2026-02-18T08:51:37.327 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:400 IOs, 176436224 bytes 2026-02-18T08:51:38.431 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:500 IOs, 249221120 bytes 2026-02-18T08:51:39.039 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:600 IOs, 290684928 bytes 2026-02-18T08:51:39.530 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:700 IOs, 316106752 bytes 2026-02-18T08:51:39.751 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ error_code=124 2026-02-18T08:51:39.751 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set -e 2026-02-18T08:51:39.751 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 124 -eq 124 ']' 2026-02-18T08:51:39.751 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:51:39.751 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_present cluster1 mirror image_10 present 2026-02-18T08:51:39.751 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:51:39.751 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:51:39.751 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_10 2026-02-18T08:51:39.751 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local state=present 2026-02-18T08:51:39.752 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image_id= 2026-02-18T08:51:39.752 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-18T08:51:39.752 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n '' 2026-02-18T08:51:39.752 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_image_id cluster1 mirror image_10 2026-02-18T08:51:39.752 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:51:39.752 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:51:39.752 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_10 2026-02-18T08:51:39.752 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster=cluster1 info mirror/image_10 2026-02-18T08:51:39.752 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-18T08:51:39.784 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image_id=120d7d7e37ba 2026-02-18T08:51:39.784 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-18T08:51:39.784 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.1 2026-02-18T08:51:39.886 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test_image_present cluster1 mirror image_10 present 120d7d7e37ba 2026-02-18T08:51:39.886 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:51:39.886 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:51:39.886 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_10 2026-02-18T08:51:39.886 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local test_state=present 2026-02-18T08:51:39.886 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image_id=120d7d7e37ba 2026-02-18T08:51:39.886 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_state=deleted 2026-02-18T08:51:39.887 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_image_id 2026-02-18T08:51:39.887 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_image_id cluster1 mirror image_10 2026-02-18T08:51:39.887 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:51:39.887 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:51:39.887 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_10 2026-02-18T08:51:39.887 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster=cluster1 info mirror/image_10 2026-02-18T08:51:39.887 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-18T08:51:39.917 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ current_image_id=120d7d7e37ba 2026-02-18T08:51:39.918 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 120d7d7e37ba 2026-02-18T08:51:39.918 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -z 120d7d7e37ba -o 120d7d7e37ba = 120d7d7e37ba 2026-02-18T08:51:39.918 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ current_state=present 2026-02-18T08:51:39.918 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test present = present 2026-02-18T08:51:39.918 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:51:39.918 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ disable_mirror cluster2 mirror image_10 2026-02-18T08:51:39.918 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:51:39.918 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:51:39.918 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_10 2026-02-18T08:51:39.918 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster=cluster2 mirror image disable mirror/image_10 2026-02-18T08:51:55.102 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.0 with skip (0.5000369089649538) > weight (0.25) 2026-02-18T08:51:55.102 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.1 with skip (0.6892070879991948) > weight (0.25) 2026-02-18T08:51:55.102 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.2 with skip (0.9180697352084909) > weight (0.25) 2026-02-18T08:51:55.102 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.3 with skip (0.5218751461632944) > weight (0.25) 2026-02-18T08:51:55.102 INFO:tasks.rbd_mirror_thrash:waiting for 83.50517619021292 secs before thrashing 2026-02-18T08:52:10.220 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:Mirroring disabled 2026-02-18T08:52:10.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_present cluster1 mirror image_10 deleted 2026-02-18T08:52:10.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:52:10.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:52:10.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_10 2026-02-18T08:52:10.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local state=deleted 2026-02-18T08:52:10.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image_id= 2026-02-18T08:52:10.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-18T08:52:10.229 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n '' 2026-02-18T08:52:10.229 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_image_id cluster1 mirror image_10 2026-02-18T08:52:10.229 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:52:10.229 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:52:10.229 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_10 2026-02-18T08:52:10.230 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster=cluster1 info mirror/image_10 2026-02-18T08:52:10.230 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-18T08:52:10.254 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd: error opening image image_10: (2) No such file or directory 2026-02-18T08:52:10.257 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image_id= 2026-02-18T08:52:10.257 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-18T08:52:10.257 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.1 2026-02-18T08:52:10.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test_image_present cluster1 mirror image_10 deleted '' 2026-02-18T08:52:10.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:52:10.360 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:52:10.360 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_10 2026-02-18T08:52:10.360 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local test_state=deleted 2026-02-18T08:52:10.360 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image_id= 2026-02-18T08:52:10.360 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_state=deleted 2026-02-18T08:52:10.360 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_image_id 2026-02-18T08:52:10.360 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_image_id cluster1 mirror image_10 2026-02-18T08:52:10.360 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:52:10.360 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:52:10.360 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_10 2026-02-18T08:52:10.360 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster=cluster1 info mirror/image_10 2026-02-18T08:52:10.361 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-18T08:52:10.382 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd: error opening image image_10: (2) No such file or directory 2026-02-18T08:52:10.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ current_image_id= 2026-02-18T08:52:10.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n '' 2026-02-18T08:52:10.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test deleted = deleted 2026-02-18T08:52:10.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:52:10.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ purge_snapshots cluster2 mirror image_10 2026-02-18T08:52:10.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:52:10.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:52:10.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_10 2026-02-18T08:52:10.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 snap purge mirror/image_10 2026-02-18T08:52:10.420 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_image_retry cluster2 mirror image_10 2026-02-18T08:52:10.420 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:52:10.420 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:52:10.420 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_10 2026-02-18T08:52:10.420 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-18T08:52:10.420 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:52:10.421 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_image cluster2 mirror image_10 2026-02-18T08:52:10.421 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:52:10.421 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:52:10.421 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_10 2026-02-18T08:52:10.421 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_10 2026-02-18T08:52:10.448 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster=cluster2 rm mirror/image_10 2026-02-18T08:52:10.575 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Removing image: 1% complete... Removing image: 2% complete... Removing image: 3% complete... Removing image: 4% complete... Removing image: 5% complete... Removing image: 6% complete... Removing image: 7% complete... Removing image: 8% complete... Removing image: 9% complete... Removing image: 10% complete... Removing image: 11% complete... Removing image: 12% complete... Removing image: 13% complete... Removing image: 14% complete... Removing image: 15% complete... Removing image: 16% complete... Removing image: 17% complete... Removing image: 18% complete... Removing image: 19% complete... Removing image: 20% complete... Removing image: 21% complete... Removing image: 22% complete... Removing image: 23% complete... Removing image: 24% complete... Removing image: 25% complete... Removing image: 26% complete... Removing image: 27% complete... Removing image: 28% complete... Removing image: 29% complete... Removing image: 30% complete... Removing image: 31% complete... Removing image: 32% complete... Removing image: 33% complete... Removing image: 34% complete... Removing image: 35% complete... Removing image: 36% complete... Removing image: 37% complete... Removing image: 38% complete... Removing image: 39% complete... Removing image: 40% complete... Removing image: 41% complete... Removing image: 42% complete... Removing image: 43% complete... Removing image: 44% complete... Removing image: 45% complete... Removing image: 46% complete... Removing image: 47% complete... Removing image: 48% complete... Removing image: 49% complete... Removing image: 50% complete... Removing image: 51% complete... Removing image: 52% complete... Removing image: 53% complete... Removing image: 54% complete... Removing image: 55% complete... Removing image: 56% complete... Removing image: 57% complete... Removing image: 58% complete... Removing image: 59% complete... Removing image: 60% complete... Removing image: 61% complete... Removing image: 62% complete... Removing image: 63% complete... Removing image: 64% complete... Removing image: 65% complete... Removing image: 66% complete... Removing image: 67% complete... Removing image: 68% complete... Removing image: 69% complete... Removing image: 70% complete... Removing image: 71% complete... Removing image: 72% complete... Removing image: 73% complete... Removing image: 74% complete... Removing image: 75% complete... Removing image: 76% complete... Removing image: 77% complete... Removing image: 78% complete... Removing image: 79% complete... Removing image: 80% complete... Removing image: 81% complete... Removing image: 82% complete... Removing image: 83% complete... Removing image: 84% complete... Removing image: 85% complete... Removing image: 86% complete... Removing image: 87% complete... Removing image: 88% complete... Removing image: 89% complete... Removing image: 90% complete... Removing image: 91% complete... Removing image: 92% complete... Removing image: 93% complete... Removing image: 94% complete... Removing image: 95% complete... Removing image: 96% complete... Removing image: 97% complete... Removing image: 98% complete... Removing image: 99% complete... Removing image: 100% complete...done. 2026-02-18T08:52:10.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:52:10.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ testlog 'TEST: check if removed images'\'' OMAP are removed' 2026-02-18T08:52:10.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ tee -a /tmp/tmp.vybh1m9yJF/rbd-mirror.test.log 2026-02-18T08:52:10.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ date '+%F %T' 2026-02-18T08:52:10.582 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ echo 2026-02-18 08:52:10 TEST: check if removed 'images'\''' OMAP are removed 2026-02-18T08:52:10.582 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:2026-02-18 08:52:10 TEST: check if removed images' OMAP are removed 2026-02-18T08:52:10.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_in_omap cluster1 mirror 2026-02-18T08:52:10.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:52:10.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:52:10.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_omap_keys cluster1 mirror rbd_mirroring status_global 2026-02-18T08:52:10.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:52:10.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:52:10.584 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local obj_name=rbd_mirroring 2026-02-18T08:52:10.584 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local filter=status_global 2026-02-18T08:52:10.584 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 2 4 4 8 8 8 16 16 32 2026-02-18T08:52:10.584 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:52:10.584 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set +e 2026-02-18T08:52:10.585 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ count_omap_keys_with_filter cluster1 mirror rbd_mirroring status_global 2026-02-18T08:52:10.585 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:52:10.585 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:52:10.585 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local obj_name=rbd_mirroring 2026-02-18T08:52:10.585 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local filter=status_global 2026-02-18T08:52:10.586 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ list_omap_keys cluster1 mirror rbd_mirroring 2026-02-18T08:52:10.586 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -c status_global 2026-02-18T08:52:10.586 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:52:10.586 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:52:10.586 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local obj_name=rbd_mirroring 2026-02-18T08:52:10.586 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rados --cluster cluster1 -p mirror listomapkeys rbd_mirroring 2026-02-18T08:52:10.612 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 0 = 0 2026-02-18T08:52:10.613 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ error_code=0 2026-02-18T08:52:10.613 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set -e 2026-02-18T08:52:10.613 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 -eq 0 ']' 2026-02-18T08:52:10.613 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:52:10.613 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_omap_keys cluster1 mirror rbd_mirroring image_ 2026-02-18T08:52:10.613 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:52:10.613 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:52:10.613 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local obj_name=rbd_mirroring 2026-02-18T08:52:10.613 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local filter=image_ 2026-02-18T08:52:10.613 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 2 4 4 8 8 8 16 16 32 2026-02-18T08:52:10.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:52:10.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set +e 2026-02-18T08:52:10.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ count_omap_keys_with_filter cluster1 mirror rbd_mirroring image_ 2026-02-18T08:52:10.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:52:10.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:52:10.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local obj_name=rbd_mirroring 2026-02-18T08:52:10.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local filter=image_ 2026-02-18T08:52:10.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ list_omap_keys cluster1 mirror rbd_mirroring 2026-02-18T08:52:10.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -c image_ 2026-02-18T08:52:10.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:52:10.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:52:10.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local obj_name=rbd_mirroring 2026-02-18T08:52:10.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rados --cluster cluster1 -p mirror listomapkeys rbd_mirroring 2026-02-18T08:52:10.637 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 0 = 0 2026-02-18T08:52:10.638 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ error_code=0 2026-02-18T08:52:10.638 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set -e 2026-02-18T08:52:10.638 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 -eq 0 ']' 2026-02-18T08:52:10.638 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:52:10.638 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_omap_keys cluster1 mirror rbd_mirror_leader image_map 2026-02-18T08:52:10.638 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:52:10.638 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:52:10.638 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local obj_name=rbd_mirror_leader 2026-02-18T08:52:10.638 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local filter=image_map 2026-02-18T08:52:10.638 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 2 4 4 8 8 8 16 16 32 2026-02-18T08:52:10.638 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:52:10.640 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set +e 2026-02-18T08:52:10.640 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ count_omap_keys_with_filter cluster1 mirror rbd_mirror_leader image_map 2026-02-18T08:52:10.640 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:52:10.640 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:52:10.640 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local obj_name=rbd_mirror_leader 2026-02-18T08:52:10.640 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local filter=image_map 2026-02-18T08:52:10.640 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ list_omap_keys cluster1 mirror rbd_mirror_leader 2026-02-18T08:52:10.640 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -c image_map 2026-02-18T08:52:10.640 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-18T08:52:10.640 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:52:10.641 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local obj_name=rbd_mirror_leader 2026-02-18T08:52:10.641 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rados --cluster cluster1 -p mirror listomapkeys rbd_mirror_leader 2026-02-18T08:52:10.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 0 = 0 2026-02-18T08:52:10.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ error_code=0 2026-02-18T08:52:10.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set -e 2026-02-18T08:52:10.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 -eq 0 ']' 2026-02-18T08:52:10.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:52:10.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_in_omap cluster2 mirror 2026-02-18T08:52:10.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:52:10.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:52:10.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_omap_keys cluster2 mirror rbd_mirroring status_global 2026-02-18T08:52:10.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:52:10.667 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:52:10.667 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local obj_name=rbd_mirroring 2026-02-18T08:52:10.667 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local filter=status_global 2026-02-18T08:52:10.667 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 2 4 4 8 8 8 16 16 32 2026-02-18T08:52:10.667 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:52:10.667 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set +e 2026-02-18T08:52:10.667 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ count_omap_keys_with_filter cluster2 mirror rbd_mirroring status_global 2026-02-18T08:52:10.667 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:52:10.667 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:52:10.667 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local obj_name=rbd_mirroring 2026-02-18T08:52:10.667 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local filter=status_global 2026-02-18T08:52:10.667 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ list_omap_keys cluster2 mirror rbd_mirroring 2026-02-18T08:52:10.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:52:10.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -c status_global 2026-02-18T08:52:10.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:52:10.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local obj_name=rbd_mirroring 2026-02-18T08:52:10.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rados --cluster cluster2 -p mirror listomapkeys rbd_mirroring 2026-02-18T08:52:10.687 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 0 = 0 2026-02-18T08:52:10.687 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ error_code=0 2026-02-18T08:52:10.687 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set -e 2026-02-18T08:52:10.687 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 -eq 0 ']' 2026-02-18T08:52:10.688 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:52:10.688 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_omap_keys cluster2 mirror rbd_mirroring image_ 2026-02-18T08:52:10.688 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:52:10.688 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:52:10.688 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local obj_name=rbd_mirroring 2026-02-18T08:52:10.688 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local filter=image_ 2026-02-18T08:52:10.688 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 2 4 4 8 8 8 16 16 32 2026-02-18T08:52:10.688 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:52:10.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set +e 2026-02-18T08:52:10.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ count_omap_keys_with_filter cluster2 mirror rbd_mirroring image_ 2026-02-18T08:52:10.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:52:10.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:52:10.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local obj_name=rbd_mirroring 2026-02-18T08:52:10.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local filter=image_ 2026-02-18T08:52:10.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ list_omap_keys cluster2 mirror rbd_mirroring 2026-02-18T08:52:10.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -c image_ 2026-02-18T08:52:10.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:52:10.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:52:10.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local obj_name=rbd_mirroring 2026-02-18T08:52:10.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rados --cluster cluster2 -p mirror listomapkeys rbd_mirroring 2026-02-18T08:52:10.711 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 0 = 0 2026-02-18T08:52:10.711 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ error_code=0 2026-02-18T08:52:10.711 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set -e 2026-02-18T08:52:10.711 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 -eq 0 ']' 2026-02-18T08:52:10.711 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:52:10.712 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_omap_keys cluster2 mirror rbd_mirror_leader image_map 2026-02-18T08:52:10.712 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:52:10.712 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-18T08:52:10.712 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local obj_name=rbd_mirror_leader 2026-02-18T08:52:10.712 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local filter=image_map 2026-02-18T08:52:10.712 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 2 4 4 8 8 8 16 16 32 2026-02-18T08:52:10.712 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-18T08:52:10.713 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set +e 2026-02-18T08:52:10.713 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ count_omap_keys_with_filter cluster2 mirror rbd_mirror_leader image_map 2026-02-18T08:52:10.713 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:52:10.714 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:52:10.714 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local obj_name=rbd_mirror_leader 2026-02-18T08:52:10.714 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local filter=image_map 2026-02-18T08:52:10.714 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ list_omap_keys cluster2 mirror rbd_mirror_leader 2026-02-18T08:52:10.714 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -c image_map 2026-02-18T08:52:10.714 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-18T08:52:10.714 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-18T08:52:10.714 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local obj_name=rbd_mirror_leader 2026-02-18T08:52:10.714 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rados --cluster cluster2 -p mirror listomapkeys rbd_mirror_leader 2026-02-18T08:52:10.735 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 0 = 0 2026-02-18T08:52:10.735 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ error_code=0 2026-02-18T08:52:10.735 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set -e 2026-02-18T08:52:10.735 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 -eq 0 ']' 2026-02-18T08:52:10.735 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-18T08:52:10.736 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cleanup 0 2026-02-18T08:52:10.736 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local error_code=0 2026-02-18T08:52:10.736 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set +e 2026-02-18T08:52:10.736 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 -ne 0 ']' 2026-02-18T08:52:10.736 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -z '' ']' 2026-02-18T08:52:10.736 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster instance 2026-02-18T08:52:10.736 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ CEPH_ARGS= 2026-02-18T08:52:10.736 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --cluster cluster1 osd pool rm mirror mirror --yes-i-really-really-mean-it 2026-02-18T08:52:11.310 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:pool 'mirror' does not exist 2026-02-18T08:52:11.320 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ CEPH_ARGS= 2026-02-18T08:52:11.320 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --cluster cluster2 osd pool rm mirror mirror --yes-i-really-really-mean-it 2026-02-18T08:52:12.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:pool 'mirror' does not exist 2026-02-18T08:52:12.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ CEPH_ARGS= 2026-02-18T08:52:12.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --cluster cluster1 osd pool rm mirror_parent mirror_parent --yes-i-really-really-mean-it 2026-02-18T08:52:13.318 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:pool 'mirror_parent' does not exist 2026-02-18T08:52:13.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ CEPH_ARGS= 2026-02-18T08:52:13.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --cluster cluster2 osd pool rm mirror_parent mirror_parent --yes-i-really-really-mean-it 2026-02-18T08:52:13.843 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.trial163.stderr:2026-02-18T08:52:13.832+0000 7fef0f823640 -1 rbd::mirror::RemotePoolPoller: 0x56407d7cedd0 handle_mirror_peer_ping: failed to ping remote mirror peer: (2) No such file or directory 2026-02-18T08:52:13.844 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.trial163.stderr:2026-02-18T08:52:13.832+0000 7fef10024640 -1 rbd::mirror::RemotePoolPoller: 0x56407d7cedd0 handle_mirror_peer_list: failed to retrieve mirror peers: (2) No such file or directory 2026-02-18T08:52:13.844 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.trial163.stderr:2026-02-18T08:52:13.832+0000 7fef10024640 -1 rbd::mirror::PoolReplayer: 0x56407e726000 handle_remote_pool_meta_updated: remote pool metadata updated unexpectedly 2026-02-18T08:52:13.846 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.trial163.stderr:2026-02-18T08:52:13.836+0000 7fef1d03e640 -1 rbd::mirror::InstanceWatcher: 0x56407f2d3180 handle_unregister_watch: error unregistering instance watcher for rbd_mirror_instance.5429 object: (2) No such file or directory 2026-02-18T08:52:13.846 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.trial163.stderr:2026-02-18T08:52:13.836+0000 7fef1d03e640 -1 rbd::mirror::InstanceWatcher: 0x56407f2d3500 handle_unregister_watch: error unregistering instance watcher for rbd_mirror_instance.5429 object: (2) No such file or directory 2026-02-18T08:52:13.846 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.trial163.stderr:2026-02-18T08:52:13.836+0000 7fef1d03e640 -1 rbd::mirror::InstanceWatcher: 0x56407f5aa380 handle_unregister_watch: error unregistering instance watcher for rbd_mirror_instance.5429 object: (2) No such file or directory 2026-02-18T08:52:13.846 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.trial163.stderr:2026-02-18T08:52:13.836+0000 7fef16831640 -1 rbd::mirror::InstanceWatcher: 0x56407f2d3180 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2026-02-18T08:52:13.846 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.trial163.stderr:2026-02-18T08:52:13.836+0000 7fef16030640 -1 rbd::mirror::InstanceWatcher: 0x56407f2d3500 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2026-02-18T08:52:13.846 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.trial163.stderr:2026-02-18T08:52:13.836+0000 7fef16831640 -1 rbd::mirror::InstanceWatcher: 0x56407f5aa380 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2026-02-18T08:52:13.847 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.trial163.stderr:2026-02-18T08:52:13.836+0000 7fef0f823640 -1 rbd::mirror::MirrorStatusUpdater 0x56407e9105a0 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-18T08:52:13.847 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.trial163.stderr:2026-02-18T08:52:13.836+0000 7fef1c83d640 -1 rbd::mirror::NamespaceReplayer: 0x56407d7e0700 handle_shut_down_remote_status_updater: error shutting remote mirror status updater down: (2) No such file or directory 2026-02-18T08:52:13.847 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.trial163.stderr:2026-02-18T08:52:13.836+0000 7fef10024640 -1 rbd::mirror::MirrorStatusUpdater 0x56407e910360 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-18T08:52:13.847 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.trial163.stderr:2026-02-18T08:52:13.836+0000 7fef1d03e640 -1 rbd::mirror::NamespaceReplayer: 0x56407e912000 handle_shut_down_remote_status_updater: error shutting remote mirror status updater down: (2) No such file or directory 2026-02-18T08:52:13.847 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.trial163.stderr:2026-02-18T08:52:13.836+0000 7fef0f823640 -1 rbd::mirror::MirrorStatusUpdater 0x56407e910480 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-18T08:52:13.847 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.trial163.stderr:2026-02-18T08:52:13.836+0000 7fef1d03e640 -1 rbd::mirror::NamespaceReplayer: 0x56407e9121c0 handle_shut_down_remote_status_updater: error shutting remote mirror status updater down: (2) No such file or directory 2026-02-18T08:52:13.847 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.trial163.stderr:2026-02-18T08:52:13.836+0000 7fef16030640 -1 rbd::mirror::MirrorStatusUpdater 0x56407e910000 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-18T08:52:13.847 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.trial163.stderr:2026-02-18T08:52:13.836+0000 7fef1c83d640 -1 rbd::mirror::NamespaceReplayer: 0x56407d7e0700 handle_shut_down_local_status_updater: error shutting local mirror status updater down: (2) No such file or directory 2026-02-18T08:52:13.847 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.trial163.stderr:2026-02-18T08:52:13.836+0000 7fef16831640 -1 rbd::mirror::MirrorStatusUpdater 0x56407e910120 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-18T08:52:13.847 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.trial163.stderr:2026-02-18T08:52:13.836+0000 7fef16831640 -1 rbd::mirror::MirrorStatusUpdater 0x56407e910240 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-18T08:52:13.847 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.trial163.stderr:2026-02-18T08:52:13.836+0000 7fef1d03e640 -1 rbd::mirror::NamespaceReplayer: 0x56407e912000 handle_shut_down_local_status_updater: error shutting local mirror status updater down: (2) No such file or directory 2026-02-18T08:52:13.847 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.trial163.stderr:2026-02-18T08:52:13.836+0000 7fef1d03e640 -1 rbd::mirror::NamespaceReplayer: 0x56407e9121c0 handle_shut_down_local_status_updater: error shutting local mirror status updater down: (2) No such file or directory 2026-02-18T08:52:13.909 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.trial163.stderr:2026-02-18T08:52:13.904+0000 7fef1d03e640 -1 rbd::mirror::LeaderWatcher: 0x56407f5fc300 handle_unregister_watch: error unregistering leader watcher for rbd_mirror_leader object: (2) No such file or directory 2026-02-18T08:52:13.911 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.trial163.stderr:2026-02-18T08:52:13.904+0000 7fef1c83d640 -1 rbd::mirror::InstanceWatcher: 0x56407f5ab500 handle_unregister_watch: error unregistering instance watcher for rbd_mirror_instance.5432 object: (2) No such file or directory 2026-02-18T08:52:13.911 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.trial163.stderr:2026-02-18T08:52:13.904+0000 7fef1c83d640 -1 rbd::mirror::InstanceWatcher: 0x56407f5abc00 handle_unregister_watch: error unregistering instance watcher for rbd_mirror_instance.5432 object: (2) No such file or directory 2026-02-18T08:52:13.911 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.trial163.stderr:2026-02-18T08:52:13.904+0000 7fef08815640 -1 rbd::mirror::InstanceWatcher: 0x56407f5ab500 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2026-02-18T08:52:13.912 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.trial163.stderr:2026-02-18T08:52:13.904+0000 7fef09016640 -1 rbd::mirror::InstanceWatcher: 0x56407f5abc00 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2026-02-18T08:52:13.913 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.trial163.stderr:2026-02-18T08:52:13.904+0000 7fef08815640 -1 rbd::mirror::MirrorStatusUpdater 0x56407ebe6900 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-18T08:52:13.913 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.trial163.stderr:2026-02-18T08:52:13.904+0000 7fef1d03e640 -1 rbd::mirror::NamespaceReplayer: 0x56407e912fc0 handle_shut_down_local_status_updater: error shutting local mirror status updater down: (2) No such file or directory 2026-02-18T08:52:13.913 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.trial163.stderr:2026-02-18T08:52:13.904+0000 7fef09016640 -1 rbd::mirror::MirrorStatusUpdater 0x56407ebe6a20 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-18T08:52:13.913 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.trial163.stderr:2026-02-18T08:52:13.904+0000 7fef1c83d640 -1 rbd::mirror::NamespaceReplayer: 0x56407e913180 handle_shut_down_local_status_updater: error shutting local mirror status updater down: (2) No such file or directory 2026-02-18T08:52:13.913 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.trial163.stderr:2026-02-18T08:52:13.904+0000 7fef1c83d640 -1 rbd::mirror::LeaderWatcher: 0x56407ec13b00 handle_unregister_watch: error unregistering leader watcher for rbd_mirror_leader object: (2) No such file or directory 2026-02-18T08:52:14.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:pool 'mirror_parent' does not exist 2026-02-18T08:52:14.197 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for cluster in "${CLUSTER1}" "${CLUSTER2}" 2026-02-18T08:52:14.197 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ stop_mirrors cluster1 2026-02-18T08:52:14.197 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-18T08:52:14.198 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local sig= 2026-02-18T08:52:14.198 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:52:14.200 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:52:14.200 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ stop_mirror cluster1:0 '' 2026-02-18T08:52:14.201 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-18T08:52:14.201 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local sig= 2026-02-18T08:52:14.201 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 1 2026-02-18T08:52:14.201 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 2026-02-18T08:52:14.201 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:52:14.201 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ stop_mirror cluster1:1 '' 2026-02-18T08:52:14.201 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-18T08:52:14.201 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local sig= 2026-02-18T08:52:14.201 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 1 2026-02-18T08:52:14.201 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 2026-02-18T08:52:14.201 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:52:14.201 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ stop_mirror cluster1:2 '' 2026-02-18T08:52:14.202 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-18T08:52:14.202 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local sig= 2026-02-18T08:52:14.202 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 1 2026-02-18T08:52:14.202 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 2026-02-18T08:52:14.202 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:52:14.202 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ stop_mirror cluster1:3 '' 2026-02-18T08:52:14.202 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-18T08:52:14.202 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local sig= 2026-02-18T08:52:14.202 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 1 2026-02-18T08:52:14.202 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 2026-02-18T08:52:14.202 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for cluster in "${CLUSTER1}" "${CLUSTER2}" 2026-02-18T08:52:14.202 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ stop_mirrors cluster2 2026-02-18T08:52:14.203 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-18T08:52:14.203 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local sig= 2026-02-18T08:52:14.203 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-18T08:52:14.204 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:52:14.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ stop_mirror cluster2:0 '' 2026-02-18T08:52:14.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2:0 2026-02-18T08:52:14.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local sig= 2026-02-18T08:52:14.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 1 2026-02-18T08:52:14.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 2026-02-18T08:52:14.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:52:14.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ stop_mirror cluster2:1 '' 2026-02-18T08:52:14.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2:1 2026-02-18T08:52:14.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local sig= 2026-02-18T08:52:14.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 1 2026-02-18T08:52:14.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 2026-02-18T08:52:14.206 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:52:14.206 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ stop_mirror cluster2:2 '' 2026-02-18T08:52:14.206 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2:2 2026-02-18T08:52:14.206 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local sig= 2026-02-18T08:52:14.206 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 1 2026-02-18T08:52:14.206 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 2026-02-18T08:52:14.206 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T08:52:14.206 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ stop_mirror cluster2:3 '' 2026-02-18T08:52:14.206 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2:3 2026-02-18T08:52:14.206 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local sig= 2026-02-18T08:52:14.206 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 1 2026-02-18T08:52:14.206 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 2026-02-18T08:52:14.206 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -z 1 ']' 2026-02-18T08:52:14.207 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test '' = /tmp/tmp.vybh1m9yJF 2026-02-18T08:52:14.207 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -Rf /tmp/tmp.vybh1m9yJF 2026-02-18T08:52:14.207 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 -eq 0 ']' 2026-02-18T08:52:14.207 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ echo OK 2026-02-18T08:52:14.207 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:OK 2026-02-18T08:52:14.208 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ exit 0 2026-02-18T08:52:14.208 INFO:teuthology.orchestra.run:Running command with timeout 3600 2026-02-18T08:52:14.209 DEBUG:teuthology.orchestra.run.trial163:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.cluster1.mirror/client.mirror/tmp 2026-02-18T08:52:14.227 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial163.stderr:2026-02-18T08:52:14.216+0000 7fbd386fd640 -1 rbd::mirror::LeaderWatcher: 0x55785dd12300 handle_notify_heartbeat: error notifying heartbeat: (2) No such file or directory, releasing leader 2026-02-18T08:52:14.227 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial163.stderr:2026-02-18T08:52:14.216+0000 7fbd31ef0640 -1 rbd::mirror::image_deleter::TrashWatcher: 0x5578667c4d80 handle_unregister_watcher: error unregistering watcher for trash directory: (2) No such file or directory 2026-02-18T08:52:14.227 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial163.stderr:2026-02-18T08:52:14.216+0000 7fbd31ef0640 -1 rbd::mirror::image_deleter::TrashWatcher: 0x55785d8018c0 handle_unregister_watcher: error unregistering watcher for trash directory: (2) No such file or directory 2026-02-18T08:52:14.227 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial163.stderr:2026-02-18T08:52:14.216+0000 7fbd31ef0640 -1 rbd::mirror::image_deleter::TrashWatcher: 0x5578667c4b40 handle_unregister_watcher: error unregistering watcher for trash directory: (2) No such file or directory 2026-02-18T08:52:14.227 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial163.stderr:2026-02-18T08:52:14.216+0000 7fbd316ef640 -1 rbd::mirror::PoolWatcher: 0x5578799821e0 operator(): error unregistering watcher for rbd_mirroring object: (2) No such file or directory 2026-02-18T08:52:14.227 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial163.stderr:2026-02-18T08:52:14.216+0000 7fbd2aee2640 -1 rbd::mirror::PoolWatcher: 0x55785be9d2c0 operator(): error unregistering watcher for rbd_mirroring object: (2) No such file or directory 2026-02-18T08:52:14.227 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial163.stderr:2026-02-18T08:52:14.216+0000 7fbd2aee2640 -1 rbd::mirror::PoolWatcher: 0x55785be9d860 operator(): error unregistering watcher for rbd_mirroring object: (2) No such file or directory 2026-02-18T08:52:14.228 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial163.stderr:2026-02-18T08:52:14.216+0000 7fbd2aee2640 -1 rbd::mirror::PoolWatcher: 0x55785df93e00 operator(): error unregistering watcher for rbd_mirroring object: (2) No such file or directory 2026-02-18T08:52:14.228 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial163.stderr:2026-02-18T08:52:14.216+0000 7fbd31ef0640 -1 rbd::mirror::PoolWatcher: 0x55785d08d0e0 operator(): error unregistering watcher for rbd_mirroring object: (2) No such file or directory 2026-02-18T08:52:14.228 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial163.stderr:2026-02-18T08:52:14.216+0000 7fbd31ef0640 -1 rbd::mirror::PoolWatcher: 0x55785d08da40 operator(): error unregistering watcher for rbd_mirroring object: (2) No such file or directory 2026-02-18T08:52:14.229 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial163.stderr:2026-02-18T08:52:14.216+0000 7fbd386fd640 -1 rbd::mirror::LeaderWatcher: 0x55785dd12300 handle_notify_lock_released: error notifying leader lock released: (2) No such file or directory 2026-02-18T08:52:14.229 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial163.stderr:2026-02-18T08:52:14.216+0000 7fbd386fd640 -1 rbd::mirror::LeaderWatcher: 0x55785dd12300 handle_release_leader_lock: error releasing lock: (2) No such file or directory 2026-02-18T08:52:14.237 INFO:tasks.workunit:Stopping ['rbd/rbd_mirror_stress.sh'] on cluster1.client.mirror... 2026-02-18T08:52:14.237 DEBUG:teuthology.orchestra.run.trial163:> sudo rm -rf -- /home/ubuntu/cephtest/workunits.list.cluster1.client.mirror /home/ubuntu/cephtest/clone.cluster1.client.mirror 2026-02-18T08:52:14.547 DEBUG:teuthology.parallel:result is None 2026-02-18T08:52:14.548 DEBUG:teuthology.orchestra.run.trial163:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.cluster1.mirror/client.mirror 2026-02-18T08:52:14.557 INFO:tasks.workunit:Deleted dir /home/ubuntu/cephtest/mnt.cluster1.mirror/client.mirror 2026-02-18T08:52:14.557 DEBUG:teuthology.orchestra.run.trial163:> rmdir -- /home/ubuntu/cephtest/mnt.cluster1.mirror 2026-02-18T08:52:14.607 INFO:tasks.workunit:Deleted artificial mount point /home/ubuntu/cephtest/mnt.cluster1.mirror/client.mirror 2026-02-18T08:52:14.608 DEBUG:teuthology.run_tasks:Unwinding manager rbd-mirror-thrash 2026-02-18T08:52:14.624 INFO:tasks.rbd_mirror_thrash:joining rbd_mirror_thrash 2026-02-18T08:52:14.624 INFO:tasks.rbd_mirror_thrash:stat['kill'] = 20 2026-02-18T08:52:14.624 INFO:tasks.rbd_mirror_thrash:done joining 2026-02-18T08:52:14.624 DEBUG:teuthology.run_tasks:Unwinding manager rbd-mirror-thrash 2026-02-18T08:52:14.639 INFO:tasks.rbd_mirror_thrash:joining rbd_mirror_thrash 2026-02-18T08:52:14.639 INFO:tasks.rbd_mirror_thrash:stat['kill'] = 16 2026-02-18T08:52:14.640 INFO:tasks.rbd_mirror_thrash:done joining 2026-02-18T08:52:14.640 DEBUG:teuthology.run_tasks:Unwinding manager rbd-mirror 2026-02-18T08:52:14.660 DEBUG:tasks.rbd_mirror.cluster2.client.mirror.3:waiting for process to exit 2026-02-18T08:52:14.661 INFO:teuthology.orchestra.run:waiting for 300 2026-02-18T08:52:14.661 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.trial163.stderr:2026-02-18T08:52:14.652+0000 7f8d066d6640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.3 (PID: 132343) UID: 1000 2026-02-18T08:52:14.663 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.trial163.stderr:2026-02-18T08:52:14.652+0000 7f8cff6c8640 -1 rbd::mirror::InstanceWatcher: 0x55ae40b38700 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2026-02-18T08:52:14.663 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.trial163.stderr:2026-02-18T08:52:14.652+0000 7f8cfeec7640 -1 rbd::mirror::InstanceWatcher: 0x55ae40b38380 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2026-02-18T08:52:14.663 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.trial163.stderr:2026-02-18T08:52:14.652+0000 7f8cff6c8640 -1 rbd::mirror::InstanceWatcher: 0x55ae40b38a80 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2026-02-18T08:52:14.664 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.trial163.stderr:2026-02-18T08:52:14.652+0000 7f8d05ed5640 -1 rbd::mirror::InstanceWatcher: 0x55ae40b38e00 handle_unregister_watch: error unregistering instance watcher for rbd_mirror_instance.14775 object: (2) No such file or directory 2026-02-18T08:52:14.664 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.trial163.stderr:2026-02-18T08:52:14.652+0000 7f8d05ed5640 -1 rbd::mirror::InstanceWatcher: 0x55ae40dda380 handle_unregister_watch: error unregistering instance watcher for rbd_mirror_instance.14775 object: (2) No such file or directory 2026-02-18T08:52:14.664 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.trial163.stderr:2026-02-18T08:52:14.652+0000 7f8cf1ead640 -1 rbd::mirror::InstanceWatcher: 0x55ae40b38e00 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2026-02-18T08:52:14.664 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.trial163.stderr:2026-02-18T08:52:14.652+0000 7f8cf16ac640 -1 rbd::mirror::InstanceWatcher: 0x55ae40dda380 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2026-02-18T08:52:14.664 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.trial163.stderr:2026-02-18T08:52:14.652+0000 7f8cf86ba640 -1 rbd::mirror::MirrorStatusUpdater 0x55ae409265a0 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-18T08:52:14.664 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.trial163.stderr:2026-02-18T08:52:14.652+0000 7f8cf86ba640 -1 rbd::mirror::MirrorStatusUpdater 0x55ae409266c0 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-18T08:52:14.664 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.trial163.stderr:2026-02-18T08:52:14.652+0000 7f8cf86ba640 -1 rbd::mirror::MirrorStatusUpdater 0x55ae4058afc0 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-18T08:52:14.664 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.trial163.stderr:2026-02-18T08:52:14.652+0000 7f8d05ed5640 -1 rbd::mirror::NamespaceReplayer: 0x55ae40924000 handle_shut_down_remote_status_updater: error shutting remote mirror status updater down: (2) No such file or directory 2026-02-18T08:52:14.664 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.trial163.stderr:2026-02-18T08:52:14.652+0000 7f8d05ed5640 -1 rbd::mirror::NamespaceReplayer: 0x55ae409241c0 handle_shut_down_remote_status_updater: error shutting remote mirror status updater down: (2) No such file or directory 2026-02-18T08:52:14.664 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.trial163.stderr:2026-02-18T08:52:14.652+0000 7f8d05ed5640 -1 rbd::mirror::NamespaceReplayer: 0x55ae40924380 handle_shut_down_remote_status_updater: error shutting remote mirror status updater down: (2) No such file or directory 2026-02-18T08:52:14.664 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.trial163.stderr:2026-02-18T08:52:14.652+0000 7f8cf1ead640 -1 rbd::mirror::MirrorStatusUpdater 0x55ae40bb6b40 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-18T08:52:14.664 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.trial163.stderr:2026-02-18T08:52:14.652+0000 7f8cf1ead640 -1 rbd::mirror::MirrorStatusUpdater 0x55ae40bb6c60 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-18T08:52:14.665 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.trial163.stderr:2026-02-18T08:52:14.652+0000 7f8d05ed5640 -1 rbd::mirror::NamespaceReplayer: 0x55ae40925340 handle_shut_down_local_status_updater: error shutting local mirror status updater down: (2) No such file or directory 2026-02-18T08:52:14.665 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.trial163.stderr:2026-02-18T08:52:14.652+0000 7f8d05ed5640 -1 rbd::mirror::NamespaceReplayer: 0x55ae40925500 handle_shut_down_local_status_updater: error shutting local mirror status updater down: (2) No such file or directory 2026-02-18T08:52:14.665 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.trial163.stderr:2026-02-18T08:52:14.652+0000 7f8cfeec7640 -1 rbd::mirror::MirrorStatusUpdater 0x55ae40926000 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-18T08:52:14.665 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.trial163.stderr:2026-02-18T08:52:14.652+0000 7f8cfeec7640 -1 rbd::mirror::MirrorStatusUpdater 0x55ae40926120 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-18T08:52:14.665 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.trial163.stderr:2026-02-18T08:52:14.652+0000 7f8d05ed5640 -1 rbd::mirror::NamespaceReplayer: 0x55ae40924000 handle_shut_down_local_status_updater: error shutting local mirror status updater down: (2) No such file or directory 2026-02-18T08:52:14.665 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.trial163.stderr:2026-02-18T08:52:14.652+0000 7f8d05ed5640 -1 rbd::mirror::NamespaceReplayer: 0x55ae409241c0 handle_shut_down_local_status_updater: error shutting local mirror status updater down: (2) No such file or directory 2026-02-18T08:52:14.665 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.trial163.stderr:2026-02-18T08:52:14.652+0000 7f8d05ed5640 -1 rbd::mirror::LeaderWatcher: 0x55ae40be1b00 handle_unregister_watch: error unregistering leader watcher for rbd_mirror_leader object: (2) No such file or directory 2026-02-18T08:52:14.666 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.trial163.stderr:2026-02-18T08:52:14.656+0000 7f8d05ed5640 -1 rbd::mirror::LeaderWatcher: 0x55ae406f8300 handle_unregister_watch: error unregistering leader watcher for rbd_mirror_leader object: (2) No such file or directory 2026-02-18T08:52:14.678 INFO:tasks.rbd_mirror.cluster2.client.mirror.3:Stopped 2026-02-18T08:52:14.678 DEBUG:teuthology.run_tasks:Unwinding manager rbd-mirror 2026-02-18T08:52:14.695 DEBUG:tasks.rbd_mirror.cluster2.client.mirror.2:waiting for process to exit 2026-02-18T08:52:14.696 INFO:teuthology.orchestra.run:waiting for 300 2026-02-18T08:52:14.720 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial163.stderr:2026-02-18T08:52:14.712+0000 7fc4745ff640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.2 (PID: 133754) UID: 1000 2026-02-18T08:52:14.723 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial163.stderr:2026-02-18T08:52:14.712+0000 7fc473dfe640 -1 rbd::mirror::InstanceWatcher: 0x560015c2a380 handle_unregister_watch: error unregistering instance watcher for rbd_mirror_instance.14893 object: (2) No such file or directory 2026-02-18T08:52:14.723 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial163.stderr:2026-02-18T08:52:14.712+0000 7fc473dfe640 -1 rbd::mirror::InstanceWatcher: 0x560015c2a700 handle_unregister_watch: error unregistering instance watcher for rbd_mirror_instance.14893 object: (2) No such file or directory 2026-02-18T08:52:14.723 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial163.stderr:2026-02-18T08:52:14.712+0000 7fc473dfe640 -1 rbd::mirror::InstanceWatcher: 0x560015c2a000 handle_unregister_watch: error unregistering instance watcher for rbd_mirror_instance.14893 object: (2) No such file or directory 2026-02-18T08:52:14.723 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial163.stderr:2026-02-18T08:52:14.712+0000 7fc473dfe640 -1 rbd::mirror::InstanceWatcher: 0x560015ed8380 handle_unregister_watch: error unregistering instance watcher for rbd_mirror_instance.14896 object: (2) No such file or directory 2026-02-18T08:52:14.723 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial163.stderr:2026-02-18T08:52:14.712+0000 7fc473dfe640 -1 rbd::mirror::InstanceWatcher: 0x560015c2aa80 handle_unregister_watch: error unregistering instance watcher for rbd_mirror_instance.14896 object: (2) No such file or directory 2026-02-18T08:52:14.723 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial163.stderr:2026-02-18T08:52:14.712+0000 7fc45fdd6640 -1 rbd::mirror::InstanceWatcher: 0x560015ed8380 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2026-02-18T08:52:14.723 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial163.stderr:2026-02-18T08:52:14.712+0000 7fc45fdd6640 -1 rbd::mirror::InstanceWatcher: 0x560015c2aa80 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2026-02-18T08:52:14.723 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial163.stderr:2026-02-18T08:52:14.712+0000 7fc46d5f1640 -1 rbd::mirror::InstanceWatcher: 0x560015c2a380 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2026-02-18T08:52:14.723 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial163.stderr:2026-02-18T08:52:14.712+0000 7fc46d5f1640 -1 rbd::mirror::InstanceWatcher: 0x560015c2a000 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2026-02-18T08:52:14.723 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial163.stderr:2026-02-18T08:52:14.712+0000 7fc46d5f1640 -1 rbd::mirror::InstanceWatcher: 0x560015c2a700 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2026-02-18T08:52:14.724 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial163.stderr:2026-02-18T08:52:14.712+0000 7fc458dc8640 -1 rbd::mirror::MirrorStatusUpdater 0x560015cfcc60 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-18T08:52:14.724 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial163.stderr:2026-02-18T08:52:14.712+0000 7fc458dc8640 -1 rbd::mirror::MirrorStatusUpdater 0x560015cfcfc0 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-18T08:52:14.724 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial163.stderr:2026-02-18T08:52:14.712+0000 7fc4735fd640 -1 rbd::mirror::NamespaceReplayer: 0x560015a56fc0 handle_shut_down_remote_status_updater: error shutting remote mirror status updater down: (2) No such file or directory 2026-02-18T08:52:14.724 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial163.stderr:2026-02-18T08:52:14.712+0000 7fc4735fd640 -1 rbd::mirror::NamespaceReplayer: 0x560015a57180 handle_shut_down_remote_status_updater: error shutting remote mirror status updater down: (2) No such file or directory 2026-02-18T08:52:14.724 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial163.stderr:2026-02-18T08:52:14.712+0000 7fc4665e3640 -1 rbd::mirror::MirrorStatusUpdater 0x560015a525a0 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-18T08:52:14.724 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial163.stderr:2026-02-18T08:52:14.712+0000 7fc4665e3640 -1 rbd::mirror::MirrorStatusUpdater 0x560015a526c0 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-18T08:52:14.724 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial163.stderr:2026-02-18T08:52:14.712+0000 7fc4665e3640 -1 rbd::mirror::MirrorStatusUpdater 0x5600156acfc0 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-18T08:52:14.724 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial163.stderr:2026-02-18T08:52:14.712+0000 7fc473dfe640 -1 rbd::mirror::NamespaceReplayer: 0x560015a56000 handle_shut_down_remote_status_updater: error shutting remote mirror status updater down: (2) No such file or directory 2026-02-18T08:52:14.724 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial163.stderr:2026-02-18T08:52:14.712+0000 7fc473dfe640 -1 rbd::mirror::NamespaceReplayer: 0x5600146c0e00 handle_shut_down_remote_status_updater: error shutting remote mirror status updater down: (2) No such file or directory 2026-02-18T08:52:14.724 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial163.stderr:2026-02-18T08:52:14.712+0000 7fc473dfe640 -1 rbd::mirror::NamespaceReplayer: 0x560015a561c0 handle_shut_down_remote_status_updater: error shutting remote mirror status updater down: (2) No such file or directory 2026-02-18T08:52:14.724 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial163.stderr:2026-02-18T08:52:14.712+0000 7fc45f5d5640 -1 rbd::mirror::MirrorStatusUpdater 0x560015cfc900 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-18T08:52:14.724 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial163.stderr:2026-02-18T08:52:14.712+0000 7fc45f5d5640 -1 rbd::mirror::MirrorStatusUpdater 0x560015cfca20 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-18T08:52:14.725 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial163.stderr:2026-02-18T08:52:14.712+0000 7fc4735fd640 -1 rbd::mirror::NamespaceReplayer: 0x560015a56fc0 handle_shut_down_local_status_updater: error shutting local mirror status updater down: (2) No such file or directory 2026-02-18T08:52:14.725 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial163.stderr:2026-02-18T08:52:14.712+0000 7fc473dfe640 -1 rbd::mirror::NamespaceReplayer: 0x560015a57180 handle_shut_down_local_status_updater: error shutting local mirror status updater down: (2) No such file or directory 2026-02-18T08:52:14.725 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial163.stderr:2026-02-18T08:52:14.712+0000 7fc46d5f1640 -1 rbd::mirror::MirrorStatusUpdater 0x560015a52120 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-18T08:52:14.725 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial163.stderr:2026-02-18T08:52:14.712+0000 7fc473dfe640 -1 rbd::mirror::NamespaceReplayer: 0x560015a56000 handle_shut_down_local_status_updater: error shutting local mirror status updater down: (2) No such file or directory 2026-02-18T08:52:14.725 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial163.stderr:2026-02-18T08:52:14.712+0000 7fc46d5f1640 -1 rbd::mirror::MirrorStatusUpdater 0x560015a52000 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-18T08:52:14.725 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial163.stderr:2026-02-18T08:52:14.712+0000 7fc46d5f1640 -1 rbd::mirror::MirrorStatusUpdater 0x560015a52240 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-18T08:52:14.725 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial163.stderr:2026-02-18T08:52:14.712+0000 7fc473dfe640 -1 rbd::mirror::NamespaceReplayer: 0x5600146c0e00 handle_shut_down_local_status_updater: error shutting local mirror status updater down: (2) No such file or directory 2026-02-18T08:52:14.725 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial163.stderr:2026-02-18T08:52:14.712+0000 7fc473dfe640 -1 rbd::mirror::NamespaceReplayer: 0x560015a561c0 handle_shut_down_local_status_updater: error shutting local mirror status updater down: (2) No such file or directory 2026-02-18T08:52:14.725 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial163.stderr:2026-02-18T08:52:14.712+0000 7fc4735fd640 -1 rbd::mirror::LeaderWatcher: 0x560015d29b00 handle_unregister_watch: error unregistering leader watcher for rbd_mirror_leader object: (2) No such file or directory 2026-02-18T08:52:14.726 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial163.stderr:2026-02-18T08:52:14.712+0000 7fbd386fd640 -1 rbd::mirror::LeaderWatcher: 0x55785d2ffb00 handle_notify_heartbeat: error notifying heartbeat: (2) No such file or directory, releasing leader 2026-02-18T08:52:14.726 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial163.stderr:2026-02-18T08:52:14.712+0000 7fbd246d5640 -1 rbd::mirror::image_deleter::TrashWatcher: 0x55785d6a9200 handle_unregister_watcher: error unregistering watcher for trash directory: (2) No such file or directory 2026-02-18T08:52:14.727 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial163.stderr:2026-02-18T08:52:14.712+0000 7fbd246d5640 -1 rbd::mirror::image_deleter::TrashWatcher: 0x55785d7a3d40 handle_unregister_watcher: error unregistering watcher for trash directory: (2) No such file or directory 2026-02-18T08:52:14.727 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial163.stderr:2026-02-18T08:52:14.712+0000 7fbd23ed4640 -1 rbd::mirror::PoolWatcher: 0x55785df923c0 operator(): error unregistering watcher for rbd_mirroring object: (2) No such file or directory 2026-02-18T08:52:14.727 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial163.stderr:2026-02-18T08:52:14.712+0000 7fbd23ed4640 -1 rbd::mirror::PoolWatcher: 0x55785df92000 operator(): error unregistering watcher for rbd_mirroring object: (2) No such file or directory 2026-02-18T08:52:14.727 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial163.stderr:2026-02-18T08:52:14.712+0000 7fbd1dec8640 -1 rbd::mirror::PoolWatcher: 0x55785df92780 operator(): error unregistering watcher for rbd_mirroring object: (2) No such file or directory 2026-02-18T08:52:14.727 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial163.stderr:2026-02-18T08:52:14.712+0000 7fbd1dec8640 -1 rbd::mirror::PoolWatcher: 0x55785df92f00 operator(): error unregistering watcher for rbd_mirroring object: (2) No such file or directory 2026-02-18T08:52:14.727 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial163.stderr:2026-02-18T08:52:14.712+0000 7fbd37efc640 -1 rbd::mirror::LeaderWatcher: 0x55785d2ffb00 handle_notify_lock_released: error notifying leader lock released: (2) No such file or directory 2026-02-18T08:52:14.727 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial163.stderr:2026-02-18T08:52:14.712+0000 7fbd37efc640 -1 rbd::mirror::LeaderWatcher: 0x55785d2ffb00 handle_release_leader_lock: error releasing lock: (2) No such file or directory 2026-02-18T08:52:14.727 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial163.stderr:2026-02-18T08:52:14.716+0000 7fc4735fd640 -1 rbd::mirror::LeaderWatcher: 0x56001580c300 handle_unregister_watch: error unregistering leader watcher for rbd_mirror_leader object: (2) No such file or directory 2026-02-18T08:52:14.738 INFO:tasks.rbd_mirror.cluster2.client.mirror.2:Stopped 2026-02-18T08:52:14.738 DEBUG:teuthology.run_tasks:Unwinding manager rbd-mirror 2026-02-18T08:52:14.758 DEBUG:tasks.rbd_mirror.cluster2.client.mirror.1:waiting for process to exit 2026-02-18T08:52:14.759 INFO:teuthology.orchestra.run:waiting for 300 2026-02-18T08:52:14.779 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.trial163.stderr:2026-02-18T08:52:14.772+0000 7f3a18ef4640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.1 (PID: 130444) UID: 1000 2026-02-18T08:52:14.783 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.trial163.stderr:2026-02-18T08:52:14.772+0000 7f3a046cb640 -1 rbd::mirror::image_deleter::TrashWatcher: 0x55ac22b8eb40 handle_unregister_watcher: error unregistering watcher for trash directory: (2) No such file or directory 2026-02-18T08:52:14.784 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.trial163.stderr:2026-02-18T08:52:14.772+0000 7f3a11ee6640 -1 rbd::mirror::image_deleter::TrashWatcher: 0x55ac22b8f440 handle_unregister_watcher: error unregistering watcher for trash directory: (2) No such file or directory 2026-02-18T08:52:14.784 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.trial163.stderr:2026-02-18T08:52:14.772+0000 7f3a046cb640 -1 rbd::mirror::image_deleter::TrashWatcher: 0x55ac22b8f8c0 handle_unregister_watcher: error unregistering watcher for trash directory: (2) No such file or directory 2026-02-18T08:52:14.784 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.trial163.stderr:2026-02-18T08:52:14.772+0000 7f3a11ee6640 -1 rbd::mirror::image_deleter::TrashWatcher: 0x55ac229f4240 handle_unregister_watcher: error unregistering watcher for trash directory: (2) No such file or directory 2026-02-18T08:52:14.784 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.trial163.stderr:2026-02-18T08:52:14.772+0000 7f3a11ee6640 -1 rbd::mirror::image_deleter::TrashWatcher: 0x55ac229f4000 handle_unregister_watcher: error unregistering watcher for trash directory: (2) No such file or directory 2026-02-18T08:52:14.784 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.trial163.stderr:2026-02-18T08:52:14.772+0000 7f3a046cb640 -1 rbd::mirror::PoolWatcher: 0x55ac223b2000 operator(): error unregistering watcher for rbd_mirroring object: (2) No such file or directory 2026-02-18T08:52:14.784 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.trial163.stderr:2026-02-18T08:52:14.772+0000 7f3a046cb640 -1 rbd::mirror::PoolWatcher: 0x55ac223b2780 operator(): error unregistering watcher for rbd_mirroring object: (2) No such file or directory 2026-02-18T08:52:14.784 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.trial163.stderr:2026-02-18T08:52:14.772+0000 7f3a116e5640 -1 rbd::mirror::PoolWatcher: 0x55ac223b30e0 operator(): error unregistering watcher for rbd_mirroring object: (2) No such file or directory 2026-02-18T08:52:14.784 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.trial163.stderr:2026-02-18T08:52:14.772+0000 7f3a116e5640 -1 rbd::mirror::PoolWatcher: 0x55ac223b34a0 operator(): error unregistering watcher for rbd_mirroring object: (2) No such file or directory 2026-02-18T08:52:14.784 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.trial163.stderr:2026-02-18T08:52:14.772+0000 7f3a116e5640 -1 rbd::mirror::PoolWatcher: 0x55ac223b32c0 operator(): error unregistering watcher for rbd_mirroring object: (2) No such file or directory 2026-02-18T08:52:14.784 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.trial163.stderr:2026-02-18T08:52:14.772+0000 7f39fdebe640 -1 rbd::mirror::PoolWatcher: 0x55ac223b21e0 operator(): error unregistering watcher for rbd_mirroring object: (2) No such file or directory 2026-02-18T08:52:14.784 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.trial163.stderr:2026-02-18T08:52:14.772+0000 7f3a0b6d9640 -1 rbd::mirror::PoolWatcher: 0x55ac223b3e00 operator(): error unregistering watcher for rbd_mirroring object: (2) No such file or directory 2026-02-18T08:52:14.785 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.trial163.stderr:2026-02-18T08:52:14.772+0000 7f39fd6bd640 -1 rbd::mirror::PoolWatcher: 0x55ac22ee01e0 operator(): error unregistering watcher for rbd_mirroring object: (2) No such file or directory 2026-02-18T08:52:14.785 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.trial163.stderr:2026-02-18T08:52:14.772+0000 7f3a0b6d9640 -1 rbd::mirror::PoolWatcher: 0x55ac223b3680 operator(): error unregistering watcher for rbd_mirroring object: (2) No such file or directory 2026-02-18T08:52:14.785 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.trial163.stderr:2026-02-18T08:52:14.772+0000 7f3a0b6d9640 -1 rbd::mirror::PoolWatcher: 0x55ac223b3860 operator(): error unregistering watcher for rbd_mirroring object: (2) No such file or directory 2026-02-18T08:52:14.785 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.trial163.stderr:2026-02-18T08:52:14.772+0000 7f3a17ef2640 -1 rbd::mirror::InstanceWatcher: 0x55ac22ba2380 handle_unregister_watch: error unregistering instance watcher for rbd_mirror_instance.14694 object: (2) No such file or directory 2026-02-18T08:52:14.785 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.trial163.stderr:2026-02-18T08:52:14.772+0000 7f3a186f3640 -1 rbd::mirror::InstanceWatcher: 0x55ac229e8000 handle_unregister_watch: error unregistering instance watcher for rbd_mirror_instance.14691 object: (2) No such file or directory 2026-02-18T08:52:14.785 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.trial163.stderr:2026-02-18T08:52:14.772+0000 7f3a186f3640 -1 rbd::mirror::InstanceWatcher: 0x55ac229e8380 handle_unregister_watch: error unregistering instance watcher for rbd_mirror_instance.14691 object: (2) No such file or directory 2026-02-18T08:52:14.785 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.trial163.stderr:2026-02-18T08:52:14.772+0000 7f3a186f3640 -1 rbd::mirror::InstanceWatcher: 0x55ac22f80380 handle_unregister_watch: error unregistering instance watcher for rbd_mirror_instance.14691 object: (2) No such file or directory 2026-02-18T08:52:14.785 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.trial163.stderr:2026-02-18T08:52:14.772+0000 7f3a17ef2640 -1 rbd::mirror::InstanceWatcher: 0x55ac229e8700 handle_unregister_watch: error unregistering instance watcher for rbd_mirror_instance.14694 object: (2) No such file or directory 2026-02-18T08:52:14.785 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.trial163.stderr:2026-02-18T08:52:14.772+0000 7f3a046cb640 -1 rbd::mirror::InstanceWatcher: 0x55ac22ba2380 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2026-02-18T08:52:14.785 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.trial163.stderr:2026-02-18T08:52:14.772+0000 7f3a116e5640 -1 rbd::mirror::InstanceWatcher: 0x55ac229e8000 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2026-02-18T08:52:14.785 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.trial163.stderr:2026-02-18T08:52:14.772+0000 7f3a116e5640 -1 rbd::mirror::InstanceWatcher: 0x55ac229e8380 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2026-02-18T08:52:14.785 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.trial163.stderr:2026-02-18T08:52:14.772+0000 7f3a116e5640 -1 rbd::mirror::InstanceWatcher: 0x55ac22f80380 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2026-02-18T08:52:14.786 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.trial163.stderr:2026-02-18T08:52:14.772+0000 7f3a03eca640 -1 rbd::mirror::InstanceWatcher: 0x55ac229e8700 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2026-02-18T08:52:14.786 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.trial163.stderr:2026-02-18T08:52:14.772+0000 7f39fdebe640 -1 rbd::mirror::MirrorStatusUpdater 0x55ac22a64d80 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-18T08:52:14.786 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.trial163.stderr:2026-02-18T08:52:14.772+0000 7f3a186f3640 -1 rbd::mirror::NamespaceReplayer: 0x55ac228d08c0 handle_shut_down_remote_status_updater: error shutting remote mirror status updater down: (2) No such file or directory 2026-02-18T08:52:14.786 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.trial163.stderr:2026-02-18T08:52:14.772+0000 7f3a0aed8640 -1 rbd::mirror::MirrorStatusUpdater 0x55ac228006c0 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-18T08:52:14.786 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.trial163.stderr:2026-02-18T08:52:14.772+0000 7f3a17ef2640 -1 rbd::mirror::NamespaceReplayer: 0x55ac21463180 handle_shut_down_remote_status_updater: error shutting remote mirror status updater down: (2) No such file or directory 2026-02-18T08:52:14.786 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.trial163.stderr:2026-02-18T08:52:14.772+0000 7f3a0b6d9640 -1 rbd::mirror::MirrorStatusUpdater 0x55ac228005a0 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-18T08:52:14.786 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.trial163.stderr:2026-02-18T08:52:14.772+0000 7f3a0b6d9640 -1 rbd::mirror::MirrorStatusUpdater 0x55ac23108000 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-18T08:52:14.786 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.trial163.stderr:2026-02-18T08:52:14.772+0000 7f39fd6bd640 -1 rbd::mirror::MirrorStatusUpdater 0x55ac22a650e0 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-18T08:52:14.786 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.trial163.stderr:2026-02-18T08:52:14.772+0000 7f3a046cb640 -1 rbd::mirror::MirrorStatusUpdater 0x55ac22a64b40 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-18T08:52:14.786 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.trial163.stderr:2026-02-18T08:52:14.772+0000 7f3a17ef2640 -1 rbd::mirror::NamespaceReplayer: 0x55ac21463340 handle_shut_down_remote_status_updater: error shutting remote mirror status updater down: (2) No such file or directory 2026-02-18T08:52:14.786 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.trial163.stderr:2026-02-18T08:52:14.772+0000 7f3a17ef2640 -1 rbd::mirror::NamespaceReplayer: 0x55ac21462700 handle_shut_down_remote_status_updater: error shutting remote mirror status updater down: (2) No such file or directory 2026-02-18T08:52:14.786 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.trial163.stderr:2026-02-18T08:52:14.772+0000 7f3a17ef2640 -1 rbd::mirror::NamespaceReplayer: 0x55ac228d0700 handle_shut_down_remote_status_updater: error shutting remote mirror status updater down: (2) No such file or directory 2026-02-18T08:52:14.786 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.trial163.stderr:2026-02-18T08:52:14.772+0000 7f3a17ef2640 -1 rbd::mirror::NamespaceReplayer: 0x55ac228d08c0 handle_shut_down_local_status_updater: error shutting local mirror status updater down: (2) No such file or directory 2026-02-18T08:52:14.787 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.trial163.stderr:2026-02-18T08:52:14.772+0000 7f3a11ee6640 -1 rbd::mirror::MirrorStatusUpdater 0x55ac22800000 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-18T08:52:14.787 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.trial163.stderr:2026-02-18T08:52:14.772+0000 7f3a11ee6640 -1 rbd::mirror::MirrorStatusUpdater 0x55ac22800120 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-18T08:52:14.787 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.trial163.stderr:2026-02-18T08:52:14.772+0000 7f3a17ef2640 -1 rbd::mirror::NamespaceReplayer: 0x55ac21463180 handle_shut_down_local_status_updater: error shutting local mirror status updater down: (2) No such file or directory 2026-02-18T08:52:14.787 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.trial163.stderr:2026-02-18T08:52:14.772+0000 7f3a11ee6640 -1 rbd::mirror::MirrorStatusUpdater 0x55ac22800240 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-18T08:52:14.787 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.trial163.stderr:2026-02-18T08:52:14.772+0000 7f3a186f3640 -1 rbd::mirror::NamespaceReplayer: 0x55ac21463340 handle_shut_down_local_status_updater: error shutting local mirror status updater down: (2) No such file or directory 2026-02-18T08:52:14.787 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.trial163.stderr:2026-02-18T08:52:14.772+0000 7f3a186f3640 -1 rbd::mirror::NamespaceReplayer: 0x55ac21462700 handle_shut_down_local_status_updater: error shutting local mirror status updater down: (2) No such file or directory 2026-02-18T08:52:14.787 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.trial163.stderr:2026-02-18T08:52:14.772+0000 7f3a03eca640 -1 rbd::mirror::MirrorStatusUpdater 0x55ac22a64a20 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-18T08:52:14.787 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.trial163.stderr:2026-02-18T08:52:14.772+0000 7f3a186f3640 -1 rbd::mirror::NamespaceReplayer: 0x55ac228d0700 handle_shut_down_local_status_updater: error shutting local mirror status updater down: (2) No such file or directory 2026-02-18T08:52:14.787 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.trial163.stderr:2026-02-18T08:52:14.772+0000 7f3a17ef2640 -1 rbd::mirror::LeaderWatcher: 0x55ac22ac5b00 handle_notify_lock_released: error notifying leader lock released: (2) No such file or directory 2026-02-18T08:52:14.787 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.trial163.stderr:2026-02-18T08:52:14.772+0000 7f3a17ef2640 -1 rbd::mirror::LeaderWatcher: 0x55ac22ac5b00 handle_shut_down_leader_lock: error shutting down leader lock: (2) No such file or directory 2026-02-18T08:52:14.787 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.trial163.stderr:2026-02-18T08:52:14.772+0000 7f3a186f3640 -1 rbd::mirror::LeaderWatcher: 0x55ac22ac5b00 handle_unregister_watch: error unregistering leader watcher for rbd_mirror_leader object: (2) No such file or directory 2026-02-18T08:52:14.788 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.trial163.stderr:2026-02-18T08:52:14.776+0000 7f3a186f3640 -1 rbd::mirror::LeaderWatcher: 0x55ac225f6300 handle_notify_lock_released: error notifying leader lock released: (2) No such file or directory 2026-02-18T08:52:14.788 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.trial163.stderr:2026-02-18T08:52:14.776+0000 7f3a186f3640 -1 rbd::mirror::LeaderWatcher: 0x55ac225f6300 handle_shut_down_leader_lock: error shutting down leader lock: (2) No such file or directory 2026-02-18T08:52:14.788 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.trial163.stderr:2026-02-18T08:52:14.776+0000 7f3a17ef2640 -1 rbd::mirror::LeaderWatcher: 0x55ac225f6300 handle_unregister_watch: error unregistering leader watcher for rbd_mirror_leader object: (2) No such file or directory 2026-02-18T08:52:14.798 INFO:tasks.rbd_mirror.cluster2.client.mirror.1:Stopped 2026-02-18T08:52:14.798 DEBUG:teuthology.run_tasks:Unwinding manager rbd-mirror 2026-02-18T08:52:14.816 DEBUG:tasks.rbd_mirror.cluster2.client.mirror.0:waiting for process to exit 2026-02-18T08:52:14.816 INFO:teuthology.orchestra.run:waiting for 300 2026-02-18T08:52:14.839 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.trial163.stderr:2026-02-18T08:52:14.832+0000 7fedfa2a4640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.0 (PID: 117736) UID: 1000 2026-02-18T08:52:14.842 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.trial163.stderr:2026-02-18T08:52:14.832+0000 7fedf9aa3640 -1 rbd::mirror::InstanceWatcher: 0x5653b9d5aa80 handle_unregister_watch: error unregistering instance watcher for rbd_mirror_instance.13572 object: (2) No such file or directory 2026-02-18T08:52:14.842 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.trial163.stderr:2026-02-18T08:52:14.832+0000 7fedf9aa3640 -1 rbd::mirror::InstanceWatcher: 0x5653b9f24380 handle_unregister_watch: error unregistering instance watcher for rbd_mirror_instance.13572 object: (2) No such file or directory 2026-02-18T08:52:14.842 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.trial163.stderr:2026-02-18T08:52:14.832+0000 7fedf92a2640 -1 rbd::mirror::InstanceWatcher: 0x5653b9d5a000 handle_unregister_watch: error unregistering instance watcher for rbd_mirror_instance.13570 object: (2) No such file or directory 2026-02-18T08:52:14.842 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.trial163.stderr:2026-02-18T08:52:14.832+0000 7fedf92a2640 -1 rbd::mirror::InstanceWatcher: 0x5653b9d5a700 handle_unregister_watch: error unregistering instance watcher for rbd_mirror_instance.13570 object: (2) No such file or directory 2026-02-18T08:52:14.842 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.trial163.stderr:2026-02-18T08:52:14.832+0000 7fedf92a2640 -1 rbd::mirror::InstanceWatcher: 0x5653b9d5a380 handle_unregister_watch: error unregistering instance watcher for rbd_mirror_instance.13570 object: (2) No such file or directory 2026-02-18T08:52:14.843 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.trial163.stderr:2026-02-18T08:52:14.832+0000 7fede5a7b640 -1 rbd::mirror::InstanceWatcher: 0x5653b9d5aa80 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2026-02-18T08:52:14.843 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.trial163.stderr:2026-02-18T08:52:14.832+0000 7fede5a7b640 -1 rbd::mirror::InstanceWatcher: 0x5653b9f24380 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2026-02-18T08:52:14.843 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.trial163.stderr:2026-02-18T08:52:14.832+0000 7fedf3296640 -1 rbd::mirror::InstanceWatcher: 0x5653b9d5a000 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2026-02-18T08:52:14.843 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.trial163.stderr:2026-02-18T08:52:14.832+0000 7fedf3296640 -1 rbd::mirror::InstanceWatcher: 0x5653b9d5a700 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2026-02-18T08:52:14.843 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.trial163.stderr:2026-02-18T08:52:14.832+0000 7fedf3296640 -1 rbd::mirror::InstanceWatcher: 0x5653b9d5a380 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2026-02-18T08:52:14.843 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.trial163.stderr:2026-02-18T08:52:14.832+0000 7feddea6d640 -1 rbd::mirror::MirrorStatusUpdater 0x5653b9d2aea0 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-18T08:52:14.843 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.trial163.stderr:2026-02-18T08:52:14.832+0000 7feddea6d640 -1 rbd::mirror::MirrorStatusUpdater 0x5653b9d2ad80 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-18T08:52:14.843 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.trial163.stderr:2026-02-18T08:52:14.832+0000 7fedf92a2640 -1 rbd::mirror::NamespaceReplayer: 0x5653b9efc1c0 handle_shut_down_remote_status_updater: error shutting remote mirror status updater down: (2) No such file or directory 2026-02-18T08:52:14.843 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.trial163.stderr:2026-02-18T08:52:14.832+0000 7fedf92a2640 -1 rbd::mirror::NamespaceReplayer: 0x5653b9efc380 handle_shut_down_remote_status_updater: error shutting remote mirror status updater down: (2) No such file or directory 2026-02-18T08:52:14.843 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.trial163.stderr:2026-02-18T08:52:14.832+0000 7fedeca89640 -1 rbd::mirror::MirrorStatusUpdater 0x5653b9a98360 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-18T08:52:14.843 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.trial163.stderr:2026-02-18T08:52:14.832+0000 7fedeca89640 -1 rbd::mirror::MirrorStatusUpdater 0x5653b9a985a0 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-18T08:52:14.843 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.trial163.stderr:2026-02-18T08:52:14.832+0000 7fedeca89640 -1 rbd::mirror::MirrorStatusUpdater 0x5653b9a98480 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-18T08:52:14.844 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.trial163.stderr:2026-02-18T08:52:14.832+0000 7fedf9aa3640 -1 rbd::mirror::NamespaceReplayer: 0x5653b86ed180 handle_shut_down_remote_status_updater: error shutting remote mirror status updater down: (2) No such file or directory 2026-02-18T08:52:14.844 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.trial163.stderr:2026-02-18T08:52:14.832+0000 7fedf9aa3640 -1 rbd::mirror::NamespaceReplayer: 0x5653b86ecfc0 handle_shut_down_remote_status_updater: error shutting remote mirror status updater down: (2) No such file or directory 2026-02-18T08:52:14.844 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.trial163.stderr:2026-02-18T08:52:14.832+0000 7fedf9aa3640 -1 rbd::mirror::NamespaceReplayer: 0x5653b86ec000 handle_shut_down_remote_status_updater: error shutting remote mirror status updater down: (2) No such file or directory 2026-02-18T08:52:14.844 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.trial163.stderr:2026-02-18T08:52:14.832+0000 7fede527a640 -1 rbd::mirror::MirrorStatusUpdater 0x5653b9d2aa20 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-18T08:52:14.844 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.trial163.stderr:2026-02-18T08:52:14.832+0000 7fede527a640 -1 rbd::mirror::MirrorStatusUpdater 0x5653b9d2ab40 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-18T08:52:14.844 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.trial163.stderr:2026-02-18T08:52:14.832+0000 7fedf92a2640 -1 rbd::mirror::NamespaceReplayer: 0x5653b9efc1c0 handle_shut_down_local_status_updater: error shutting local mirror status updater down: (2) No such file or directory 2026-02-18T08:52:14.844 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.trial163.stderr:2026-02-18T08:52:14.832+0000 7fedf92a2640 -1 rbd::mirror::NamespaceReplayer: 0x5653b9efc380 handle_shut_down_local_status_updater: error shutting local mirror status updater down: (2) No such file or directory 2026-02-18T08:52:14.844 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.trial163.stderr:2026-02-18T08:52:14.832+0000 7fedf3296640 -1 rbd::mirror::MirrorStatusUpdater 0x5653b9a98000 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-18T08:52:14.844 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.trial163.stderr:2026-02-18T08:52:14.832+0000 7fedf3296640 -1 rbd::mirror::MirrorStatusUpdater 0x5653b9a98120 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-18T08:52:14.844 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.trial163.stderr:2026-02-18T08:52:14.832+0000 7fedf3296640 -1 rbd::mirror::MirrorStatusUpdater 0x5653b9a98240 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-18T08:52:14.844 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.trial163.stderr:2026-02-18T08:52:14.832+0000 7fedf9aa3640 -1 rbd::mirror::NamespaceReplayer: 0x5653b86ed180 handle_shut_down_local_status_updater: error shutting local mirror status updater down: (2) No such file or directory 2026-02-18T08:52:14.844 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.trial163.stderr:2026-02-18T08:52:14.832+0000 7fedf9aa3640 -1 rbd::mirror::NamespaceReplayer: 0x5653b86ecfc0 handle_shut_down_local_status_updater: error shutting local mirror status updater down: (2) No such file or directory 2026-02-18T08:52:14.845 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.trial163.stderr:2026-02-18T08:52:14.832+0000 7fedf9aa3640 -1 rbd::mirror::NamespaceReplayer: 0x5653b86ec000 handle_shut_down_local_status_updater: error shutting local mirror status updater down: (2) No such file or directory 2026-02-18T08:52:14.845 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.trial163.stderr:2026-02-18T08:52:14.832+0000 7fedf92a2640 -1 rbd::mirror::LeaderWatcher: 0x5653b9d19b00 handle_unregister_watch: error unregistering leader watcher for rbd_mirror_leader object: (2) No such file or directory 2026-02-18T08:52:14.845 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.trial163.stderr:2026-02-18T08:52:14.832+0000 7fedf92a2640 -1 rbd::mirror::LeaderWatcher: 0x5653b9738300 handle_unregister_watch: error unregistering leader watcher for rbd_mirror_leader object: (2) No such file or directory 2026-02-18T08:52:14.857 INFO:tasks.rbd_mirror.cluster2.client.mirror.0:Stopped 2026-02-18T08:52:14.857 DEBUG:teuthology.run_tasks:Unwinding manager rbd-mirror 2026-02-18T08:52:14.875 DEBUG:tasks.rbd_mirror.cluster1.client.mirror.3:waiting for process to exit 2026-02-18T08:52:14.875 INFO:teuthology.orchestra.run:waiting for 300 2026-02-18T08:52:14.904 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial163.stderr:2026-02-18T08:52:14.896+0000 7fbd38efe640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.3 (PID: 88634) UID: 1000 2026-02-18T08:52:14.905 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial163.stderr:2026-02-18T08:52:14.896+0000 7fbd37efc640 -1 rbd::mirror::InstanceWatcher: 0x55785da13880 handle_unregister_watch: error unregistering instance watcher for rbd_mirror_instance.4624 object: (2) No such file or directory 2026-02-18T08:52:14.905 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial163.stderr:2026-02-18T08:52:14.896+0000 7fbd386fd640 -1 rbd::mirror::InstanceWatcher: 0x55785d4c0380 handle_unregister_watch: error unregistering instance watcher for rbd_mirror_instance.4624 object: (2) No such file or directory 2026-02-18T08:52:14.906 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial163.stderr:2026-02-18T08:52:14.896+0000 7fbd246d5640 -1 rbd::mirror::InstanceWatcher: 0x55785da13880 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2026-02-18T08:52:14.906 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial163.stderr:2026-02-18T08:52:14.896+0000 7fbd23ed4640 -1 rbd::mirror::InstanceWatcher: 0x55785d4c0380 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2026-02-18T08:52:14.906 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial163.stderr:2026-02-18T08:52:14.896+0000 7fbd1d6c7640 -1 rbd::mirror::MirrorStatusUpdater 0x55785d4eaa20 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-18T08:52:14.906 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial163.stderr:2026-02-18T08:52:14.896+0000 7fbd37efc640 -1 rbd::mirror::NamespaceReplayer: 0x55785d1cc700 handle_shut_down_remote_status_updater: error shutting remote mirror status updater down: (2) No such file or directory 2026-02-18T08:52:14.906 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial163.stderr:2026-02-18T08:52:14.896+0000 7fbd1dec8640 -1 rbd::mirror::MirrorStatusUpdater 0x55785d4eab40 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-18T08:52:14.906 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial163.stderr:2026-02-18T08:52:14.896+0000 7fbd386fd640 -1 rbd::mirror::NamespaceReplayer: 0x55785d1cc8c0 handle_shut_down_remote_status_updater: error shutting remote mirror status updater down: (2) No such file or directory 2026-02-18T08:52:14.906 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial163.stderr:2026-02-18T08:52:14.896+0000 7fbd23ed4640 -1 rbd::mirror::MirrorStatusUpdater 0x55785d4ea6c0 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-18T08:52:14.906 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial163.stderr:2026-02-18T08:52:14.896+0000 7fbd23ed4640 -1 rbd::mirror::MirrorStatusUpdater 0x55785d4ea7e0 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-18T08:52:14.906 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial163.stderr:2026-02-18T08:52:14.896+0000 7fbd37efc640 -1 rbd::mirror::NamespaceReplayer: 0x55785d1cc700 handle_shut_down_local_status_updater: error shutting local mirror status updater down: (2) No such file or directory 2026-02-18T08:52:14.906 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial163.stderr:2026-02-18T08:52:14.896+0000 7fbd386fd640 -1 rbd::mirror::NamespaceReplayer: 0x55785d1cc8c0 handle_shut_down_local_status_updater: error shutting local mirror status updater down: (2) No such file or directory 2026-02-18T08:52:14.906 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial163.stderr:2026-02-18T08:52:14.896+0000 7fbd386fd640 -1 rbd::mirror::LeaderWatcher: 0x55785d2ffb00 handle_unregister_watch: error unregistering leader watcher for rbd_mirror_leader object: (2) No such file or directory 2026-02-18T08:52:14.908 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial163.stderr:2026-02-18T08:52:14.896+0000 7fbd386fd640 -1 rbd::mirror::InstanceWatcher: 0x55785d9ce380 handle_unregister_watch: error unregistering instance watcher for rbd_mirror_instance.4621 object: (2) No such file or directory 2026-02-18T08:52:14.908 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial163.stderr:2026-02-18T08:52:14.896+0000 7fbd37efc640 -1 rbd::mirror::InstanceWatcher: 0x55785da13500 handle_unregister_watch: error unregistering instance watcher for rbd_mirror_instance.4621 object: (2) No such file or directory 2026-02-18T08:52:14.909 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial163.stderr:2026-02-18T08:52:14.896+0000 7fbd37efc640 -1 rbd::mirror::InstanceWatcher: 0x55785d9cf180 handle_unregister_watch: error unregistering instance watcher for rbd_mirror_instance.4621 object: (2) No such file or directory 2026-02-18T08:52:14.909 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial163.stderr:2026-02-18T08:52:14.896+0000 7fbd316ef640 -1 rbd::mirror::InstanceWatcher: 0x55785d9ce380 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2026-02-18T08:52:14.909 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial163.stderr:2026-02-18T08:52:14.896+0000 7fbd31ef0640 -1 rbd::mirror::InstanceWatcher: 0x55785d9cf180 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2026-02-18T08:52:14.909 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial163.stderr:2026-02-18T08:52:14.896+0000 7fbd31ef0640 -1 rbd::mirror::InstanceWatcher: 0x55785da13500 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2026-02-18T08:52:14.909 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial163.stderr:2026-02-18T08:52:14.896+0000 7fbd2b6e3640 -1 rbd::mirror::MirrorStatusUpdater 0x55785cfd05a0 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-18T08:52:14.909 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial163.stderr:2026-02-18T08:52:14.896+0000 7fbd2b6e3640 -1 rbd::mirror::MirrorStatusUpdater 0x55785cfd06c0 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-18T08:52:14.909 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial163.stderr:2026-02-18T08:52:14.896+0000 7fbd2b6e3640 -1 rbd::mirror::MirrorStatusUpdater 0x55785cfd07e0 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-18T08:52:14.909 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial163.stderr:2026-02-18T08:52:14.896+0000 7fbd386fd640 -1 rbd::mirror::NamespaceReplayer: 0x55785bee6000 handle_shut_down_remote_status_updater: error shutting remote mirror status updater down: (2) No such file or directory 2026-02-18T08:52:14.909 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial163.stderr:2026-02-18T08:52:14.896+0000 7fbd386fd640 -1 rbd::mirror::NamespaceReplayer: 0x55785bee6a80 handle_shut_down_remote_status_updater: error shutting remote mirror status updater down: (2) No such file or directory 2026-02-18T08:52:14.909 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial163.stderr:2026-02-18T08:52:14.896+0000 7fbd386fd640 -1 rbd::mirror::NamespaceReplayer: 0x55785bee6700 handle_shut_down_remote_status_updater: error shutting remote mirror status updater down: (2) No such file or directory 2026-02-18T08:52:14.909 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial163.stderr:2026-02-18T08:52:14.896+0000 7fbd316ef640 -1 rbd::mirror::MirrorStatusUpdater 0x55785cfd0240 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-18T08:52:14.910 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial163.stderr:2026-02-18T08:52:14.896+0000 7fbd316ef640 -1 rbd::mirror::MirrorStatusUpdater 0x55785cfd0360 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-18T08:52:14.910 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial163.stderr:2026-02-18T08:52:14.896+0000 7fbd37efc640 -1 rbd::mirror::NamespaceReplayer: 0x55785bee6000 handle_shut_down_local_status_updater: error shutting local mirror status updater down: (2) No such file or directory 2026-02-18T08:52:14.910 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial163.stderr:2026-02-18T08:52:14.896+0000 7fbd31ef0640 -1 rbd::mirror::MirrorStatusUpdater 0x55785cfd0480 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-18T08:52:14.910 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial163.stderr:2026-02-18T08:52:14.896+0000 7fbd37efc640 -1 rbd::mirror::NamespaceReplayer: 0x55785bee6700 handle_shut_down_local_status_updater: error shutting local mirror status updater down: (2) No such file or directory 2026-02-18T08:52:14.910 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial163.stderr:2026-02-18T08:52:14.896+0000 7fbd37efc640 -1 rbd::mirror::NamespaceReplayer: 0x55785bee6a80 handle_shut_down_local_status_updater: error shutting local mirror status updater down: (2) No such file or directory 2026-02-18T08:52:14.910 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial163.stderr:2026-02-18T08:52:14.900+0000 7fbd386fd640 -1 rbd::mirror::LeaderWatcher: 0x55785dd12300 handle_unregister_watch: error unregistering leader watcher for rbd_mirror_leader object: (2) No such file or directory 2026-02-18T08:52:14.926 INFO:tasks.rbd_mirror.cluster1.client.mirror.3:Stopped 2026-02-18T08:52:14.926 DEBUG:teuthology.run_tasks:Unwinding manager rbd-mirror 2026-02-18T08:52:14.944 DEBUG:tasks.rbd_mirror.cluster1.client.mirror.2:waiting for process to exit 2026-02-18T08:52:14.944 INFO:teuthology.orchestra.run:waiting for 300 2026-02-18T08:52:14.971 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.trial163.stderr:2026-02-18T08:52:14.964+0000 7fef1d83f640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.2 (PID: 134128) UID: 1000 2026-02-18T08:52:14.981 INFO:tasks.rbd_mirror.cluster1.client.mirror.2:Stopped 2026-02-18T08:52:14.981 DEBUG:teuthology.run_tasks:Unwinding manager rbd-mirror 2026-02-18T08:52:14.992 DEBUG:tasks.rbd_mirror.cluster1.client.mirror.1:waiting for process to exit 2026-02-18T08:52:14.992 INFO:teuthology.orchestra.run:waiting for 300 2026-02-18T08:52:15.023 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.trial163.stderr:2026-02-18T08:52:15.016+0000 7fa304e23640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.1 (PID: 122131) UID: 1000 2026-02-18T08:52:15.026 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.trial163.stderr:2026-02-18T08:52:15.016+0000 7fa303e21640 -1 rbd::mirror::InstanceWatcher: 0x55ae64ab2700 handle_unregister_watch: error unregistering instance watcher for rbd_mirror_instance.4968 object: (2) No such file or directory 2026-02-18T08:52:15.026 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.trial163.stderr:2026-02-18T08:52:15.016+0000 7fa303e21640 -1 rbd::mirror::InstanceWatcher: 0x55ae64ab2380 handle_unregister_watch: error unregistering instance watcher for rbd_mirror_instance.4968 object: (2) No such file or directory 2026-02-18T08:52:15.027 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.trial163.stderr:2026-02-18T08:52:15.016+0000 7fa303e21640 -1 rbd::mirror::InstanceWatcher: 0x55ae64fb1180 handle_unregister_watch: error unregistering instance watcher for rbd_mirror_instance.4965 object: (2) No such file or directory 2026-02-18T08:52:15.027 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.trial163.stderr:2026-02-18T08:52:15.016+0000 7fa304622640 -1 rbd::mirror::InstanceWatcher: 0x55ae64618000 handle_unregister_watch: error unregistering instance watcher for rbd_mirror_instance.4965 object: (2) No such file or directory 2026-02-18T08:52:15.027 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.trial163.stderr:2026-02-18T08:52:15.016+0000 7fa303e21640 -1 rbd::mirror::InstanceWatcher: 0x55ae64619880 handle_unregister_watch: error unregistering instance watcher for rbd_mirror_instance.4965 object: (2) No such file or directory 2026-02-18T08:52:15.027 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.trial163.stderr:2026-02-18T08:52:15.016+0000 7fa2f05fa640 -1 rbd::mirror::InstanceWatcher: 0x55ae64ab2700 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2026-02-18T08:52:15.027 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.trial163.stderr:2026-02-18T08:52:15.016+0000 7fa2efdf9640 -1 rbd::mirror::InstanceWatcher: 0x55ae64ab2380 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2026-02-18T08:52:15.027 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.trial163.stderr:2026-02-18T08:52:15.016+0000 7fa2fde15640 -1 rbd::mirror::InstanceWatcher: 0x55ae64fb1180 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2026-02-18T08:52:15.027 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.trial163.stderr:2026-02-18T08:52:15.016+0000 7fa2fd614640 -1 rbd::mirror::InstanceWatcher: 0x55ae64618000 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2026-02-18T08:52:15.027 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.trial163.stderr:2026-02-18T08:52:15.016+0000 7fa2e95ec640 -1 rbd::mirror::MirrorStatusUpdater 0x55ae64878480 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-18T08:52:15.027 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.trial163.stderr:2026-02-18T08:52:15.016+0000 7fa2e95ec640 -1 rbd::mirror::MirrorStatusUpdater 0x55ae64620900 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-18T08:52:15.027 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.trial163.stderr:2026-02-18T08:52:15.016+0000 7fa304622640 -1 rbd::mirror::NamespaceReplayer: 0x55ae647848c0 handle_shut_down_remote_status_updater: error shutting remote mirror status updater down: (2) No such file or directory 2026-02-18T08:52:15.027 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.trial163.stderr:2026-02-18T08:52:15.016+0000 7fa304622640 -1 rbd::mirror::NamespaceReplayer: 0x55ae64784a80 handle_shut_down_remote_status_updater: error shutting remote mirror status updater down: (2) No such file or directory 2026-02-18T08:52:15.027 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.trial163.stderr:2026-02-18T08:52:15.016+0000 7fa2fde15640 -1 rbd::mirror::InstanceWatcher: 0x55ae64619880 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2026-02-18T08:52:15.028 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.trial163.stderr:2026-02-18T08:52:15.016+0000 7fa2f05fa640 -1 rbd::mirror::MirrorStatusUpdater 0x55ae64878d80 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-18T08:52:15.028 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.trial163.stderr:2026-02-18T08:52:15.016+0000 7fa303e21640 -1 rbd::mirror::NamespaceReplayer: 0x55ae647848c0 handle_shut_down_local_status_updater: error shutting local mirror status updater down: (2) No such file or directory 2026-02-18T08:52:15.028 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.trial163.stderr:2026-02-18T08:52:15.016+0000 7fa2f7608640 -1 rbd::mirror::MirrorStatusUpdater 0x55ae64620360 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-18T08:52:15.028 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.trial163.stderr:2026-02-18T08:52:15.016+0000 7fa2f7608640 -1 rbd::mirror::MirrorStatusUpdater 0x55ae646205a0 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-18T08:52:15.028 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.trial163.stderr:2026-02-18T08:52:15.016+0000 7fa2f7608640 -1 rbd::mirror::MirrorStatusUpdater 0x55ae64620480 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-18T08:52:15.028 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.trial163.stderr:2026-02-18T08:52:15.016+0000 7fa303e21640 -1 rbd::mirror::NamespaceReplayer: 0x55ae634c8fc0 handle_shut_down_remote_status_updater: error shutting remote mirror status updater down: (2) No such file or directory 2026-02-18T08:52:15.028 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.trial163.stderr:2026-02-18T08:52:15.016+0000 7fa2efdf9640 -1 rbd::mirror::MirrorStatusUpdater 0x55ae64878ea0 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-18T08:52:15.028 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.trial163.stderr:2026-02-18T08:52:15.016+0000 7fa303e21640 -1 rbd::mirror::NamespaceReplayer: 0x55ae634c8000 handle_shut_down_remote_status_updater: error shutting remote mirror status updater down: (2) No such file or directory 2026-02-18T08:52:15.028 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.trial163.stderr:2026-02-18T08:52:15.016+0000 7fa303e21640 -1 rbd::mirror::NamespaceReplayer: 0x55ae634c8700 handle_shut_down_remote_status_updater: error shutting remote mirror status updater down: (2) No such file or directory 2026-02-18T08:52:15.028 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.trial163.stderr:2026-02-18T08:52:15.016+0000 7fa303e21640 -1 rbd::mirror::NamespaceReplayer: 0x55ae64784a80 handle_shut_down_local_status_updater: error shutting local mirror status updater down: (2) No such file or directory 2026-02-18T08:52:15.028 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.trial163.stderr:2026-02-18T08:52:15.016+0000 7fa2fd614640 -1 rbd::mirror::MirrorStatusUpdater 0x55ae64620000 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-18T08:52:15.028 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.trial163.stderr:2026-02-18T08:52:15.016+0000 7fa303e21640 -1 rbd::mirror::NamespaceReplayer: 0x55ae634c8fc0 handle_shut_down_local_status_updater: error shutting local mirror status updater down: (2) No such file or directory 2026-02-18T08:52:15.028 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.trial163.stderr:2026-02-18T08:52:15.016+0000 7fa2fde15640 -1 rbd::mirror::MirrorStatusUpdater 0x55ae64620120 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-18T08:52:15.029 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.trial163.stderr:2026-02-18T08:52:15.016+0000 7fa303e21640 -1 rbd::mirror::NamespaceReplayer: 0x55ae634c8000 handle_shut_down_local_status_updater: error shutting local mirror status updater down: (2) No such file or directory 2026-02-18T08:52:15.029 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.trial163.stderr:2026-02-18T08:52:15.016+0000 7fa2fd614640 -1 rbd::mirror::MirrorStatusUpdater 0x55ae64620240 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-18T08:52:15.029 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.trial163.stderr:2026-02-18T08:52:15.016+0000 7fa304622640 -1 rbd::mirror::NamespaceReplayer: 0x55ae634c8700 handle_shut_down_local_status_updater: error shutting local mirror status updater down: (2) No such file or directory 2026-02-18T08:52:15.029 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.trial163.stderr:2026-02-18T08:52:15.016+0000 7fa304622640 -1 rbd::mirror::LeaderWatcher: 0x55ae648e1b00 handle_unregister_watch: error unregistering leader watcher for rbd_mirror_leader object: (2) No such file or directory 2026-02-18T08:52:15.029 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.trial163.stderr:2026-02-18T08:52:15.020+0000 7fa304622640 -1 rbd::mirror::LeaderWatcher: 0x55ae652f2300 handle_unregister_watch: error unregistering leader watcher for rbd_mirror_leader object: (2) No such file or directory 2026-02-18T08:52:15.042 INFO:tasks.rbd_mirror.cluster1.client.mirror.1:Stopped 2026-02-18T08:52:15.042 DEBUG:teuthology.run_tasks:Unwinding manager rbd-mirror 2026-02-18T08:52:15.062 DEBUG:tasks.rbd_mirror.cluster1.client.mirror.0:waiting for process to exit 2026-02-18T08:52:15.062 INFO:teuthology.orchestra.run:waiting for 300 2026-02-18T08:52:15.083 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.trial163.stderr:2026-02-18T08:52:15.076+0000 7f6adac83640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.0 (PID: 116275) UID: 1000 2026-02-18T08:52:15.086 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.trial163.stderr:2026-02-18T08:52:15.076+0000 7f6ad9c81640 -1 rbd::mirror::InstanceWatcher: 0x562c06b95c00 handle_unregister_watch: error unregistering instance watcher for rbd_mirror_instance.4903 object: (2) No such file or directory 2026-02-18T08:52:15.086 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.trial163.stderr:2026-02-18T08:52:15.076+0000 7f6ada482640 -1 rbd::mirror::InstanceWatcher: 0x562c067c1180 handle_unregister_watch: error unregistering instance watcher for rbd_mirror_instance.4901 object: (2) No such file or directory 2026-02-18T08:52:15.087 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.trial163.stderr:2026-02-18T08:52:15.076+0000 7f6ada482640 -1 rbd::mirror::InstanceWatcher: 0x562c062b2380 handle_unregister_watch: error unregistering instance watcher for rbd_mirror_instance.4903 object: (2) No such file or directory 2026-02-18T08:52:15.087 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.trial163.stderr:2026-02-18T08:52:15.076+0000 7f6ad9c81640 -1 rbd::mirror::InstanceWatcher: 0x562c06b95880 handle_unregister_watch: error unregistering instance watcher for rbd_mirror_instance.4901 object: (2) No such file or directory 2026-02-18T08:52:15.087 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.trial163.stderr:2026-02-18T08:52:15.076+0000 7f6ad9c81640 -1 rbd::mirror::InstanceWatcher: 0x562c067c1500 handle_unregister_watch: error unregistering instance watcher for rbd_mirror_instance.4901 object: (2) No such file or directory 2026-02-18T08:52:15.087 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.trial163.stderr:2026-02-18T08:52:15.076+0000 7f6ac645a640 -1 rbd::mirror::InstanceWatcher: 0x562c06b95c00 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2026-02-18T08:52:15.087 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.trial163.stderr:2026-02-18T08:52:15.076+0000 7f6ac645a640 -1 rbd::mirror::InstanceWatcher: 0x562c062b2380 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2026-02-18T08:52:15.087 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.trial163.stderr:2026-02-18T08:52:15.076+0000 7f6ad3474640 -1 rbd::mirror::InstanceWatcher: 0x562c067c1180 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2026-02-18T08:52:15.087 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.trial163.stderr:2026-02-18T08:52:15.076+0000 7f6abf44c640 -1 rbd::mirror::MirrorStatusUpdater 0x562c06077320 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-18T08:52:15.087 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.trial163.stderr:2026-02-18T08:52:15.076+0000 7f6abf44c640 -1 rbd::mirror::MirrorStatusUpdater 0x562c06076d80 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-18T08:52:15.087 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.trial163.stderr:2026-02-18T08:52:15.076+0000 7f6ada482640 -1 rbd::mirror::NamespaceReplayer: 0x562c05e16fc0 handle_shut_down_remote_status_updater: error shutting remote mirror status updater down: (2) No such file or directory 2026-02-18T08:52:15.087 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.trial163.stderr:2026-02-18T08:52:15.076+0000 7f6ad3c75640 -1 rbd::mirror::InstanceWatcher: 0x562c06b95880 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2026-02-18T08:52:15.087 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.trial163.stderr:2026-02-18T08:52:15.076+0000 7f6ad3c75640 -1 rbd::mirror::InstanceWatcher: 0x562c067c1500 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2026-02-18T08:52:15.088 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.trial163.stderr:2026-02-18T08:52:15.076+0000 7f6ada482640 -1 rbd::mirror::NamespaceReplayer: 0x562c05e17180 handle_shut_down_remote_status_updater: error shutting remote mirror status updater down: (2) No such file or directory 2026-02-18T08:52:15.088 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.trial163.stderr:2026-02-18T08:52:15.076+0000 7f6acd468640 -1 rbd::mirror::MirrorStatusUpdater 0x562c05db05a0 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-18T08:52:15.088 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.trial163.stderr:2026-02-18T08:52:15.076+0000 7f6ad9c81640 -1 rbd::mirror::NamespaceReplayer: 0x562c05e16000 handle_shut_down_remote_status_updater: error shutting remote mirror status updater down: (2) No such file or directory 2026-02-18T08:52:15.088 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.trial163.stderr:2026-02-18T08:52:15.076+0000 7f6ac5c59640 -1 rbd::mirror::MirrorStatusUpdater 0x562c06076a20 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-18T08:52:15.088 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.trial163.stderr:2026-02-18T08:52:15.076+0000 7f6ac5c59640 -1 rbd::mirror::MirrorStatusUpdater 0x562c06076b40 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-18T08:52:15.088 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.trial163.stderr:2026-02-18T08:52:15.076+0000 7f6ada482640 -1 rbd::mirror::NamespaceReplayer: 0x562c05e16fc0 handle_shut_down_local_status_updater: error shutting local mirror status updater down: (2) No such file or directory 2026-02-18T08:52:15.088 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.trial163.stderr:2026-02-18T08:52:15.076+0000 7f6ada482640 -1 rbd::mirror::NamespaceReplayer: 0x562c05e17180 handle_shut_down_local_status_updater: error shutting local mirror status updater down: (2) No such file or directory 2026-02-18T08:52:15.088 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.trial163.stderr:2026-02-18T08:52:15.076+0000 7f6accc67640 -1 rbd::mirror::MirrorStatusUpdater 0x562c05db06c0 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-18T08:52:15.088 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.trial163.stderr:2026-02-18T08:52:15.076+0000 7f6accc67640 -1 rbd::mirror::MirrorStatusUpdater 0x562c05db07e0 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-18T08:52:15.088 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.trial163.stderr:2026-02-18T08:52:15.076+0000 7f6ad9c81640 -1 rbd::mirror::NamespaceReplayer: 0x562c05e16380 handle_shut_down_remote_status_updater: error shutting remote mirror status updater down: (2) No such file or directory 2026-02-18T08:52:15.088 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.trial163.stderr:2026-02-18T08:52:15.076+0000 7f6ad3474640 -1 rbd::mirror::MirrorStatusUpdater 0x562c05db0240 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-18T08:52:15.088 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.trial163.stderr:2026-02-18T08:52:15.076+0000 7f6ad9c81640 -1 rbd::mirror::NamespaceReplayer: 0x562c05e161c0 handle_shut_down_remote_status_updater: error shutting remote mirror status updater down: (2) No such file or directory 2026-02-18T08:52:15.089 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.trial163.stderr:2026-02-18T08:52:15.076+0000 7f6ad9c81640 -1 rbd::mirror::NamespaceReplayer: 0x562c05e16000 handle_shut_down_local_status_updater: error shutting local mirror status updater down: (2) No such file or directory 2026-02-18T08:52:15.089 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.trial163.stderr:2026-02-18T08:52:15.076+0000 7f6ad3c75640 -1 rbd::mirror::MirrorStatusUpdater 0x562c05db0480 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-18T08:52:15.089 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.trial163.stderr:2026-02-18T08:52:15.076+0000 7f6ad9c81640 -1 rbd::mirror::NamespaceReplayer: 0x562c05e16380 handle_shut_down_local_status_updater: error shutting local mirror status updater down: (2) No such file or directory 2026-02-18T08:52:15.089 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.trial163.stderr:2026-02-18T08:52:15.076+0000 7f6ad9c81640 -1 rbd::mirror::LeaderWatcher: 0x562c060fbb00 handle_unregister_watch: error unregistering leader watcher for rbd_mirror_leader object: (2) No such file or directory 2026-02-18T08:52:15.089 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.trial163.stderr:2026-02-18T08:52:15.076+0000 7f6ad3474640 -1 rbd::mirror::MirrorStatusUpdater 0x562c05db0360 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-18T08:52:15.089 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.trial163.stderr:2026-02-18T08:52:15.076+0000 7f6ada482640 -1 rbd::mirror::NamespaceReplayer: 0x562c05e161c0 handle_shut_down_local_status_updater: error shutting local mirror status updater down: (2) No such file or directory 2026-02-18T08:52:15.089 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.trial163.stderr:2026-02-18T08:52:15.080+0000 7f6ada482640 -1 rbd::mirror::LeaderWatcher: 0x562c069cc300 handle_unregister_watch: error unregistering leader watcher for rbd_mirror_leader object: (2) No such file or directory 2026-02-18T08:52:15.104 INFO:tasks.rbd_mirror.cluster1.client.mirror.0:Stopped 2026-02-18T08:52:15.104 DEBUG:teuthology.run_tasks:Unwinding manager ceph 2026-02-18T08:52:15.122 INFO:tasks.ceph.ceph_manager.cluster2:waiting for clean 2026-02-18T08:52:15.123 DEBUG:teuthology.orchestra.run.trial163:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 pg dump --format=json 2026-02-18T08:52:15.298 INFO:teuthology.orchestra.run.trial163.stdout: 2026-02-18T08:52:15.299 INFO:teuthology.orchestra.run.trial163.stderr:dumped all 2026-02-18T08:52:15.309 INFO:teuthology.orchestra.run.trial163.stdout:{"pg_ready":true,"pg_map":{"version":1699,"stamp":"2026-02-18T08:52:14.909897+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590387,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":103,"num_write_kb":1630,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":75,"ondisk_log_size":75,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":274,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":6081256,"kb_used_data":986808,"kb_used_omap":852,"kb_used_meta":5093547,"kb_avail":277034264,"statfs":{"total":289910292480,"available":283683086336,"internally_reserved":0,"allocated":1010491392,"data_stored":1000695212,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":873413,"internal_metadata":5215792187},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":8,"apply_latency_ms":8,"commit_latency_ns":8000000,"apply_latency_ns":8000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-787431672,"num_objects":-193,"num_object_clones":0,"num_object_copies":-386,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-193,"num_whiteouts":0,"num_read":-2429475,"num_read_kb":-92852858,"num_write":-2333992,"num_write_kb":-91441114,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":-1,"num_bytes_recovered":0,"num_keys_recovered":-3,"num_objects_omap":-19,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"6.000829"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":849,"reported_epoch":419,"state":"active+clean","last_fresh":"2026-02-18T08:52:10.122679+0000","last_change":"2026-02-18T08:09:39.904504+0000","last_active":"2026-02-18T08:52:10.122679+0000","last_peered":"2026-02-18T08:52:10.122679+0000","last_clean":"2026-02-18T08:52:10.122679+0000","last_became_active":"2026-02-18T08:09:37.900809+0000","last_became_peered":"2026-02-18T08:09:37.900809+0000","last_unstale":"2026-02-18T08:52:10.122679+0000","last_undegraded":"2026-02-18T08:52:10.122679+0000","last_fullsized":"2026-02-18T08:52:10.122679+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-18T08:09:36.890905+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-18T08:09:36.890905+0000","last_clean_scrub_stamp":"2026-02-18T08:09:36.890905+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-19T18:39:43.531542+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00068838299999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":849,"reported_epoch":419,"state":"active+clean","last_fresh":"2026-02-18T08:52:10.121896+0000","last_change":"2026-02-18T08:09:39.904331+0000","last_active":"2026-02-18T08:52:10.121896+0000","last_peered":"2026-02-18T08:52:10.121896+0000","last_clean":"2026-02-18T08:52:10.121896+0000","last_became_active":"2026-02-18T08:09:37.900594+0000","last_became_peered":"2026-02-18T08:09:37.900594+0000","last_unstale":"2026-02-18T08:52:10.121896+0000","last_undegraded":"2026-02-18T08:52:10.121896+0000","last_fullsized":"2026-02-18T08:52:10.121896+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-18T08:09:36.890905+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-18T08:09:36.890905+0000","last_clean_scrub_stamp":"2026-02-18T08:09:36.890905+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-19T19:59:21.557650+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00053109200000000004,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":850,"reported_epoch":419,"state":"active+clean","last_fresh":"2026-02-18T08:52:10.122610+0000","last_change":"2026-02-18T08:09:39.904488+0000","last_active":"2026-02-18T08:52:10.122610+0000","last_peered":"2026-02-18T08:52:10.122610+0000","last_clean":"2026-02-18T08:52:10.122610+0000","last_became_active":"2026-02-18T08:09:37.900657+0000","last_became_peered":"2026-02-18T08:09:37.900657+0000","last_unstale":"2026-02-18T08:52:10.122610+0000","last_undegraded":"2026-02-18T08:52:10.122610+0000","last_fullsized":"2026-02-18T08:52:10.122610+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-18T08:09:36.890905+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-18T08:09:36.890905+0000","last_clean_scrub_stamp":"2026-02-18T08:09:36.890905+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-19T16:39:19.623528+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00067508900000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":850,"reported_epoch":419,"state":"active+clean","last_fresh":"2026-02-18T08:52:10.120474+0000","last_change":"2026-02-18T08:09:39.904269+0000","last_active":"2026-02-18T08:52:10.120474+0000","last_peered":"2026-02-18T08:52:10.120474+0000","last_clean":"2026-02-18T08:52:10.120474+0000","last_became_active":"2026-02-18T08:09:37.901625+0000","last_became_peered":"2026-02-18T08:09:37.901625+0000","last_unstale":"2026-02-18T08:52:10.120474+0000","last_undegraded":"2026-02-18T08:52:10.120474+0000","last_fullsized":"2026-02-18T08:52:10.120474+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-18T08:09:36.890905+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-18T08:09:36.890905+0000","last_clean_scrub_stamp":"2026-02-18T08:09:36.890905+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-19T14:48:52.058850+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00056007599999999995,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"14'2","reported_seq":852,"reported_epoch":419,"state":"active+clean","last_fresh":"2026-02-18T08:52:10.120653+0000","last_change":"2026-02-18T08:09:39.902339+0000","last_active":"2026-02-18T08:52:10.120653+0000","last_peered":"2026-02-18T08:52:10.120653+0000","last_clean":"2026-02-18T08:52:10.120653+0000","last_became_active":"2026-02-18T08:09:37.900679+0000","last_became_peered":"2026-02-18T08:09:37.900679+0000","last_unstale":"2026-02-18T08:52:10.120653+0000","last_undegraded":"2026-02-18T08:52:10.120653+0000","last_fullsized":"2026-02-18T08:52:10.120653+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-18T08:09:36.890905+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-18T08:09:36.890905+0000","last_clean_scrub_stamp":"2026-02-18T08:09:36.890905+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-19T14:23:03.463004+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00035240299999999999,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":846,"reported_epoch":418,"state":"active+clean","last_fresh":"2026-02-18T08:51:44.191169+0000","last_change":"2026-02-18T08:09:40.252366+0000","last_active":"2026-02-18T08:51:44.191169+0000","last_peered":"2026-02-18T08:51:44.191169+0000","last_clean":"2026-02-18T08:51:44.191169+0000","last_became_active":"2026-02-18T08:09:37.900553+0000","last_became_peered":"2026-02-18T08:09:37.900553+0000","last_unstale":"2026-02-18T08:51:44.191169+0000","last_undegraded":"2026-02-18T08:51:44.191169+0000","last_fullsized":"2026-02-18T08:51:44.191169+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-18T08:09:36.890905+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-18T08:09:36.890905+0000","last_clean_scrub_stamp":"2026-02-18T08:09:36.890905+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-19T19:45:29.501652+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000311546,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":847,"reported_epoch":418,"state":"active+clean","last_fresh":"2026-02-18T08:51:44.191596+0000","last_change":"2026-02-18T08:09:40.252404+0000","last_active":"2026-02-18T08:51:44.191596+0000","last_peered":"2026-02-18T08:51:44.191596+0000","last_clean":"2026-02-18T08:51:44.191596+0000","last_became_active":"2026-02-18T08:09:37.900556+0000","last_became_peered":"2026-02-18T08:09:37.900556+0000","last_unstale":"2026-02-18T08:51:44.191596+0000","last_undegraded":"2026-02-18T08:51:44.191596+0000","last_fullsized":"2026-02-18T08:51:44.191596+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-18T08:09:36.890905+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-18T08:09:36.890905+0000","last_clean_scrub_stamp":"2026-02-18T08:09:36.890905+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-19T15:36:20.626321+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032881799999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.3","version":"12'1","reported_seq":851,"reported_epoch":419,"state":"active+clean","last_fresh":"2026-02-18T08:52:10.121435+0000","last_change":"2026-02-18T08:09:39.904803+0000","last_active":"2026-02-18T08:52:10.121435+0000","last_peered":"2026-02-18T08:52:10.121435+0000","last_clean":"2026-02-18T08:52:10.121435+0000","last_became_active":"2026-02-18T08:09:37.900195+0000","last_became_peered":"2026-02-18T08:09:37.900195+0000","last_unstale":"2026-02-18T08:52:10.121435+0000","last_undegraded":"2026-02-18T08:52:10.121435+0000","last_fullsized":"2026-02-18T08:52:10.121435+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-18T08:09:36.890905+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-18T08:09:36.890905+0000","last_clean_scrub_stamp":"2026-02-18T08:09:36.890905+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-19T09:59:37.426943+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00096736000000000001,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"10'72","reported_seq":940,"reported_epoch":419,"state":"active+clean","last_fresh":"2026-02-18T08:52:10.120080+0000","last_change":"2026-02-18T08:09:36.049944+0000","last_active":"2026-02-18T08:52:10.120080+0000","last_peered":"2026-02-18T08:52:10.120080+0000","last_clean":"2026-02-18T08:52:10.120080+0000","last_became_active":"2026-02-18T08:09:36.049266+0000","last_became_peered":"2026-02-18T08:09:36.049266+0000","last_unstale":"2026-02-18T08:52:10.120080+0000","last_undegraded":"2026-02-18T08:52:10.120080+0000","last_fullsized":"2026-02-18T08:52:10.120080+0000","mapping_epoch":9,"log_start":"0'0","ondisk_log_start":"0'0","created":9,"last_epoch_clean":10,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-18T08:09:34.886217+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-18T08:09:34.886217+0000","last_clean_scrub_stamp":"2026-02-18T08:09:34.886217+0000","objects_scrubbed":0,"log_size":72,"log_dups_size":0,"ondisk_log_size":72,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-19T08:12:47.784849+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":38,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":16,"acting":16,"num_store_stats":3},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1187840,"data_stored":1180736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":72,"ondisk_log_size":72,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":2,"up_from":8,"seq":34359738884,"num_pgs":81,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":2059748,"kb_used_data":300564,"kb_used_omap":288,"kb_used_meta":1758879,"kb_avail":92312092,"statfs":{"total":96636764160,"available":94527582208,"internally_reserved":0,"allocated":307777536,"data_stored":304511645,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":295679,"internal_metadata":1801092353},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":8,"seq":34359738884,"num_pgs":98,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":2000988,"kb_used_data":352204,"kb_used_omap":335,"kb_used_meta":1648432,"kb_avail":92370852,"statfs":{"total":96636764160,"available":94587752448,"internally_reserved":0,"allocated":360656896,"data_stored":357436022,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":343633,"internal_metadata":1687994799},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[]},{"osd":0,"up_from":8,"seq":34359738885,"num_pgs":95,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":2020520,"kb_used_data":334040,"kb_used_omap":228,"kb_used_meta":1686235,"kb_avail":92351320,"statfs":{"total":96636764160,"available":94567751680,"internally_reserved":0,"allocated":342056960,"data_stored":338747545,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":234101,"internal_metadata":1726705035},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":5,"apply_latency_ms":5,"commit_latency_ns":5000000,"apply_latency_ns":5000000},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2026-02-18T08:52:15.311 DEBUG:teuthology.orchestra.run.trial163:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 pg dump --format=json 2026-02-18T08:52:15.462 INFO:teuthology.orchestra.run.trial163.stdout: 2026-02-18T08:52:15.463 INFO:teuthology.orchestra.run.trial163.stderr:dumped all 2026-02-18T08:52:15.475 INFO:teuthology.orchestra.run.trial163.stdout:{"pg_ready":true,"pg_map":{"version":1699,"stamp":"2026-02-18T08:52:14.909897+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590387,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":103,"num_write_kb":1630,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":75,"ondisk_log_size":75,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":274,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":6081256,"kb_used_data":986808,"kb_used_omap":852,"kb_used_meta":5093547,"kb_avail":277034264,"statfs":{"total":289910292480,"available":283683086336,"internally_reserved":0,"allocated":1010491392,"data_stored":1000695212,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":873413,"internal_metadata":5215792187},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":8,"apply_latency_ms":8,"commit_latency_ns":8000000,"apply_latency_ns":8000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-787431672,"num_objects":-193,"num_object_clones":0,"num_object_copies":-386,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-193,"num_whiteouts":0,"num_read":-2429475,"num_read_kb":-92852858,"num_write":-2333992,"num_write_kb":-91441114,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":-1,"num_bytes_recovered":0,"num_keys_recovered":-3,"num_objects_omap":-19,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"6.000829"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":849,"reported_epoch":419,"state":"active+clean","last_fresh":"2026-02-18T08:52:10.122679+0000","last_change":"2026-02-18T08:09:39.904504+0000","last_active":"2026-02-18T08:52:10.122679+0000","last_peered":"2026-02-18T08:52:10.122679+0000","last_clean":"2026-02-18T08:52:10.122679+0000","last_became_active":"2026-02-18T08:09:37.900809+0000","last_became_peered":"2026-02-18T08:09:37.900809+0000","last_unstale":"2026-02-18T08:52:10.122679+0000","last_undegraded":"2026-02-18T08:52:10.122679+0000","last_fullsized":"2026-02-18T08:52:10.122679+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-18T08:09:36.890905+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-18T08:09:36.890905+0000","last_clean_scrub_stamp":"2026-02-18T08:09:36.890905+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-19T18:39:43.531542+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00068838299999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":849,"reported_epoch":419,"state":"active+clean","last_fresh":"2026-02-18T08:52:10.121896+0000","last_change":"2026-02-18T08:09:39.904331+0000","last_active":"2026-02-18T08:52:10.121896+0000","last_peered":"2026-02-18T08:52:10.121896+0000","last_clean":"2026-02-18T08:52:10.121896+0000","last_became_active":"2026-02-18T08:09:37.900594+0000","last_became_peered":"2026-02-18T08:09:37.900594+0000","last_unstale":"2026-02-18T08:52:10.121896+0000","last_undegraded":"2026-02-18T08:52:10.121896+0000","last_fullsized":"2026-02-18T08:52:10.121896+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-18T08:09:36.890905+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-18T08:09:36.890905+0000","last_clean_scrub_stamp":"2026-02-18T08:09:36.890905+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-19T19:59:21.557650+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00053109200000000004,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":850,"reported_epoch":419,"state":"active+clean","last_fresh":"2026-02-18T08:52:10.122610+0000","last_change":"2026-02-18T08:09:39.904488+0000","last_active":"2026-02-18T08:52:10.122610+0000","last_peered":"2026-02-18T08:52:10.122610+0000","last_clean":"2026-02-18T08:52:10.122610+0000","last_became_active":"2026-02-18T08:09:37.900657+0000","last_became_peered":"2026-02-18T08:09:37.900657+0000","last_unstale":"2026-02-18T08:52:10.122610+0000","last_undegraded":"2026-02-18T08:52:10.122610+0000","last_fullsized":"2026-02-18T08:52:10.122610+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-18T08:09:36.890905+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-18T08:09:36.890905+0000","last_clean_scrub_stamp":"2026-02-18T08:09:36.890905+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-19T16:39:19.623528+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00067508900000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":850,"reported_epoch":419,"state":"active+clean","last_fresh":"2026-02-18T08:52:10.120474+0000","last_change":"2026-02-18T08:09:39.904269+0000","last_active":"2026-02-18T08:52:10.120474+0000","last_peered":"2026-02-18T08:52:10.120474+0000","last_clean":"2026-02-18T08:52:10.120474+0000","last_became_active":"2026-02-18T08:09:37.901625+0000","last_became_peered":"2026-02-18T08:09:37.901625+0000","last_unstale":"2026-02-18T08:52:10.120474+0000","last_undegraded":"2026-02-18T08:52:10.120474+0000","last_fullsized":"2026-02-18T08:52:10.120474+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-18T08:09:36.890905+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-18T08:09:36.890905+0000","last_clean_scrub_stamp":"2026-02-18T08:09:36.890905+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-19T14:48:52.058850+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00056007599999999995,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"14'2","reported_seq":852,"reported_epoch":419,"state":"active+clean","last_fresh":"2026-02-18T08:52:10.120653+0000","last_change":"2026-02-18T08:09:39.902339+0000","last_active":"2026-02-18T08:52:10.120653+0000","last_peered":"2026-02-18T08:52:10.120653+0000","last_clean":"2026-02-18T08:52:10.120653+0000","last_became_active":"2026-02-18T08:09:37.900679+0000","last_became_peered":"2026-02-18T08:09:37.900679+0000","last_unstale":"2026-02-18T08:52:10.120653+0000","last_undegraded":"2026-02-18T08:52:10.120653+0000","last_fullsized":"2026-02-18T08:52:10.120653+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-18T08:09:36.890905+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-18T08:09:36.890905+0000","last_clean_scrub_stamp":"2026-02-18T08:09:36.890905+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-19T14:23:03.463004+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00035240299999999999,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":846,"reported_epoch":418,"state":"active+clean","last_fresh":"2026-02-18T08:51:44.191169+0000","last_change":"2026-02-18T08:09:40.252366+0000","last_active":"2026-02-18T08:51:44.191169+0000","last_peered":"2026-02-18T08:51:44.191169+0000","last_clean":"2026-02-18T08:51:44.191169+0000","last_became_active":"2026-02-18T08:09:37.900553+0000","last_became_peered":"2026-02-18T08:09:37.900553+0000","last_unstale":"2026-02-18T08:51:44.191169+0000","last_undegraded":"2026-02-18T08:51:44.191169+0000","last_fullsized":"2026-02-18T08:51:44.191169+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-18T08:09:36.890905+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-18T08:09:36.890905+0000","last_clean_scrub_stamp":"2026-02-18T08:09:36.890905+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-19T19:45:29.501652+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000311546,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":847,"reported_epoch":418,"state":"active+clean","last_fresh":"2026-02-18T08:51:44.191596+0000","last_change":"2026-02-18T08:09:40.252404+0000","last_active":"2026-02-18T08:51:44.191596+0000","last_peered":"2026-02-18T08:51:44.191596+0000","last_clean":"2026-02-18T08:51:44.191596+0000","last_became_active":"2026-02-18T08:09:37.900556+0000","last_became_peered":"2026-02-18T08:09:37.900556+0000","last_unstale":"2026-02-18T08:51:44.191596+0000","last_undegraded":"2026-02-18T08:51:44.191596+0000","last_fullsized":"2026-02-18T08:51:44.191596+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-18T08:09:36.890905+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-18T08:09:36.890905+0000","last_clean_scrub_stamp":"2026-02-18T08:09:36.890905+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-19T15:36:20.626321+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032881799999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.3","version":"12'1","reported_seq":851,"reported_epoch":419,"state":"active+clean","last_fresh":"2026-02-18T08:52:10.121435+0000","last_change":"2026-02-18T08:09:39.904803+0000","last_active":"2026-02-18T08:52:10.121435+0000","last_peered":"2026-02-18T08:52:10.121435+0000","last_clean":"2026-02-18T08:52:10.121435+0000","last_became_active":"2026-02-18T08:09:37.900195+0000","last_became_peered":"2026-02-18T08:09:37.900195+0000","last_unstale":"2026-02-18T08:52:10.121435+0000","last_undegraded":"2026-02-18T08:52:10.121435+0000","last_fullsized":"2026-02-18T08:52:10.121435+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-18T08:09:36.890905+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-18T08:09:36.890905+0000","last_clean_scrub_stamp":"2026-02-18T08:09:36.890905+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-19T09:59:37.426943+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00096736000000000001,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"10'72","reported_seq":940,"reported_epoch":419,"state":"active+clean","last_fresh":"2026-02-18T08:52:10.120080+0000","last_change":"2026-02-18T08:09:36.049944+0000","last_active":"2026-02-18T08:52:10.120080+0000","last_peered":"2026-02-18T08:52:10.120080+0000","last_clean":"2026-02-18T08:52:10.120080+0000","last_became_active":"2026-02-18T08:09:36.049266+0000","last_became_peered":"2026-02-18T08:09:36.049266+0000","last_unstale":"2026-02-18T08:52:10.120080+0000","last_undegraded":"2026-02-18T08:52:10.120080+0000","last_fullsized":"2026-02-18T08:52:10.120080+0000","mapping_epoch":9,"log_start":"0'0","ondisk_log_start":"0'0","created":9,"last_epoch_clean":10,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-18T08:09:34.886217+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-18T08:09:34.886217+0000","last_clean_scrub_stamp":"2026-02-18T08:09:34.886217+0000","objects_scrubbed":0,"log_size":72,"log_dups_size":0,"ondisk_log_size":72,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-19T08:12:47.784849+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":38,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":16,"acting":16,"num_store_stats":3},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1187840,"data_stored":1180736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":72,"ondisk_log_size":72,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":2,"up_from":8,"seq":34359738884,"num_pgs":81,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":2059748,"kb_used_data":300564,"kb_used_omap":288,"kb_used_meta":1758879,"kb_avail":92312092,"statfs":{"total":96636764160,"available":94527582208,"internally_reserved":0,"allocated":307777536,"data_stored":304511645,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":295679,"internal_metadata":1801092353},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":8,"seq":34359738884,"num_pgs":98,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":2000988,"kb_used_data":352204,"kb_used_omap":335,"kb_used_meta":1648432,"kb_avail":92370852,"statfs":{"total":96636764160,"available":94587752448,"internally_reserved":0,"allocated":360656896,"data_stored":357436022,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":343633,"internal_metadata":1687994799},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[]},{"osd":0,"up_from":8,"seq":34359738885,"num_pgs":95,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":2020520,"kb_used_data":334040,"kb_used_omap":228,"kb_used_meta":1686235,"kb_avail":92351320,"statfs":{"total":96636764160,"available":94567751680,"internally_reserved":0,"allocated":342056960,"data_stored":338747545,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":234101,"internal_metadata":1726705035},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":5,"apply_latency_ms":5,"commit_latency_ns":5000000,"apply_latency_ns":5000000},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2026-02-18T08:52:15.476 INFO:tasks.ceph.ceph_manager.cluster2:clean! 2026-02-18T08:52:15.476 DEBUG:teuthology.orchestra.run.trial163:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 pg dump --format=json 2026-02-18T08:52:15.634 INFO:teuthology.orchestra.run.trial163.stdout: 2026-02-18T08:52:15.634 INFO:teuthology.orchestra.run.trial163.stderr:dumped all 2026-02-18T08:52:15.645 INFO:teuthology.orchestra.run.trial163.stdout:{"pg_ready":true,"pg_map":{"version":1699,"stamp":"2026-02-18T08:52:14.909897+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590387,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":103,"num_write_kb":1630,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":75,"ondisk_log_size":75,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":274,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":6081256,"kb_used_data":986808,"kb_used_omap":852,"kb_used_meta":5093547,"kb_avail":277034264,"statfs":{"total":289910292480,"available":283683086336,"internally_reserved":0,"allocated":1010491392,"data_stored":1000695212,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":873413,"internal_metadata":5215792187},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":8,"apply_latency_ms":8,"commit_latency_ns":8000000,"apply_latency_ns":8000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-787431672,"num_objects":-193,"num_object_clones":0,"num_object_copies":-386,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-193,"num_whiteouts":0,"num_read":-2429475,"num_read_kb":-92852858,"num_write":-2333992,"num_write_kb":-91441114,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":-1,"num_bytes_recovered":0,"num_keys_recovered":-3,"num_objects_omap":-19,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"6.000829"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":849,"reported_epoch":419,"state":"active+clean","last_fresh":"2026-02-18T08:52:10.122679+0000","last_change":"2026-02-18T08:09:39.904504+0000","last_active":"2026-02-18T08:52:10.122679+0000","last_peered":"2026-02-18T08:52:10.122679+0000","last_clean":"2026-02-18T08:52:10.122679+0000","last_became_active":"2026-02-18T08:09:37.900809+0000","last_became_peered":"2026-02-18T08:09:37.900809+0000","last_unstale":"2026-02-18T08:52:10.122679+0000","last_undegraded":"2026-02-18T08:52:10.122679+0000","last_fullsized":"2026-02-18T08:52:10.122679+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-18T08:09:36.890905+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-18T08:09:36.890905+0000","last_clean_scrub_stamp":"2026-02-18T08:09:36.890905+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-19T18:39:43.531542+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00068838299999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":849,"reported_epoch":419,"state":"active+clean","last_fresh":"2026-02-18T08:52:10.121896+0000","last_change":"2026-02-18T08:09:39.904331+0000","last_active":"2026-02-18T08:52:10.121896+0000","last_peered":"2026-02-18T08:52:10.121896+0000","last_clean":"2026-02-18T08:52:10.121896+0000","last_became_active":"2026-02-18T08:09:37.900594+0000","last_became_peered":"2026-02-18T08:09:37.900594+0000","last_unstale":"2026-02-18T08:52:10.121896+0000","last_undegraded":"2026-02-18T08:52:10.121896+0000","last_fullsized":"2026-02-18T08:52:10.121896+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-18T08:09:36.890905+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-18T08:09:36.890905+0000","last_clean_scrub_stamp":"2026-02-18T08:09:36.890905+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-19T19:59:21.557650+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00053109200000000004,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":850,"reported_epoch":419,"state":"active+clean","last_fresh":"2026-02-18T08:52:10.122610+0000","last_change":"2026-02-18T08:09:39.904488+0000","last_active":"2026-02-18T08:52:10.122610+0000","last_peered":"2026-02-18T08:52:10.122610+0000","last_clean":"2026-02-18T08:52:10.122610+0000","last_became_active":"2026-02-18T08:09:37.900657+0000","last_became_peered":"2026-02-18T08:09:37.900657+0000","last_unstale":"2026-02-18T08:52:10.122610+0000","last_undegraded":"2026-02-18T08:52:10.122610+0000","last_fullsized":"2026-02-18T08:52:10.122610+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-18T08:09:36.890905+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-18T08:09:36.890905+0000","last_clean_scrub_stamp":"2026-02-18T08:09:36.890905+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-19T16:39:19.623528+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00067508900000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":850,"reported_epoch":419,"state":"active+clean","last_fresh":"2026-02-18T08:52:10.120474+0000","last_change":"2026-02-18T08:09:39.904269+0000","last_active":"2026-02-18T08:52:10.120474+0000","last_peered":"2026-02-18T08:52:10.120474+0000","last_clean":"2026-02-18T08:52:10.120474+0000","last_became_active":"2026-02-18T08:09:37.901625+0000","last_became_peered":"2026-02-18T08:09:37.901625+0000","last_unstale":"2026-02-18T08:52:10.120474+0000","last_undegraded":"2026-02-18T08:52:10.120474+0000","last_fullsized":"2026-02-18T08:52:10.120474+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-18T08:09:36.890905+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-18T08:09:36.890905+0000","last_clean_scrub_stamp":"2026-02-18T08:09:36.890905+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-19T14:48:52.058850+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00056007599999999995,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"14'2","reported_seq":852,"reported_epoch":419,"state":"active+clean","last_fresh":"2026-02-18T08:52:10.120653+0000","last_change":"2026-02-18T08:09:39.902339+0000","last_active":"2026-02-18T08:52:10.120653+0000","last_peered":"2026-02-18T08:52:10.120653+0000","last_clean":"2026-02-18T08:52:10.120653+0000","last_became_active":"2026-02-18T08:09:37.900679+0000","last_became_peered":"2026-02-18T08:09:37.900679+0000","last_unstale":"2026-02-18T08:52:10.120653+0000","last_undegraded":"2026-02-18T08:52:10.120653+0000","last_fullsized":"2026-02-18T08:52:10.120653+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-18T08:09:36.890905+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-18T08:09:36.890905+0000","last_clean_scrub_stamp":"2026-02-18T08:09:36.890905+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-19T14:23:03.463004+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00035240299999999999,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":846,"reported_epoch":418,"state":"active+clean","last_fresh":"2026-02-18T08:51:44.191169+0000","last_change":"2026-02-18T08:09:40.252366+0000","last_active":"2026-02-18T08:51:44.191169+0000","last_peered":"2026-02-18T08:51:44.191169+0000","last_clean":"2026-02-18T08:51:44.191169+0000","last_became_active":"2026-02-18T08:09:37.900553+0000","last_became_peered":"2026-02-18T08:09:37.900553+0000","last_unstale":"2026-02-18T08:51:44.191169+0000","last_undegraded":"2026-02-18T08:51:44.191169+0000","last_fullsized":"2026-02-18T08:51:44.191169+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-18T08:09:36.890905+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-18T08:09:36.890905+0000","last_clean_scrub_stamp":"2026-02-18T08:09:36.890905+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-19T19:45:29.501652+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000311546,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":847,"reported_epoch":418,"state":"active+clean","last_fresh":"2026-02-18T08:51:44.191596+0000","last_change":"2026-02-18T08:09:40.252404+0000","last_active":"2026-02-18T08:51:44.191596+0000","last_peered":"2026-02-18T08:51:44.191596+0000","last_clean":"2026-02-18T08:51:44.191596+0000","last_became_active":"2026-02-18T08:09:37.900556+0000","last_became_peered":"2026-02-18T08:09:37.900556+0000","last_unstale":"2026-02-18T08:51:44.191596+0000","last_undegraded":"2026-02-18T08:51:44.191596+0000","last_fullsized":"2026-02-18T08:51:44.191596+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-18T08:09:36.890905+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-18T08:09:36.890905+0000","last_clean_scrub_stamp":"2026-02-18T08:09:36.890905+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-19T15:36:20.626321+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032881799999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.3","version":"12'1","reported_seq":851,"reported_epoch":419,"state":"active+clean","last_fresh":"2026-02-18T08:52:10.121435+0000","last_change":"2026-02-18T08:09:39.904803+0000","last_active":"2026-02-18T08:52:10.121435+0000","last_peered":"2026-02-18T08:52:10.121435+0000","last_clean":"2026-02-18T08:52:10.121435+0000","last_became_active":"2026-02-18T08:09:37.900195+0000","last_became_peered":"2026-02-18T08:09:37.900195+0000","last_unstale":"2026-02-18T08:52:10.121435+0000","last_undegraded":"2026-02-18T08:52:10.121435+0000","last_fullsized":"2026-02-18T08:52:10.121435+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-18T08:09:36.890905+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-18T08:09:36.890905+0000","last_clean_scrub_stamp":"2026-02-18T08:09:36.890905+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-19T09:59:37.426943+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00096736000000000001,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"10'72","reported_seq":940,"reported_epoch":419,"state":"active+clean","last_fresh":"2026-02-18T08:52:10.120080+0000","last_change":"2026-02-18T08:09:36.049944+0000","last_active":"2026-02-18T08:52:10.120080+0000","last_peered":"2026-02-18T08:52:10.120080+0000","last_clean":"2026-02-18T08:52:10.120080+0000","last_became_active":"2026-02-18T08:09:36.049266+0000","last_became_peered":"2026-02-18T08:09:36.049266+0000","last_unstale":"2026-02-18T08:52:10.120080+0000","last_undegraded":"2026-02-18T08:52:10.120080+0000","last_fullsized":"2026-02-18T08:52:10.120080+0000","mapping_epoch":9,"log_start":"0'0","ondisk_log_start":"0'0","created":9,"last_epoch_clean":10,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-18T08:09:34.886217+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-18T08:09:34.886217+0000","last_clean_scrub_stamp":"2026-02-18T08:09:34.886217+0000","objects_scrubbed":0,"log_size":72,"log_dups_size":0,"ondisk_log_size":72,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-19T08:12:47.784849+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":38,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":16,"acting":16,"num_store_stats":3},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1187840,"data_stored":1180736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":72,"ondisk_log_size":72,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":2,"up_from":8,"seq":34359738884,"num_pgs":81,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":2059748,"kb_used_data":300564,"kb_used_omap":288,"kb_used_meta":1758879,"kb_avail":92312092,"statfs":{"total":96636764160,"available":94527582208,"internally_reserved":0,"allocated":307777536,"data_stored":304511645,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":295679,"internal_metadata":1801092353},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":8,"seq":34359738884,"num_pgs":98,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":2000988,"kb_used_data":352204,"kb_used_omap":335,"kb_used_meta":1648432,"kb_avail":92370852,"statfs":{"total":96636764160,"available":94587752448,"internally_reserved":0,"allocated":360656896,"data_stored":357436022,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":343633,"internal_metadata":1687994799},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[]},{"osd":0,"up_from":8,"seq":34359738885,"num_pgs":95,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":2020520,"kb_used_data":334040,"kb_used_omap":228,"kb_used_meta":1686235,"kb_avail":92351320,"statfs":{"total":96636764160,"available":94567751680,"internally_reserved":0,"allocated":342056960,"data_stored":338747545,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":234101,"internal_metadata":1726705035},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":5,"apply_latency_ms":5,"commit_latency_ns":5000000,"apply_latency_ns":5000000},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2026-02-18T08:52:15.646 DEBUG:teuthology.orchestra.run.trial163:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 osd dump --format=json 2026-02-18T08:52:15.802 INFO:teuthology.orchestra.run.trial163.stdout: 2026-02-18T08:52:15.802 INFO:teuthology.orchestra.run.trial163.stdout:{"epoch":421,"fsid":"892b71e6-8019-418b-97e7-5a7b63a9d866","created":"2026-02-18T08:09:28.873380+0000","modified":"2026-02-18T08:52:14.130745+0000","last_up_change":"2026-02-18T08:09:33.883650+0000","last_in_change":"2026-02-18T08:09:29.510528+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":4,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":4,"max_osd":3,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"tentacle","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2026-02-18T08:09:34.514392+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"11","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"nonprimary_shards":"{}","options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":2.9900000095367432,"score_stable":2.9900000095367432,"optimal_score":0.67000001668930054,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2026-02-18T08:09:36.663412+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"14","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":2,"snap_epoch":14,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"nonprimary_shards":"{}","options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_type":"Fair distribution","score_acting":1.8799999952316284,"score_stable":1.8799999952316284,"optimal_score":1,"raw_score_acting":1.8799999952316284,"raw_score_stable":1.8799999952316284,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"6e23dc77-aa8c-4655-9ca3-7777a9882c2c","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":25,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.163:6800","nonce":49861742},{"type":"v1","addr":"10.20.193.163:6801","nonce":49861742}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.163:6802","nonce":49861742},{"type":"v1","addr":"10.20.193.163:6803","nonce":49861742}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.163:6806","nonce":49861742},{"type":"v1","addr":"10.20.193.163:6807","nonce":49861742}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.163:6804","nonce":49861742},{"type":"v1","addr":"10.20.193.163:6805","nonce":49861742}]},"public_addr":"10.20.193.163:6801/49861742","cluster_addr":"10.20.193.163:6803/49861742","heartbeat_back_addr":"10.20.193.163:6807/49861742","heartbeat_front_addr":"10.20.193.163:6805/49861742","state":["exists","up"]},{"osd":1,"uuid":"e04d91f9-3710-4d1d-9161-4c60f2c21444","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":25,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.163:6808","nonce":1532368640},{"type":"v1","addr":"10.20.193.163:6809","nonce":1532368640}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.163:6810","nonce":1532368640},{"type":"v1","addr":"10.20.193.163:6811","nonce":1532368640}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.163:6814","nonce":1532368640},{"type":"v1","addr":"10.20.193.163:6815","nonce":1532368640}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.163:6812","nonce":1532368640},{"type":"v1","addr":"10.20.193.163:6813","nonce":1532368640}]},"public_addr":"10.20.193.163:6809/1532368640","cluster_addr":"10.20.193.163:6811/1532368640","heartbeat_back_addr":"10.20.193.163:6815/1532368640","heartbeat_front_addr":"10.20.193.163:6813/1532368640","state":["exists","up"]},{"osd":2,"uuid":"4d746e5b-ce93-4d1b-89d7-e25977e57605","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":25,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.163:6816","nonce":1781709176},{"type":"v1","addr":"10.20.193.163:6817","nonce":1781709176}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.163:6818","nonce":1781709176},{"type":"v1","addr":"10.20.193.163:6819","nonce":1781709176}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.163:6822","nonce":1781709176},{"type":"v1","addr":"10.20.193.163:6823","nonce":1781709176}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.163:6820","nonce":1781709176},{"type":"v1","addr":"10.20.193.163:6821","nonce":1781709176}]},"public_addr":"10.20.193.163:6817/1781709176","cluster_addr":"10.20.193.163:6819/1781709176","heartbeat_back_addr":"10.20.193.163:6823/1781709176","heartbeat_front_addr":"10.20.193.163:6821/1781709176","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4544132024016699391,"old_weight":0,"last_purged_snaps_scrub":"2026-02-18T08:09:31.969837+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4544132024016699391,"old_weight":0,"last_purged_snaps_scrub":"2026-02-18T08:09:32.050520+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4544132024016699391,"old_weight":0,"last_purged_snaps_scrub":"2026-02-18T08:09:32.221607+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"10.20.193.163:0/1150092237":"2026-02-18T09:52:09.661615+0000","10.20.193.163:0/208203554":"2026-02-18T09:51:33.792416+0000","10.20.193.163:0/3852312173":"2026-02-18T09:50:52.575797+0000","10.20.193.163:0/3304963939":"2026-02-18T09:49:31.214807+0000","10.20.193.163:0/2231122543":"2026-02-18T09:48:09.642691+0000","10.20.193.163:0/1988166407":"2026-02-18T09:47:32.434120+0000","10.20.193.163:0/2624806624":"2026-02-18T09:46:53.640716+0000","10.20.193.163:0/3414276762":"2026-02-18T09:46:12.086004+0000","10.20.193.163:0/1793271187":"2026-02-18T09:44:25.037503+0000","10.20.193.163:0/2260824474":"2026-02-18T09:38:10.613696+0000","10.20.193.163:0/613239226":"2026-02-18T09:15:35.673550+0000","10.20.193.163:0/1406974797":"2026-02-18T09:38:01.349028+0000","10.20.193.163:0/1451832559":"2026-02-18T09:37:47.241183+0000","10.20.193.163:0/2606039982":"2026-02-18T09:35:58.835890+0000","10.20.193.163:0/4213881607":"2026-02-18T09:39:59.333790+0000","10.20.193.163:0/635937698":"2026-02-18T09:37:39.911370+0000","10.20.193.163:0/3178064416":"2026-02-18T09:37:31.579916+0000","10.20.193.163:0/3134588468":"2026-02-18T09:43:50.922759+0000","10.20.193.163:0/1365419947":"2026-02-18T09:36:26.182625+0000","10.20.193.163:0/1063446783":"2026-02-18T09:11:31.704712+0000","10.20.193.163:0/2689227962":"2026-02-18T09:36:17.977706+0000","10.20.193.163:0/1704021602":"2026-02-18T09:39:09.992220+0000","10.20.193.163:0/3316411344":"2026-02-18T09:35:21.186664+0000","10.20.193.163:0/1470019256":"2026-02-18T09:37:16.506249+0000","10.20.193.163:0/950338269":"2026-02-18T09:42:41.618998+0000","10.20.193.163:0/1798082081":"2026-02-18T09:36:36.229890+0000","10.20.193.163:0/2314676158":"2026-02-18T09:23:44.843486+0000","10.20.193.163:0/3914458165":"2026-02-18T09:38:39.946497+0000","10.20.193.163:0/3124717104":"2026-02-18T09:42:57.850581+0000","10.20.193.163:0/3423700380":"2026-02-18T09:36:45.616167+0000","10.20.193.163:0/1662554142":"2026-02-18T09:13:58.692817+0000","10.20.193.163:0/517450783":"2026-02-18T09:22:10.626449+0000","10.20.193.163:0/4287540463":"2026-02-18T09:38:26.808043+0000","10.20.193.163:0/2696441206":"2026-02-18T09:19:38.833235+0000","10.20.193.163:0/1294776808":"2026-02-18T09:35:00.713087+0000","10.20.193.163:0/1596045593":"2026-02-18T09:39:20.551767+0000","10.20.193.163:0/3340369409":"2026-02-18T09:43:16.368674+0000","10.20.193.163:0/3977820359":"2026-02-18T09:40:07.802467+0000","10.20.193.163:0/3400068666":"2026-02-18T09:34:46.013569+0000","10.20.193.163:0/327921100":"2026-02-18T09:16:51.134369+0000","10.20.193.163:0/3097969815":"2026-02-18T09:38:49.606722+0000","10.20.193.163:0/1501648775":"2026-02-18T09:35:35.352081+0000","10.20.193.163:0/4179906352":"2026-02-18T09:42:05.522121+0000","10.20.193.163:0/2612224757":"2026-02-18T09:12:47.460848+0000","10.20.193.163:0/1370326951":"2026-02-18T09:44:05.223496+0000","10.20.193.163:0/3425363587":"2026-02-18T09:38:59.865910+0000","10.20.193.163:0/2624612519":"2026-02-18T09:39:45.263838+0000","10.20.193.163:0/1239137096":"2026-02-18T09:41:15.367278+0000","10.20.193.163:0/711127850":"2026-02-18T09:40:22.306623+0000","10.20.193.163:0/1944341406":"2026-02-18T09:50:07.985694+0000","10.20.193.163:0/3586197916":"2026-02-18T09:41:34.579433+0000","10.20.193.163:0/1282250203":"2026-02-18T09:40:33.675862+0000","10.20.193.163:0/3658566221":"2026-02-18T09:34:34.986137+0000","10.20.193.163:0/1600147852":"2026-02-18T09:40:41.827340+0000","10.20.193.163:0/2628424331":"2026-02-18T09:40:52.297171+0000","10.20.193.163:0/3333761731":"2026-02-18T09:42:27.358793+0000","10.20.193.163:0/4193702289":"2026-02-18T09:40:58.630310+0000","10.20.193.163:0/4019567456":"2026-02-18T09:35:13.941449+0000","10.20.193.163:0/298966498":"2026-02-18T09:18:29.755062+0000","10.20.193.163:0/227556613":"2026-02-18T09:21:08.827107+0000","10.20.193.163:0/1755592904":"2026-02-18T09:41:24.045158+0000","10.20.193.163:0/3277223743":"2026-02-18T09:42:09.984499+0000","10.20.193.163:0/847015797":"2026-02-18T09:42:51.914340+0000","10.20.193.163:0/2946450159":"2026-02-18T09:43:10.053044+0000","10.20.193.163:0/335183446":"2026-02-18T09:39:32.209861+0000","10.20.193.163:0/2292242540":"2026-02-18T09:43:40.922426+0000","10.20.193.163:0/706907924":"2026-02-18T09:48:53.370792+0000","10.20.193.163:0/2954074548":"2026-02-18T09:43:29.606272+0000","10.20.193.163:0/189498349":"2026-02-18T09:35:46.626240+0000","10.20.193.163:0/368989962":"2026-02-18T09:44:13.954260+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"isa","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2026-02-18T08:52:16.816 INFO:tasks.ceph:Scrubbing cluster2.osd.0 2026-02-18T08:52:16.817 DEBUG:teuthology.orchestra.run.trial163:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 tell osd.0 config set osd_debug_deep_scrub_sleep 0 2026-02-18T08:52:16.898 INFO:teuthology.orchestra.run.trial163.stdout:{ 2026-02-18T08:52:16.899 INFO:teuthology.orchestra.run.trial163.stdout: "success": "osd_debug_deep_scrub_sleep = '' (not observed, change may require restart) " 2026-02-18T08:52:16.899 INFO:teuthology.orchestra.run.trial163.stdout:} 2026-02-18T08:52:16.910 DEBUG:teuthology.orchestra.run.trial163:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 osd deep-scrub 0 2026-02-18T08:52:17.066 INFO:teuthology.orchestra.run.trial163.stderr:instructed osd(s) 0 to deep-scrub 2026-02-18T08:52:17.079 INFO:tasks.ceph:Scrubbing cluster2.osd.1 2026-02-18T08:52:17.079 DEBUG:teuthology.orchestra.run.trial163:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 tell osd.1 config set osd_debug_deep_scrub_sleep 0 2026-02-18T08:52:17.158 INFO:teuthology.orchestra.run.trial163.stdout:{ 2026-02-18T08:52:17.158 INFO:teuthology.orchestra.run.trial163.stdout: "success": "osd_debug_deep_scrub_sleep = '' (not observed, change may require restart) " 2026-02-18T08:52:17.158 INFO:teuthology.orchestra.run.trial163.stdout:} 2026-02-18T08:52:17.168 DEBUG:teuthology.orchestra.run.trial163:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 osd deep-scrub 1 2026-02-18T08:52:17.326 INFO:teuthology.orchestra.run.trial163.stderr:instructed osd(s) 1 to deep-scrub 2026-02-18T08:52:17.339 INFO:tasks.ceph:Scrubbing cluster2.osd.2 2026-02-18T08:52:17.339 DEBUG:teuthology.orchestra.run.trial163:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 tell osd.2 config set osd_debug_deep_scrub_sleep 0 2026-02-18T08:52:17.422 INFO:teuthology.orchestra.run.trial163.stdout:{ 2026-02-18T08:52:17.422 INFO:teuthology.orchestra.run.trial163.stdout: "success": "osd_debug_deep_scrub_sleep = '' (not observed, change may require restart) osd_delete_sleep = '' osd_delete_sleep_hdd = '' osd_delete_sleep_hybrid = '' osd_delete_sleep_ssd = '' osd_max_backfills = '' osd_recovery_max_active = '' osd_recovery_max_active_hdd = '' osd_recovery_max_active_ssd = '' osd_recovery_sleep = '' osd_recovery_sleep_degraded = '' osd_recovery_sleep_degraded_hdd = '' osd_recovery_sleep_degraded_hybrid = '' osd_recovery_sleep_degraded_ssd = '' osd_recovery_sleep_hdd = '' osd_recovery_sleep_hybrid = '' osd_recovery_sleep_ssd = '' osd_scrub_sleep = '' osd_snap_trim_sleep = '' osd_snap_trim_sleep_hdd = '' osd_snap_trim_sleep_hybrid = '' osd_snap_trim_sleep_ssd = '' " 2026-02-18T08:52:17.422 INFO:teuthology.orchestra.run.trial163.stdout:} 2026-02-18T08:52:17.432 DEBUG:teuthology.orchestra.run.trial163:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 osd deep-scrub 2 2026-02-18T08:52:17.592 INFO:teuthology.orchestra.run.trial163.stderr:instructed osd(s) 2 to deep-scrub 2026-02-18T08:52:17.604 DEBUG:teuthology.orchestra.run.trial163:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 pg dump --format=json 2026-02-18T08:52:17.761 INFO:teuthology.orchestra.run.trial163.stdout: 2026-02-18T08:52:17.761 INFO:teuthology.orchestra.run.trial163.stderr:dumped all 2026-02-18T08:52:17.774 INFO:teuthology.orchestra.run.trial163.stdout:{"pg_ready":true,"pg_map":{"version":1700,"stamp":"2026-02-18T08:52:16.910082+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590387,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":103,"num_write_kb":1630,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":75,"ondisk_log_size":75,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":179,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":5146364,"kb_used_data":15820,"kb_used_omap":863,"kb_used_meta":5129632,"kb_avail":277969156,"statfs":{"total":289910292480,"available":284640415744,"internally_reserved":0,"allocated":16199680,"data_stored":8102806,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":884614,"internal_metadata":5252743290},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":8,"apply_latency_ms":8,"commit_latency_ns":8000000,"apply_latency_ns":8000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-787431672,"num_objects":-193,"num_object_clones":0,"num_object_copies":-386,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-193,"num_whiteouts":0,"num_read":-2429483,"num_read_kb":-92852866,"num_write":-2333994,"num_write_kb":-91441116,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":-1,"num_bytes_recovered":0,"num_keys_recovered":-3,"num_objects_omap":-19,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"6.790013"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":853,"reported_epoch":421,"state":"active+clean","last_fresh":"2026-02-18T08:52:14.351997+0000","last_change":"2026-02-18T08:09:39.904504+0000","last_active":"2026-02-18T08:52:14.351997+0000","last_peered":"2026-02-18T08:52:14.351997+0000","last_clean":"2026-02-18T08:52:14.351997+0000","last_became_active":"2026-02-18T08:09:37.900809+0000","last_became_peered":"2026-02-18T08:09:37.900809+0000","last_unstale":"2026-02-18T08:52:14.351997+0000","last_undegraded":"2026-02-18T08:52:14.351997+0000","last_fullsized":"2026-02-18T08:52:14.351997+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-18T08:09:36.890905+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-18T08:09:36.890905+0000","last_clean_scrub_stamp":"2026-02-18T08:09:36.890905+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-19T18:39:43.531542+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00068838299999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":853,"reported_epoch":421,"state":"active+clean","last_fresh":"2026-02-18T08:52:14.351526+0000","last_change":"2026-02-18T08:09:39.904331+0000","last_active":"2026-02-18T08:52:14.351526+0000","last_peered":"2026-02-18T08:52:14.351526+0000","last_clean":"2026-02-18T08:52:14.351526+0000","last_became_active":"2026-02-18T08:09:37.900594+0000","last_became_peered":"2026-02-18T08:09:37.900594+0000","last_unstale":"2026-02-18T08:52:14.351526+0000","last_undegraded":"2026-02-18T08:52:14.351526+0000","last_fullsized":"2026-02-18T08:52:14.351526+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-18T08:09:36.890905+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-18T08:09:36.890905+0000","last_clean_scrub_stamp":"2026-02-18T08:09:36.890905+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-19T19:59:21.557650+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00053109200000000004,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":854,"reported_epoch":421,"state":"active+clean","last_fresh":"2026-02-18T08:52:14.353041+0000","last_change":"2026-02-18T08:09:39.904488+0000","last_active":"2026-02-18T08:52:14.353041+0000","last_peered":"2026-02-18T08:52:14.353041+0000","last_clean":"2026-02-18T08:52:14.353041+0000","last_became_active":"2026-02-18T08:09:37.900657+0000","last_became_peered":"2026-02-18T08:09:37.900657+0000","last_unstale":"2026-02-18T08:52:14.353041+0000","last_undegraded":"2026-02-18T08:52:14.353041+0000","last_fullsized":"2026-02-18T08:52:14.353041+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-18T08:09:36.890905+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-18T08:09:36.890905+0000","last_clean_scrub_stamp":"2026-02-18T08:09:36.890905+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-19T16:39:19.623528+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00067508900000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":854,"reported_epoch":421,"state":"active+clean","last_fresh":"2026-02-18T08:52:14.349447+0000","last_change":"2026-02-18T08:09:39.904269+0000","last_active":"2026-02-18T08:52:14.349447+0000","last_peered":"2026-02-18T08:52:14.349447+0000","last_clean":"2026-02-18T08:52:14.349447+0000","last_became_active":"2026-02-18T08:09:37.901625+0000","last_became_peered":"2026-02-18T08:09:37.901625+0000","last_unstale":"2026-02-18T08:52:14.349447+0000","last_undegraded":"2026-02-18T08:52:14.349447+0000","last_fullsized":"2026-02-18T08:52:14.349447+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-18T08:09:36.890905+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-18T08:09:36.890905+0000","last_clean_scrub_stamp":"2026-02-18T08:09:36.890905+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-19T14:48:52.058850+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00056007599999999995,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"14'2","reported_seq":856,"reported_epoch":421,"state":"active+clean","last_fresh":"2026-02-18T08:52:14.138507+0000","last_change":"2026-02-18T08:09:39.902339+0000","last_active":"2026-02-18T08:52:14.138507+0000","last_peered":"2026-02-18T08:52:14.138507+0000","last_clean":"2026-02-18T08:52:14.138507+0000","last_became_active":"2026-02-18T08:09:37.900679+0000","last_became_peered":"2026-02-18T08:09:37.900679+0000","last_unstale":"2026-02-18T08:52:14.138507+0000","last_undegraded":"2026-02-18T08:52:14.138507+0000","last_fullsized":"2026-02-18T08:52:14.138507+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-18T08:09:36.890905+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-18T08:09:36.890905+0000","last_clean_scrub_stamp":"2026-02-18T08:09:36.890905+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-19T14:23:03.463004+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00035240299999999999,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":852,"reported_epoch":421,"state":"active+clean","last_fresh":"2026-02-18T08:52:14.722082+0000","last_change":"2026-02-18T08:09:40.252366+0000","last_active":"2026-02-18T08:52:14.722082+0000","last_peered":"2026-02-18T08:52:14.722082+0000","last_clean":"2026-02-18T08:52:14.722082+0000","last_became_active":"2026-02-18T08:09:37.900553+0000","last_became_peered":"2026-02-18T08:09:37.900553+0000","last_unstale":"2026-02-18T08:52:14.722082+0000","last_undegraded":"2026-02-18T08:52:14.722082+0000","last_fullsized":"2026-02-18T08:52:14.722082+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-18T08:09:36.890905+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-18T08:09:36.890905+0000","last_clean_scrub_stamp":"2026-02-18T08:09:36.890905+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-19T19:45:29.501652+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000311546,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":853,"reported_epoch":421,"state":"active+clean","last_fresh":"2026-02-18T08:52:14.722400+0000","last_change":"2026-02-18T08:09:40.252404+0000","last_active":"2026-02-18T08:52:14.722400+0000","last_peered":"2026-02-18T08:52:14.722400+0000","last_clean":"2026-02-18T08:52:14.722400+0000","last_became_active":"2026-02-18T08:09:37.900556+0000","last_became_peered":"2026-02-18T08:09:37.900556+0000","last_unstale":"2026-02-18T08:52:14.722400+0000","last_undegraded":"2026-02-18T08:52:14.722400+0000","last_fullsized":"2026-02-18T08:52:14.722400+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-18T08:09:36.890905+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-18T08:09:36.890905+0000","last_clean_scrub_stamp":"2026-02-18T08:09:36.890905+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-19T15:36:20.626321+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032881799999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.3","version":"12'1","reported_seq":855,"reported_epoch":421,"state":"active+clean","last_fresh":"2026-02-18T08:52:14.351801+0000","last_change":"2026-02-18T08:09:39.904803+0000","last_active":"2026-02-18T08:52:14.351801+0000","last_peered":"2026-02-18T08:52:14.351801+0000","last_clean":"2026-02-18T08:52:14.351801+0000","last_became_active":"2026-02-18T08:09:37.900195+0000","last_became_peered":"2026-02-18T08:09:37.900195+0000","last_unstale":"2026-02-18T08:52:14.351801+0000","last_undegraded":"2026-02-18T08:52:14.351801+0000","last_fullsized":"2026-02-18T08:52:14.351801+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-18T08:09:36.890905+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-18T08:09:36.890905+0000","last_clean_scrub_stamp":"2026-02-18T08:09:36.890905+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-19T09:59:37.426943+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00096736000000000001,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"10'72","reported_seq":944,"reported_epoch":421,"state":"active+clean","last_fresh":"2026-02-18T08:52:14.352902+0000","last_change":"2026-02-18T08:09:36.049944+0000","last_active":"2026-02-18T08:52:14.352902+0000","last_peered":"2026-02-18T08:52:14.352902+0000","last_clean":"2026-02-18T08:52:14.352902+0000","last_became_active":"2026-02-18T08:09:36.049266+0000","last_became_peered":"2026-02-18T08:09:36.049266+0000","last_unstale":"2026-02-18T08:52:14.352902+0000","last_undegraded":"2026-02-18T08:52:14.352902+0000","last_fullsized":"2026-02-18T08:52:14.352902+0000","mapping_epoch":9,"log_start":"0'0","ondisk_log_start":"0'0","created":9,"last_epoch_clean":10,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-18T08:09:34.886217+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-18T08:09:34.886217+0000","last_clean_scrub_stamp":"2026-02-18T08:09:34.886217+0000","objects_scrubbed":0,"log_size":72,"log_dups_size":0,"ondisk_log_size":72,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-19T08:12:47.784849+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":38,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":16,"acting":16,"num_store_stats":3},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1187840,"data_stored":1180736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":72,"ondisk_log_size":72,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":2,"up_from":8,"seq":34359738885,"num_pgs":75,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":1782116,"kb_used_data":4884,"kb_used_omap":291,"kb_used_meta":1776924,"kb_avail":92589724,"statfs":{"total":96636764160,"available":94811877376,"internally_reserved":0,"allocated":5001216,"data_stored":2307344,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":298465,"internal_metadata":1819570719},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[]},{"osd":1,"up_from":8,"seq":34359738885,"num_pgs":56,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":1654252,"kb_used_data":5468,"kb_used_omap":339,"kb_used_meta":1648428,"kb_avail":92717588,"statfs":{"total":96636764160,"available":94942810112,"internally_reserved":0,"allocated":5599232,"data_stored":2897731,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":347998,"internal_metadata":1687990434},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[]},{"osd":0,"up_from":8,"seq":34359738886,"num_pgs":48,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":1709996,"kb_used_data":5468,"kb_used_omap":232,"kb_used_meta":1704279,"kb_avail":92661844,"statfs":{"total":96636764160,"available":94885728256,"internally_reserved":0,"allocated":5599232,"data_stored":2897731,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":238151,"internal_metadata":1745182137},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2026-02-18T08:52:17.779 INFO:tasks.ceph:pgid 2.7 last_scrub_stamp 2026-02-18T08:09:36.890905+0000 time.struct_time(tm_year=2026, tm_mon=2, tm_mday=18, tm_hour=8, tm_min=9, tm_sec=36, tm_wday=2, tm_yday=49, tm_isdst=-1) <= time.struct_time(tm_year=2026, tm_mon=2, tm_mday=18, tm_hour=8, tm_min=52, tm_sec=15, tm_wday=2, tm_yday=49, tm_isdst=0) 2026-02-18T08:52:17.779 INFO:tasks.ceph:pgid 2.6 last_scrub_stamp 2026-02-18T08:09:36.890905+0000 time.struct_time(tm_year=2026, tm_mon=2, tm_mday=18, tm_hour=8, tm_min=9, tm_sec=36, tm_wday=2, tm_yday=49, tm_isdst=-1) <= time.struct_time(tm_year=2026, tm_mon=2, tm_mday=18, tm_hour=8, tm_min=52, tm_sec=15, tm_wday=2, tm_yday=49, tm_isdst=0) 2026-02-18T08:52:17.779 INFO:tasks.ceph:pgid 2.5 last_scrub_stamp 2026-02-18T08:09:36.890905+0000 time.struct_time(tm_year=2026, tm_mon=2, tm_mday=18, tm_hour=8, tm_min=9, tm_sec=36, tm_wday=2, tm_yday=49, tm_isdst=-1) <= time.struct_time(tm_year=2026, tm_mon=2, tm_mday=18, tm_hour=8, tm_min=52, tm_sec=15, tm_wday=2, tm_yday=49, tm_isdst=0) 2026-02-18T08:52:17.779 INFO:tasks.ceph:pgid 2.4 last_scrub_stamp 2026-02-18T08:09:36.890905+0000 time.struct_time(tm_year=2026, tm_mon=2, tm_mday=18, tm_hour=8, tm_min=9, tm_sec=36, tm_wday=2, tm_yday=49, tm_isdst=-1) <= time.struct_time(tm_year=2026, tm_mon=2, tm_mday=18, tm_hour=8, tm_min=52, tm_sec=15, tm_wday=2, tm_yday=49, tm_isdst=0) 2026-02-18T08:52:17.779 INFO:tasks.ceph:pgid 2.2 last_scrub_stamp 2026-02-18T08:09:36.890905+0000 time.struct_time(tm_year=2026, tm_mon=2, tm_mday=18, tm_hour=8, tm_min=9, tm_sec=36, tm_wday=2, tm_yday=49, tm_isdst=-1) <= time.struct_time(tm_year=2026, tm_mon=2, tm_mday=18, tm_hour=8, tm_min=52, tm_sec=15, tm_wday=2, tm_yday=49, tm_isdst=0) 2026-02-18T08:52:17.780 INFO:tasks.ceph:pgid 2.1 last_scrub_stamp 2026-02-18T08:09:36.890905+0000 time.struct_time(tm_year=2026, tm_mon=2, tm_mday=18, tm_hour=8, tm_min=9, tm_sec=36, tm_wday=2, tm_yday=49, tm_isdst=-1) <= time.struct_time(tm_year=2026, tm_mon=2, tm_mday=18, tm_hour=8, tm_min=52, tm_sec=15, tm_wday=2, tm_yday=49, tm_isdst=0) 2026-02-18T08:52:17.780 INFO:tasks.ceph:pgid 2.0 last_scrub_stamp 2026-02-18T08:09:36.890905+0000 time.struct_time(tm_year=2026, tm_mon=2, tm_mday=18, tm_hour=8, tm_min=9, tm_sec=36, tm_wday=2, tm_yday=49, tm_isdst=-1) <= time.struct_time(tm_year=2026, tm_mon=2, tm_mday=18, tm_hour=8, tm_min=52, tm_sec=15, tm_wday=2, tm_yday=49, tm_isdst=0) 2026-02-18T08:52:17.780 INFO:tasks.ceph:pgid 2.3 last_scrub_stamp 2026-02-18T08:09:36.890905+0000 time.struct_time(tm_year=2026, tm_mon=2, tm_mday=18, tm_hour=8, tm_min=9, tm_sec=36, tm_wday=2, tm_yday=49, tm_isdst=-1) <= time.struct_time(tm_year=2026, tm_mon=2, tm_mday=18, tm_hour=8, tm_min=52, tm_sec=15, tm_wday=2, tm_yday=49, tm_isdst=0) 2026-02-18T08:52:17.780 INFO:tasks.ceph:pgid 1.0 last_scrub_stamp 2026-02-18T08:09:34.886217+0000 time.struct_time(tm_year=2026, tm_mon=2, tm_mday=18, tm_hour=8, tm_min=9, tm_sec=34, tm_wday=2, tm_yday=49, tm_isdst=-1) <= time.struct_time(tm_year=2026, tm_mon=2, tm_mday=18, tm_hour=8, tm_min=52, tm_sec=15, tm_wday=2, tm_yday=49, tm_isdst=0) 2026-02-18T08:52:17.780 INFO:tasks.ceph:Still waiting for all pgs to be scrubbed. 2026-02-18T08:52:37.781 DEBUG:teuthology.orchestra.run.trial163:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 pg dump --format=json 2026-02-18T08:52:38.052 INFO:teuthology.orchestra.run.trial163.stderr:2026-02-18T08:52:38.047+0000 7fecb88ab640 0 --2- 10.20.193.163:0/2479587358 >> [v2:10.20.193.163:3300/0,v1:10.20.193.163:6789/0] conn(0x7fecb4093950 0x7fecb4059d60 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request get_initial_auth_request returned -2 2026-02-18T08:52:38.144 INFO:teuthology.orchestra.run.trial163.stdout: 2026-02-18T08:52:38.144 INFO:teuthology.orchestra.run.trial163.stderr:dumped all 2026-02-18T08:52:38.155 INFO:teuthology.orchestra.run.trial163.stdout:{"pg_ready":true,"pg_map":{"version":1710,"stamp":"2026-02-18T08:52:36.911696+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590387,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":103,"num_write_kb":1630,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":79,"ondisk_log_size":79,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":18,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":5146364,"kb_used_data":15820,"kb_used_omap":863,"kb_used_meta":5129632,"kb_avail":277969156,"statfs":{"total":289910292480,"available":284640415744,"internally_reserved":0,"allocated":16199680,"data_stored":8102806,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":884614,"internal_metadata":5252743290},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.000944"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":861,"reported_epoch":421,"state":"active+clean","last_fresh":"2026-02-18T08:52:22.346221+0000","last_change":"2026-02-18T08:52:22.346221+0000","last_active":"2026-02-18T08:52:22.346221+0000","last_peered":"2026-02-18T08:52:22.346221+0000","last_clean":"2026-02-18T08:52:22.346221+0000","last_became_active":"2026-02-18T08:09:37.900809+0000","last_became_peered":"2026-02-18T08:09:37.900809+0000","last_unstale":"2026-02-18T08:52:22.346221+0000","last_undegraded":"2026-02-18T08:52:22.346221+0000","last_fullsized":"2026-02-18T08:52:22.346221+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-18T08:52:22.346141+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-18T08:52:22.346141+0000","last_clean_scrub_stamp":"2026-02-18T08:52:22.346141+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-19T13:58:15.515964+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00068838299999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":861,"reported_epoch":421,"state":"active+clean","last_fresh":"2026-02-18T08:52:21.386173+0000","last_change":"2026-02-18T08:52:21.386173+0000","last_active":"2026-02-18T08:52:21.386173+0000","last_peered":"2026-02-18T08:52:21.386173+0000","last_clean":"2026-02-18T08:52:21.386173+0000","last_became_active":"2026-02-18T08:09:37.900594+0000","last_became_peered":"2026-02-18T08:09:37.900594+0000","last_unstale":"2026-02-18T08:52:21.386173+0000","last_undegraded":"2026-02-18T08:52:21.386173+0000","last_fullsized":"2026-02-18T08:52:21.386173+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-18T08:52:21.386048+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-18T08:52:21.386048+0000","last_clean_scrub_stamp":"2026-02-18T08:52:21.386048+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-19T10:51:08.780504+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00053109200000000004,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":862,"reported_epoch":421,"state":"active+clean","last_fresh":"2026-02-18T08:52:20.432615+0000","last_change":"2026-02-18T08:52:20.432615+0000","last_active":"2026-02-18T08:52:20.432615+0000","last_peered":"2026-02-18T08:52:20.432615+0000","last_clean":"2026-02-18T08:52:20.432615+0000","last_became_active":"2026-02-18T08:09:37.900657+0000","last_became_peered":"2026-02-18T08:09:37.900657+0000","last_unstale":"2026-02-18T08:52:20.432615+0000","last_undegraded":"2026-02-18T08:52:20.432615+0000","last_fullsized":"2026-02-18T08:52:20.432615+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-18T08:52:20.432548+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-18T08:52:20.432548+0000","last_clean_scrub_stamp":"2026-02-18T08:52:20.432548+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2026-02-19T10:52:12.705610+0000","scrub_duration":4,"objects_trimmed":0,"snaptrim_duration":0.00067508900000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":862,"reported_epoch":421,"state":"active+clean","last_fresh":"2026-02-18T08:52:18.481278+0000","last_change":"2026-02-18T08:52:18.481278+0000","last_active":"2026-02-18T08:52:18.481278+0000","last_peered":"2026-02-18T08:52:18.481278+0000","last_clean":"2026-02-18T08:52:18.481278+0000","last_became_active":"2026-02-18T08:09:37.901625+0000","last_became_peered":"2026-02-18T08:09:37.901625+0000","last_unstale":"2026-02-18T08:52:18.481278+0000","last_undegraded":"2026-02-18T08:52:18.481278+0000","last_fullsized":"2026-02-18T08:52:18.481278+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-18T08:52:18.481141+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-18T08:52:18.481141+0000","last_clean_scrub_stamp":"2026-02-18T08:52:18.481141+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-19T16:38:12.342948+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00056007599999999995,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"421'3","reported_seq":865,"reported_epoch":421,"state":"active+clean","last_fresh":"2026-02-18T08:52:17.231552+0000","last_change":"2026-02-18T08:52:17.231552+0000","last_active":"2026-02-18T08:52:17.231552+0000","last_peered":"2026-02-18T08:52:17.231552+0000","last_clean":"2026-02-18T08:52:17.231552+0000","last_became_active":"2026-02-18T08:09:37.900679+0000","last_became_peered":"2026-02-18T08:09:37.900679+0000","last_unstale":"2026-02-18T08:52:17.231552+0000","last_undegraded":"2026-02-18T08:52:17.231552+0000","last_fullsized":"2026-02-18T08:52:17.231552+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"421'3","last_scrub_stamp":"2026-02-18T08:52:17.231506+0000","last_deep_scrub":"421'3","last_deep_scrub_stamp":"2026-02-18T08:52:17.231506+0000","last_clean_scrub_stamp":"2026-02-18T08:52:17.231506+0000","objects_scrubbed":1,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-19T09:27:04.864340+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00035240299999999999,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":860,"reported_epoch":421,"state":"active+clean","last_fresh":"2026-02-18T08:52:18.729944+0000","last_change":"2026-02-18T08:52:18.729944+0000","last_active":"2026-02-18T08:52:18.729944+0000","last_peered":"2026-02-18T08:52:18.729944+0000","last_clean":"2026-02-18T08:52:18.729944+0000","last_became_active":"2026-02-18T08:09:37.900553+0000","last_became_peered":"2026-02-18T08:09:37.900553+0000","last_unstale":"2026-02-18T08:52:18.729944+0000","last_undegraded":"2026-02-18T08:52:18.729944+0000","last_fullsized":"2026-02-18T08:52:18.729944+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-18T08:52:18.729868+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-18T08:52:18.729868+0000","last_clean_scrub_stamp":"2026-02-18T08:52:18.729868+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-19T15:43:56.261419+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000311546,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":861,"reported_epoch":421,"state":"active+clean","last_fresh":"2026-02-18T08:52:17.754733+0000","last_change":"2026-02-18T08:52:17.754733+0000","last_active":"2026-02-18T08:52:17.754733+0000","last_peered":"2026-02-18T08:52:17.754733+0000","last_clean":"2026-02-18T08:52:17.754733+0000","last_became_active":"2026-02-18T08:09:37.900556+0000","last_became_peered":"2026-02-18T08:09:37.900556+0000","last_unstale":"2026-02-18T08:52:17.754733+0000","last_undegraded":"2026-02-18T08:52:17.754733+0000","last_fullsized":"2026-02-18T08:52:17.754733+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-18T08:52:17.754671+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-18T08:52:17.754671+0000","last_clean_scrub_stamp":"2026-02-18T08:52:17.754671+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-19T10:47:20.576055+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032881799999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.3","version":"421'2","reported_seq":864,"reported_epoch":421,"state":"active+clean","last_fresh":"2026-02-18T08:52:17.472183+0000","last_change":"2026-02-18T08:52:17.472183+0000","last_active":"2026-02-18T08:52:17.472183+0000","last_peered":"2026-02-18T08:52:17.472183+0000","last_clean":"2026-02-18T08:52:17.472183+0000","last_became_active":"2026-02-18T08:09:37.900195+0000","last_became_peered":"2026-02-18T08:09:37.900195+0000","last_unstale":"2026-02-18T08:52:17.472183+0000","last_undegraded":"2026-02-18T08:52:17.472183+0000","last_fullsized":"2026-02-18T08:52:17.472183+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"421'2","last_scrub_stamp":"2026-02-18T08:52:17.472118+0000","last_deep_scrub":"421'2","last_deep_scrub_stamp":"2026-02-18T08:52:17.472118+0000","last_clean_scrub_stamp":"2026-02-18T08:52:17.472118+0000","objects_scrubbed":1,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2026-02-19T13:27:24.256380+0000","scrub_duration":4,"objects_trimmed":0,"snaptrim_duration":0.00096736000000000001,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"421'74","reported_seq":954,"reported_epoch":421,"state":"active+clean","last_fresh":"2026-02-18T08:52:19.482207+0000","last_change":"2026-02-18T08:52:19.482207+0000","last_active":"2026-02-18T08:52:19.482207+0000","last_peered":"2026-02-18T08:52:19.482207+0000","last_clean":"2026-02-18T08:52:19.482207+0000","last_became_active":"2026-02-18T08:09:36.049266+0000","last_became_peered":"2026-02-18T08:09:36.049266+0000","last_unstale":"2026-02-18T08:52:19.482207+0000","last_undegraded":"2026-02-18T08:52:19.482207+0000","last_fullsized":"2026-02-18T08:52:19.482207+0000","mapping_epoch":9,"log_start":"0'0","ondisk_log_start":"0'0","created":9,"last_epoch_clean":10,"parent":"0.0","parent_split_bits":0,"last_scrub":"421'74","last_scrub_stamp":"2026-02-18T08:52:19.482167+0000","last_deep_scrub":"421'74","last_deep_scrub_stamp":"2026-02-18T08:52:19.482167+0000","last_clean_scrub_stamp":"2026-02-18T08:52:19.482167+0000","objects_scrubbed":2,"log_size":74,"log_dups_size":0,"ondisk_log_size":74,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2026-02-19T16:36:55.970969+0000","scrub_duration":4,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":38,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":5,"ondisk_log_size":5,"up":16,"acting":16,"num_store_stats":3},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1187840,"data_stored":1180736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":74,"ondisk_log_size":74,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":2,"up_from":8,"seq":34359738889,"num_pgs":3,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":1782116,"kb_used_data":4884,"kb_used_omap":291,"kb_used_meta":1776924,"kb_avail":92589724,"statfs":{"total":96636764160,"available":94811877376,"internally_reserved":0,"allocated":5001216,"data_stored":2307344,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":298465,"internal_metadata":1819570719},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":8,"seq":34359738889,"num_pgs":9,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":1654252,"kb_used_data":5468,"kb_used_omap":339,"kb_used_meta":1648428,"kb_avail":92717588,"statfs":{"total":96636764160,"available":94942810112,"internally_reserved":0,"allocated":5599232,"data_stored":2897731,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":347998,"internal_metadata":1687990434},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":8,"seq":34359738890,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":1709996,"kb_used_data":5468,"kb_used_omap":232,"kb_used_meta":1704279,"kb_avail":92661844,"statfs":{"total":96636764160,"available":94885728256,"internally_reserved":0,"allocated":5599232,"data_stored":2897731,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":238151,"internal_metadata":1745182137},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2026-02-18T08:52:38.157 DEBUG:teuthology.orchestra.run.trial163:> sudo ceph --cluster cluster2 config set global mon_health_to_clog false 2026-02-18T08:52:38.328 INFO:teuthology.misc:Shutting down mds daemons... 2026-02-18T08:52:38.329 INFO:teuthology.misc:Shutting down osd daemons... 2026-02-18T08:52:38.329 DEBUG:tasks.ceph.cluster2.osd.0:waiting for process to exit 2026-02-18T08:52:38.329 INFO:teuthology.orchestra.run:waiting for 300 2026-02-18T08:52:41.482 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon cluster2.osd.0 is failed for ~0s 2026-02-18T08:52:44.431 INFO:tasks.ceph.cluster2.osd.0:Stopped 2026-02-18T08:52:44.431 DEBUG:tasks.ceph.cluster2.osd.1:waiting for process to exit 2026-02-18T08:52:44.431 INFO:teuthology.orchestra.run:waiting for 300 2026-02-18T08:52:46.785 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon cluster2.osd.1 is failed for ~0s 2026-02-18T08:52:46.785 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon cluster2.osd.0 has been restored 2026-02-18T08:52:50.533 INFO:tasks.ceph.cluster2.osd.1:Stopped 2026-02-18T08:52:50.534 DEBUG:tasks.ceph.cluster2.osd.2:waiting for process to exit 2026-02-18T08:52:50.534 INFO:teuthology.orchestra.run:waiting for 300 2026-02-18T08:52:51.988 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon cluster2.osd.2 is failed for ~0s 2026-02-18T08:52:51.988 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon cluster2.osd.1 has been restored 2026-02-18T08:52:56.636 INFO:tasks.ceph.cluster2.osd.2:Stopped 2026-02-18T08:52:56.636 INFO:teuthology.misc:Shutting down mgr daemons... 2026-02-18T08:52:56.637 DEBUG:tasks.ceph.cluster2.mgr.x:waiting for process to exit 2026-02-18T08:52:56.637 INFO:teuthology.orchestra.run:waiting for 300 2026-02-18T08:52:56.692 INFO:tasks.ceph.cluster2.mgr.x:Stopped 2026-02-18T08:52:56.692 INFO:teuthology.misc:Shutting down mon daemons... 2026-02-18T08:52:56.692 DEBUG:tasks.ceph.cluster2.mon.a:waiting for process to exit 2026-02-18T08:52:56.692 INFO:teuthology.orchestra.run:waiting for 300 2026-02-18T08:52:56.709 INFO:tasks.ceph.cluster2.mon.a:Stopped 2026-02-18T08:52:56.709 INFO:tasks.ceph:Checking cluster log for badness... 2026-02-18T08:52:56.709 DEBUG:teuthology.orchestra.run.trial163:> sudo egrep '\[ERR\]|\[WRN\]|\[SEC\]' /var/log/ceph/cluster2.log | egrep -v '\(MDS_ALL_DOWN\)' | egrep -v '\(MDS_UP_LESS_THAN_MAX\)' | egrep -v '\(OSD_SLOW_PING_TIME' | head -n 1 2026-02-18T08:52:56.761 INFO:tasks.ceph:Unmounting /var/lib/ceph/osd/cluster2-0 on ubuntu@trial163.front.sepia.ceph.com 2026-02-18T08:52:56.761 DEBUG:teuthology.orchestra.run.trial163:> sync && sudo umount -f /var/lib/ceph/osd/cluster2-0 2026-02-18T08:52:56.989 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon cluster2.osd.2 has been restored 2026-02-18T08:52:57.225 INFO:tasks.ceph:Unmounting /var/lib/ceph/osd/cluster2-1 on ubuntu@trial163.front.sepia.ceph.com 2026-02-18T08:52:57.226 DEBUG:teuthology.orchestra.run.trial163:> sync && sudo umount -f /var/lib/ceph/osd/cluster2-1 2026-02-18T08:52:57.579 INFO:tasks.ceph:Unmounting /var/lib/ceph/osd/cluster2-2 on ubuntu@trial163.front.sepia.ceph.com 2026-02-18T08:52:57.579 DEBUG:teuthology.orchestra.run.trial163:> sync && sudo umount -f /var/lib/ceph/osd/cluster2-2 2026-02-18T08:52:57.945 INFO:tasks.ceph:Archiving mon data... 2026-02-18T08:52:57.954 DEBUG:teuthology.misc:Transferring archived files from trial163:/var/lib/ceph/mon/cluster2-a to /home/teuthworker/mnt/teuthology/yuriw-2026-02-17_21:26:40-rbd-20.2.1_2.17.26-baseline_2-distro-default-trial/54948/data/cluster2.mon.a.tgz 2026-02-18T08:52:57.954 DEBUG:teuthology.orchestra.run.trial163:> mktemp 2026-02-18T08:52:57.969 INFO:teuthology.orchestra.run.trial163.stdout:/tmp/tmp.wrzh5yrAQk 2026-02-18T08:52:57.969 DEBUG:teuthology.orchestra.run.trial163:> sudo tar cz -f - -C /var/lib/ceph/mon/cluster2-a -- . > /tmp/tmp.wrzh5yrAQk 2026-02-18T08:52:58.085 DEBUG:teuthology.orchestra.run.trial163:> sudo chmod 0666 /tmp/tmp.wrzh5yrAQk 2026-02-18T08:52:58.145 DEBUG:teuthology.orchestra.remote:trial163:/tmp/tmp.wrzh5yrAQk is 425KB 2026-02-18T08:52:58.201 DEBUG:teuthology.orchestra.run.trial163:> rm -fr /tmp/tmp.wrzh5yrAQk 2026-02-18T08:52:58.206 INFO:tasks.ceph:Cleaning ceph cluster... 2026-02-18T08:52:58.206 DEBUG:teuthology.orchestra.run.trial034:> sudo rm -rf -- /etc/ceph/cluster2.conf /etc/ceph/cluster2.keyring /home/ubuntu/cephtest/cluster2.data /home/ubuntu/cephtest/cluster2.monmap /home/ubuntu/cephtest/../*.pid 2026-02-18T08:52:58.209 DEBUG:teuthology.orchestra.run.trial163:> sudo rm -rf -- /etc/ceph/cluster2.conf /etc/ceph/cluster2.keyring /home/ubuntu/cephtest/cluster2.data /home/ubuntu/cephtest/cluster2.monmap /home/ubuntu/cephtest/../*.pid 2026-02-18T08:52:58.259 DEBUG:teuthology.run_tasks:Unwinding manager ceph 2026-02-18T08:52:58.285 INFO:tasks.ceph.ceph_manager.cluster1:waiting for clean 2026-02-18T08:52:58.286 DEBUG:teuthology.orchestra.run.trial034:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 pg dump --format=json 2026-02-18T08:52:58.443 INFO:teuthology.orchestra.run.trial034.stdout: 2026-02-18T08:52:58.444 INFO:teuthology.orchestra.run.trial034.stderr:dumped all 2026-02-18T08:52:58.454 INFO:teuthology.orchestra.run.trial034.stdout:{"pg_ready":true,"pg_map":{"version":1652,"stamp":"2026-02-18T08:52:56.590196+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590387,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":103,"num_write_kb":1630,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":75,"ondisk_log_size":75,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":18,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":3854272,"kb_used_data":9424,"kb_used_omap":3284,"kb_used_meta":3841515,"kb_avail":279261248,"statfs":{"total":289910292480,"available":285963517952,"internally_reserved":0,"allocated":9650176,"data_stored":5653783,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3363724,"internal_metadata":3933711476},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.001054"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":696,"reported_epoch":342,"state":"active+clean","last_fresh":"2026-02-18T08:52:13.270425+0000","last_change":"2026-02-18T08:09:17.206844+0000","last_active":"2026-02-18T08:52:13.270425+0000","last_peered":"2026-02-18T08:52:13.270425+0000","last_clean":"2026-02-18T08:52:13.270425+0000","last_became_active":"2026-02-18T08:09:15.204488+0000","last_became_peered":"2026-02-18T08:09:15.204488+0000","last_unstale":"2026-02-18T08:52:13.270425+0000","last_undegraded":"2026-02-18T08:52:13.270425+0000","last_fullsized":"2026-02-18T08:52:13.270425+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-18T08:09:14.193205+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-18T08:09:14.193205+0000","last_clean_scrub_stamp":"2026-02-18T08:09:14.193205+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-19T08:17:13.769843+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00094066400000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":696,"reported_epoch":342,"state":"active+clean","last_fresh":"2026-02-18T08:52:13.271166+0000","last_change":"2026-02-18T08:09:17.206676+0000","last_active":"2026-02-18T08:52:13.271166+0000","last_peered":"2026-02-18T08:52:13.271166+0000","last_clean":"2026-02-18T08:52:13.271166+0000","last_became_active":"2026-02-18T08:09:15.204451+0000","last_became_peered":"2026-02-18T08:09:15.204451+0000","last_unstale":"2026-02-18T08:52:13.271166+0000","last_undegraded":"2026-02-18T08:52:13.271166+0000","last_fullsized":"2026-02-18T08:52:13.271166+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-18T08:09:14.193205+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-18T08:09:14.193205+0000","last_clean_scrub_stamp":"2026-02-18T08:09:14.193205+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-19T10:15:15.160787+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00075490900000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":696,"reported_epoch":342,"state":"active+clean","last_fresh":"2026-02-18T08:52:13.270441+0000","last_change":"2026-02-18T08:09:17.206497+0000","last_active":"2026-02-18T08:52:13.270441+0000","last_peered":"2026-02-18T08:52:13.270441+0000","last_clean":"2026-02-18T08:52:13.270441+0000","last_became_active":"2026-02-18T08:09:15.204532+0000","last_became_peered":"2026-02-18T08:09:15.204532+0000","last_unstale":"2026-02-18T08:52:13.270441+0000","last_undegraded":"2026-02-18T08:52:13.270441+0000","last_fullsized":"2026-02-18T08:52:13.270441+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-18T08:09:14.193205+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-18T08:09:14.193205+0000","last_clean_scrub_stamp":"2026-02-18T08:09:14.193205+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-19T17:38:17.949604+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00059731699999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":696,"reported_epoch":342,"state":"active+clean","last_fresh":"2026-02-18T08:52:13.269046+0000","last_change":"2026-02-18T08:09:17.206457+0000","last_active":"2026-02-18T08:52:13.269046+0000","last_peered":"2026-02-18T08:52:13.269046+0000","last_clean":"2026-02-18T08:52:13.269046+0000","last_became_active":"2026-02-18T08:09:15.204567+0000","last_became_peered":"2026-02-18T08:09:15.204567+0000","last_unstale":"2026-02-18T08:52:13.269046+0000","last_undegraded":"2026-02-18T08:52:13.269046+0000","last_fullsized":"2026-02-18T08:52:13.269046+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-18T08:09:14.193205+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-18T08:09:14.193205+0000","last_clean_scrub_stamp":"2026-02-18T08:09:14.193205+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-19T08:39:51.433130+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00062999799999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"14'2","reported_seq":698,"reported_epoch":342,"state":"active+clean","last_fresh":"2026-02-18T08:52:13.268570+0000","last_change":"2026-02-18T08:09:17.204700+0000","last_active":"2026-02-18T08:52:13.268570+0000","last_peered":"2026-02-18T08:52:13.268570+0000","last_clean":"2026-02-18T08:52:13.268570+0000","last_became_active":"2026-02-18T08:09:15.204550+0000","last_became_peered":"2026-02-18T08:09:15.204550+0000","last_unstale":"2026-02-18T08:52:13.268570+0000","last_undegraded":"2026-02-18T08:52:13.268570+0000","last_fullsized":"2026-02-18T08:52:13.268570+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-18T08:09:14.193205+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-18T08:09:14.193205+0000","last_clean_scrub_stamp":"2026-02-18T08:09:14.193205+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-19T14:33:42.672738+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00095267599999999996,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":696,"reported_epoch":342,"state":"active+clean","last_fresh":"2026-02-18T08:52:13.265935+0000","last_change":"2026-02-18T08:09:17.603054+0000","last_active":"2026-02-18T08:52:13.265935+0000","last_peered":"2026-02-18T08:52:13.265935+0000","last_clean":"2026-02-18T08:52:13.265935+0000","last_became_active":"2026-02-18T08:09:15.204585+0000","last_became_peered":"2026-02-18T08:09:15.204585+0000","last_unstale":"2026-02-18T08:52:13.265935+0000","last_undegraded":"2026-02-18T08:52:13.265935+0000","last_fullsized":"2026-02-18T08:52:13.265935+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-18T08:09:14.193205+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-18T08:09:14.193205+0000","last_clean_scrub_stamp":"2026-02-18T08:09:14.193205+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-19T19:11:28.258653+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00029873299999999997,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":697,"reported_epoch":342,"state":"active+clean","last_fresh":"2026-02-18T08:52:13.266739+0000","last_change":"2026-02-18T08:09:17.603088+0000","last_active":"2026-02-18T08:52:13.266739+0000","last_peered":"2026-02-18T08:52:13.266739+0000","last_clean":"2026-02-18T08:52:13.266739+0000","last_became_active":"2026-02-18T08:09:15.204604+0000","last_became_peered":"2026-02-18T08:09:15.204604+0000","last_unstale":"2026-02-18T08:52:13.266739+0000","last_undegraded":"2026-02-18T08:52:13.266739+0000","last_fullsized":"2026-02-18T08:52:13.266739+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-18T08:09:14.193205+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-18T08:09:14.193205+0000","last_clean_scrub_stamp":"2026-02-18T08:09:14.193205+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-19T14:02:23.790633+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00033665399999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.3","version":"12'1","reported_seq":697,"reported_epoch":342,"state":"active+clean","last_fresh":"2026-02-18T08:52:13.270011+0000","last_change":"2026-02-18T08:09:17.206665+0000","last_active":"2026-02-18T08:52:13.270011+0000","last_peered":"2026-02-18T08:52:13.270011+0000","last_clean":"2026-02-18T08:52:13.270011+0000","last_became_active":"2026-02-18T08:09:15.204115+0000","last_became_peered":"2026-02-18T08:09:15.204115+0000","last_unstale":"2026-02-18T08:52:13.270011+0000","last_undegraded":"2026-02-18T08:52:13.270011+0000","last_fullsized":"2026-02-18T08:52:13.270011+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-18T08:09:14.193205+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-18T08:09:14.193205+0000","last_clean_scrub_stamp":"2026-02-18T08:09:14.193205+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-19T15:05:10.251315+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00077986500000000003,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"10'72","reported_seq":786,"reported_epoch":342,"state":"active+clean","last_fresh":"2026-02-18T08:52:13.267476+0000","last_change":"2026-02-18T08:09:13.198212+0000","last_active":"2026-02-18T08:52:13.267476+0000","last_peered":"2026-02-18T08:52:13.267476+0000","last_clean":"2026-02-18T08:52:13.267476+0000","last_became_active":"2026-02-18T08:09:13.196393+0000","last_became_peered":"2026-02-18T08:09:13.196393+0000","last_unstale":"2026-02-18T08:52:13.267476+0000","last_undegraded":"2026-02-18T08:52:13.267476+0000","last_fullsized":"2026-02-18T08:52:13.267476+0000","mapping_epoch":9,"log_start":"0'0","ondisk_log_start":"0'0","created":9,"last_epoch_clean":10,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-18T08:09:12.188963+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-18T08:09:12.188963+0000","last_clean_scrub_stamp":"2026-02-18T08:09:12.188963+0000","objects_scrubbed":0,"log_size":72,"log_dups_size":0,"ondisk_log_size":72,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-19T11:44:22.114942+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":38,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":16,"acting":16,"num_store_stats":3},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1187840,"data_stored":1180736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":72,"ondisk_log_size":72,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":2,"up_from":8,"seq":34359738897,"num_pgs":3,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":1503376,"kb_used_data":2752,"kb_used_omap":1394,"kb_used_meta":1499213,"kb_avail":92868464,"statfs":{"total":96636764160,"available":95097307136,"internally_reserved":0,"allocated":2818048,"data_stored":1491003,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1428061,"internal_metadata":1535194531},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":8,"seq":34359738895,"num_pgs":9,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":981656,"kb_used_data":3336,"kb_used_omap":585,"kb_used_meta":977718,"kb_avail":93390184,"statfs":{"total":96636764160,"available":95631548416,"internally_reserved":0,"allocated":3416064,"data_stored":2081390,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":599194,"internal_metadata":1001184102},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":8,"seq":34359738895,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":1369240,"kb_used_data":3336,"kb_used_omap":1305,"kb_used_meta":1364582,"kb_avail":93002600,"statfs":{"total":96636764160,"available":95234662400,"internally_reserved":0,"allocated":3416064,"data_stored":2081390,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1336469,"internal_metadata":1397332843},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2026-02-18T08:52:58.456 DEBUG:teuthology.orchestra.run.trial034:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 pg dump --format=json 2026-02-18T08:52:58.615 INFO:teuthology.orchestra.run.trial034.stdout: 2026-02-18T08:52:58.615 INFO:teuthology.orchestra.run.trial034.stderr:dumped all 2026-02-18T08:52:58.627 INFO:teuthology.orchestra.run.trial034.stdout:{"pg_ready":true,"pg_map":{"version":1653,"stamp":"2026-02-18T08:52:58.590368+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590387,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":103,"num_write_kb":1630,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":75,"ondisk_log_size":75,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":18,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":3854272,"kb_used_data":9424,"kb_used_omap":3284,"kb_used_meta":3841515,"kb_avail":279261248,"statfs":{"total":289910292480,"available":285963517952,"internally_reserved":0,"allocated":9650176,"data_stored":5653783,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3363724,"internal_metadata":3933711476},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.001062"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":696,"reported_epoch":342,"state":"active+clean","last_fresh":"2026-02-18T08:52:13.270425+0000","last_change":"2026-02-18T08:09:17.206844+0000","last_active":"2026-02-18T08:52:13.270425+0000","last_peered":"2026-02-18T08:52:13.270425+0000","last_clean":"2026-02-18T08:52:13.270425+0000","last_became_active":"2026-02-18T08:09:15.204488+0000","last_became_peered":"2026-02-18T08:09:15.204488+0000","last_unstale":"2026-02-18T08:52:13.270425+0000","last_undegraded":"2026-02-18T08:52:13.270425+0000","last_fullsized":"2026-02-18T08:52:13.270425+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-18T08:09:14.193205+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-18T08:09:14.193205+0000","last_clean_scrub_stamp":"2026-02-18T08:09:14.193205+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-19T08:17:13.769843+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00094066400000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":696,"reported_epoch":342,"state":"active+clean","last_fresh":"2026-02-18T08:52:13.271166+0000","last_change":"2026-02-18T08:09:17.206676+0000","last_active":"2026-02-18T08:52:13.271166+0000","last_peered":"2026-02-18T08:52:13.271166+0000","last_clean":"2026-02-18T08:52:13.271166+0000","last_became_active":"2026-02-18T08:09:15.204451+0000","last_became_peered":"2026-02-18T08:09:15.204451+0000","last_unstale":"2026-02-18T08:52:13.271166+0000","last_undegraded":"2026-02-18T08:52:13.271166+0000","last_fullsized":"2026-02-18T08:52:13.271166+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-18T08:09:14.193205+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-18T08:09:14.193205+0000","last_clean_scrub_stamp":"2026-02-18T08:09:14.193205+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-19T10:15:15.160787+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00075490900000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":696,"reported_epoch":342,"state":"active+clean","last_fresh":"2026-02-18T08:52:13.270441+0000","last_change":"2026-02-18T08:09:17.206497+0000","last_active":"2026-02-18T08:52:13.270441+0000","last_peered":"2026-02-18T08:52:13.270441+0000","last_clean":"2026-02-18T08:52:13.270441+0000","last_became_active":"2026-02-18T08:09:15.204532+0000","last_became_peered":"2026-02-18T08:09:15.204532+0000","last_unstale":"2026-02-18T08:52:13.270441+0000","last_undegraded":"2026-02-18T08:52:13.270441+0000","last_fullsized":"2026-02-18T08:52:13.270441+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-18T08:09:14.193205+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-18T08:09:14.193205+0000","last_clean_scrub_stamp":"2026-02-18T08:09:14.193205+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-19T17:38:17.949604+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00059731699999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":696,"reported_epoch":342,"state":"active+clean","last_fresh":"2026-02-18T08:52:13.269046+0000","last_change":"2026-02-18T08:09:17.206457+0000","last_active":"2026-02-18T08:52:13.269046+0000","last_peered":"2026-02-18T08:52:13.269046+0000","last_clean":"2026-02-18T08:52:13.269046+0000","last_became_active":"2026-02-18T08:09:15.204567+0000","last_became_peered":"2026-02-18T08:09:15.204567+0000","last_unstale":"2026-02-18T08:52:13.269046+0000","last_undegraded":"2026-02-18T08:52:13.269046+0000","last_fullsized":"2026-02-18T08:52:13.269046+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-18T08:09:14.193205+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-18T08:09:14.193205+0000","last_clean_scrub_stamp":"2026-02-18T08:09:14.193205+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-19T08:39:51.433130+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00062999799999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"14'2","reported_seq":698,"reported_epoch":342,"state":"active+clean","last_fresh":"2026-02-18T08:52:13.268570+0000","last_change":"2026-02-18T08:09:17.204700+0000","last_active":"2026-02-18T08:52:13.268570+0000","last_peered":"2026-02-18T08:52:13.268570+0000","last_clean":"2026-02-18T08:52:13.268570+0000","last_became_active":"2026-02-18T08:09:15.204550+0000","last_became_peered":"2026-02-18T08:09:15.204550+0000","last_unstale":"2026-02-18T08:52:13.268570+0000","last_undegraded":"2026-02-18T08:52:13.268570+0000","last_fullsized":"2026-02-18T08:52:13.268570+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-18T08:09:14.193205+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-18T08:09:14.193205+0000","last_clean_scrub_stamp":"2026-02-18T08:09:14.193205+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-19T14:33:42.672738+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00095267599999999996,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":696,"reported_epoch":342,"state":"active+clean","last_fresh":"2026-02-18T08:52:13.265935+0000","last_change":"2026-02-18T08:09:17.603054+0000","last_active":"2026-02-18T08:52:13.265935+0000","last_peered":"2026-02-18T08:52:13.265935+0000","last_clean":"2026-02-18T08:52:13.265935+0000","last_became_active":"2026-02-18T08:09:15.204585+0000","last_became_peered":"2026-02-18T08:09:15.204585+0000","last_unstale":"2026-02-18T08:52:13.265935+0000","last_undegraded":"2026-02-18T08:52:13.265935+0000","last_fullsized":"2026-02-18T08:52:13.265935+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-18T08:09:14.193205+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-18T08:09:14.193205+0000","last_clean_scrub_stamp":"2026-02-18T08:09:14.193205+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-19T19:11:28.258653+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00029873299999999997,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":697,"reported_epoch":342,"state":"active+clean","last_fresh":"2026-02-18T08:52:13.266739+0000","last_change":"2026-02-18T08:09:17.603088+0000","last_active":"2026-02-18T08:52:13.266739+0000","last_peered":"2026-02-18T08:52:13.266739+0000","last_clean":"2026-02-18T08:52:13.266739+0000","last_became_active":"2026-02-18T08:09:15.204604+0000","last_became_peered":"2026-02-18T08:09:15.204604+0000","last_unstale":"2026-02-18T08:52:13.266739+0000","last_undegraded":"2026-02-18T08:52:13.266739+0000","last_fullsized":"2026-02-18T08:52:13.266739+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-18T08:09:14.193205+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-18T08:09:14.193205+0000","last_clean_scrub_stamp":"2026-02-18T08:09:14.193205+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-19T14:02:23.790633+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00033665399999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.3","version":"12'1","reported_seq":697,"reported_epoch":342,"state":"active+clean","last_fresh":"2026-02-18T08:52:13.270011+0000","last_change":"2026-02-18T08:09:17.206665+0000","last_active":"2026-02-18T08:52:13.270011+0000","last_peered":"2026-02-18T08:52:13.270011+0000","last_clean":"2026-02-18T08:52:13.270011+0000","last_became_active":"2026-02-18T08:09:15.204115+0000","last_became_peered":"2026-02-18T08:09:15.204115+0000","last_unstale":"2026-02-18T08:52:13.270011+0000","last_undegraded":"2026-02-18T08:52:13.270011+0000","last_fullsized":"2026-02-18T08:52:13.270011+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-18T08:09:14.193205+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-18T08:09:14.193205+0000","last_clean_scrub_stamp":"2026-02-18T08:09:14.193205+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-19T15:05:10.251315+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00077986500000000003,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"10'72","reported_seq":786,"reported_epoch":342,"state":"active+clean","last_fresh":"2026-02-18T08:52:13.267476+0000","last_change":"2026-02-18T08:09:13.198212+0000","last_active":"2026-02-18T08:52:13.267476+0000","last_peered":"2026-02-18T08:52:13.267476+0000","last_clean":"2026-02-18T08:52:13.267476+0000","last_became_active":"2026-02-18T08:09:13.196393+0000","last_became_peered":"2026-02-18T08:09:13.196393+0000","last_unstale":"2026-02-18T08:52:13.267476+0000","last_undegraded":"2026-02-18T08:52:13.267476+0000","last_fullsized":"2026-02-18T08:52:13.267476+0000","mapping_epoch":9,"log_start":"0'0","ondisk_log_start":"0'0","created":9,"last_epoch_clean":10,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-18T08:09:12.188963+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-18T08:09:12.188963+0000","last_clean_scrub_stamp":"2026-02-18T08:09:12.188963+0000","objects_scrubbed":0,"log_size":72,"log_dups_size":0,"ondisk_log_size":72,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-19T11:44:22.114942+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":38,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":16,"acting":16,"num_store_stats":3},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1187840,"data_stored":1180736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":72,"ondisk_log_size":72,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":2,"up_from":8,"seq":34359738898,"num_pgs":3,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":1503376,"kb_used_data":2752,"kb_used_omap":1394,"kb_used_meta":1499213,"kb_avail":92868464,"statfs":{"total":96636764160,"available":95097307136,"internally_reserved":0,"allocated":2818048,"data_stored":1491003,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1428061,"internal_metadata":1535194531},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":8,"seq":34359738896,"num_pgs":9,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":981656,"kb_used_data":3336,"kb_used_omap":585,"kb_used_meta":977718,"kb_avail":93390184,"statfs":{"total":96636764160,"available":95631548416,"internally_reserved":0,"allocated":3416064,"data_stored":2081390,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":599194,"internal_metadata":1001184102},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":8,"seq":34359738895,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":1369240,"kb_used_data":3336,"kb_used_omap":1305,"kb_used_meta":1364582,"kb_avail":93002600,"statfs":{"total":96636764160,"available":95234662400,"internally_reserved":0,"allocated":3416064,"data_stored":2081390,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1336469,"internal_metadata":1397332843},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2026-02-18T08:52:58.628 INFO:tasks.ceph.ceph_manager.cluster1:clean! 2026-02-18T08:52:58.629 DEBUG:teuthology.orchestra.run.trial034:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 pg dump --format=json 2026-02-18T08:52:58.781 INFO:teuthology.orchestra.run.trial034.stdout: 2026-02-18T08:52:58.782 INFO:teuthology.orchestra.run.trial034.stderr:dumped all 2026-02-18T08:52:58.794 INFO:teuthology.orchestra.run.trial034.stdout:{"pg_ready":true,"pg_map":{"version":1653,"stamp":"2026-02-18T08:52:58.590368+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590387,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":103,"num_write_kb":1630,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":75,"ondisk_log_size":75,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":18,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":3854272,"kb_used_data":9424,"kb_used_omap":3284,"kb_used_meta":3841515,"kb_avail":279261248,"statfs":{"total":289910292480,"available":285963517952,"internally_reserved":0,"allocated":9650176,"data_stored":5653783,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3363724,"internal_metadata":3933711476},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.001062"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":696,"reported_epoch":342,"state":"active+clean","last_fresh":"2026-02-18T08:52:13.270425+0000","last_change":"2026-02-18T08:09:17.206844+0000","last_active":"2026-02-18T08:52:13.270425+0000","last_peered":"2026-02-18T08:52:13.270425+0000","last_clean":"2026-02-18T08:52:13.270425+0000","last_became_active":"2026-02-18T08:09:15.204488+0000","last_became_peered":"2026-02-18T08:09:15.204488+0000","last_unstale":"2026-02-18T08:52:13.270425+0000","last_undegraded":"2026-02-18T08:52:13.270425+0000","last_fullsized":"2026-02-18T08:52:13.270425+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-18T08:09:14.193205+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-18T08:09:14.193205+0000","last_clean_scrub_stamp":"2026-02-18T08:09:14.193205+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-19T08:17:13.769843+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00094066400000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":696,"reported_epoch":342,"state":"active+clean","last_fresh":"2026-02-18T08:52:13.271166+0000","last_change":"2026-02-18T08:09:17.206676+0000","last_active":"2026-02-18T08:52:13.271166+0000","last_peered":"2026-02-18T08:52:13.271166+0000","last_clean":"2026-02-18T08:52:13.271166+0000","last_became_active":"2026-02-18T08:09:15.204451+0000","last_became_peered":"2026-02-18T08:09:15.204451+0000","last_unstale":"2026-02-18T08:52:13.271166+0000","last_undegraded":"2026-02-18T08:52:13.271166+0000","last_fullsized":"2026-02-18T08:52:13.271166+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-18T08:09:14.193205+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-18T08:09:14.193205+0000","last_clean_scrub_stamp":"2026-02-18T08:09:14.193205+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-19T10:15:15.160787+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00075490900000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":696,"reported_epoch":342,"state":"active+clean","last_fresh":"2026-02-18T08:52:13.270441+0000","last_change":"2026-02-18T08:09:17.206497+0000","last_active":"2026-02-18T08:52:13.270441+0000","last_peered":"2026-02-18T08:52:13.270441+0000","last_clean":"2026-02-18T08:52:13.270441+0000","last_became_active":"2026-02-18T08:09:15.204532+0000","last_became_peered":"2026-02-18T08:09:15.204532+0000","last_unstale":"2026-02-18T08:52:13.270441+0000","last_undegraded":"2026-02-18T08:52:13.270441+0000","last_fullsized":"2026-02-18T08:52:13.270441+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-18T08:09:14.193205+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-18T08:09:14.193205+0000","last_clean_scrub_stamp":"2026-02-18T08:09:14.193205+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-19T17:38:17.949604+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00059731699999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":696,"reported_epoch":342,"state":"active+clean","last_fresh":"2026-02-18T08:52:13.269046+0000","last_change":"2026-02-18T08:09:17.206457+0000","last_active":"2026-02-18T08:52:13.269046+0000","last_peered":"2026-02-18T08:52:13.269046+0000","last_clean":"2026-02-18T08:52:13.269046+0000","last_became_active":"2026-02-18T08:09:15.204567+0000","last_became_peered":"2026-02-18T08:09:15.204567+0000","last_unstale":"2026-02-18T08:52:13.269046+0000","last_undegraded":"2026-02-18T08:52:13.269046+0000","last_fullsized":"2026-02-18T08:52:13.269046+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-18T08:09:14.193205+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-18T08:09:14.193205+0000","last_clean_scrub_stamp":"2026-02-18T08:09:14.193205+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-19T08:39:51.433130+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00062999799999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"14'2","reported_seq":698,"reported_epoch":342,"state":"active+clean","last_fresh":"2026-02-18T08:52:13.268570+0000","last_change":"2026-02-18T08:09:17.204700+0000","last_active":"2026-02-18T08:52:13.268570+0000","last_peered":"2026-02-18T08:52:13.268570+0000","last_clean":"2026-02-18T08:52:13.268570+0000","last_became_active":"2026-02-18T08:09:15.204550+0000","last_became_peered":"2026-02-18T08:09:15.204550+0000","last_unstale":"2026-02-18T08:52:13.268570+0000","last_undegraded":"2026-02-18T08:52:13.268570+0000","last_fullsized":"2026-02-18T08:52:13.268570+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-18T08:09:14.193205+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-18T08:09:14.193205+0000","last_clean_scrub_stamp":"2026-02-18T08:09:14.193205+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-19T14:33:42.672738+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00095267599999999996,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":696,"reported_epoch":342,"state":"active+clean","last_fresh":"2026-02-18T08:52:13.265935+0000","last_change":"2026-02-18T08:09:17.603054+0000","last_active":"2026-02-18T08:52:13.265935+0000","last_peered":"2026-02-18T08:52:13.265935+0000","last_clean":"2026-02-18T08:52:13.265935+0000","last_became_active":"2026-02-18T08:09:15.204585+0000","last_became_peered":"2026-02-18T08:09:15.204585+0000","last_unstale":"2026-02-18T08:52:13.265935+0000","last_undegraded":"2026-02-18T08:52:13.265935+0000","last_fullsized":"2026-02-18T08:52:13.265935+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-18T08:09:14.193205+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-18T08:09:14.193205+0000","last_clean_scrub_stamp":"2026-02-18T08:09:14.193205+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-19T19:11:28.258653+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00029873299999999997,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":697,"reported_epoch":342,"state":"active+clean","last_fresh":"2026-02-18T08:52:13.266739+0000","last_change":"2026-02-18T08:09:17.603088+0000","last_active":"2026-02-18T08:52:13.266739+0000","last_peered":"2026-02-18T08:52:13.266739+0000","last_clean":"2026-02-18T08:52:13.266739+0000","last_became_active":"2026-02-18T08:09:15.204604+0000","last_became_peered":"2026-02-18T08:09:15.204604+0000","last_unstale":"2026-02-18T08:52:13.266739+0000","last_undegraded":"2026-02-18T08:52:13.266739+0000","last_fullsized":"2026-02-18T08:52:13.266739+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-18T08:09:14.193205+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-18T08:09:14.193205+0000","last_clean_scrub_stamp":"2026-02-18T08:09:14.193205+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-19T14:02:23.790633+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00033665399999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.3","version":"12'1","reported_seq":697,"reported_epoch":342,"state":"active+clean","last_fresh":"2026-02-18T08:52:13.270011+0000","last_change":"2026-02-18T08:09:17.206665+0000","last_active":"2026-02-18T08:52:13.270011+0000","last_peered":"2026-02-18T08:52:13.270011+0000","last_clean":"2026-02-18T08:52:13.270011+0000","last_became_active":"2026-02-18T08:09:15.204115+0000","last_became_peered":"2026-02-18T08:09:15.204115+0000","last_unstale":"2026-02-18T08:52:13.270011+0000","last_undegraded":"2026-02-18T08:52:13.270011+0000","last_fullsized":"2026-02-18T08:52:13.270011+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-18T08:09:14.193205+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-18T08:09:14.193205+0000","last_clean_scrub_stamp":"2026-02-18T08:09:14.193205+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-19T15:05:10.251315+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00077986500000000003,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"10'72","reported_seq":786,"reported_epoch":342,"state":"active+clean","last_fresh":"2026-02-18T08:52:13.267476+0000","last_change":"2026-02-18T08:09:13.198212+0000","last_active":"2026-02-18T08:52:13.267476+0000","last_peered":"2026-02-18T08:52:13.267476+0000","last_clean":"2026-02-18T08:52:13.267476+0000","last_became_active":"2026-02-18T08:09:13.196393+0000","last_became_peered":"2026-02-18T08:09:13.196393+0000","last_unstale":"2026-02-18T08:52:13.267476+0000","last_undegraded":"2026-02-18T08:52:13.267476+0000","last_fullsized":"2026-02-18T08:52:13.267476+0000","mapping_epoch":9,"log_start":"0'0","ondisk_log_start":"0'0","created":9,"last_epoch_clean":10,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-18T08:09:12.188963+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-18T08:09:12.188963+0000","last_clean_scrub_stamp":"2026-02-18T08:09:12.188963+0000","objects_scrubbed":0,"log_size":72,"log_dups_size":0,"ondisk_log_size":72,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-19T11:44:22.114942+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":38,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":16,"acting":16,"num_store_stats":3},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1187840,"data_stored":1180736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":72,"ondisk_log_size":72,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":2,"up_from":8,"seq":34359738898,"num_pgs":3,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":1503376,"kb_used_data":2752,"kb_used_omap":1394,"kb_used_meta":1499213,"kb_avail":92868464,"statfs":{"total":96636764160,"available":95097307136,"internally_reserved":0,"allocated":2818048,"data_stored":1491003,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1428061,"internal_metadata":1535194531},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":8,"seq":34359738896,"num_pgs":9,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":981656,"kb_used_data":3336,"kb_used_omap":585,"kb_used_meta":977718,"kb_avail":93390184,"statfs":{"total":96636764160,"available":95631548416,"internally_reserved":0,"allocated":3416064,"data_stored":2081390,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":599194,"internal_metadata":1001184102},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":8,"seq":34359738895,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":1369240,"kb_used_data":3336,"kb_used_omap":1305,"kb_used_meta":1364582,"kb_avail":93002600,"statfs":{"total":96636764160,"available":95234662400,"internally_reserved":0,"allocated":3416064,"data_stored":2081390,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1336469,"internal_metadata":1397332843},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2026-02-18T08:52:58.796 DEBUG:teuthology.orchestra.run.trial034:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 osd dump --format=json 2026-02-18T08:52:58.962 INFO:teuthology.orchestra.run.trial034.stdout: 2026-02-18T08:52:58.962 INFO:teuthology.orchestra.run.trial034.stdout:{"epoch":342,"fsid":"538b5201-9b84-4a73-9eef-5646b3360b9d","created":"2026-02-18T08:09:06.049413+0000","modified":"2026-02-18T08:52:13.260334+0000","last_up_change":"2026-02-18T08:09:11.186536+0000","last_in_change":"2026-02-18T08:09:06.672265+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":4,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":4,"max_osd":3,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"tentacle","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2026-02-18T08:09:12.184009+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"11","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"nonprimary_shards":"{}","options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":2.9900000095367432,"score_stable":2.9900000095367432,"optimal_score":0.67000001668930054,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2026-02-18T08:09:14.017646+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"14","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":2,"snap_epoch":14,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"nonprimary_shards":"{}","options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_type":"Fair distribution","score_acting":1.8799999952316284,"score_stable":1.8799999952316284,"optimal_score":1,"raw_score_acting":1.8799999952316284,"raw_score_stable":1.8799999952316284,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"6ca6cdc0-6879-42a3-a4d6-eee71da49395","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":31,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.34:6808","nonce":3130378423},{"type":"v1","addr":"10.20.193.34:6809","nonce":3130378423}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.34:6810","nonce":3130378423},{"type":"v1","addr":"10.20.193.34:6811","nonce":3130378423}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.34:6814","nonce":3130378423},{"type":"v1","addr":"10.20.193.34:6815","nonce":3130378423}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.34:6812","nonce":3130378423},{"type":"v1","addr":"10.20.193.34:6813","nonce":3130378423}]},"public_addr":"10.20.193.34:6809/3130378423","cluster_addr":"10.20.193.34:6811/3130378423","heartbeat_back_addr":"10.20.193.34:6815/3130378423","heartbeat_front_addr":"10.20.193.34:6813/3130378423","state":["exists","up"]},{"osd":1,"uuid":"eb801757-b69b-4914-a101-65b314371e7c","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":32,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.34:6800","nonce":2597388232},{"type":"v1","addr":"10.20.193.34:6801","nonce":2597388232}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.34:6802","nonce":2597388232},{"type":"v1","addr":"10.20.193.34:6803","nonce":2597388232}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.34:6806","nonce":2597388232},{"type":"v1","addr":"10.20.193.34:6807","nonce":2597388232}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.34:6804","nonce":2597388232},{"type":"v1","addr":"10.20.193.34:6805","nonce":2597388232}]},"public_addr":"10.20.193.34:6801/2597388232","cluster_addr":"10.20.193.34:6803/2597388232","heartbeat_back_addr":"10.20.193.34:6807/2597388232","heartbeat_front_addr":"10.20.193.34:6805/2597388232","state":["exists","up"]},{"osd":2,"uuid":"8cff614b-3827-4a60-8c9f-a83bed4c4d45","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":32,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.34:6816","nonce":1006015704},{"type":"v1","addr":"10.20.193.34:6817","nonce":1006015704}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.34:6818","nonce":1006015704},{"type":"v1","addr":"10.20.193.34:6819","nonce":1006015704}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.34:6822","nonce":1006015704},{"type":"v1","addr":"10.20.193.34:6823","nonce":1006015704}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.34:6820","nonce":1006015704},{"type":"v1","addr":"10.20.193.34:6821","nonce":1006015704}]},"public_addr":"10.20.193.34:6817/1006015704","cluster_addr":"10.20.193.34:6819/1006015704","heartbeat_back_addr":"10.20.193.34:6823/1006015704","heartbeat_front_addr":"10.20.193.34:6821/1006015704","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4544132024016699391,"old_weight":0,"last_purged_snaps_scrub":"2026-02-18T08:09:09.648501+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4544132024016699391,"old_weight":0,"last_purged_snaps_scrub":"2026-02-18T08:09:09.755125+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4544132024016699391,"old_weight":0,"last_purged_snaps_scrub":"2026-02-18T08:09:09.681000+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"isa","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2026-02-18T08:52:59.974 INFO:tasks.ceph:Scrubbing cluster1.osd.0 2026-02-18T08:52:59.975 DEBUG:teuthology.orchestra.run.trial034:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 tell osd.0 config set osd_debug_deep_scrub_sleep 0 2026-02-18T08:53:00.053 INFO:teuthology.orchestra.run.trial034.stdout:{ 2026-02-18T08:53:00.053 INFO:teuthology.orchestra.run.trial034.stdout: "success": "osd_debug_deep_scrub_sleep = '' (not observed, change may require restart) " 2026-02-18T08:53:00.053 INFO:teuthology.orchestra.run.trial034.stdout:} 2026-02-18T08:53:00.061 DEBUG:teuthology.orchestra.run.trial034:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 osd deep-scrub 0 2026-02-18T08:53:00.222 INFO:teuthology.orchestra.run.trial034.stderr:instructed osd(s) 0 to deep-scrub 2026-02-18T08:53:00.233 INFO:tasks.ceph:Scrubbing cluster1.osd.1 2026-02-18T08:53:00.233 DEBUG:teuthology.orchestra.run.trial034:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 tell osd.1 config set osd_debug_deep_scrub_sleep 0 2026-02-18T08:53:00.317 INFO:teuthology.orchestra.run.trial034.stdout:{ 2026-02-18T08:53:00.317 INFO:teuthology.orchestra.run.trial034.stdout: "success": "osd_debug_deep_scrub_sleep = '' (not observed, change may require restart) " 2026-02-18T08:53:00.317 INFO:teuthology.orchestra.run.trial034.stdout:} 2026-02-18T08:53:00.326 DEBUG:teuthology.orchestra.run.trial034:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 osd deep-scrub 1 2026-02-18T08:53:00.487 INFO:teuthology.orchestra.run.trial034.stderr:instructed osd(s) 1 to deep-scrub 2026-02-18T08:53:00.499 INFO:tasks.ceph:Scrubbing cluster1.osd.2 2026-02-18T08:53:00.499 DEBUG:teuthology.orchestra.run.trial034:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 tell osd.2 config set osd_debug_deep_scrub_sleep 0 2026-02-18T08:53:00.582 INFO:teuthology.orchestra.run.trial034.stdout:{ 2026-02-18T08:53:00.582 INFO:teuthology.orchestra.run.trial034.stdout: "success": "osd_debug_deep_scrub_sleep = '' (not observed, change may require restart) " 2026-02-18T08:53:00.582 INFO:teuthology.orchestra.run.trial034.stdout:} 2026-02-18T08:53:00.591 DEBUG:teuthology.orchestra.run.trial034:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 osd deep-scrub 2 2026-02-18T08:53:00.751 INFO:teuthology.orchestra.run.trial034.stderr:instructed osd(s) 2 to deep-scrub 2026-02-18T08:53:00.763 DEBUG:teuthology.orchestra.run.trial034:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 pg dump --format=json 2026-02-18T08:53:00.928 INFO:teuthology.orchestra.run.trial034.stdout: 2026-02-18T08:53:00.928 INFO:teuthology.orchestra.run.trial034.stderr:dumped all 2026-02-18T08:53:00.938 INFO:teuthology.orchestra.run.trial034.stdout:{"pg_ready":true,"pg_map":{"version":1654,"stamp":"2026-02-18T08:53:00.590601+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590387,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":103,"num_write_kb":1630,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":75,"ondisk_log_size":75,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":18,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":3854272,"kb_used_data":9424,"kb_used_omap":3284,"kb_used_meta":3841515,"kb_avail":279261248,"statfs":{"total":289910292480,"available":285963517952,"internally_reserved":0,"allocated":9650176,"data_stored":5653783,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3363724,"internal_metadata":3933711476},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.001120"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":696,"reported_epoch":342,"state":"active+clean","last_fresh":"2026-02-18T08:52:13.270425+0000","last_change":"2026-02-18T08:09:17.206844+0000","last_active":"2026-02-18T08:52:13.270425+0000","last_peered":"2026-02-18T08:52:13.270425+0000","last_clean":"2026-02-18T08:52:13.270425+0000","last_became_active":"2026-02-18T08:09:15.204488+0000","last_became_peered":"2026-02-18T08:09:15.204488+0000","last_unstale":"2026-02-18T08:52:13.270425+0000","last_undegraded":"2026-02-18T08:52:13.270425+0000","last_fullsized":"2026-02-18T08:52:13.270425+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-18T08:09:14.193205+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-18T08:09:14.193205+0000","last_clean_scrub_stamp":"2026-02-18T08:09:14.193205+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-19T08:17:13.769843+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00094066400000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":696,"reported_epoch":342,"state":"active+clean","last_fresh":"2026-02-18T08:52:13.271166+0000","last_change":"2026-02-18T08:09:17.206676+0000","last_active":"2026-02-18T08:52:13.271166+0000","last_peered":"2026-02-18T08:52:13.271166+0000","last_clean":"2026-02-18T08:52:13.271166+0000","last_became_active":"2026-02-18T08:09:15.204451+0000","last_became_peered":"2026-02-18T08:09:15.204451+0000","last_unstale":"2026-02-18T08:52:13.271166+0000","last_undegraded":"2026-02-18T08:52:13.271166+0000","last_fullsized":"2026-02-18T08:52:13.271166+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-18T08:09:14.193205+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-18T08:09:14.193205+0000","last_clean_scrub_stamp":"2026-02-18T08:09:14.193205+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-19T10:15:15.160787+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00075490900000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":696,"reported_epoch":342,"state":"active+clean","last_fresh":"2026-02-18T08:52:13.270441+0000","last_change":"2026-02-18T08:09:17.206497+0000","last_active":"2026-02-18T08:52:13.270441+0000","last_peered":"2026-02-18T08:52:13.270441+0000","last_clean":"2026-02-18T08:52:13.270441+0000","last_became_active":"2026-02-18T08:09:15.204532+0000","last_became_peered":"2026-02-18T08:09:15.204532+0000","last_unstale":"2026-02-18T08:52:13.270441+0000","last_undegraded":"2026-02-18T08:52:13.270441+0000","last_fullsized":"2026-02-18T08:52:13.270441+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-18T08:09:14.193205+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-18T08:09:14.193205+0000","last_clean_scrub_stamp":"2026-02-18T08:09:14.193205+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-19T17:38:17.949604+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00059731699999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":696,"reported_epoch":342,"state":"active+clean","last_fresh":"2026-02-18T08:52:13.269046+0000","last_change":"2026-02-18T08:09:17.206457+0000","last_active":"2026-02-18T08:52:13.269046+0000","last_peered":"2026-02-18T08:52:13.269046+0000","last_clean":"2026-02-18T08:52:13.269046+0000","last_became_active":"2026-02-18T08:09:15.204567+0000","last_became_peered":"2026-02-18T08:09:15.204567+0000","last_unstale":"2026-02-18T08:52:13.269046+0000","last_undegraded":"2026-02-18T08:52:13.269046+0000","last_fullsized":"2026-02-18T08:52:13.269046+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-18T08:09:14.193205+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-18T08:09:14.193205+0000","last_clean_scrub_stamp":"2026-02-18T08:09:14.193205+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-19T08:39:51.433130+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00062999799999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"14'2","reported_seq":698,"reported_epoch":342,"state":"active+clean","last_fresh":"2026-02-18T08:52:13.268570+0000","last_change":"2026-02-18T08:09:17.204700+0000","last_active":"2026-02-18T08:52:13.268570+0000","last_peered":"2026-02-18T08:52:13.268570+0000","last_clean":"2026-02-18T08:52:13.268570+0000","last_became_active":"2026-02-18T08:09:15.204550+0000","last_became_peered":"2026-02-18T08:09:15.204550+0000","last_unstale":"2026-02-18T08:52:13.268570+0000","last_undegraded":"2026-02-18T08:52:13.268570+0000","last_fullsized":"2026-02-18T08:52:13.268570+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-18T08:09:14.193205+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-18T08:09:14.193205+0000","last_clean_scrub_stamp":"2026-02-18T08:09:14.193205+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-19T14:33:42.672738+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00095267599999999996,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":696,"reported_epoch":342,"state":"active+clean","last_fresh":"2026-02-18T08:52:13.265935+0000","last_change":"2026-02-18T08:09:17.603054+0000","last_active":"2026-02-18T08:52:13.265935+0000","last_peered":"2026-02-18T08:52:13.265935+0000","last_clean":"2026-02-18T08:52:13.265935+0000","last_became_active":"2026-02-18T08:09:15.204585+0000","last_became_peered":"2026-02-18T08:09:15.204585+0000","last_unstale":"2026-02-18T08:52:13.265935+0000","last_undegraded":"2026-02-18T08:52:13.265935+0000","last_fullsized":"2026-02-18T08:52:13.265935+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-18T08:09:14.193205+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-18T08:09:14.193205+0000","last_clean_scrub_stamp":"2026-02-18T08:09:14.193205+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-19T19:11:28.258653+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00029873299999999997,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":697,"reported_epoch":342,"state":"active+clean","last_fresh":"2026-02-18T08:52:13.266739+0000","last_change":"2026-02-18T08:09:17.603088+0000","last_active":"2026-02-18T08:52:13.266739+0000","last_peered":"2026-02-18T08:52:13.266739+0000","last_clean":"2026-02-18T08:52:13.266739+0000","last_became_active":"2026-02-18T08:09:15.204604+0000","last_became_peered":"2026-02-18T08:09:15.204604+0000","last_unstale":"2026-02-18T08:52:13.266739+0000","last_undegraded":"2026-02-18T08:52:13.266739+0000","last_fullsized":"2026-02-18T08:52:13.266739+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-18T08:09:14.193205+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-18T08:09:14.193205+0000","last_clean_scrub_stamp":"2026-02-18T08:09:14.193205+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-19T14:02:23.790633+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00033665399999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.3","version":"12'1","reported_seq":697,"reported_epoch":342,"state":"active+clean","last_fresh":"2026-02-18T08:52:13.270011+0000","last_change":"2026-02-18T08:09:17.206665+0000","last_active":"2026-02-18T08:52:13.270011+0000","last_peered":"2026-02-18T08:52:13.270011+0000","last_clean":"2026-02-18T08:52:13.270011+0000","last_became_active":"2026-02-18T08:09:15.204115+0000","last_became_peered":"2026-02-18T08:09:15.204115+0000","last_unstale":"2026-02-18T08:52:13.270011+0000","last_undegraded":"2026-02-18T08:52:13.270011+0000","last_fullsized":"2026-02-18T08:52:13.270011+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-18T08:09:14.193205+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-18T08:09:14.193205+0000","last_clean_scrub_stamp":"2026-02-18T08:09:14.193205+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-19T15:05:10.251315+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00077986500000000003,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"10'72","reported_seq":786,"reported_epoch":342,"state":"active+clean","last_fresh":"2026-02-18T08:52:13.267476+0000","last_change":"2026-02-18T08:09:13.198212+0000","last_active":"2026-02-18T08:52:13.267476+0000","last_peered":"2026-02-18T08:52:13.267476+0000","last_clean":"2026-02-18T08:52:13.267476+0000","last_became_active":"2026-02-18T08:09:13.196393+0000","last_became_peered":"2026-02-18T08:09:13.196393+0000","last_unstale":"2026-02-18T08:52:13.267476+0000","last_undegraded":"2026-02-18T08:52:13.267476+0000","last_fullsized":"2026-02-18T08:52:13.267476+0000","mapping_epoch":9,"log_start":"0'0","ondisk_log_start":"0'0","created":9,"last_epoch_clean":10,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-18T08:09:12.188963+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-18T08:09:12.188963+0000","last_clean_scrub_stamp":"2026-02-18T08:09:12.188963+0000","objects_scrubbed":0,"log_size":72,"log_dups_size":0,"ondisk_log_size":72,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-19T11:44:22.114942+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":38,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":16,"acting":16,"num_store_stats":3},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1187840,"data_stored":1180736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":72,"ondisk_log_size":72,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":2,"up_from":8,"seq":34359738898,"num_pgs":3,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":1503376,"kb_used_data":2752,"kb_used_omap":1394,"kb_used_meta":1499213,"kb_avail":92868464,"statfs":{"total":96636764160,"available":95097307136,"internally_reserved":0,"allocated":2818048,"data_stored":1491003,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1428061,"internal_metadata":1535194531},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":8,"seq":34359738896,"num_pgs":9,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":981656,"kb_used_data":3336,"kb_used_omap":585,"kb_used_meta":977718,"kb_avail":93390184,"statfs":{"total":96636764160,"available":95631548416,"internally_reserved":0,"allocated":3416064,"data_stored":2081390,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":599194,"internal_metadata":1001184102},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":8,"seq":34359738895,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":1369240,"kb_used_data":3336,"kb_used_omap":1305,"kb_used_meta":1364582,"kb_avail":93002600,"statfs":{"total":96636764160,"available":95234662400,"internally_reserved":0,"allocated":3416064,"data_stored":2081390,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1336469,"internal_metadata":1397332843},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2026-02-18T08:53:00.940 INFO:tasks.ceph:pgid 2.7 last_scrub_stamp 2026-02-18T08:09:14.193205+0000 time.struct_time(tm_year=2026, tm_mon=2, tm_mday=18, tm_hour=8, tm_min=9, tm_sec=14, tm_wday=2, tm_yday=49, tm_isdst=-1) <= time.struct_time(tm_year=2026, tm_mon=2, tm_mday=18, tm_hour=8, tm_min=52, tm_sec=58, tm_wday=2, tm_yday=49, tm_isdst=0) 2026-02-18T08:53:00.940 INFO:tasks.ceph:pgid 2.6 last_scrub_stamp 2026-02-18T08:09:14.193205+0000 time.struct_time(tm_year=2026, tm_mon=2, tm_mday=18, tm_hour=8, tm_min=9, tm_sec=14, tm_wday=2, tm_yday=49, tm_isdst=-1) <= time.struct_time(tm_year=2026, tm_mon=2, tm_mday=18, tm_hour=8, tm_min=52, tm_sec=58, tm_wday=2, tm_yday=49, tm_isdst=0) 2026-02-18T08:53:00.940 INFO:tasks.ceph:pgid 2.5 last_scrub_stamp 2026-02-18T08:09:14.193205+0000 time.struct_time(tm_year=2026, tm_mon=2, tm_mday=18, tm_hour=8, tm_min=9, tm_sec=14, tm_wday=2, tm_yday=49, tm_isdst=-1) <= time.struct_time(tm_year=2026, tm_mon=2, tm_mday=18, tm_hour=8, tm_min=52, tm_sec=58, tm_wday=2, tm_yday=49, tm_isdst=0) 2026-02-18T08:53:00.940 INFO:tasks.ceph:pgid 2.4 last_scrub_stamp 2026-02-18T08:09:14.193205+0000 time.struct_time(tm_year=2026, tm_mon=2, tm_mday=18, tm_hour=8, tm_min=9, tm_sec=14, tm_wday=2, tm_yday=49, tm_isdst=-1) <= time.struct_time(tm_year=2026, tm_mon=2, tm_mday=18, tm_hour=8, tm_min=52, tm_sec=58, tm_wday=2, tm_yday=49, tm_isdst=0) 2026-02-18T08:53:00.941 INFO:tasks.ceph:pgid 2.2 last_scrub_stamp 2026-02-18T08:09:14.193205+0000 time.struct_time(tm_year=2026, tm_mon=2, tm_mday=18, tm_hour=8, tm_min=9, tm_sec=14, tm_wday=2, tm_yday=49, tm_isdst=-1) <= time.struct_time(tm_year=2026, tm_mon=2, tm_mday=18, tm_hour=8, tm_min=52, tm_sec=58, tm_wday=2, tm_yday=49, tm_isdst=0) 2026-02-18T08:53:00.941 INFO:tasks.ceph:pgid 2.1 last_scrub_stamp 2026-02-18T08:09:14.193205+0000 time.struct_time(tm_year=2026, tm_mon=2, tm_mday=18, tm_hour=8, tm_min=9, tm_sec=14, tm_wday=2, tm_yday=49, tm_isdst=-1) <= time.struct_time(tm_year=2026, tm_mon=2, tm_mday=18, tm_hour=8, tm_min=52, tm_sec=58, tm_wday=2, tm_yday=49, tm_isdst=0) 2026-02-18T08:53:00.941 INFO:tasks.ceph:pgid 2.0 last_scrub_stamp 2026-02-18T08:09:14.193205+0000 time.struct_time(tm_year=2026, tm_mon=2, tm_mday=18, tm_hour=8, tm_min=9, tm_sec=14, tm_wday=2, tm_yday=49, tm_isdst=-1) <= time.struct_time(tm_year=2026, tm_mon=2, tm_mday=18, tm_hour=8, tm_min=52, tm_sec=58, tm_wday=2, tm_yday=49, tm_isdst=0) 2026-02-18T08:53:00.941 INFO:tasks.ceph:pgid 2.3 last_scrub_stamp 2026-02-18T08:09:14.193205+0000 time.struct_time(tm_year=2026, tm_mon=2, tm_mday=18, tm_hour=8, tm_min=9, tm_sec=14, tm_wday=2, tm_yday=49, tm_isdst=-1) <= time.struct_time(tm_year=2026, tm_mon=2, tm_mday=18, tm_hour=8, tm_min=52, tm_sec=58, tm_wday=2, tm_yday=49, tm_isdst=0) 2026-02-18T08:53:00.941 INFO:tasks.ceph:pgid 1.0 last_scrub_stamp 2026-02-18T08:09:12.188963+0000 time.struct_time(tm_year=2026, tm_mon=2, tm_mday=18, tm_hour=8, tm_min=9, tm_sec=12, tm_wday=2, tm_yday=49, tm_isdst=-1) <= time.struct_time(tm_year=2026, tm_mon=2, tm_mday=18, tm_hour=8, tm_min=52, tm_sec=58, tm_wday=2, tm_yday=49, tm_isdst=0) 2026-02-18T08:53:00.941 INFO:tasks.ceph:Still waiting for all pgs to be scrubbed. 2026-02-18T08:53:20.942 DEBUG:teuthology.orchestra.run.trial034:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 pg dump --format=json 2026-02-18T08:53:21.097 INFO:teuthology.orchestra.run.trial034.stdout: 2026-02-18T08:53:21.097 INFO:teuthology.orchestra.run.trial034.stderr:dumped all 2026-02-18T08:53:21.109 INFO:teuthology.orchestra.run.trial034.stdout:{"pg_ready":true,"pg_map":{"version":1664,"stamp":"2026-02-18T08:53:20.592399+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590387,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":103,"num_write_kb":1630,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":79,"ondisk_log_size":79,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":18,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":3854272,"kb_used_data":9424,"kb_used_omap":3284,"kb_used_meta":3841515,"kb_avail":279261248,"statfs":{"total":289910292480,"available":285963517952,"internally_reserved":0,"allocated":9650176,"data_stored":5653783,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3363724,"internal_metadata":3933711476},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.000966"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":704,"reported_epoch":342,"state":"active+clean","last_fresh":"2026-02-18T08:53:06.139064+0000","last_change":"2026-02-18T08:53:06.139064+0000","last_active":"2026-02-18T08:53:06.139064+0000","last_peered":"2026-02-18T08:53:06.139064+0000","last_clean":"2026-02-18T08:53:06.139064+0000","last_became_active":"2026-02-18T08:09:15.204488+0000","last_became_peered":"2026-02-18T08:09:15.204488+0000","last_unstale":"2026-02-18T08:53:06.139064+0000","last_undegraded":"2026-02-18T08:53:06.139064+0000","last_fullsized":"2026-02-18T08:53:06.139064+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-18T08:53:06.138976+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-18T08:53:06.138976+0000","last_clean_scrub_stamp":"2026-02-18T08:53:06.138976+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-19T19:43:32.949781+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00094066400000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":704,"reported_epoch":342,"state":"active+clean","last_fresh":"2026-02-18T08:53:05.142505+0000","last_change":"2026-02-18T08:53:05.142505+0000","last_active":"2026-02-18T08:53:05.142505+0000","last_peered":"2026-02-18T08:53:05.142505+0000","last_clean":"2026-02-18T08:53:05.142505+0000","last_became_active":"2026-02-18T08:09:15.204451+0000","last_became_peered":"2026-02-18T08:09:15.204451+0000","last_unstale":"2026-02-18T08:53:05.142505+0000","last_undegraded":"2026-02-18T08:53:05.142505+0000","last_fullsized":"2026-02-18T08:53:05.142505+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-18T08:53:05.142415+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-18T08:53:05.142415+0000","last_clean_scrub_stamp":"2026-02-18T08:53:05.142415+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-19T19:01:39.700219+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00075490900000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":705,"reported_epoch":342,"state":"active+clean","last_fresh":"2026-02-18T08:53:04.158378+0000","last_change":"2026-02-18T08:53:04.158320+0000","last_active":"2026-02-18T08:53:04.158378+0000","last_peered":"2026-02-18T08:53:04.158378+0000","last_clean":"2026-02-18T08:53:04.158378+0000","last_became_active":"2026-02-18T08:09:15.204532+0000","last_became_peered":"2026-02-18T08:09:15.204532+0000","last_unstale":"2026-02-18T08:53:04.158378+0000","last_undegraded":"2026-02-18T08:53:04.158378+0000","last_fullsized":"2026-02-18T08:53:04.158378+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-18T08:53:04.158251+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-18T08:53:04.158251+0000","last_clean_scrub_stamp":"2026-02-18T08:53:04.158251+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-19T17:44:00.392697+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00059731699999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":705,"reported_epoch":342,"state":"active+clean","last_fresh":"2026-02-18T08:53:03.171551+0000","last_change":"2026-02-18T08:53:03.171551+0000","last_active":"2026-02-18T08:53:03.171551+0000","last_peered":"2026-02-18T08:53:03.171551+0000","last_clean":"2026-02-18T08:53:03.171551+0000","last_became_active":"2026-02-18T08:09:15.204567+0000","last_became_peered":"2026-02-18T08:09:15.204567+0000","last_unstale":"2026-02-18T08:53:03.171551+0000","last_undegraded":"2026-02-18T08:53:03.171551+0000","last_fullsized":"2026-02-18T08:53:03.171551+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-18T08:53:03.171485+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-18T08:53:03.171485+0000","last_clean_scrub_stamp":"2026-02-18T08:53:03.171485+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-19T10:13:09.810373+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00062999799999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"342'3","reported_seq":708,"reported_epoch":342,"state":"active+clean","last_fresh":"2026-02-18T08:53:00.446413+0000","last_change":"2026-02-18T08:53:00.446366+0000","last_active":"2026-02-18T08:53:00.446413+0000","last_peered":"2026-02-18T08:53:00.446413+0000","last_clean":"2026-02-18T08:53:00.446413+0000","last_became_active":"2026-02-18T08:09:15.204550+0000","last_became_peered":"2026-02-18T08:09:15.204550+0000","last_unstale":"2026-02-18T08:53:00.446413+0000","last_undegraded":"2026-02-18T08:53:00.446413+0000","last_fullsized":"2026-02-18T08:53:00.446413+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"342'3","last_scrub_stamp":"2026-02-18T08:53:00.446307+0000","last_deep_scrub":"342'3","last_deep_scrub_stamp":"2026-02-18T08:53:00.446307+0000","last_clean_scrub_stamp":"2026-02-18T08:53:00.446307+0000","objects_scrubbed":1,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-19T10:06:01.604018+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00095267599999999996,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":705,"reported_epoch":342,"state":"active+clean","last_fresh":"2026-02-18T08:53:01.136411+0000","last_change":"2026-02-18T08:53:01.136336+0000","last_active":"2026-02-18T08:53:01.136411+0000","last_peered":"2026-02-18T08:53:01.136411+0000","last_clean":"2026-02-18T08:53:01.136411+0000","last_became_active":"2026-02-18T08:09:15.204585+0000","last_became_peered":"2026-02-18T08:09:15.204585+0000","last_unstale":"2026-02-18T08:53:01.136411+0000","last_undegraded":"2026-02-18T08:53:01.136411+0000","last_fullsized":"2026-02-18T08:53:01.136411+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-18T08:53:01.136239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-18T08:53:01.136239+0000","last_clean_scrub_stamp":"2026-02-18T08:53:01.136239+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2026-02-19T20:24:30.245277+0000","scrub_duration":4,"objects_trimmed":0,"snaptrim_duration":0.00029873299999999997,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":705,"reported_epoch":342,"state":"active+clean","last_fresh":"2026-02-18T08:53:02.108213+0000","last_change":"2026-02-18T08:53:02.108213+0000","last_active":"2026-02-18T08:53:02.108213+0000","last_peered":"2026-02-18T08:53:02.108213+0000","last_clean":"2026-02-18T08:53:02.108213+0000","last_became_active":"2026-02-18T08:09:15.204604+0000","last_became_peered":"2026-02-18T08:09:15.204604+0000","last_unstale":"2026-02-18T08:53:02.108213+0000","last_undegraded":"2026-02-18T08:53:02.108213+0000","last_fullsized":"2026-02-18T08:53:02.108213+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-18T08:53:02.108106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-18T08:53:02.108106+0000","last_clean_scrub_stamp":"2026-02-18T08:53:02.108106+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2026-02-19T20:52:10.341438+0000","scrub_duration":4,"objects_trimmed":0,"snaptrim_duration":0.00033665399999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.3","version":"342'2","reported_seq":707,"reported_epoch":342,"state":"active+clean","last_fresh":"2026-02-18T08:53:01.210955+0000","last_change":"2026-02-18T08:53:01.210877+0000","last_active":"2026-02-18T08:53:01.210955+0000","last_peered":"2026-02-18T08:53:01.210955+0000","last_clean":"2026-02-18T08:53:01.210955+0000","last_became_active":"2026-02-18T08:09:15.204115+0000","last_became_peered":"2026-02-18T08:09:15.204115+0000","last_unstale":"2026-02-18T08:53:01.210955+0000","last_undegraded":"2026-02-18T08:53:01.210955+0000","last_fullsized":"2026-02-18T08:53:01.210955+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"342'2","last_scrub_stamp":"2026-02-18T08:53:01.210780+0000","last_deep_scrub":"342'2","last_deep_scrub_stamp":"2026-02-18T08:53:01.210780+0000","last_clean_scrub_stamp":"2026-02-18T08:53:01.210780+0000","objects_scrubbed":1,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-19T09:20:51.071730+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00077986500000000003,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"342'74","reported_seq":796,"reported_epoch":342,"state":"active+clean","last_fresh":"2026-02-18T08:53:02.169969+0000","last_change":"2026-02-18T08:53:02.169969+0000","last_active":"2026-02-18T08:53:02.169969+0000","last_peered":"2026-02-18T08:53:02.169969+0000","last_clean":"2026-02-18T08:53:02.169969+0000","last_became_active":"2026-02-18T08:09:13.196393+0000","last_became_peered":"2026-02-18T08:09:13.196393+0000","last_unstale":"2026-02-18T08:53:02.169969+0000","last_undegraded":"2026-02-18T08:53:02.169969+0000","last_fullsized":"2026-02-18T08:53:02.169969+0000","mapping_epoch":9,"log_start":"0'0","ondisk_log_start":"0'0","created":9,"last_epoch_clean":10,"parent":"0.0","parent_split_bits":0,"last_scrub":"342'74","last_scrub_stamp":"2026-02-18T08:53:02.169915+0000","last_deep_scrub":"342'74","last_deep_scrub_stamp":"2026-02-18T08:53:02.169915+0000","last_clean_scrub_stamp":"2026-02-18T08:53:02.169915+0000","objects_scrubbed":2,"log_size":74,"log_dups_size":0,"ondisk_log_size":74,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2026-02-19T17:33:45.539853+0000","scrub_duration":4,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":38,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":5,"ondisk_log_size":5,"up":16,"acting":16,"num_store_stats":3},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1187840,"data_stored":1180736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":74,"ondisk_log_size":74,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":2,"up_from":8,"seq":34359738902,"num_pgs":3,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":1503376,"kb_used_data":2752,"kb_used_omap":1394,"kb_used_meta":1499213,"kb_avail":92868464,"statfs":{"total":96636764160,"available":95097307136,"internally_reserved":0,"allocated":2818048,"data_stored":1491003,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1428061,"internal_metadata":1535194531},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":8,"seq":34359738900,"num_pgs":9,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":981656,"kb_used_data":3336,"kb_used_omap":585,"kb_used_meta":977718,"kb_avail":93390184,"statfs":{"total":96636764160,"available":95631548416,"internally_reserved":0,"allocated":3416064,"data_stored":2081390,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":599194,"internal_metadata":1001184102},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":8,"seq":34359738899,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":1369240,"kb_used_data":3336,"kb_used_omap":1305,"kb_used_meta":1364582,"kb_avail":93002600,"statfs":{"total":96636764160,"available":95234662400,"internally_reserved":0,"allocated":3416064,"data_stored":2081390,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1336469,"internal_metadata":1397332843},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2026-02-18T08:53:21.111 DEBUG:teuthology.orchestra.run.trial034:> sudo ceph --cluster cluster1 config set global mon_health_to_clog false 2026-02-18T08:53:21.283 INFO:teuthology.misc:Shutting down mds daemons... 2026-02-18T08:53:21.283 INFO:teuthology.misc:Shutting down osd daemons... 2026-02-18T08:53:21.284 DEBUG:tasks.ceph.cluster1.osd.0:waiting for process to exit 2026-02-18T08:53:21.284 INFO:teuthology.orchestra.run:waiting for 300 2026-02-18T08:53:24.880 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon cluster1.osd.0 is failed for ~0s 2026-02-18T08:53:27.385 INFO:tasks.ceph.cluster1.osd.0:Stopped 2026-02-18T08:53:27.386 DEBUG:tasks.ceph.cluster1.osd.1:waiting for process to exit 2026-02-18T08:53:27.386 INFO:teuthology.orchestra.run:waiting for 300 2026-02-18T08:53:30.183 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon cluster1.osd.1 is failed for ~0s 2026-02-18T08:53:30.183 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon cluster1.osd.0 has been restored 2026-02-18T08:53:33.487 INFO:tasks.ceph.cluster1.osd.1:Stopped 2026-02-18T08:53:33.488 DEBUG:tasks.ceph.cluster1.osd.2:waiting for process to exit 2026-02-18T08:53:33.488 INFO:teuthology.orchestra.run:waiting for 300 2026-02-18T08:53:35.385 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon cluster1.osd.2 is failed for ~0s 2026-02-18T08:53:35.386 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon cluster1.osd.1 has been restored 2026-02-18T08:53:39.590 INFO:tasks.ceph.cluster1.osd.2:Stopped 2026-02-18T08:53:39.590 INFO:teuthology.misc:Shutting down mgr daemons... 2026-02-18T08:53:39.590 DEBUG:tasks.ceph.cluster1.mgr.x:waiting for process to exit 2026-02-18T08:53:39.591 INFO:teuthology.orchestra.run:waiting for 300 2026-02-18T08:53:39.620 INFO:tasks.ceph.cluster1.mgr.x:Stopped 2026-02-18T08:53:39.621 INFO:teuthology.misc:Shutting down mon daemons... 2026-02-18T08:53:39.621 DEBUG:tasks.ceph.cluster1.mon.a:waiting for process to exit 2026-02-18T08:53:39.621 INFO:teuthology.orchestra.run:waiting for 300 2026-02-18T08:53:39.677 INFO:tasks.ceph.cluster1.mon.a:Stopped 2026-02-18T08:53:39.677 INFO:tasks.ceph:Checking cluster log for badness... 2026-02-18T08:53:39.678 DEBUG:teuthology.orchestra.run.trial034:> sudo egrep '\[ERR\]|\[WRN\]|\[SEC\]' /var/log/ceph/cluster1.log | egrep -v '\(MDS_ALL_DOWN\)' | egrep -v '\(MDS_UP_LESS_THAN_MAX\)' | egrep -v '\(OSD_SLOW_PING_TIME' | head -n 1 2026-02-18T08:53:39.727 INFO:tasks.ceph:Unmounting /var/lib/ceph/osd/cluster1-0 on ubuntu@trial034.front.sepia.ceph.com 2026-02-18T08:53:39.728 DEBUG:teuthology.orchestra.run.trial034:> sync && sudo umount -f /var/lib/ceph/osd/cluster1-0 2026-02-18T08:53:40.386 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon cluster1.osd.2 has been restored 2026-02-18T08:53:40.674 INFO:tasks.ceph:Unmounting /var/lib/ceph/osd/cluster1-1 on ubuntu@trial034.front.sepia.ceph.com 2026-02-18T08:53:40.675 DEBUG:teuthology.orchestra.run.trial034:> sync && sudo umount -f /var/lib/ceph/osd/cluster1-1 2026-02-18T08:53:41.491 INFO:tasks.ceph:Unmounting /var/lib/ceph/osd/cluster1-2 on ubuntu@trial034.front.sepia.ceph.com 2026-02-18T08:53:41.492 DEBUG:teuthology.orchestra.run.trial034:> sync && sudo umount -f /var/lib/ceph/osd/cluster1-2 2026-02-18T08:53:42.360 INFO:tasks.ceph:Archiving mon data... 2026-02-18T08:53:42.360 DEBUG:teuthology.misc:Transferring archived files from trial034:/var/lib/ceph/mon/cluster1-a to /home/teuthworker/mnt/teuthology/yuriw-2026-02-17_21:26:40-rbd-20.2.1_2.17.26-baseline_2-distro-default-trial/54948/data/cluster1.mon.a.tgz 2026-02-18T08:53:42.360 DEBUG:teuthology.orchestra.run.trial034:> mktemp 2026-02-18T08:53:42.365 INFO:teuthology.orchestra.run.trial034.stdout:/tmp/tmp.33PmyesmWg 2026-02-18T08:53:42.366 DEBUG:teuthology.orchestra.run.trial034:> sudo tar cz -f - -C /var/lib/ceph/mon/cluster1-a -- . > /tmp/tmp.33PmyesmWg 2026-02-18T08:53:42.463 DEBUG:teuthology.orchestra.run.trial034:> sudo chmod 0666 /tmp/tmp.33PmyesmWg 2026-02-18T08:53:42.519 DEBUG:teuthology.orchestra.remote:trial034:/tmp/tmp.33PmyesmWg is 359KB 2026-02-18T08:53:42.575 DEBUG:teuthology.orchestra.run.trial034:> rm -fr /tmp/tmp.33PmyesmWg 2026-02-18T08:53:42.579 INFO:tasks.ceph:Cleaning ceph cluster... 2026-02-18T08:53:42.579 DEBUG:teuthology.orchestra.run.trial034:> sudo rm -rf -- /etc/ceph/cluster1.conf /etc/ceph/cluster1.keyring /home/ubuntu/cephtest/cluster1.data /home/ubuntu/cephtest/cluster1.monmap /home/ubuntu/cephtest/../*.pid 2026-02-18T08:53:42.627 DEBUG:teuthology.orchestra.run.trial163:> sudo rm -rf -- /etc/ceph/cluster1.conf /etc/ceph/cluster1.keyring /home/ubuntu/cephtest/cluster1.data /home/ubuntu/cephtest/cluster1.monmap /home/ubuntu/cephtest/../*.pid 2026-02-18T08:53:42.677 INFO:teuthology.util.scanner:summary_data or yaml_file is empty! 2026-02-18T08:53:42.682 INFO:teuthology.util.scanner:summary_data or yaml_file is empty! 2026-02-18T08:53:42.683 INFO:tasks.ceph:Archiving crash dumps... 2026-02-18T08:53:42.686 DEBUG:teuthology.misc:Transferring archived files from trial034:/var/lib/ceph/crash to /home/teuthworker/mnt/teuthology/yuriw-2026-02-17_21:26:40-rbd-20.2.1_2.17.26-baseline_2-distro-default-trial/54948/remote/trial034/crash 2026-02-18T08:53:42.687 DEBUG:teuthology.orchestra.run.trial034:> sudo tar c -f - -C /var/lib/ceph/crash -- . 2026-02-18T08:53:42.726 DEBUG:teuthology.misc:Transferring archived files from trial163:/var/lib/ceph/crash to /home/teuthworker/mnt/teuthology/yuriw-2026-02-17_21:26:40-rbd-20.2.1_2.17.26-baseline_2-distro-default-trial/54948/remote/trial163/crash 2026-02-18T08:53:42.727 DEBUG:teuthology.orchestra.run.trial163:> sudo tar c -f - -C /var/lib/ceph/crash -- . 2026-02-18T08:53:42.737 INFO:tasks.ceph:Compressing logs... 2026-02-18T08:53:42.738 DEBUG:teuthology.orchestra.run.trial034:> time sudo find /var/log/ceph -name '*.log' -print0 | sudo xargs --max-args=1 --max-procs=0 --verbose -0 --no-run-if-empty -- gzip -5 --verbose -- 2026-02-18T08:53:42.775 DEBUG:teuthology.orchestra.run.trial163:> time sudo find /var/log/ceph -name '*.log' -print0 | sudo xargs --max-args=1 --max-procs=0 --verbose -0 --no-run-if-empty -- gzip -5 --verbose -- 2026-02-18T08:53:42.782 INFO:teuthology.orchestra.run.trial034.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.11499.log 2026-02-18T08:53:42.784 INFO:teuthology.orchestra.run.trial034.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.15351.log 2026-02-18T08:53:42.784 INFO:teuthology.orchestra.run.trial034.stderr:/var/log/ceph/cluster1-client.admin.11499.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.15195.log 2026-02-18T08:53:42.784 INFO:teuthology.orchestra.run.trial034.stderr: 26.0% -- replaced with /var/log/ceph/cluster1-client.admin.11499.log.gz 2026-02-18T08:53:42.784 INFO:teuthology.orchestra.run.trial034.stderr:/var/log/ceph/cluster1-client.admin.15351.log: gzip -5 --verbose -- /var/log/ceph/cluster1-mon.a.log 2026-02-18T08:53:42.785 INFO:teuthology.orchestra.run.trial034.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.15351.log.gz 2026-02-18T08:53:42.785 INFO:teuthology.orchestra.run.trial034.stderr:/var/log/ceph/cluster1-client.admin.15195.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.15195.log.gz 2026-02-18T08:53:42.785 INFO:teuthology.orchestra.run.trial034.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.14340.log 2026-02-18T08:53:42.785 INFO:teuthology.orchestra.run.trial034.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-osd.1.log 2026-02-18T08:53:42.785 INFO:teuthology.orchestra.run.trial034.stderr:/var/log/ceph/cluster1-mon.a.log: /var/log/ceph/cluster1-client.admin.14340.log: gzip -5 --verbose -- /var/log/ceph/cluster1.audit.log 2026-02-18T08:53:42.785 INFO:teuthology.orchestra.run.trial034.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.14340.log.gz 2026-02-18T08:53:42.785 INFO:teuthology.orchestra.run.trial034.stderr:/var/log/ceph/cluster1-osd.1.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.11362.log 2026-02-18T08:53:42.803 INFO:teuthology.orchestra.run.trial034.stderr:/var/log/ceph/cluster1.audit.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.14110.log 2026-02-18T08:53:42.803 INFO:teuthology.orchestra.run.trial034.stderr:/var/log/ceph/cluster1-client.admin.11362.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.14155.log 2026-02-18T08:53:42.803 INFO:teuthology.orchestra.run.trial034.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.11362.log.gz 2026-02-18T08:53:42.803 INFO:teuthology.orchestra.run.trial034.stderr:/var/log/ceph/cluster1-client.admin.14110.log: gzip -5 --verbose -- /var/log/ceph/cluster1-mgr.x.log 2026-02-18T08:53:42.804 INFO:teuthology.orchestra.run.trial034.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.14110.log.gz 2026-02-18T08:53:42.804 INFO:teuthology.orchestra.run.trial034.stderr:/var/log/ceph/cluster1-client.admin.14155.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.15273.log 2026-02-18T08:53:42.804 INFO:teuthology.orchestra.run.trial034.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.14155.log.gz 2026-02-18T08:53:42.804 INFO:teuthology.orchestra.run.trial034.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-mgr.x.log 2026-02-18T08:53:42.804 INFO:teuthology.orchestra.run.trial034.stderr:/var/log/ceph/cluster1-mgr.x.log: /var/log/ceph/cluster1-client.admin.15273.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.15273.log.gz 2026-02-18T08:53:42.804 INFO:teuthology.orchestra.run.trial034.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.14493.log 2026-02-18T08:53:42.804 INFO:teuthology.orchestra.run.trial034.stderr:/var/log/ceph/cluster2-mgr.x.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.15403.log 2026-02-18T08:53:42.804 INFO:teuthology.orchestra.run.trial034.stderr: 93.2% -- replaced with /var/log/ceph/cluster1.audit.log.gz 2026-02-18T08:53:42.804 INFO:teuthology.orchestra.run.trial034.stderr:/var/log/ceph/cluster1-client.admin.14493.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.14233.log 2026-02-18T08:53:42.804 INFO:teuthology.orchestra.run.trial034.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.14493.log.gz 2026-02-18T08:53:42.805 INFO:teuthology.orchestra.run.trial034.stderr:/var/log/ceph/cluster1-client.admin.15403.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.15403.log.gz 2026-02-18T08:53:42.805 INFO:teuthology.orchestra.run.trial034.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1.log 2026-02-18T08:53:42.805 INFO:teuthology.orchestra.run.trial034.stderr:/var/log/ceph/cluster1-client.admin.14233.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.14233.log.gz 2026-02-18T08:53:42.805 INFO:teuthology.orchestra.run.trial034.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.15455.log 2026-02-18T08:53:42.805 INFO:teuthology.orchestra.run.trial034.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.11338.log 2026-02-18T08:53:42.805 INFO:teuthology.orchestra.run.trial034.stderr:/var/log/ceph/cluster1.log: /var/log/ceph/cluster1-client.admin.15455.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.15455.log.gz 2026-02-18T08:53:42.805 INFO:teuthology.orchestra.run.trial034.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.15247.log 2026-02-18T08:53:42.805 INFO:teuthology.orchestra.run.trial034.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.14441.log 2026-02-18T08:53:42.805 INFO:teuthology.orchestra.run.trial034.stderr:/var/log/ceph/cluster1-client.admin.11338.log: /var/log/ceph/cluster1-client.admin.15247.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.15247.log.gz 2026-02-18T08:53:42.805 INFO:teuthology.orchestra.run.trial034.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.15221.log 2026-02-18T08:53:42.805 INFO:teuthology.orchestra.run.trial034.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.11338.log.gz 2026-02-18T08:53:42.806 INFO:teuthology.orchestra.run.trial034.stderr:/var/log/ceph/cluster1-client.admin.14441.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.15505.log 2026-02-18T08:53:42.806 INFO:teuthology.orchestra.run.trial034.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.14441.log.gz 2026-02-18T08:53:42.806 INFO:teuthology.orchestra.run.trial034.stderr:/var/log/ceph/cluster1-client.admin.15221.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.15221.log.gz 2026-02-18T08:53:42.806 INFO:teuthology.orchestra.run.trial034.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.11421.log 2026-02-18T08:53:42.806 INFO:teuthology.orchestra.run.trial034.stderr:/var/log/ceph/cluster1-client.admin.15505.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.15505.log.gz 2026-02-18T08:53:42.806 INFO:teuthology.orchestra.run.trial034.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.15429.log 2026-02-18T08:53:42.806 INFO:teuthology.orchestra.run.trial034.stderr:/var/log/ceph/cluster1-client.admin.11421.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.15377.log 2026-02-18T08:53:42.806 INFO:teuthology.orchestra.run.trial034.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.11421.log.gz 2026-02-18T08:53:42.806 INFO:teuthology.orchestra.run.trial034.stderr: 87.7% -- replaced with /var/log/ceph/cluster1.log.gz 2026-02-18T08:53:42.806 INFO:teuthology.orchestra.run.trial034.stderr:/var/log/ceph/cluster1-client.admin.15429.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.14181.log 2026-02-18T08:53:42.806 INFO:teuthology.orchestra.run.trial034.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.15429.log.gz 2026-02-18T08:53:42.807 INFO:teuthology.orchestra.run.trial034.stderr:/var/log/ceph/cluster1-client.admin.15377.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.14467.log 2026-02-18T08:53:42.807 INFO:teuthology.orchestra.run.trial034.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.15377.log.gz 2026-02-18T08:53:42.807 INFO:teuthology.orchestra.run.trial034.stderr:gzip/var/log/ceph/cluster1-client.admin.14181.log: -5 --verbose -- /var/log/ceph/cluster1-client.admin.14267.log 2026-02-18T08:53:42.807 INFO:teuthology.orchestra.run.trial034.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.14181.log.gz 2026-02-18T08:53:42.807 INFO:teuthology.orchestra.run.trial034.stderr:/var/log/ceph/cluster1-client.admin.14467.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.15481.log 2026-02-18T08:53:42.807 INFO:teuthology.orchestra.run.trial034.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.14467.log.gz 2026-02-18T08:53:42.807 INFO:teuthology.orchestra.run.trial034.stderr:/var/log/ceph/cluster1-client.admin.14267.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.14085.log 2026-02-18T08:53:42.807 INFO:teuthology.orchestra.run.trial034.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.14267.log.gz 2026-02-18T08:53:42.807 INFO:teuthology.orchestra.run.trial034.stderr:/var/log/ceph/cluster1-client.admin.15481.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.15481.log.gz 2026-02-18T08:53:42.807 INFO:teuthology.orchestra.run.trial034.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.14341.log 2026-02-18T08:53:42.807 INFO:teuthology.orchestra.run.trial034.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.11224.log 2026-02-18T08:53:42.808 INFO:teuthology.orchestra.run.trial034.stderr:/var/log/ceph/cluster1-client.admin.14085.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.14085.log.gz 2026-02-18T08:53:42.808 INFO:teuthology.orchestra.run.trial034.stderr:/var/log/ceph/cluster1-client.admin.14341.log: 0.0%gzip -- replaced with /var/log/ceph/cluster1-client.admin.14341.log.gz 2026-02-18T08:53:42.808 INFO:teuthology.orchestra.run.trial034.stderr: -5 --verbose -- /var/log/ceph/cluster1-client.admin.14207.log 2026-02-18T08:53:42.808 INFO:teuthology.orchestra.run.trial034.stderr:/var/log/ceph/cluster1-client.admin.11224.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.14265.log 2026-02-18T08:53:42.808 INFO:teuthology.orchestra.run.trial034.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.11224.log.gz 2026-02-18T08:53:42.808 INFO:teuthology.orchestra.run.trial034.stderr:gzip -5 --verbose -- /var/log/ceph/ceph.tmp-client.admin.7605.log 2026-02-18T08:53:42.808 INFO:teuthology.orchestra.run.trial034.stderr:/var/log/ceph/cluster1-client.admin.14207.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.14207.log.gz 2026-02-18T08:53:42.808 INFO:teuthology.orchestra.run.trial034.stderr:/var/log/ceph/cluster1-client.admin.14265.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.14265.log.gz 2026-02-18T08:53:42.808 INFO:teuthology.orchestra.run.trial034.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.11386.log 2026-02-18T08:53:42.808 INFO:teuthology.orchestra.run.trial034.stderr:/var/log/ceph/ceph.tmp-client.admin.7605.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.14135.log 2026-02-18T08:53:42.809 INFO:teuthology.orchestra.run.trial034.stderr: 0.0% -- replaced with /var/log/ceph/ceph.tmp-client.admin.7605.log.gz 2026-02-18T08:53:42.809 INFO:teuthology.orchestra.run.trial034.stderr:/var/log/ceph/cluster1-client.admin.11386.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.11386.log.gz 2026-02-18T08:53:42.809 INFO:teuthology.orchestra.run.trial034.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.15325.log 2026-02-18T08:53:42.809 INFO:teuthology.orchestra.run.trial034.stderr:/var/log/ceph/cluster1-client.admin.14135.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.11296.log 2026-02-18T08:53:42.809 INFO:teuthology.orchestra.run.trial034.stderr: 80.4% -- replaced with /var/log/ceph/cluster1-client.admin.14135.log.gz 2026-02-18T08:53:42.809 INFO:teuthology.orchestra.run.trial034.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-osd.0.log 2026-02-18T08:53:42.809 INFO:teuthology.orchestra.run.trial034.stderr:/var/log/ceph/cluster1-client.admin.15325.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.15325.log.gz 2026-02-18T08:53:42.809 INFO:teuthology.orchestra.run.trial034.stderr:/var/log/ceph/cluster1-client.admin.11296.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.11296.log.gz 2026-02-18T08:53:42.809 INFO:teuthology.orchestra.run.trial034.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.14347.log 2026-02-18T08:53:42.809 INFO:teuthology.orchestra.run.trial034.stderr:/var/log/ceph/cluster1-osd.0.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.15299.log 2026-02-18T08:53:42.809 INFO:teuthology.orchestra.run.trial034.stderr:/var/log/ceph/cluster1-client.admin.14347.log: gzip -5 --verbose -- /var/log/ceph/cluster1-osd.2.log 2026-02-18T08:53:42.810 INFO:teuthology.orchestra.run.trial034.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.14347.log.gz 2026-02-18T08:53:42.810 INFO:teuthology.orchestra.run.trial034.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.14266.log 2026-02-18T08:53:42.810 INFO:teuthology.orchestra.run.trial034.stderr:/var/log/ceph/cluster1-client.admin.15299.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.15299.log.gz 2026-02-18T08:53:42.810 INFO:teuthology.orchestra.run.trial034.stderr:/var/log/ceph/cluster1-osd.2.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.14415.log 2026-02-18T08:53:42.810 INFO:teuthology.orchestra.run.trial034.stderr:/var/log/ceph/cluster1-client.admin.14266.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.14266.log.gz 2026-02-18T08:53:42.810 INFO:teuthology.orchestra.run.trial034.stderr:/var/log/ceph/cluster1-client.admin.14415.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.14415.log.gz 2026-02-18T08:53:42.838 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.36329.log 2026-02-18T08:53:42.838 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.56534.log 2026-02-18T08:53:42.838 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.36329.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.14237.log 2026-02-18T08:53:42.838 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.56534.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.113370.log 2026-02-18T08:53:42.838 INFO:teuthology.orchestra.run.trial163.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.36329.log.gz 2026-02-18T08:53:42.838 INFO:teuthology.orchestra.run.trial163.stderr: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.56534.log.gz 2026-02-18T08:53:42.838 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.admin.14237.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.32105.log 2026-02-18T08:53:42.838 INFO:teuthology.orchestra.run.trial163.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.14237.log.gz 2026-02-18T08:53:42.839 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.113370.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.77102.log 2026-02-18T08:53:42.839 INFO:teuthology.orchestra.run.trial163.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.113370.log.gz 2026-02-18T08:53:42.839 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.32105.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.51634.log 2026-02-18T08:53:42.839 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.77102.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.59499.log 2026-02-18T08:53:42.839 INFO:teuthology.orchestra.run.trial163.stderr: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.32105.log.gz 2026-02-18T08:53:42.839 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.51634.log: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.77102.log.gz 2026-02-18T08:53:42.839 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.63410.log 2026-02-18T08:53:42.839 INFO:teuthology.orchestra.run.trial163.stderr: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.51634.log.gz 2026-02-18T08:53:42.839 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.98363.log 2026-02-18T08:53:42.839 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.59499.log: /var/log/ceph/cluster2-client.mirror.63410.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.67274.log 2026-02-18T08:53:42.839 INFO:teuthology.orchestra.run.trial163.stderr: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.63410.log.gz 2026-02-18T08:53:42.839 INFO:teuthology.orchestra.run.trial163.stderr: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.59499.log.gz 2026-02-18T08:53:42.839 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.47745.log 2026-02-18T08:53:42.839 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.98363.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.123881.log 2026-02-18T08:53:42.839 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.67274.log: 88.4% -- replaced with /var/log/ceph/cluster1-client.mirror.98363.log.gz 2026-02-18T08:53:42.839 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.47745.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.24998.log 2026-02-18T08:53:42.839 INFO:teuthology.orchestra.run.trial163.stderr: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.67274.log.gz 2026-02-18T08:53:42.839 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.123881.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.22001.log 2026-02-18T08:53:42.839 INFO:teuthology.orchestra.run.trial163.stderr: 88.3% -- replaced with /var/log/ceph/cluster1-client.mirror.123881.log.gz 2026-02-18T08:53:42.839 INFO:teuthology.orchestra.run.trial163.stderr: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.47745.log.gz 2026-02-18T08:53:42.839 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.24998.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.87992.log 2026-02-18T08:53:42.839 INFO:teuthology.orchestra.run.trial163.stderr: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.24998.log.gz 2026-02-18T08:53:42.839 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.22001.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.95406.log 2026-02-18T08:53:42.839 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.87992.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.74019.log 2026-02-18T08:53:42.839 INFO:teuthology.orchestra.run.trial163.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.22001.log.gz 2026-02-18T08:53:42.840 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.95406.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.112876.log 2026-02-18T08:53:42.840 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.74019.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.128278.log 2026-02-18T08:53:42.840 INFO:teuthology.orchestra.run.trial163.stderr: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.74019.log.gz 2026-02-18T08:53:42.840 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.112876.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.39165.log 2026-02-18T08:53:42.840 INFO:teuthology.orchestra.run.trial163.stderr: 88.3% -- replaced with /var/log/ceph/cluster1-client.mirror.112876.log.gz 2026-02-18T08:53:42.840 INFO:teuthology.orchestra.run.trial163.stderr: 92.6% -- replaced with /var/log/ceph/cluster2-client.mirror.95406.log.gz 2026-02-18T08:53:42.840 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.128278.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.18273.log 2026-02-18T08:53:42.840 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.39165.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.126533.log 2026-02-18T08:53:42.840 INFO:teuthology.orchestra.run.trial163.stderr: 89.4% -- replaced with /var/log/ceph/cluster2-client.mirror.128278.log.gz 2026-02-18T08:53:42.840 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.107167.log 2026-02-18T08:53:42.840 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.18273.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.39165.log.gz 2026-02-18T08:53:42.840 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.126533.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.22187.log 2026-02-18T08:53:42.840 INFO:teuthology.orchestra.run.trial163.stderr: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.126533.log.gz 2026-02-18T08:53:42.840 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.53734.log 2026-02-18T08:53:42.840 INFO:teuthology.orchestra.run.trial163.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.18273.log.gz 2026-02-18T08:53:42.840 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.107167.log: /var/log/ceph/cluster2-client.mirror.22187.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.116722.log 2026-02-18T08:53:42.840 INFO:teuthology.orchestra.run.trial163.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.107167.log.gz 2026-02-18T08:53:42.840 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.53734.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.67519.log 2026-02-18T08:53:42.840 INFO:teuthology.orchestra.run.trial163.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.22187.log.gz 2026-02-18T08:53:42.840 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.116722.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.95381.log 2026-02-18T08:53:42.840 INFO:teuthology.orchestra.run.trial163.stderr: 87.9% -- replaced with /var/log/ceph/cluster2-client.mirror.116722.log.gz 2026-02-18T08:53:42.840 INFO:teuthology.orchestra.run.trial163.stderr: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.53734.log.gz 2026-02-18T08:53:42.840 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.67519.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.38514.log 2026-02-18T08:53:42.840 INFO:teuthology.orchestra.run.trial163.stderr: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.67519.log.gz 2026-02-18T08:53:42.840 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.95381.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.114114.log 2026-02-18T08:53:42.841 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.0.87008.log 2026-02-18T08:53:42.841 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.38514.log: 88.3% -- replaced with /var/log/ceph/cluster1-client.mirror.95381.log.gz 2026-02-18T08:53:42.841 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.114114.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.114114.log.gz 2026-02-18T08:53:42.841 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.110559.log 2026-02-18T08:53:42.841 INFO:teuthology.orchestra.run.trial163.stderr: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.38514.log.gz 2026-02-18T08:53:42.841 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.0.87008.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.119783.log 2026-02-18T08:53:42.841 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.96751.log 2026-02-18T08:53:42.841 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.110559.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.56149.log 2026-02-18T08:53:42.841 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.119783.log: /var/log/ceph/cluster1-client.mirror.96751.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.120839.log 2026-02-18T08:53:42.841 INFO:teuthology.orchestra.run.trial163.stderr: 88.2% -- replaced with /var/log/ceph/cluster1-client.mirror.96751.log.gz 2026-02-18T08:53:42.841 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.56149.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.126624.log 2026-02-18T08:53:42.841 INFO:teuthology.orchestra.run.trial163.stderr: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.56149.log.gz 2026-02-18T08:53:42.841 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.20197.log 2026-02-18T08:53:42.841 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.120839.log: /var/log/ceph/cluster2-client.mirror.126624.log: 92.3% -- replaced with /var/log/ceph/cluster1-client.mirror.110559.log.gz 2026-02-18T08:53:42.841 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.36050.log 2026-02-18T08:53:42.841 INFO:teuthology.orchestra.run.trial163.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.120839.log.gz 2026-02-18T08:53:42.841 INFO:teuthology.orchestra.run.trial163.stderr: 92.4% -- replaced with /var/log/ceph/cluster2-client.mirror.119783.log.gz 2026-02-18T08:53:42.841 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.20197.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.78887.log 2026-02-18T08:53:42.841 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.36050.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.81166.log 2026-02-18T08:53:42.841 INFO:teuthology.orchestra.run.trial163.stderr: 90.7% -- replaced with /var/log/ceph/cluster2-client.mirror.126624.log.gz 2026-02-18T08:53:42.841 INFO:teuthology.orchestra.run.trial163.stderr: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.20197.log.gz 2026-02-18T08:53:42.841 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.90144.log 2026-02-18T08:53:42.841 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.78887.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.36050.log.gz 2026-02-18T08:53:42.841 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.81166.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.50269.log 2026-02-18T08:53:42.841 INFO:teuthology.orchestra.run.trial163.stderr: 89.3% -- replaced with /var/log/ceph/cluster2-client.mirror.78887.log.gz 2026-02-18T08:53:42.841 INFO:teuthology.orchestra.run.trial163.stderr: 89.2% -- replaced with /var/log/ceph/cluster2-client.mirror.81166.log.gz 2026-02-18T08:53:42.842 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.90144.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.103827.log 2026-02-18T08:53:42.842 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.123996.log 2026-02-18T08:53:42.842 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.50269.log: /var/log/ceph/cluster2-client.mirror.103827.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.90280.log 2026-02-18T08:53:42.842 INFO:teuthology.orchestra.run.trial163.stderr: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.50269.log.gz 2026-02-18T08:53:42.842 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.14423.log 2026-02-18T08:53:42.842 INFO:teuthology.orchestra.run.trial163.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.103827.log.gz 2026-02-18T08:53:42.842 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.123996.log: /var/log/ceph/cluster2-client.mirror.90280.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.130960.log 2026-02-18T08:53:42.842 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.admin.14423.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.14423.log.gz 2026-02-18T08:53:42.842 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.127220.log 2026-02-18T08:53:42.842 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.114946.log 2026-02-18T08:53:42.842 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.130960.log: /var/log/ceph/cluster2-client.mirror.127220.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.79622.log 2026-02-18T08:53:42.842 INFO:teuthology.orchestra.run.trial163.stderr: 86.8% -- replaced with /var/log/ceph/cluster1-client.mirror.130960.log.gz 2026-02-18T08:53:42.842 INFO:teuthology.orchestra.run.trial163.stderr: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.127220.log.gz 2026-02-18T08:53:42.842 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.114946.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.120151.log 2026-02-18T08:53:42.842 INFO:teuthology.orchestra.run.trial163.stderr: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.114946.log.gz 2026-02-18T08:53:42.842 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.17715.log 2026-02-18T08:53:42.842 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.79622.log: /var/log/ceph/cluster2-client.mirror.120151.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.36707.log 2026-02-18T08:53:42.842 INFO:teuthology.orchestra.run.trial163.stderr: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.79622.log.gz 2026-02-18T08:53:42.842 INFO:teuthology.orchestra.run.trial163.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.120151.log.gz 2026-02-18T08:53:42.842 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.17715.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.133415.log 2026-02-18T08:53:42.842 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.36707.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.109985.log 2026-02-18T08:53:42.842 INFO:teuthology.orchestra.run.trial163.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.17715.log.gz 2026-02-18T08:53:42.842 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.133415.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.55055.log 2026-02-18T08:53:42.842 INFO:teuthology.orchestra.run.trial163.stderr: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.133415.log.gz 2026-02-18T08:53:42.842 INFO:teuthology.orchestra.run.trial163.stderr: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.36707.log.gz 2026-02-18T08:53:42.842 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.109985.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.83405.log 2026-02-18T08:53:42.843 INFO:teuthology.orchestra.run.trial163.stderr: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.109985.log.gz 2026-02-18T08:53:42.843 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.55055.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.134833.log 2026-02-18T08:53:42.843 INFO:teuthology.orchestra.run.trial163.stderr: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.55055.log.gz 2026-02-18T08:53:42.843 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.130758.log 2026-02-18T08:53:42.843 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.83405.log: /var/log/ceph/cluster2-client.mirror.134833.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.103916.log 2026-02-18T08:53:42.843 INFO:teuthology.orchestra.run.trial163.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.134833.log.gz 2026-02-18T08:53:42.843 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.53209.log 2026-02-18T08:53:42.843 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.130758.log: /var/log/ceph/cluster2-client.mirror.103916.log: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.130758.log.gz 2026-02-18T08:53:42.843 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 87.9% --verbose -- replaced with /var/log/ceph/cluster2-client.mirror.103916.log.gz -- 2026-02-18T08:53:42.843 INFO:teuthology.orchestra.run.trial163.stderr: /var/log/ceph/cluster2-client.mirror.84709.log 2026-02-18T08:53:42.843 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.53209.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.112599.log 2026-02-18T08:53:42.843 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.84709.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.52369.log 2026-02-18T08:53:42.843 INFO:teuthology.orchestra.run.trial163.stderr: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.53209.log.gz 2026-02-18T08:53:42.843 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.112599.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.119997.log 2026-02-18T08:53:42.843 INFO:teuthology.orchestra.run.trial163.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.112599.log.gz 2026-02-18T08:53:42.843 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.52369.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.114191.log 2026-02-18T08:53:42.843 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.119997.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.119997.log.gz 2026-02-18T08:53:42.843 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.101521.log 2026-02-18T08:53:42.843 INFO:teuthology.orchestra.run.trial163.stderr: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.52369.log.gz 2026-02-18T08:53:42.843 INFO:teuthology.orchestra.run.trial163.stderr: 93.5% -- replaced with /var/log/ceph/cluster1-client.mirror.83405.log.gz 2026-02-18T08:53:42.843 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose/var/log/ceph/cluster1-client.mirror.114191.log: -- /var/log/ceph/cluster2-client.mirror.40467.log 2026-02-18T08:53:42.843 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.101521.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.56254.log 2026-02-18T08:53:42.843 INFO:teuthology.orchestra.run.trial163.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.101521.log.gz 2026-02-18T08:53:42.843 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.40467.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.39386.log 2026-02-18T08:53:42.843 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.56254.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.40467.log.gz 2026-02-18T08:53:42.843 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.101750.log 2026-02-18T08:53:42.843 INFO:teuthology.orchestra.run.trial163.stderr: 93.5% -- replaced with /var/log/ceph/cluster2-client.mirror.84709.log.gz 2026-02-18T08:53:42.843 INFO:teuthology.orchestra.run.trial163.stderr: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.56254.log.gz 2026-02-18T08:53:42.844 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.39386.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.55834.log 2026-02-18T08:53:42.844 INFO:teuthology.orchestra.run.trial163.stderr: 92.7% -- replaced with /var/log/ceph/cluster1-client.mirror.114191.log.gz 2026-02-18T08:53:42.844 INFO:teuthology.orchestra.run.trial163.stderr: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.39386.log.gz 2026-02-18T08:53:42.844 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.134020.log 2026-02-18T08:53:42.844 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.101750.log: /var/log/ceph/cluster2-client.mirror.55834.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.69819.log 2026-02-18T08:53:42.844 INFO:teuthology.orchestra.run.trial163.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.101750.log.gz 2026-02-18T08:53:42.844 INFO:teuthology.orchestra.run.trial163.stderr: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.55834.log.gz 2026-02-18T08:53:42.844 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.134020.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.50724.log 2026-02-18T08:53:42.844 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.118177.log 2026-02-18T08:53:42.844 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.69819.log: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.69819.log.gz 2026-02-18T08:53:42.844 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.50724.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.25835.log 2026-02-18T08:53:42.844 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.118177.log: 88.2%gzip -- replaced with /var/log/ceph/cluster2-client.mirror.118177.log.gz -5 2026-02-18T08:53:42.844 INFO:teuthology.orchestra.run.trial163.stderr: --verbose -- /var/log/ceph/cluster2-client.mirror.27228.log 2026-02-18T08:53:42.844 INFO:teuthology.orchestra.run.trial163.stderr: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.50724.log.gz 2026-02-18T08:53:42.844 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.25835.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.87791.log 2026-02-18T08:53:42.844 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.27228.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.23396.log 2026-02-18T08:53:42.844 INFO:teuthology.orchestra.run.trial163.stderr: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.25835.log.gz 2026-02-18T08:53:42.844 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.87791.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.78117.log 2026-02-18T08:53:42.844 INFO:teuthology.orchestra.run.trial163.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.27228.log.gz 2026-02-18T08:53:42.844 INFO:teuthology.orchestra.run.trial163.stderr: 84.7% -- replaced with /var/log/ceph/cluster2-client.mirror.87791.log.gz 2026-02-18T08:53:42.844 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.14344.log 2026-02-18T08:53:42.844 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.23396.log: /var/log/ceph/cluster2-client.mirror.78117.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.126866.log 2026-02-18T08:53:42.844 INFO:teuthology.orchestra.run.trial163.stderr: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.78117.log.gz 2026-02-18T08:53:42.844 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.admin.14344.log: 88.3% 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.14344.log.gz -- replaced with /var/log/ceph/cluster2-client.mirror.23396.log.gz 2026-02-18T08:53:42.844 INFO:teuthology.orchestra.run.trial163.stderr: 2026-02-18T08:53:42.844 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.122762.log 2026-02-18T08:53:42.844 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.29309.log 2026-02-18T08:53:42.844 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.126866.log: /var/log/ceph/cluster2-client.mirror.122762.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.0.86666.log 2026-02-18T08:53:42.845 INFO:teuthology.orchestra.run.trial163.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.122762.log.gz 2026-02-18T08:53:42.845 INFO:teuthology.orchestra.run.trial163.stderr: 89.3% -- replaced with /var/log/ceph/cluster2-client.mirror.126866.log.gz 2026-02-18T08:53:42.845 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.29309.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.128107.log 2026-02-18T08:53:42.845 INFO:teuthology.orchestra.run.trial163.stderr: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.29309.log.gz 2026-02-18T08:53:42.845 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.0.86666.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.126227.log 2026-02-18T08:53:42.845 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.128107.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.135367.log 2026-02-18T08:53:42.845 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.126227.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.34713.log 2026-02-18T08:53:42.845 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.admin.135367.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.58839.log 2026-02-18T08:53:42.845 INFO:teuthology.orchestra.run.trial163.stderr: 90.8% -- replaced with /var/log/ceph/cluster2-client.mirror.128107.log.gz 2026-02-18T08:53:42.845 INFO:teuthology.orchestra.run.trial163.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.135367.log.gz 2026-02-18T08:53:42.845 INFO:teuthology.orchestra.run.trial163.stderr: 89.2% -- replaced with /var/log/ceph/cluster2-client.mirror.126227.log.gz 2026-02-18T08:53:42.845 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.98517.log 2026-02-18T08:53:42.845 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.34713.log: /var/log/ceph/cluster1-client.admin.58839.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.73074.log 2026-02-18T08:53:42.845 INFO:teuthology.orchestra.run.trial163.stderr: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.34713.log.gz 2026-02-18T08:53:42.845 INFO:teuthology.orchestra.run.trial163.stderr: 89.1% -- replaced with /var/log/ceph/cluster1-client.admin.58839.log.gz 2026-02-18T08:53:42.845 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.38979.log 2026-02-18T08:53:42.845 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.98517.log: /var/log/ceph/cluster2-client.mirror.73074.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.98517.log.gz 2026-02-18T08:53:42.845 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.135159.log 2026-02-18T08:53:42.845 INFO:teuthology.orchestra.run.trial163.stderr: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.73074.log.gz 2026-02-18T08:53:42.845 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.38979.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.70554.log 2026-02-18T08:53:42.845 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.admin.135159.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.135159.log.gz 2026-02-18T08:53:42.845 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.92240.log 2026-02-18T08:53:42.845 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.70554.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.38979.log.gz 2026-02-18T08:53:42.845 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.119726.log 2026-02-18T08:53:42.845 INFO:teuthology.orchestra.run.trial163.stderr: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.70554.log.gz 2026-02-18T08:53:42.845 INFO:teuthology.orchestra.run.trial163.stderr: 94.1% -- replaced with /var/log/ceph/cluster2-client.mirror.0.87008.log.gz 2026-02-18T08:53:42.845 INFO:teuthology.orchestra.run.trial163.stderr:gzip/var/log/ceph/cluster2-client.mirror.92240.log: -5 --verbose -- /var/log/ceph/cluster2-client.mirror.127624.log 2026-02-18T08:53:42.845 INFO:teuthology.orchestra.run.trial163.stderr: 1.3% -- replaced with /var/log/ceph/cluster2-client.mirror.92240.log.gz 2026-02-18T08:53:42.846 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.119726.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.96137.log 2026-02-18T08:53:42.846 INFO:teuthology.orchestra.run.trial163.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.119726.log.gz 2026-02-18T08:53:42.846 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.127624.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.59744.log 2026-02-18T08:53:42.846 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.96137.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.11479.log 2026-02-18T08:53:42.846 INFO:teuthology.orchestra.run.trial163.stderr: 89.4% -- replaced with /var/log/ceph/cluster2-client.mirror.127624.log.gz 2026-02-18T08:53:42.846 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.59744.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.78222.log 2026-02-18T08:53:42.846 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose/var/log/ceph/cluster2-client.admin.11479.log: -- /var/log/ceph/cluster2-client.mirror.96441.log 2026-02-18T08:53:42.846 INFO:teuthology.orchestra.run.trial163.stderr: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.59744.log.gz 2026-02-18T08:53:42.846 INFO:teuthology.orchestra.run.trial163.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.11479.log.gz 2026-02-18T08:53:42.846 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.78222.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.24233.log 2026-02-18T08:53:42.846 INFO:teuthology.orchestra.run.trial163.stderr: 92.7% -- replaced with /var/log/ceph/cluster1-client.mirror.96137.log.gz 2026-02-18T08:53:42.846 INFO:teuthology.orchestra.run.trial163.stderr: 89.2% -- replaced with /var/log/ceph/cluster2-client.mirror.78222.log.gz 2026-02-18T08:53:42.846 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.96441.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.106037.log 2026-02-18T08:53:42.846 INFO:teuthology.orchestra.run.trial163.stderr: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.96441.log.gz 2026-02-18T08:53:42.846 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.24233.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.83237.log 2026-02-18T08:53:42.846 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.109368.log 2026-02-18T08:53:42.846 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.106037.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.24233.log.gz 2026-02-18T08:53:42.846 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.83237.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.134306.log 2026-02-18T08:53:42.846 INFO:teuthology.orchestra.run.trial163.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.106037.log.gz 2026-02-18T08:53:42.846 INFO:teuthology.orchestra.run.trial163.stderr: 88.9%/var/log/ceph/cluster2-client.mirror.109368.log: -- replaced with /var/log/ceph/cluster2-client.mirror.83237.log.gz 2026-02-18T08:53:42.846 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.74618.log 2026-02-18T08:53:42.846 INFO:teuthology.orchestra.run.trial163.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.109368.log.gz 2026-02-18T08:53:42.846 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.134306.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.84729.log 2026-02-18T08:53:42.846 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.74618.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.89784.log 2026-02-18T08:53:42.846 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.84729.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.24920.log 2026-02-18T08:53:42.846 INFO:teuthology.orchestra.run.trial163.stderr: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.74618.log.gz 2026-02-18T08:53:42.846 INFO:teuthology.orchestra.run.trial163.stderr: 91.8% -- replaced with /var/log/ceph/cluster2-client.mirror.134306.log.gz 2026-02-18T08:53:42.846 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.89784.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.130172.log 2026-02-18T08:53:42.846 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.admin.24920.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.126037.log 2026-02-18T08:53:42.846 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.130172.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.123250.log 2026-02-18T08:53:42.847 INFO:teuthology.orchestra.run.trial163.stderr: 87.9% -- replaced with /var/log/ceph/cluster1-client.admin.24920.log.gz 2026-02-18T08:53:42.847 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose 86.9% -- /var/log/ceph/cluster2-client.mirror.109135.log 2026-02-18T08:53:42.847 INFO:teuthology.orchestra.run.trial163.stderr: -- replaced with /var/log/ceph/cluster1-client.mirror.130172.log.gz 2026-02-18T08:53:42.847 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.126037.log: /var/log/ceph/cluster2-client.mirror.123250.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.108069.log 2026-02-18T08:53:42.847 INFO:teuthology.orchestra.run.trial163.stderr: 89.4% -- replaced with /var/log/ceph/cluster2-client.mirror.126037.log.gz 2026-02-18T08:53:42.847 INFO:teuthology.orchestra.run.trial163.stderr: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.123250.log.gz 2026-02-18T08:53:42.847 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.109135.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.26856.log 2026-02-18T08:53:42.847 INFO:teuthology.orchestra.run.trial163.stderr: 93.5% -- replaced with /var/log/ceph/cluster1-client.mirror.84729.log.gz 2026-02-18T08:53:42.847 INFO:teuthology.orchestra.run.trial163.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.109135.log.gz 2026-02-18T08:53:42.847 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.125966.log 2026-02-18T08:53:42.847 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.108069.log: /var/log/ceph/cluster2-client.mirror.26856.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.56779.log 2026-02-18T08:53:42.847 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.125966.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.99818.log 2026-02-18T08:53:42.847 INFO:teuthology.orchestra.run.trial163.stderr: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.26856.log.gz 2026-02-18T08:53:42.847 INFO:teuthology.orchestra.run.trial163.stderr: 89.2% -- replaced with /var/log/ceph/cluster2-client.mirror.125966.log.gz 2026-02-18T08:53:42.847 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.56779.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.41680.log 2026-02-18T08:53:42.847 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.99818.log: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.56779.log.gz 2026-02-18T08:53:42.847 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.103358.log 2026-02-18T08:53:42.847 INFO:teuthology.orchestra.run.trial163.stderr: 88.1% -- replaced with /var/log/ceph/cluster1-client.mirror.99818.log.gz 2026-02-18T08:53:42.847 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.41680.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.53629.log 2026-02-18T08:53:42.847 INFO:teuthology.orchestra.run.trial163.stderr: 92.7% -- replaced with /var/log/ceph/cluster2-client.mirror.108069.log.gz 2026-02-18T08:53:42.847 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.103358.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.41680.log.gz 2026-02-18T08:53:42.847 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.113755.log 2026-02-18T08:53:42.847 INFO:teuthology.orchestra.run.trial163.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.103358.log.gz 2026-02-18T08:53:42.847 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.53629.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.32291.log 2026-02-18T08:53:42.847 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.113755.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.34058.log 2026-02-18T08:53:42.847 INFO:teuthology.orchestra.run.trial163.stderr: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.53629.log.gz 2026-02-18T08:53:42.847 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.32291.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.100895.log 2026-02-18T08:53:42.847 INFO:teuthology.orchestra.run.trial163.stderr: 95.2% -- replaced with /var/log/ceph/cluster1-client.mirror.89784.log.gz 2026-02-18T08:53:42.847 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.34058.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.32291.log.gz 2026-02-18T08:53:42.848 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.68446.log 2026-02-18T08:53:42.848 INFO:teuthology.orchestra.run.trial163.stderr: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.34058.log.gz 2026-02-18T08:53:42.848 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.100895.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.65498.log 2026-02-18T08:53:42.848 INFO:teuthology.orchestra.run.trial163.stderr: 92.6% -- replaced with /var/log/ceph/cluster1-client.mirror.113755.log.gz 2026-02-18T08:53:42.848 INFO:teuthology.orchestra.run.trial163.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.100895.log.gz 2026-02-18T08:53:42.848 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.68446.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.18738.log 2026-02-18T08:53:42.848 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.128825.log 2026-02-18T08:53:42.848 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.65498.log: /var/log/ceph/cluster2-client.mirror.18738.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.23268.log 2026-02-18T08:53:42.848 INFO:teuthology.orchestra.run.trial163.stderr: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.65498.log.gz 2026-02-18T08:53:42.848 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.128825.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.58646.log 2026-02-18T08:53:42.848 INFO:teuthology.orchestra.run.trial163.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.18738.log.gz 2026-02-18T08:53:42.848 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.23268.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.44875.log 2026-02-18T08:53:42.848 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.58646.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.22652.log 2026-02-18T08:53:42.848 INFO:teuthology.orchestra.run.trial163.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.23268.log.gz 2026-02-18T08:53:42.848 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.44875.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.88636.log 2026-02-18T08:53:42.848 INFO:teuthology.orchestra.run.trial163.stderr: 90.7% -- replaced with /var/log/ceph/cluster2-client.mirror.128825.log.gz 2026-02-18T08:53:42.848 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.22652.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.97846.log 2026-02-18T08:53:42.848 INFO:teuthology.orchestra.run.trial163.stderr: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.44875.log.gz 2026-02-18T08:53:42.848 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.88636.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.90967.log 2026-02-18T08:53:42.848 INFO:teuthology.orchestra.run.trial163.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.22652.log.gz 2026-02-18T08:53:42.848 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.97846.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.57514.log 2026-02-18T08:53:42.848 INFO:teuthology.orchestra.run.trial163.stderr: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.97846.log.gz 2026-02-18T08:53:42.848 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.94836.log 2026-02-18T08:53:42.848 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.90967.log: /var/log/ceph/cluster2-client.mirror.57514.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.72444.log 2026-02-18T08:53:42.848 INFO:teuthology.orchestra.run.trial163.stderr: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.57514.log.gz 2026-02-18T08:53:42.848 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.94836.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.54819.log 2026-02-18T08:53:42.848 INFO:teuthology.orchestra.run.trial163.stderr: 88.1% -- replaced with /var/log/ceph/cluster1-client.mirror.94836.log.gz 2026-02-18T08:53:42.848 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.72444.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.66535.log 2026-02-18T08:53:42.848 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.54819.log: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.72444.log.gz 2026-02-18T08:53:42.849 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.75388.log 2026-02-18T08:53:42.849 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.66535.log: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.54819.log.gz 2026-02-18T08:53:42.849 INFO:teuthology.orchestra.run.trial163.stderr: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.66535.log.gz 2026-02-18T08:53:42.849 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.42132.log 2026-02-18T08:53:42.849 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.56639.log 2026-02-18T08:53:42.849 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.75388.log: /var/log/ceph/cluster2-client.mirror.42132.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.21222.log 2026-02-18T08:53:42.849 INFO:teuthology.orchestra.run.trial163.stderr: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.75388.log.gz 2026-02-18T08:53:42.849 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.56639.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.35771.log 2026-02-18T08:53:42.849 INFO:teuthology.orchestra.run.trial163.stderr: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.56639.log.gz 2026-02-18T08:53:42.849 INFO:teuthology.orchestra.run.trial163.stderr: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.42132.log.gz 2026-02-18T08:53:42.849 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.21222.log: 97.2% -- replaced with /var/log/ceph/cluster2-client.mirror.58646.log.gz 2026-02-18T08:53:42.849 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.33070.log 2026-02-18T08:53:42.849 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.35771.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.21222.log.gz 2026-02-18T08:53:42.849 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.124523.log 2026-02-18T08:53:42.849 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.33070.log: 88.5%gzip -- replaced with /var/log/ceph/cluster2-client.mirror.35771.log.gz 2026-02-18T08:53:42.849 INFO:teuthology.orchestra.run.trial163.stderr: -5 --verbose -- /var/log/ceph/cluster2-client.mirror.75073.log 2026-02-18T08:53:42.849 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.124523.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.49565.log 2026-02-18T08:53:42.849 INFO:teuthology.orchestra.run.trial163.stderr: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.33070.log.gz 2026-02-18T08:53:42.849 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.75073.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.130037.log 2026-02-18T08:53:42.849 INFO:teuthology.orchestra.run.trial163.stderr: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.75073.log.gz 2026-02-18T08:53:42.849 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.49565.log: 92.5% -- replaced with /var/log/ceph/cluster1-client.mirror.124523.log.gz 2026-02-18T08:53:42.849 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.36614.log 2026-02-18T08:53:42.849 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.130037.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.130582.log 2026-02-18T08:53:42.849 INFO:teuthology.orchestra.run.trial163.stderr: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.49565.log.gz 2026-02-18T08:53:42.849 INFO:teuthology.orchestra.run.trial163.stderr: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.130037.log.gz 2026-02-18T08:53:42.850 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.36614.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.15672.log 2026-02-18T08:53:42.855 INFO:teuthology.orchestra.run.trial163.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.36614.log.gz 2026-02-18T08:53:42.855 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.130582.log: 87.0% -- replaced with /var/log/ceph/cluster1-client.mirror.130582.log.gz 2026-02-18T08:53:42.855 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.128896.log 2026-02-18T08:53:42.855 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.41490.log 2026-02-18T08:53:42.855 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.15672.log: /var/log/ceph/cluster2-client.mirror.128896.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.96476.log 2026-02-18T08:53:42.855 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.41490.log: 84.8% -- replaced with /var/log/ceph/cluster2-client.mirror.15672.log.gz 2026-02-18T08:53:42.855 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.102418.log 2026-02-18T08:53:42.855 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.96476.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.41490.log.gz 2026-02-18T08:53:42.855 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.127030.log 2026-02-18T08:53:42.855 INFO:teuthology.orchestra.run.trial163.stderr: 90.8% -- replaced with /var/log/ceph/cluster2-client.mirror.128896.log.gz 2026-02-18T08:53:42.855 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.102418.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.96476.log.gz 2026-02-18T08:53:42.855 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.82881.log 2026-02-18T08:53:42.855 INFO:teuthology.orchestra.run.trial163.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.102418.log.gz 2026-02-18T08:53:42.855 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.127030.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.104040.log 2026-02-18T08:53:42.855 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.82881.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.127030.log.gz 2026-02-18T08:53:42.855 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.20605.log 2026-02-18T08:53:42.855 INFO:teuthology.orchestra.run.trial163.stderr: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.82881.log.gz 2026-02-18T08:53:42.855 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.104040.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.94332.log 2026-02-18T08:53:42.855 INFO:teuthology.orchestra.run.trial163.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.104040.log.gz 2026-02-18T08:53:42.855 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.89504.log 2026-02-18T08:53:42.855 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.20605.log: /var/log/ceph/cluster2-client.mirror.94332.log: gzip 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.94332.log.gz -5 2026-02-18T08:53:42.855 INFO:teuthology.orchestra.run.trial163.stderr: --verbose -- /var/log/ceph/cluster2-client.mirror.126249.log 2026-02-18T08:53:42.855 INFO:teuthology.orchestra.run.trial163.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.20605.log.gz 2026-02-18T08:53:42.855 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.72304.log 2026-02-18T08:53:42.855 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.89504.log: /var/log/ceph/cluster2-client.mirror.126249.log: 84.8% -- replaced with /var/log/ceph/cluster2-client.mirror.89504.log.gz 2026-02-18T08:53:42.855 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.108776.log 2026-02-18T08:53:42.855 INFO:teuthology.orchestra.run.trial163.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.126249.log.gz 2026-02-18T08:53:42.855 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.72304.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.115045.log 2026-02-18T08:53:42.855 INFO:teuthology.orchestra.run.trial163.stderr: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.72304.log.gz 2026-02-18T08:53:42.856 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.108776.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.25126.log 2026-02-18T08:53:42.856 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.115045.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.99072.log 2026-02-18T08:53:42.856 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.25126.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.132062.log 2026-02-18T08:53:42.856 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.99072.log: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.25126.log.gz 2026-02-18T08:53:42.856 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.50479.log 2026-02-18T08:53:42.856 INFO:teuthology.orchestra.run.trial163.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.99072.log.gz 2026-02-18T08:53:42.856 INFO:teuthology.orchestra.run.trial163.stderr: 92.0%/var/log/ceph/cluster1-client.mirror.132062.log: -- replaced with /var/log/ceph/cluster2-client.mirror.108776.log.gz 2026-02-18T08:53:42.856 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.98000.log 2026-02-18T08:53:42.856 INFO:teuthology.orchestra.run.trial163.stderr: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.132062.log.gz 2026-02-18T08:53:42.856 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.50479.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.56744.log 2026-02-18T08:53:42.856 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.98000.log: gzip 88.8% -5 -- replaced with /var/log/ceph/cluster2-client.mirror.50479.log.gz --verbose 2026-02-18T08:53:42.856 INFO:teuthology.orchestra.run.trial163.stderr: -- /var/log/ceph/cluster2-client.mirror.76332.log 2026-02-18T08:53:42.856 INFO:teuthology.orchestra.run.trial163.stderr: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.98000.log.gz 2026-02-18T08:53:42.856 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.56744.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.62465.log 2026-02-18T08:53:42.856 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.76332.log: gzip 88.8% -5 -- replaced with /var/log/ceph/cluster2-client.mirror.56744.log.gz --verbose 2026-02-18T08:53:42.856 INFO:teuthology.orchestra.run.trial163.stderr: -- /var/log/ceph/cluster2-client.mirror.121591.log 2026-02-18T08:53:42.856 INFO:teuthology.orchestra.run.trial163.stderr: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.76332.log.gz 2026-02-18T08:53:42.856 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.62465.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.101661.log 2026-02-18T08:53:42.856 INFO:teuthology.orchestra.run.trial163.stderr: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.62465.log.gz 2026-02-18T08:53:42.856 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.121591.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.128308.log 2026-02-18T08:53:42.856 INFO:teuthology.orchestra.run.trial163.stderr: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.121591.log.gz 2026-02-18T08:53:42.856 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.101661.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.45258.log 2026-02-18T08:53:42.856 INFO:teuthology.orchestra.run.trial163.stderr: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.101661.log.gz 2026-02-18T08:53:42.856 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.134902.log 2026-02-18T08:53:42.856 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.128308.log: /var/log/ceph/cluster2-client.mirror.45258.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.128308.log.gz 2026-02-18T08:53:42.856 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.19579.log 2026-02-18T08:53:42.856 INFO:teuthology.orchestra.run.trial163.stderr: 85.0% -- replaced with /var/log/ceph/cluster2-client.mirror.45258.log.gz 2026-02-18T08:53:42.856 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.37956.log 2026-02-18T08:53:42.856 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.19579.log: /var/log/ceph/cluster1-client.mirror.134902.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.39758.log 2026-02-18T08:53:42.856 INFO:teuthology.orchestra.run.trial163.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.mirror.134902.log.gz 2026-02-18T08:53:42.856 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.37956.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.64250.log 2026-02-18T08:53:42.856 INFO:teuthology.orchestra.run.trial163.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.19579.log.gz 2026-02-18T08:53:42.856 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.39758.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.114712.log 2026-02-18T08:53:42.856 INFO:teuthology.orchestra.run.trial163.stderr: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.37956.log.gz 2026-02-18T08:53:42.856 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose/var/log/ceph/cluster2-client.mirror.64250.log: -- /var/log/ceph/cluster2-client.mirror.49950.log 2026-02-18T08:53:42.856 INFO:teuthology.orchestra.run.trial163.stderr: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.39758.log.gz 2026-02-18T08:53:42.857 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.114712.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.30895.log 88.9% 2026-02-18T08:53:42.857 INFO:teuthology.orchestra.run.trial163.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.64250.log.gz 2026-02-18T08:53:42.857 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.49950.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.114712.log.gz 2026-02-18T08:53:42.857 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.89370.log 2026-02-18T08:53:42.857 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.30895.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.93659.log 2026-02-18T08:53:42.857 INFO:teuthology.orchestra.run.trial163.stderr: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.49950.log.gz 2026-02-18T08:53:42.857 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.89370.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.30895.log.gz 2026-02-18T08:53:42.857 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.51949.log 2026-02-18T08:53:42.857 INFO:teuthology.orchestra.run.trial163.stderr: 84.8% -- replaced with /var/log/ceph/cluster2-client.mirror.89370.log.gz 2026-02-18T08:53:42.857 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.93659.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.75178.log 2026-02-18T08:53:42.857 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.51949.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.93659.log.gz 2026-02-18T08:53:42.857 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.15258.log 2026-02-18T08:53:42.857 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.75178.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.126888.log 2026-02-18T08:53:42.857 INFO:teuthology.orchestra.run.trial163.stderr: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.51949.log.gz 2026-02-18T08:53:42.857 INFO:teuthology.orchestra.run.trial163.stderr: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.75178.log.gz 2026-02-18T08:53:42.857 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.15258.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.127050.log 2026-02-18T08:53:42.857 INFO:teuthology.orchestra.run.trial163.stderr: 63.5% -- replaced with /var/log/ceph/cluster1-client.mirror.15258.log.gz 2026-02-18T08:53:42.857 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.126888.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.14827.log 2026-02-18T08:53:42.857 INFO:teuthology.orchestra.run.trial163.stderr: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.126888.log.gz 2026-02-18T08:53:42.857 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.127050.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.42924.log 2026-02-18T08:53:42.857 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.admin.14827.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.0.66319.log 2026-02-18T08:53:42.857 INFO:teuthology.orchestra.run.trial163.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.14827.log.gz 2026-02-18T08:53:42.857 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.42924.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.124891.log 2026-02-18T08:53:42.857 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.0.66319.log: 90.7% -- replaced with /var/log/ceph/cluster2-client.mirror.127050.log.gz 2026-02-18T08:53:42.857 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.116164.log 2026-02-18T08:53:42.857 INFO:teuthology.orchestra.run.trial163.stderr: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.42924.log.gz 2026-02-18T08:53:42.857 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose/var/log/ceph/cluster2-client.mirror.124891.log: -- /var/log/ceph/cluster2-client.mirror.113643.log 2026-02-18T08:53:42.857 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.116164.log: 88.3%gzip -- replaced with /var/log/ceph/cluster2-client.mirror.124891.log.gz -5 2026-02-18T08:53:42.857 INFO:teuthology.orchestra.run.trial163.stderr: --verbose -- /var/log/ceph/cluster2-client.admin.14269.log 2026-02-18T08:53:42.857 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.113643.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.90779.log 2026-02-18T08:53:42.857 INFO:teuthology.orchestra.run.trial163.stderr: 92.9% -- replaced with /var/log/ceph/cluster2-client.mirror.0.66319.log.gz 2026-02-18T08:53:42.857 INFO:teuthology.orchestra.run.trial163.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.113643.log.gz 2026-02-18T08:53:42.857 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.admin.14269.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.42552.log 2026-02-18T08:53:42.857 INFO:teuthology.orchestra.run.trial163.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.14269.log.gz 2026-02-18T08:53:42.857 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.90779.log: 92.4% -- replaced with /var/log/ceph/cluster2-client.mirror.116164.log.gz 2026-02-18T08:53:42.857 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.30687.log 2026-02-18T08:53:42.858 INFO:teuthology.orchestra.run.trial163.stderr: 83.1% -- replaced with /var/log/ceph/cluster2-client.mirror.90779.log.gz 2026-02-18T08:53:42.858 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.42552.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.110322.log 2026-02-18T08:53:42.858 INFO:teuthology.orchestra.run.trial163.stderr: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.42552.log.gz 2026-02-18T08:53:42.858 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.30687.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.22152.log 2026-02-18T08:53:42.859 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.110322.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.99616.log 2026-02-18T08:53:42.859 INFO:teuthology.orchestra.run.trial163.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.110322.log.gz 2026-02-18T08:53:42.859 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.22152.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.41901.log 2026-02-18T08:53:42.859 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.99616.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.89420.log 2026-02-18T08:53:42.859 INFO:teuthology.orchestra.run.trial163.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.22152.log.gz 2026-02-18T08:53:42.859 INFO:teuthology.orchestra.run.trial163.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.99616.log.gz 2026-02-18T08:53:42.859 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.122716.log 2026-02-18T08:53:42.860 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.41901.log: /var/log/ceph/cluster2-client.mirror.89420.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.100814.log 2026-02-18T08:53:42.860 INFO:teuthology.orchestra.run.trial163.stderr: 84.8% -- replaced with /var/log/ceph/cluster2-client.mirror.89420.log.gz 2026-02-18T08:53:42.860 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.122716.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.80847.log 2026-02-18T08:53:42.860 INFO:teuthology.orchestra.run.trial163.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.41901.log.gz 2026-02-18T08:53:42.860 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.100814.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.122716.log.gz 2026-02-18T08:53:42.860 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.46001.log 2026-02-18T08:53:42.860 INFO:teuthology.orchestra.run.trial163.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.100814.log.gz 2026-02-18T08:53:42.860 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.63900.log 2026-02-18T08:53:42.860 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.80847.log: /var/log/ceph/cluster1-client.mirror.46001.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.23175.log 2026-02-18T08:53:42.860 INFO:teuthology.orchestra.run.trial163.stderr: 89.2% -- replaced with /var/log/ceph/cluster2-client.mirror.80847.log.gz 2026-02-18T08:53:42.860 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.63900.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.73599.log 2026-02-18T08:53:42.860 INFO:teuthology.orchestra.run.trial163.stderr: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.63900.log.gz 2026-02-18T08:53:42.860 INFO:teuthology.orchestra.run.trial163.stderr: 88.4% -- replaced with /var/log/ceph/cluster1-client.mirror.46001.log.gz 2026-02-18T08:53:42.860 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.23175.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.117396.log 2026-02-18T08:53:42.860 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.124694.log 2026-02-18T08:53:42.860 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.73599.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.23175.log.gz 2026-02-18T08:53:42.860 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.117396.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.131650.log 2026-02-18T08:53:42.860 INFO:teuthology.orchestra.run.trial163.stderr: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.73599.log.gz 2026-02-18T08:53:42.860 INFO:teuthology.orchestra.run.trial163.stderr: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.117396.log.gz 2026-02-18T08:53:42.860 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.124694.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.28251.log 2026-02-18T08:53:42.860 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.131650.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.124694.log.gz 2026-02-18T08:53:42.860 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.41118.log 2026-02-18T08:53:42.861 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.28251.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.58469.log 2026-02-18T08:53:42.861 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.41118.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.28251.log.gz 2026-02-18T08:53:42.861 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.11362.log 2026-02-18T08:53:42.861 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.129969.log 2026-02-18T08:53:42.861 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.58469.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.41118.log.gz 2026-02-18T08:53:42.861 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.admin.11362.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.11362.log.gz 2026-02-18T08:53:42.861 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.114627.log 2026-02-18T08:53:42.861 INFO:teuthology.orchestra.run.trial163.stderr: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.58469.log.gz 2026-02-18T08:53:42.861 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.129969.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.98752.log 2026-02-18T08:53:42.861 INFO:teuthology.orchestra.run.trial163.stderr: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.129969.log.gz 2026-02-18T08:53:42.861 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.114627.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.61905.log 2026-02-18T08:53:42.861 INFO:teuthology.orchestra.run.trial163.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.114627.log.gz 2026-02-18T08:53:42.862 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.98752.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.117438.log 2026-02-18T08:53:42.862 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.61905.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.43668.log 2026-02-18T08:53:42.862 INFO:teuthology.orchestra.run.trial163.stderr: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.61905.log.gz 2026-02-18T08:53:42.862 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.117438.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.130397.log 2026-02-18T08:53:42.862 INFO:teuthology.orchestra.run.trial163.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.117438.log.gz 2026-02-18T08:53:42.862 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.43668.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.71674.log 2026-02-18T08:53:42.863 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.130397.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.126298.log 2026-02-18T08:53:42.863 INFO:teuthology.orchestra.run.trial163.stderr: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.43668.log.gz 2026-02-18T08:53:42.863 INFO:teuthology.orchestra.run.trial163.stderr: 87.0% -- replaced with /var/log/ceph/cluster1-client.mirror.130397.log.gz 2026-02-18T08:53:42.863 INFO:teuthology.orchestra.run.trial163.stderr: 92.3% -- replaced with /var/log/ceph/cluster2-client.mirror.98752.log.gz 2026-02-18T08:53:42.863 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.71674.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.15360.log 2026-02-18T08:53:42.863 INFO:teuthology.orchestra.run.trial163.stderr: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.71674.log.gz 2026-02-18T08:53:42.863 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.126298.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.108044.log 2026-02-18T08:53:42.864 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.21966.log 2026-02-18T08:53:42.864 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.15360.log: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.126298.log.gz 2026-02-18T08:53:42.864 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.108044.log: 68.6% -- replaced with /var/log/ceph/cluster1-client.mirror.15360.log.gz 2026-02-18T08:53:42.864 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.84377.log 2026-02-18T08:53:42.864 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.21966.log: 88.2% -- replaced with /var/log/ceph/cluster1-client.mirror.108044.log.gz 2026-02-18T08:53:42.864 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.42738.log 2026-02-18T08:53:42.864 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.84377.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.58898.log 2026-02-18T08:53:42.865 INFO:teuthology.orchestra.run.trial163.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.21966.log.gz 2026-02-18T08:53:42.865 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.42738.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.57584.log 2026-02-18T08:53:42.865 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.58898.log: gzip 88.3% -5 -- replaced with /var/log/ceph/cluster2-client.mirror.42738.log.gz --verbose 2026-02-18T08:53:42.865 INFO:teuthology.orchestra.run.trial163.stderr: -- /var/log/ceph/cluster2-client.mirror.126391.log 2026-02-18T08:53:42.865 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.57584.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.114507.log 2026-02-18T08:53:42.866 INFO:teuthology.orchestra.run.trial163.stderr: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.57584.log.gz 2026-02-18T08:53:42.866 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.126391.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.40781.log 2026-02-18T08:53:42.866 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.114507.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.97965.log 2026-02-18T08:53:42.866 INFO:teuthology.orchestra.run.trial163.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.126391.log.gz 2026-02-18T08:53:42.866 INFO:teuthology.orchestra.run.trial163.stderr: 87.4% -- replaced with /var/log/ceph/cluster2-client.mirror.114507.log.gz 2026-02-18T08:53:42.866 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.40781.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.50689.log 2026-02-18T08:53:42.867 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.97965.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.47850.log 2026-02-18T08:53:42.867 INFO:teuthology.orchestra.run.trial163.stderr: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.40781.log.gz 2026-02-18T08:53:42.867 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.50689.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.97965.log.gz 2026-02-18T08:53:42.867 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.69609.log 2026-02-18T08:53:42.867 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.47850.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.23954.log 2026-02-18T08:53:42.867 INFO:teuthology.orchestra.run.trial163.stderr: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.50689.log.gz 2026-02-18T08:53:42.868 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.69609.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.56044.log 2026-02-18T08:53:42.868 INFO:teuthology.orchestra.run.trial163.stderr: 93.4% -- replaced with /var/log/ceph/cluster1-client.mirror.84377.log.gz 2026-02-18T08:53:42.868 INFO:teuthology.orchestra.run.trial163.stderr: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.47850.log.gz 2026-02-18T08:53:42.868 INFO:teuthology.orchestra.run.trial163.stderr: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.69609.log.gz 2026-02-18T08:53:42.868 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.23954.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.98077.log 2026-02-18T08:53:42.868 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.56044.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.23954.log.gz 2026-02-18T08:53:42.868 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/ceph.tmp-client.admin.7699.log 2026-02-18T08:53:42.868 INFO:teuthology.orchestra.run.trial163.stderr: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.56044.log.gz 2026-02-18T08:53:42.868 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.98077.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.125429.log 2026-02-18T08:53:42.869 INFO:teuthology.orchestra.run.trial163.stderr: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.98077.log.gz 2026-02-18T08:53:42.869 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.58733.log 2026-02-18T08:53:42.869 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/ceph.tmp-client.admin.7699.log: 0.0% -- replaced with /var/log/ceph/ceph.tmp-client.admin.7699.log.gz 2026-02-18T08:53:42.869 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.125429.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.80357.log 2026-02-18T08:53:42.869 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.58733.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.84924.log 2026-02-18T08:53:42.869 INFO:teuthology.orchestra.run.trial163.stderr: 78.0% -- replaced with /var/log/ceph/cluster2-client.mirror.58733.log.gz 2026-02-18T08:53:42.869 INFO:teuthology.orchestra.run.trial163.stderr: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.125429.log.gz 2026-02-18T08:53:42.870 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.80357.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.18610.log 2026-02-18T08:53:42.870 INFO:teuthology.orchestra.run.trial163.stderr: 89.2% -- replaced with /var/log/ceph/cluster2-client.mirror.80357.log.gz 2026-02-18T08:53:42.870 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.84924.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.55454.log 2026-02-18T08:53:42.870 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.18610.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.126724.log 2026-02-18T08:53:42.870 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.55454.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.86428.log 2026-02-18T08:53:42.871 INFO:teuthology.orchestra.run.trial163.stderr: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.18610.log.gz 2026-02-18T08:53:42.871 INFO:teuthology.orchestra.run.trial163.stderr: 90.5% -- replaced with /var/log/ceph/cluster1-client.mirror.55454.log.gz 2026-02-18T08:53:42.871 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.126724.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.130735.log 2026-02-18T08:53:42.871 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.86428.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.85227.log 2026-02-18T08:53:42.871 INFO:teuthology.orchestra.run.trial163.stderr: 89.3% -- replaced with /var/log/ceph/cluster2-client.mirror.126724.log.gz 2026-02-18T08:53:42.871 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.130735.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.14752.log 2026-02-18T08:53:42.872 INFO:teuthology.orchestra.run.trial163.stderr: 86.8% -- replaced with /var/log/ceph/cluster1-client.mirror.130735.log.gz 2026-02-18T08:53:42.872 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.68660.log 2026-02-18T08:53:42.872 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.85227.log: /var/log/ceph/cluster1-client.admin.14752.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.54784.log 2026-02-18T08:53:42.872 INFO:teuthology.orchestra.run.trial163.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.14752.log.gz 2026-02-18T08:53:42.872 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.68660.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.132924.log 2026-02-18T08:53:42.872 INFO:teuthology.orchestra.run.trial163.stderr: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.68660.log.gz 2026-02-18T08:53:42.872 INFO:teuthology.orchestra.run.trial163.stderr: 89.5% -- replaced with /var/log/ceph/cluster2-client.mirror.85227.log.gz 2026-02-18T08:53:42.873 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.54784.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.133440.log 2026-02-18T08:53:42.873 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.132924.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.79517.log 2026-02-18T08:53:42.873 INFO:teuthology.orchestra.run.trial163.stderr: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.54784.log.gz 2026-02-18T08:53:42.873 INFO:teuthology.orchestra.run.trial163.stderr: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.132924.log.gz 2026-02-18T08:53:42.873 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.133440.log: 93.6% -- replaced with /var/log/ceph/cluster2-client.mirror.84924.log.gz 2026-02-18T08:53:42.873 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-osd.0.log 2026-02-18T08:53:42.874 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.79517.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.128470.log 2026-02-18T08:53:42.874 INFO:teuthology.orchestra.run.trial163.stderr: 89.2% -- replaced with /var/log/ceph/cluster2-client.mirror.79517.log.gz 2026-02-18T08:53:42.874 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-osd.0.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.98119.log 2026-02-18T08:53:42.874 INFO:teuthology.orchestra.run.trial163.stderr: 91.3% -- replaced with /var/log/ceph/cluster2-client.mirror.133440.log.gz 2026-02-18T08:53:42.874 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.128470.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.83573.log 2026-02-18T08:53:42.874 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.98119.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.32849.log 2026-02-18T08:53:42.875 INFO:teuthology.orchestra.run.trial163.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.98119.log.gz 2026-02-18T08:53:42.875 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.83573.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.58380.log 2026-02-18T08:53:42.875 INFO:teuthology.orchestra.run.trial163.stderr: 89.2% -- replaced with /var/log/ceph/cluster2-client.mirror.83573.log.gz 2026-02-18T08:53:42.875 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.14159.log/var/log/ceph/cluster2-client.mirror.32849.log: 2026-02-18T08:53:42.875 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.58380.log: 90.6% -- replaced with /var/log/ceph/cluster2-client.mirror.128470.log.gz 2026-02-18T08:53:42.875 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.32942.log 2026-02-18T08:53:42.876 INFO:teuthology.orchestra.run.trial163.stderr: 88.5% 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.32849.log.gz 2026-02-18T08:53:42.876 INFO:teuthology.orchestra.run.trial163.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.58380.log.gz 2026-02-18T08:53:42.876 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.admin.14159.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.97167.log 2026-02-18T08:53:42.876 INFO:teuthology.orchestra.run.trial163.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.14159.log.gz 2026-02-18T08:53:42.876 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.32942.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.91378.log 2026-02-18T08:53:42.876 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.97167.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.129204.log 2026-02-18T08:53:42.876 INFO:teuthology.orchestra.run.trial163.stderr: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.32942.log.gz 2026-02-18T08:53:42.876 INFO:teuthology.orchestra.run.trial163.stderr: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.97167.log.gz 2026-02-18T08:53:42.876 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.91378.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.37249.log 2026-02-18T08:53:42.877 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.129204.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.23454.log 2026-02-18T08:53:42.877 INFO:teuthology.orchestra.run.trial163.stderr: 86.8% -- replaced with /var/log/ceph/cluster1-client.mirror.129204.log.gz 2026-02-18T08:53:42.877 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.37249.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.77522.log 2026-02-18T08:53:42.878 INFO:teuthology.orchestra.run.trial163.stderr:gzip 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.37249.log.gz 2026-02-18T08:53:42.878 INFO:teuthology.orchestra.run.trial163.stderr: -5/var/log/ceph/cluster2-client.mirror.23454.log: --verbose -- /var/log/ceph/cluster2-client.mirror.31081.log 2026-02-18T08:53:42.878 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.77522.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.87487.log 2026-02-18T08:53:42.878 INFO:teuthology.orchestra.run.trial163.stderr: 89.2% -- replaced with /var/log/ceph/cluster2-client.mirror.77522.log.gz 2026-02-18T08:53:42.878 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.31081.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.23454.log.gz 2026-02-18T08:53:42.878 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.133847.log 2026-02-18T08:53:42.878 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.87487.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.23640.log 2026-02-18T08:53:42.878 INFO:teuthology.orchestra.run.trial163.stderr: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.31081.log.gz 2026-02-18T08:53:42.879 INFO:teuthology.orchestra.run.trial163.stderr: 84.8% -- replaced with /var/log/ceph/cluster2-client.mirror.87487.log.gz 2026-02-18T08:53:42.879 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.133847.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.27135.log 2026-02-18T08:53:42.879 INFO:teuthology.orchestra.run.trial163.stderr: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.133847.log.gz 2026-02-18T08:53:42.879 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.23640.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.85143.log 2026-02-18T08:53:42.879 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.27135.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.130329.log 2026-02-18T08:53:42.879 INFO:teuthology.orchestra.run.trial163.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.23640.log.gz 2026-02-18T08:53:42.880 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.85143.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.125988.log 2026-02-18T08:53:42.880 INFO:teuthology.orchestra.run.trial163.stderr: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.27135.log.gz 2026-02-18T08:53:42.880 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.130329.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.65463.log 2026-02-18T08:53:42.880 INFO:teuthology.orchestra.run.trial163.stderr: 87.0% -- replaced with /var/log/ceph/cluster1-client.mirror.130329.log.gz 2026-02-18T08:53:42.880 INFO:teuthology.orchestra.run.trial163.stderr: 89.5% -- replaced with /var/log/ceph/cluster2-client.mirror.85143.log.gz 2026-02-18T08:53:42.880 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.52929.log 2026-02-18T08:53:42.881 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.125988.log: /var/log/ceph/cluster2-client.mirror.65463.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.84752.log 2026-02-18T08:53:42.881 INFO:teuthology.orchestra.run.trial163.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.125988.log.gz 2026-02-18T08:53:42.881 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.52929.log: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.65463.log.gz 2026-02-18T08:53:42.881 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.75760.log 2026-02-18T08:53:42.881 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.84752.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.20943.log 2026-02-18T08:53:42.881 INFO:teuthology.orchestra.run.trial163.stderr: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.52929.log.gz 2026-02-18T08:53:42.881 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.75760.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.52614.log 2026-02-18T08:53:42.881 INFO:teuthology.orchestra.run.trial163.stderr: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.75760.log.gz 2026-02-18T08:53:42.882 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.20943.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.49740.log 2026-02-18T08:53:42.882 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.52614.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.50090.log 2026-02-18T08:53:42.882 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.49740.log: gzip 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.20943.log.gz 2026-02-18T08:53:42.882 INFO:teuthology.orchestra.run.trial163.stderr: -5 --verbose -- /var/log/ceph/cluster2-client.mirror.83024.log 2026-02-18T08:53:42.882 INFO:teuthology.orchestra.run.trial163.stderr: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.52614.log.gz 2026-02-18T08:53:42.883 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.50090.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.104005.log 2026-02-18T08:53:42.883 INFO:teuthology.orchestra.run.trial163.stderr: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.49740.log.gz 2026-02-18T08:53:42.883 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.73284.log 2026-02-18T08:53:42.883 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.83024.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.50090.log.gz 2026-02-18T08:53:42.883 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.104005.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.71044.log 2026-02-18T08:53:42.883 INFO:teuthology.orchestra.run.trial163.stderr: 89.2% -- replaced with /var/log/ceph/cluster2-client.mirror.83024.log.gz 2026-02-18T08:53:42.883 INFO:teuthology.orchestra.run.trial163.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.104005.log.gz 2026-02-18T08:53:42.883 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.73284.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.30847.log 2026-02-18T08:53:42.884 INFO:teuthology.orchestra.run.trial163.stderr: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.73284.log.gz 2026-02-18T08:53:42.884 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.71044.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.106916.log 2026-02-18T08:53:42.884 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.30847.log: 89.1%gzip -- replaced with /var/log/ceph/cluster2-client.mirror.71044.log.gz -5 2026-02-18T08:53:42.884 INFO:teuthology.orchestra.run.trial163.stderr: --verbose -- /var/log/ceph/cluster2-client.mirror.68044.log 2026-02-18T08:53:42.884 INFO:teuthology.orchestra.run.trial163.stderr: 93.5% -- replaced with /var/log/ceph/cluster2-client.mirror.84752.log.gz 2026-02-18T08:53:42.884 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.106916.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.11620.log 2026-02-18T08:53:42.884 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.68044.log: 88.2% -- replaced with /var/log/ceph/cluster1-client.mirror.106916.log.gz 2026-02-18T08:53:42.885 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.37770.log 2026-02-18T08:53:42.885 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.admin.11620.log: 89.0%gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.60904.log 2026-02-18T08:53:42.885 INFO:teuthology.orchestra.run.trial163.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.68044.log.gz 2026-02-18T08:53:42.885 INFO:teuthology.orchestra.run.trial163.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.11620.log.gz 2026-02-18T08:53:42.886 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.37770.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.118761.log 2026-02-18T08:53:42.886 INFO:teuthology.orchestra.run.trial163.stderr: 92.2% -- replaced with /var/log/ceph/cluster2-client.mirror.91378.log.gz 2026-02-18T08:53:42.886 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.60904.log: gzip 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.37770.log.gz 2026-02-18T08:53:42.886 INFO:teuthology.orchestra.run.trial163.stderr: -5 --verbose -- /var/log/ceph/cluster2-client.mirror.108308.log 2026-02-18T08:53:42.886 INFO:teuthology.orchestra.run.trial163.stderr: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.60904.log.gz 2026-02-18T08:53:42.886 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.118761.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.20046.log 2026-02-18T08:53:42.886 INFO:teuthology.orchestra.run.trial163.stderr: 88.3% -- replaced with /var/log/ceph/cluster1-client.mirror.118761.log.gz 2026-02-18T08:53:42.886 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.108308.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.72619.log 2026-02-18T08:53:42.887 INFO:teuthology.orchestra.run.trial163.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.108308.log.gz 2026-02-18T08:53:42.887 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.20046.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.103014.log 2026-02-18T08:53:42.887 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5/var/log/ceph/cluster2-client.mirror.72619.log: --verbose -- /var/log/ceph/cluster2-client.admin.14271.log 2026-02-18T08:53:42.887 INFO:teuthology.orchestra.run.trial163.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.20046.log.gz 2026-02-18T08:53:42.887 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.103014.log: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.72619.log.gz 2026-02-18T08:53:42.887 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.86186.log 2026-02-18T08:53:42.888 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.admin.14271.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.49110.log 2026-02-18T08:53:42.888 INFO:teuthology.orchestra.run.trial163.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.14271.log.gz 2026-02-18T08:53:42.888 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.86186.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.63305.log 2026-02-18T08:53:42.888 INFO:teuthology.orchestra.run.trial163.stderr: 84.7% -- replaced with /var/log/ceph/cluster2-client.mirror.86186.log.gz 2026-02-18T08:53:42.888 INFO:teuthology.orchestra.run.trial163.stderr: 92.5% -- replaced with /var/log/ceph/cluster1-client.mirror.103014.log.gz 2026-02-18T08:53:42.888 INFO:teuthology.orchestra.run.trial163.stderr:gzip/var/log/ceph/cluster2-client.mirror.49110.log: -5 --verbose -- /var/log/ceph/cluster2-client.mirror.126320.log 2026-02-18T08:53:42.888 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.63305.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.76542.log 2026-02-18T08:53:42.889 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.126320.log: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.63305.log.gz 2026-02-18T08:53:42.889 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.83732.log 2026-02-18T08:53:42.889 INFO:teuthology.orchestra.run.trial163.stderr: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.49110.log.gz 2026-02-18T08:53:42.889 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.76542.log: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.126320.log.gz 2026-02-18T08:53:42.889 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.78432.log 2026-02-18T08:53:42.889 INFO:teuthology.orchestra.run.trial163.stderr: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.76542.log.gz 2026-02-18T08:53:42.889 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.83732.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.89036.log 2026-02-18T08:53:42.889 INFO:teuthology.orchestra.run.trial163.stderr: 89.2% -- replaced with /var/log/ceph/cluster2-client.mirror.83732.log.gz 2026-02-18T08:53:42.890 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.130127.log 2026-02-18T08:53:42.890 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.78432.log: /var/log/ceph/cluster2-client.mirror.89036.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.19389.log 2026-02-18T08:53:42.890 INFO:teuthology.orchestra.run.trial163.stderr: 89.2% -- replaced with /var/log/ceph/cluster2-client.mirror.78432.log.gz 2026-02-18T08:53:42.890 INFO:teuthology.orchestra.run.trial163.stderr: 84.7% -- replaced with /var/log/ceph/cluster2-client.mirror.89036.log.gz 2026-02-18T08:53:42.890 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.130127.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.85073.log 2026-02-18T08:53:42.890 INFO:teuthology.orchestra.run.trial163.stderr: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.130127.log.gz 2026-02-18T08:53:42.891 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.19389.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.15531.log 2026-02-18T08:53:42.891 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.85073.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.19389.log.gz 2026-02-18T08:53:42.891 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.53244.log 2026-02-18T08:53:42.891 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.67799.log 2026-02-18T08:53:42.892 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.15531.log: /var/log/ceph/cluster2-client.mirror.53244.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.83910.log 2026-02-18T08:53:42.892 INFO:teuthology.orchestra.run.trial163.stderr: 4.7% -- replaced with /var/log/ceph/cluster2-client.mirror.15531.log.gz 2026-02-18T08:53:42.892 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.67799.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.18703.log 2026-02-18T08:53:42.892 INFO:teuthology.orchestra.run.trial163.stderr: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.67799.log.gz 2026-02-18T08:53:42.892 INFO:teuthology.orchestra.run.trial163.stderr: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.53244.log.gz 2026-02-18T08:53:42.892 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.83910.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.132894.log 2026-02-18T08:53:42.892 INFO:teuthology.orchestra.run.trial163.stderr: 89.2% -- replaced with /var/log/ceph/cluster2-client.mirror.83910.log.gz 2026-02-18T08:53:42.892 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.18703.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.46797.log 2026-02-18T08:53:42.893 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.132894.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.117977.log 2026-02-18T08:53:42.893 INFO:teuthology.orchestra.run.trial163.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.18703.log.gz 2026-02-18T08:53:42.893 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.46797.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.54049.log 2026-02-18T08:53:42.893 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.117977.log: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.46797.log.gz 2026-02-18T08:53:42.893 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.124775.log 2026-02-18T08:53:42.894 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.54049.log: 93.5% -- replaced with /var/log/ceph/cluster1-client.mirror.85073.log.gz 2026-02-18T08:53:42.894 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.124775.log: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.54049.log.gz 2026-02-18T08:53:42.894 INFO:teuthology.orchestra.run.trial163.stderr: 92.6% -- replaced with /var/log/ceph/cluster2-client.mirror.117977.log.gz 2026-02-18T08:53:42.894 INFO:teuthology.orchestra.run.trial163.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.124775.log.gz 2026-02-18T08:53:42.894 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.59219.log 2026-02-18T08:53:42.895 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.95233.log 2026-02-18T08:53:42.895 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.59219.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.70974.log 2026-02-18T08:53:42.895 INFO:teuthology.orchestra.run.trial163.stderr: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.59219.log.gz 2026-02-18T08:53:42.895 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.85500.log 2026-02-18T08:53:42.896 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.95233.log: /var/log/ceph/cluster2-client.mirror.70974.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.15224.log 2026-02-18T08:53:42.896 INFO:teuthology.orchestra.run.trial163.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.95233.log.gz 2026-02-18T08:53:42.896 INFO:teuthology.orchestra.run.trial163.stderr: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.70974.log.gz 2026-02-18T08:53:42.896 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.85500.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.127125.log 2026-02-18T08:53:42.896 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.admin.15224.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.59954.log 2026-02-18T08:53:42.896 INFO:teuthology.orchestra.run.trial163.stderr: 89.2% -- replaced with /var/log/ceph/cluster2-client.mirror.85500.log.gz 2026-02-18T08:53:42.896 INFO:teuthology.orchestra.run.trial163.stderr: 80.4% -- replaced with /var/log/ceph/cluster1-client.admin.15224.log.gz 2026-02-18T08:53:42.897 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.127125.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.125220.log 2026-02-18T08:53:42.897 INFO:teuthology.orchestra.run.trial163.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.127125.log.gz 2026-02-18T08:53:42.897 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.59954.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.96522.log 2026-02-18T08:53:42.897 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.125220.log: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.59954.log.gz 2026-02-18T08:53:42.897 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.33628.log 2026-02-18T08:53:42.897 INFO:teuthology.orchestra.run.trial163.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.125220.log.gz 2026-02-18T08:53:42.898 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.96522.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.55366.log 2026-02-18T08:53:42.898 INFO:teuthology.orchestra.run.trial163.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.96522.log.gz 2026-02-18T08:53:42.898 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.130825.log 2026-02-18T08:53:42.898 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.33628.log: /var/log/ceph/cluster1-client.admin.55366.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.17343.log 2026-02-18T08:53:42.898 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.130825.log: 88.8% -- replaced with /var/log/ceph/cluster1-client.admin.55366.log.gz 2026-02-18T08:53:42.898 INFO:teuthology.orchestra.run.trial163.stderr: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.33628.log.gz 2026-02-18T08:53:42.899 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- 86.8% /var/log/ceph/cluster2-client.mirror.47990.log 2026-02-18T08:53:42.899 INFO:teuthology.orchestra.run.trial163.stderr: -- replaced with /var/log/ceph/cluster1-client.mirror.130825.log.gz 2026-02-18T08:53:42.899 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.17343.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.61835.log 2026-02-18T08:53:42.899 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.47990.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.76367.log 2026-02-18T08:53:42.899 INFO:teuthology.orchestra.run.trial163.stderr: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.17343.log.gz 2026-02-18T08:53:42.900 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.61835.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.115452.log 2026-02-18T08:53:42.900 INFO:teuthology.orchestra.run.trial163.stderr: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.47990.log.gz 2026-02-18T08:53:42.900 INFO:teuthology.orchestra.run.trial163.stderr: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.61835.log.gz 2026-02-18T08:53:42.900 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.76367.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.75671.log 2026-02-18T08:53:42.900 INFO:teuthology.orchestra.run.trial163.stderr: 89.2% -- replaced with /var/log/ceph/cluster2-client.mirror.76367.log.gz 2026-02-18T08:53:42.900 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.100976.log 2026-02-18T08:53:42.900 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.115452.log: /var/log/ceph/cluster2-client.mirror.75671.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.86323.log 2026-02-18T08:53:42.901 INFO:teuthology.orchestra.run.trial163.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.115452.log.gz 2026-02-18T08:53:42.901 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.100976.log: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.75671.log.gz 2026-02-18T08:53:42.901 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.76682.log 2026-02-18T08:53:42.901 INFO:teuthology.orchestra.run.trial163.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.100976.log.gz 2026-02-18T08:53:42.901 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.86323.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.87727.log 2026-02-18T08:53:42.901 INFO:teuthology.orchestra.run.trial163.stderr: 84.8% -- replaced with /var/log/ceph/cluster2-client.mirror.86323.log.gz 2026-02-18T08:53:42.901 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.76682.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.20512.log 2026-02-18T08:53:42.902 INFO:teuthology.orchestra.run.trial163.stderr: 89.2% -- replaced with /var/log/ceph/cluster2-client.mirror.76682.log.gz 2026-02-18T08:53:42.902 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.87727.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.114422.log 2026-02-18T08:53:42.902 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.20512.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.110684.log 2026-02-18T08:53:42.902 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.114422.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.20512.log.gz 2026-02-18T08:53:42.903 INFO:teuthology.orchestra.run.trial163.stderr:gzip 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.114422.log.gz 2026-02-18T08:53:42.903 INFO:teuthology.orchestra.run.trial163.stderr: -5 --verbose -- /var/log/ceph/cluster2-client.mirror.128136.log 2026-02-18T08:53:42.903 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.110684.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.110684.log.gz 2026-02-18T08:53:42.903 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.128136.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.34341.log 2026-02-18T08:53:42.904 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.21722.log 2026-02-18T08:53:42.904 INFO:teuthology.orchestra.run.trial163.stderr: 89.4%/var/log/ceph/cluster2-client.mirror.34341.log: -- replaced with /var/log/ceph/cluster2-client.mirror.128136.log.gz 2026-02-18T08:53:42.904 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.40874.log 2026-02-18T08:53:42.904 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.21722.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.23861.log 2026-02-18T08:53:42.904 INFO:teuthology.orchestra.run.trial163.stderr: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.34341.log.gz 2026-02-18T08:53:42.905 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.40874.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.21722.log.gz 2026-02-18T08:53:42.905 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.57374.log 2026-02-18T08:53:42.905 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.23861.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.40874.log.gz 2026-02-18T08:53:42.905 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.74089.log 2026-02-18T08:53:42.905 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.57374.log: 88.3%gzip -- replaced with /var/log/ceph/cluster2-client.mirror.23861.log.gz 2026-02-18T08:53:42.905 INFO:teuthology.orchestra.run.trial163.stderr: -5 --verbose -- /var/log/ceph/cluster2-client.mirror.62745.log 2026-02-18T08:53:42.905 INFO:teuthology.orchestra.run.trial163.stderr: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.57374.log.gz 2026-02-18T08:53:42.906 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.74089.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.129609.log 2026-02-18T08:53:42.906 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.62745.log: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.74089.log.gz 2026-02-18T08:53:42.906 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.122376.log 2026-02-18T08:53:42.906 INFO:teuthology.orchestra.run.trial163.stderr: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.62745.log.gz 2026-02-18T08:53:42.906 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.129609.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.48095.log 2026-02-18T08:53:42.906 INFO:teuthology.orchestra.run.trial163.stderr: 87.0% -- replaced with /var/log/ceph/cluster1-client.mirror.129609.log.gz 2026-02-18T08:53:42.907 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.122376.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.131734.log 2026-02-18T08:53:42.907 INFO:teuthology.orchestra.run.trial163.stderr: 88.1% -- replaced with /var/log/ceph/cluster1-client.mirror.122376.log.gz 2026-02-18T08:53:42.907 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.48095.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.40688.log 2026-02-18T08:53:42.907 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.131734.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.114861.log 2026-02-18T08:53:42.908 INFO:teuthology.orchestra.run.trial163.stderr: 88.8%/var/log/ceph/cluster2-client.mirror.40688.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.125004.log 2026-02-18T08:53:42.908 INFO:teuthology.orchestra.run.trial163.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.48095.log.gz 2026-02-18T08:53:42.908 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.114861.log: 88.5% 91.4%gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.111540.log 2026-02-18T08:53:42.908 INFO:teuthology.orchestra.run.trial163.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.131734.log.gz 2026-02-18T08:53:42.908 INFO:teuthology.orchestra.run.trial163.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.40688.log.gz 2026-02-18T08:53:42.909 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.125004.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.114861.log.gz 2026-02-18T08:53:42.909 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.111416.log 2026-02-18T08:53:42.909 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.111540.log: 88.1% -- replaced with /var/log/ceph/cluster1-client.mirror.125004.log.gz 2026-02-18T08:53:42.909 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.24797.log 2026-02-18T08:53:42.909 INFO:teuthology.orchestra.run.trial163.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.111540.log.gz 2026-02-18T08:53:42.909 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.111416.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.97717.log 2026-02-18T08:53:42.909 INFO:teuthology.orchestra.run.trial163.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.111416.log.gz 2026-02-18T08:53:42.909 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.24797.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.44447.log 2026-02-18T08:53:42.910 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.97717.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.22059.log 2026-02-18T08:53:42.910 INFO:teuthology.orchestra.run.trial163.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.97717.log.gz 2026-02-18T08:53:42.910 INFO:teuthology.orchestra.run.trial163.stderr:gzip/var/log/ceph/cluster2-client.mirror.44447.log: -5 --verbose -- /var/log/ceph/cluster2-client.mirror.19953.log 2026-02-18T08:53:42.911 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.22059.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.22059.log.gz 2026-02-18T08:53:42.911 INFO:teuthology.orchestra.run.trial163.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.44447.log.gz 2026-02-18T08:53:42.911 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.19953.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.48830.log 2026-02-18T08:53:42.912 INFO:teuthology.orchestra.run.trial163.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.19953.log.gz 2026-02-18T08:53:42.912 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.118468.log 2026-02-18T08:53:42.912 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.48830.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.106884.log 2026-02-18T08:53:42.913 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.118468.log: gzip 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.48830.log.gz 2026-02-18T08:53:42.913 INFO:teuthology.orchestra.run.trial163.stderr: -5 --verbose -- /var/log/ceph/cluster2-client.mirror.87340.log 2026-02-18T08:53:42.913 INFO:teuthology.orchestra.run.trial163.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.118468.log.gz 2026-02-18T08:53:42.913 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.106884.log: 96.2% -- replaced with /var/log/ceph/cluster2-client.mirror.24797.log.gz 2026-02-18T08:53:42.913 INFO:teuthology.orchestra.run.trial163.stderr:gzip 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.106884.log.gz 2026-02-18T08:53:42.913 INFO:teuthology.orchestra.run.trial163.stderr: -5 --verbose -- /var/log/ceph/cluster2-client.mirror.49005.log 2026-02-18T08:53:42.914 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.87340.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.72829.log 2026-02-18T08:53:42.914 INFO:teuthology.orchestra.run.trial163.stderr: 84.7% -- replaced with /var/log/ceph/cluster2-client.mirror.87340.log.gz 2026-02-18T08:53:42.914 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.49005.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.112769.log 2026-02-18T08:53:42.914 INFO:teuthology.orchestra.run.trial163.stderr: 88.7%/var/log/ceph/cluster2-client.mirror.72829.log: -- replaced with /var/log/ceph/cluster2-client.mirror.49005.log.gz 2026-02-18T08:53:42.914 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.22838.log 2026-02-18T08:53:42.914 INFO:teuthology.orchestra.run.trial163.stderr: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.72829.log.gz 2026-02-18T08:53:42.915 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.61079.log 2026-02-18T08:53:42.915 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.112769.log: /var/log/ceph/cluster2-client.mirror.22838.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.112769.log.gz 2026-02-18T08:53:42.915 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.126008.log 2026-02-18T08:53:42.915 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.61079.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.80112.log 2026-02-18T08:53:42.915 INFO:teuthology.orchestra.run.trial163.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.22838.log.gz 2026-02-18T08:53:42.915 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.126008.log: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.61079.log.gz 2026-02-18T08:53:42.915 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.116904.log 2026-02-18T08:53:42.916 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.80112.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.128087.log 2026-02-18T08:53:42.916 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.116904.log: 89.2% -- replaced with /var/log/ceph/cluster2-client.mirror.80112.log.gz 2026-02-18T08:53:42.916 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.50899.log 2026-02-18T08:53:42.917 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.128087.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.87219.log 2026-02-18T08:53:42.917 INFO:teuthology.orchestra.run.trial163.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.128087.log.gz 2026-02-18T08:53:42.917 INFO:teuthology.orchestra.run.trial163.stderr: 90.6% -- replaced with /var/log/ceph/cluster2-client.mirror.126008.log.gz 2026-02-18T08:53:42.917 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.50899.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.131682.log 2026-02-18T08:53:42.918 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.87219.log: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.50899.log.gz 2026-02-18T08:53:42.918 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.102902.log 2026-02-18T08:53:42.918 INFO:teuthology.orchestra.run.trial163.stderr: 84.8% -- replaced with /var/log/ceph/cluster2-client.mirror.87219.log.gz/var/log/ceph/cluster1-client.mirror.131682.log: 2026-02-18T08:53:42.918 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.76962.log 2026-02-18T08:53:42.918 INFO:teuthology.orchestra.run.trial163.stderr: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.131682.log.gz 2026-02-18T08:53:42.918 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.102902.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.83148.log 2026-02-18T08:53:42.918 INFO:teuthology.orchestra.run.trial163.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.102902.log.gz 2026-02-18T08:53:42.918 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.76962.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.92721.log 2026-02-18T08:53:42.919 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.83148.log: gzip 89.2% -5 --verbose -- /var/log/ceph/cluster2-client.mirror.103323.log 2026-02-18T08:53:42.919 INFO:teuthology.orchestra.run.trial163.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.76962.log.gz 2026-02-18T08:53:42.919 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.92721.log: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.83148.log.gz 2026-02-18T08:53:42.919 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.82935.log 2026-02-18T08:53:42.919 INFO:teuthology.orchestra.run.trial163.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.92721.log.gz 2026-02-18T08:53:42.920 INFO:teuthology.orchestra.run.trial034.stderr: 91.1% -- replaced with /var/log/ceph/cluster1-mgr.x.log.gz 2026-02-18T08:53:42.920 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.103323.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.109950.log 2026-02-18T08:53:42.920 INFO:teuthology.orchestra.run.trial163.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.103323.log.gz 2026-02-18T08:53:42.920 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.82935.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.105263.log 2026-02-18T08:53:42.920 INFO:teuthology.orchestra.run.trial163.stderr: 89.2% -- replaced with /var/log/ceph/cluster2-client.mirror.82935.log.gz 2026-02-18T08:53:42.920 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.109950.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.129497.log 2026-02-18T08:53:42.921 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.105263.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.109950.log.gz 2026-02-18T08:53:42.921 INFO:teuthology.orchestra.run.trial163.stderr: 92.5% -- replaced with /var/log/ceph/cluster2-client.mirror.88636.log.gz 2026-02-18T08:53:42.921 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.66430.log 2026-02-18T08:53:42.921 INFO:teuthology.orchestra.run.trial163.stderr: 87.9% -- replaced with /var/log/ceph/cluster2-client.mirror.105263.log.gz 2026-02-18T08:53:42.921 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.129497.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.36294.log 2026-02-18T08:53:42.921 INFO:teuthology.orchestra.run.trial163.stderr: 87.0% -- replaced with /var/log/ceph/cluster1-client.mirror.129497.log.gz 2026-02-18T08:53:42.921 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.66430.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.74303.log 2026-02-18T08:53:42.922 INFO:teuthology.orchestra.run.trial163.stderr: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.66430.log.gz 2026-02-18T08:53:42.922 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.36294.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.36236.log 2026-02-18T08:53:42.922 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.74303.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.108424.log 2026-02-18T08:53:42.922 INFO:teuthology.orchestra.run.trial163.stderr: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.36294.log.gz 2026-02-18T08:53:42.922 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.36236.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.31640.log 2026-02-18T08:53:42.922 INFO:teuthology.orchestra.run.trial163.stderr: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.74303.log.gz 2026-02-18T08:53:42.923 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.108424.log: gzip -5 --verbose 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.36236.log.gz 2026-02-18T08:53:42.923 INFO:teuthology.orchestra.run.trial163.stderr: -- /var/log/ceph/cluster2-client.mirror.121974.log 2026-02-18T08:53:42.923 INFO:teuthology.orchestra.run.trial163.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.108424.log.gz 2026-02-18T08:53:42.923 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.31640.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.17866.log 2026-02-18T08:53:42.923 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.121974.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.31640.log.gz 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.121974.log.gz 2026-02-18T08:53:42.923 INFO:teuthology.orchestra.run.trial163.stderr: 2026-02-18T08:53:42.924 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.58970.log 2026-02-18T08:53:42.924 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.17866.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.127740.log 2026-02-18T08:53:42.924 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.58970.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.17866.log.gz 2026-02-18T08:53:42.924 INFO:teuthology.orchestra.run.trial163.stderr: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.58970.log.gz 2026-02-18T08:53:42.924 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.130194.log 2026-02-18T08:53:42.925 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.127740.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.74653.log 2026-02-18T08:53:42.925 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.130194.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.2.134130.log 2026-02-18T08:53:42.925 INFO:teuthology.orchestra.run.trial163.stderr: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.130194.log.gz 2026-02-18T08:53:42.925 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.74653.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.34841.log 2026-02-18T08:53:42.926 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.2.134130.log: gzip 89.0% -5 --verbose -- /var/log/ceph/cluster2-client.mirror.84228.log 2026-02-18T08:53:42.926 INFO:teuthology.orchestra.run.trial163.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.74653.log.gz 2026-02-18T08:53:42.926 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.34841.log: 90.8% -- replaced with /var/log/ceph/cluster2-client.mirror.127740.log.gz 2026-02-18T08:53:42.926 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.58870.log 2026-02-18T08:53:42.926 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.84228.log: gzip -5 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.34841.log.gz 2026-02-18T08:53:42.926 INFO:teuthology.orchestra.run.trial163.stderr: --verbose -- /var/log/ceph/cluster2-client.mirror.22617.log 2026-02-18T08:53:42.927 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.admin.58870.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.127945.log 2026-02-18T08:53:42.927 INFO:teuthology.orchestra.run.trial163.stderr: 88.1% -- replaced with /var/log/ceph/cluster1-client.admin.58870.log.gz/var/log/ceph/cluster2-client.mirror.22617.log: 2026-02-18T08:53:42.927 INFO:teuthology.orchestra.run.trial163.stderr: 92.5% -- replaced with /var/log/ceph/cluster1-client.mirror.2.134130.log.gz 2026-02-18T08:53:42.927 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.110447.log 2026-02-18T08:53:42.927 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.127945.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.93208.log 2026-02-18T08:53:42.928 INFO:teuthology.orchestra.run.trial163.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.127945.log.gz 2026-02-18T08:53:42.928 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.110447.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.116789.log 2026-02-18T08:53:42.928 INFO:teuthology.orchestra.run.trial163.stderr: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.22617.log.gz 2026-02-18T08:53:42.928 INFO:teuthology.orchestra.run.trial163.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.110447.log.gz 2026-02-18T08:53:42.928 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.93208.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.66129.log 2026-02-18T08:53:42.929 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.116789.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.128399.log 2026-02-18T08:53:42.929 INFO:teuthology.orchestra.run.trial163.stderr: 88.1% -- replaced with /var/log/ceph/cluster1-client.mirror.116789.log.gz 2026-02-18T08:53:42.929 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.66129.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.51214.log 2026-02-18T08:53:42.929 INFO:teuthology.orchestra.run.trial163.stderr: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.66129.log.gz 2026-02-18T08:53:42.929 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.128399.log: 93.0% -- replaced with /var/log/ceph/cluster2-client.mirror.84228.log.gz 2026-02-18T08:53:42.930 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.71569.log 2026-02-18T08:53:42.930 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.51214.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.134610.log 2026-02-18T08:53:42.930 INFO:teuthology.orchestra.run.trial163.stderr: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.51214.log.gz 2026-02-18T08:53:42.931 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.71569.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.33128.log 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.71569.log.gz 2026-02-18T08:53:42.931 INFO:teuthology.orchestra.run.trial163.stderr: 2026-02-18T08:53:42.931 INFO:teuthology.orchestra.run.trial163.stderr: 90.6% -- replaced with /var/log/ceph/cluster2-client.mirror.128399.log.gz 2026-02-18T08:53:42.931 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.134610.log: gzip 83.5% -- replaced with /var/log/ceph/cluster2-client.mirror.134610.log.gz 2026-02-18T08:53:42.931 INFO:teuthology.orchestra.run.trial163.stderr: -5 --verbose -- /var/log/ceph/cluster2-client.mirror.127840.log 2026-02-18T08:53:42.931 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.33128.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.42504.log 2026-02-18T08:53:42.932 INFO:teuthology.orchestra.run.trial163.stderr: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.33128.log.gz 2026-02-18T08:53:42.932 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.127840.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.131479.log 2026-02-18T08:53:42.932 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.42504.log: 89.3% -- replaced with /var/log/ceph/cluster2-client.mirror.127840.log.gz 2026-02-18T08:53:42.932 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.128805.log 2026-02-18T08:53:42.933 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.131479.log: 87.0%gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.47955.log 2026-02-18T08:53:42.933 INFO:teuthology.orchestra.run.trial163.stderr: -- replaced with /var/log/ceph/cluster1-client.mirror.131479.log.gz 2026-02-18T08:53:42.933 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.128805.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.118258.log 2026-02-18T08:53:42.933 INFO:teuthology.orchestra.run.trial163.stderr: 88.4%/var/log/ceph/cluster2-client.mirror.47955.log: -- replaced with /var/log/ceph/cluster2-client.mirror.128805.log.gz 2026-02-18T08:53:42.934 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.21687.log 2026-02-18T08:53:42.934 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.118258.log: gzip -5 --verbose 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.118258.log.gz 2026-02-18T08:53:42.934 INFO:teuthology.orchestra.run.trial163.stderr: -- /var/log/ceph/cluster2-client.mirror.105087.log 2026-02-18T08:53:42.934 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.21687.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.47955.log.gz 2026-02-18T08:53:42.934 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.119179.log 2026-02-18T08:53:42.935 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.105087.log: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.21687.log.gz 2026-02-18T08:53:42.935 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.107519.log 2026-02-18T08:53:42.935 INFO:teuthology.orchestra.run.trial163.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.105087.log.gz 2026-02-18T08:53:42.935 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.119179.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.61174.log 2026-02-18T08:53:42.935 INFO:teuthology.orchestra.run.trial163.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.119179.log.gz 2026-02-18T08:53:42.936 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.107519.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.104183.log 2026-02-18T08:53:42.936 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.61174.log: 77.9% -- replaced with /var/log/ceph/cluster1-client.mirror.61174.log.gz 2026-02-18T08:53:42.936 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.78152.log 2026-02-18T08:53:42.936 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.104183.log: 92.6%gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.100498.log 2026-02-18T08:53:42.937 INFO:teuthology.orchestra.run.trial163.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.107519.log.gz 2026-02-18T08:53:42.937 INFO:teuthology.orchestra.run.trial163.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.104183.log.gz 2026-02-18T08:53:42.937 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.78152.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.38363.log 2026-02-18T08:53:42.937 INFO:teuthology.orchestra.run.trial163.stderr: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.78152.log.gzgzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.132345.log 2026-02-18T08:53:42.937 INFO:teuthology.orchestra.run.trial163.stderr: 2026-02-18T08:53:42.937 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.100498.log: /var/log/ceph/cluster2-client.mirror.38363.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.93481.log 2026-02-18T08:53:42.938 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.132345.log: 88.2% -- replaced with /var/log/ceph/cluster1-client.mirror.100498.log.gz 2026-02-18T08:53:42.938 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.125937.log 2026-02-18T08:53:42.938 INFO:teuthology.orchestra.run.trial163.stderr: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.38363.log.gz 2026-02-18T08:53:42.938 INFO:teuthology.orchestra.run.trial163.stderr: 88.6% -- replaced with /var/log/ceph/cluster1-client.mirror.132345.log.gz 2026-02-18T08:53:42.938 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.93481.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.94046.log 2026-02-18T08:53:42.938 INFO:teuthology.orchestra.run.trial163.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.93481.log.gz 2026-02-18T08:53:42.938 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.75038.log 2026-02-18T08:53:42.939 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.125937.log: /var/log/ceph/cluster2-client.mirror.94046.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.117284.log 2026-02-18T08:53:42.939 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.75038.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.94046.log.gz 2026-02-18T08:53:42.939 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.63060.log 2026-02-18T08:53:42.939 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.117284.log: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.75038.log.gz 2026-02-18T08:53:42.940 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.16860.log 2026-02-18T08:53:42.940 INFO:teuthology.orchestra.run.trial163.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.117284.log.gz 2026-02-18T08:53:42.940 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.63060.log: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.63060.log.gz 2026-02-18T08:53:42.940 INFO:teuthology.orchestra.run.trial163.stderr: 90.7% -- replaced with /var/log/ceph/cluster2-client.mirror.125937.log.gz 2026-02-18T08:53:42.940 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.93570.log 2026-02-18T08:53:42.941 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.2.133756.log 2026-02-18T08:53:42.941 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.admin.16860.log: /var/log/ceph/cluster2-client.mirror.93570.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.28751.log 2026-02-18T08:53:42.941 INFO:teuthology.orchestra.run.trial163.stderr: 87.9% -- replaced with /var/log/ceph/cluster2-client.mirror.93570.log.gz 2026-02-18T08:53:42.941 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.2.133756.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.38270.log 2026-02-18T08:53:42.942 INFO:teuthology.orchestra.run.trial163.stderr: 87.2%/var/log/ceph/cluster2-client.mirror.28751.log: -- replaced with /var/log/ceph/cluster1-client.admin.16860.log.gz 2026-02-18T08:53:42.942 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.55425.log 2026-02-18T08:53:42.942 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.38270.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.28751.log.gz 2026-02-18T08:53:42.942 INFO:teuthology.orchestra.run.trial163.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.38270.log.gz 2026-02-18T08:53:42.942 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.127433.log 2026-02-18T08:53:42.943 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.55425.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.42055.log 2026-02-18T08:53:42.943 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.96719.log 2026-02-18T08:53:42.943 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.127433.log: 93.0% -- replaced with /var/log/ceph/cluster2-client.mirror.2.133756.log.gz/var/log/ceph/cluster2-client.mirror.42055.log: 2026-02-18T08:53:42.943 INFO:teuthology.orchestra.run.trial163.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.127433.log.gz 2026-02-18T08:53:42.944 INFO:teuthology.orchestra.run.trial163.stderr: 88.5%gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.126178.log 2026-02-18T08:53:42.944 INFO:teuthology.orchestra.run.trial163.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.42055.log.gz 2026-02-18T08:53:42.944 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.73249.log 2026-02-18T08:53:42.944 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.96719.log: /var/log/ceph/cluster2-client.mirror.126178.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.77732.log 2026-02-18T08:53:42.945 INFO:teuthology.orchestra.run.trial163.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.96719.log.gz 2026-02-18T08:53:42.945 INFO:teuthology.orchestra.run.trial163.stderr: 88.2%/var/log/ceph/cluster2-client.mirror.73249.log: -- replaced with /var/log/ceph/cluster2-client.mirror.126178.log.gz 2026-02-18T08:53:42.945 INFO:teuthology.orchestra.run.trial163.stderr: 89.1%gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.24140.log 2026-02-18T08:53:42.945 INFO:teuthology.orchestra.run.trial163.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.73249.log.gz 2026-02-18T08:53:42.945 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.112943.log 2026-02-18T08:53:42.946 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.77732.log: /var/log/ceph/cluster2-client.mirror.24140.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.133007.log 2026-02-18T08:53:42.946 INFO:teuthology.orchestra.run.trial163.stderr: 89.2% -- replaced with /var/log/ceph/cluster2-client.mirror.77732.log.gz 2026-02-18T08:53:42.946 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.112943.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.24140.log.gz 2026-02-18T08:53:42.946 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.129474.log 2026-02-18T08:53:42.946 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.133007.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.112206.log 2026-02-18T08:53:42.947 INFO:teuthology.orchestra.run.trial163.stderr: 85.8% -- replaced with /var/log/ceph/cluster2-client.mirror.133007.log.gz 2026-02-18T08:53:42.947 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.129474.log: 86.8% -- replaced with /var/log/ceph/cluster1-client.mirror.129474.log.gz 2026-02-18T08:53:42.947 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.104094.log 2026-02-18T08:53:42.948 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.112206.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.112206.log.gz 2026-02-18T08:53:42.948 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.95704.log 2026-02-18T08:53:42.948 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.104094.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.108684.log 2026-02-18T08:53:42.949 INFO:teuthology.orchestra.run.trial163.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.104094.log.gz 2026-02-18T08:53:42.949 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.95704.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.95704.log.gz 2026-02-18T08:53:42.949 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.102337.log 2026-02-18T08:53:42.949 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.1.44683.log 2026-02-18T08:53:42.950 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.108684.log: /var/log/ceph/cluster2-client.mirror.102337.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.108684.log.gz 2026-02-18T08:53:42.950 INFO:teuthology.orchestra.run.trial163.stderr: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.102337.log.gz 2026-02-18T08:53:42.950 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.78992.log 2026-02-18T08:53:42.950 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.60164.log 2026-02-18T08:53:42.951 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.1.44683.log: /var/log/ceph/cluster2-client.mirror.78992.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.20978.log 2026-02-18T08:53:42.951 INFO:teuthology.orchestra.run.trial163.stderr: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.78992.log.gz 2026-02-18T08:53:42.960 INFO:teuthology.orchestra.run.trial163.stderr: 95.8% -- replaced with /var/log/ceph/cluster2-client.mirror.116904.log.gz/var/log/ceph/cluster2-client.mirror.60164.log: 2026-02-18T08:53:42.960 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.108957.log 2026-02-18T08:53:42.960 INFO:teuthology.orchestra.run.trial163.stderr: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.60164.log.gz 2026-02-18T08:53:42.960 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.20978.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.43703.log 2026-02-18T08:53:42.960 INFO:teuthology.orchestra.run.trial163.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.20978.log.gz/var/log/ceph/cluster2-client.mirror.108957.log: 2026-02-18T08:53:42.960 INFO:teuthology.orchestra.run.trial163.stderr: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.108957.log.gz 2026-02-18T08:53:42.961 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.94496.log 2026-02-18T08:53:42.961 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.43703.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.20850.log 2026-02-18T08:53:42.962 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.94496.log: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.43703.log.gz 2026-02-18T08:53:42.962 INFO:teuthology.orchestra.run.trial163.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.94496.log.gz 2026-02-18T08:53:42.962 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.120789.log 2026-02-18T08:53:42.962 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.20850.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.49250.log 2026-02-18T08:53:42.962 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.120789.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.20850.log.gz 2026-02-18T08:53:42.963 INFO:teuthology.orchestra.run.trial163.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.120789.log.gz 2026-02-18T08:53:42.963 INFO:teuthology.orchestra.run.trial163.stderr: 93.7% -- replaced with /var/log/ceph/cluster2-client.mirror.1.44683.log.gz 2026-02-18T08:53:42.963 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.14727.log 2026-02-18T08:53:42.963 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.49250.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.84881.log 2026-02-18T08:53:42.963 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.admin.14727.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.14727.log.gz 2026-02-18T08:53:42.964 INFO:teuthology.orchestra.run.trial163.stderr: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.49250.log.gz 2026-02-18T08:53:42.964 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.55344.log 2026-02-18T08:53:42.964 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.84881.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.55694.log 2026-02-18T08:53:42.965 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.55344.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.28809.log 2026-02-18T08:53:42.965 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.55694.log: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.55694.log.gz 2026-02-18T08:53:42.967 INFO:teuthology.orchestra.run.trial163.stderr: 93.5% -- replaced with /var/log/ceph/cluster2-client.mirror.84881.log.gz 2026-02-18T08:53:42.979 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.132085.log 2026-02-18T08:53:42.980 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.28809.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.125255.log 2026-02-18T08:53:42.980 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.132085.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.101011.log 2026-02-18T08:53:42.980 INFO:teuthology.orchestra.run.trial163.stderr: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.28809.log.gz 2026-02-18T08:53:42.980 INFO:teuthology.orchestra.run.trial163.stderr: 86.8% -- replaced with /var/log/ceph/cluster1-client.mirror.132085.log.gz 2026-02-18T08:53:42.980 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.125255.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.133699.log 2026-02-18T08:53:42.981 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.101011.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.125255.log.gz 2026-02-18T08:53:42.981 INFO:teuthology.orchestra.run.trial163.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.101011.log.gz 2026-02-18T08:53:42.981 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.86110.log 2026-02-18T08:53:42.982 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.45488.log 2026-02-18T08:53:42.982 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.133699.log: /var/log/ceph/cluster2-client.mirror.86110.log: 84.7% -- replaced with /var/log/ceph/cluster2-client.mirror.86110.log.gz 2026-02-18T08:53:42.982 INFO:teuthology.orchestra.run.trial163.stderr: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.133699.log.gz 2026-02-18T08:53:42.982 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.67414.log 2026-02-18T08:53:42.983 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.45488.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.55729.log 2026-02-18T08:53:42.983 INFO:teuthology.orchestra.run.trial163.stderr: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.45488.log.gz 2026-02-18T08:53:42.983 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.67414.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.17308.log 2026-02-18T08:53:42.984 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.55729.log: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.67414.log.gz 2026-02-18T08:53:42.984 INFO:teuthology.orchestra.run.trial163.stderr: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.55729.log.gz 2026-02-18T08:53:42.984 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.82356.log 2026-02-18T08:53:42.984 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.82146.log 2026-02-18T08:53:42.985 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.17308.log: /var/log/ceph/cluster2-client.mirror.82356.log: 89.2% -- replaced with /var/log/ceph/cluster2-client.mirror.82356.log.gz 2026-02-18T08:53:42.985 INFO:teuthology.orchestra.run.trial163.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.17308.log.gz 2026-02-18T08:53:42.985 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.130014.log 2026-02-18T08:53:42.986 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.93957.log 2026-02-18T08:53:42.986 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.82146.log: /var/log/ceph/cluster1-client.mirror.130014.log: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.82146.log.gz 86.7% -- replaced with /var/log/ceph/cluster1-client.mirror.130014.log.gz 2026-02-18T08:53:42.986 INFO:teuthology.orchestra.run.trial163.stderr: 2026-02-18T08:53:42.986 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.23547.log 2026-02-18T08:53:42.987 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.93957.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.18459.log 2026-02-18T08:53:42.987 INFO:teuthology.orchestra.run.trial163.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.93957.log.gz 2026-02-18T08:53:42.987 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.23547.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.57199.log 2026-02-18T08:53:42.988 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.18459.log: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.23547.log.gz 2026-02-18T08:53:42.988 INFO:teuthology.orchestra.run.trial163.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.18459.log.gz 2026-02-18T08:53:42.988 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.28472.log 2026-02-18T08:53:42.989 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.57199.log: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.57199.log.gz 2026-02-18T08:53:42.989 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.1.91378.log 2026-02-18T08:53:42.990 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.28472.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.28472.log.gz 2026-02-18T08:53:42.990 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.115509.log 2026-02-18T08:53:42.991 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.1.91378.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.45880.log 2026-02-18T08:53:42.993 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.115509.log: 91.8% -- replaced with /var/log/ceph/cluster2-client.mirror.115509.log.gz 2026-02-18T08:53:42.998 INFO:teuthology.orchestra.run.trial034.stderr: 91.5% -- replaced with /var/log/ceph/cluster2-mgr.x.log.gz 2026-02-18T08:53:42.999 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.41525.log 2026-02-18T08:53:43.000 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.45880.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.120906.log 2026-02-18T08:53:43.000 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.41525.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.106348.log 2026-02-18T08:53:43.001 INFO:teuthology.orchestra.run.trial163.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.41525.log.gz 2026-02-18T08:53:43.001 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.120906.log: 88.3% -- replaced with /var/log/ceph/cluster1-client.mirror.120906.log.gz 2026-02-18T08:53:43.002 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.131456.log 2026-02-18T08:53:43.002 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.106348.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.106348.log.gz 2026-02-18T08:53:43.003 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.22094.log 2026-02-18T08:53:43.003 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.131456.log: 87.0% -- replaced with /var/log/ceph/cluster1-client.mirror.131456.log.gz 2026-02-18T08:53:43.004 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.130668.log 2026-02-18T08:53:43.005 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.22094.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.22094.log.gz 2026-02-18T08:53:43.005 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.47322.log 2026-02-18T08:53:43.005 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.130668.log: 87.0% -- replaced with /var/log/ceph/cluster1-client.mirror.130668.log.gz 2026-02-18T08:53:43.006 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.130848.log 2026-02-18T08:53:43.006 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.47322.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.47322.log.gz 2026-02-18T08:53:43.007 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.85640.log 2026-02-18T08:53:43.007 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.130848.log: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.130848.log.gz 2026-02-18T08:53:43.007 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.52579.log 2026-02-18T08:53:43.008 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.85640.log: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.85640.log.gz 2026-02-18T08:53:43.008 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.37063.log 2026-02-18T08:53:43.009 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.52579.log: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.52579.log.gz 2026-02-18T08:53:43.009 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.30297.log 2026-02-18T08:53:43.010 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.37063.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.15565.log 2026-02-18T08:53:43.010 INFO:teuthology.orchestra.run.trial163.stderr: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.37063.log.gz 2026-02-18T08:53:43.011 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.30297.log: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.30297.log.gz 2026-02-18T08:53:43.011 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.134812.log 2026-02-18T08:53:43.011 INFO:teuthology.orchestra.run.trial163.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.45880.log.gz 2026-02-18T08:53:43.012 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.15565.log: 4.7% -- replaced with /var/log/ceph/cluster2-client.mirror.15565.log.gz 2026-02-18T08:53:43.012 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.60269.log 2026-02-18T08:53:43.013 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.134812.log: 86.8% -- replaced with /var/log/ceph/cluster1-client.mirror.134812.log.gz 2026-02-18T08:53:43.013 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.74124.log 2026-02-18T08:53:43.013 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.60269.log: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.60269.log.gz 2026-02-18T08:53:43.014 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.46069.log 2026-02-18T08:53:43.014 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.74124.log: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.74124.log.gz 2026-02-18T08:53:43.015 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.79062.log 2026-02-18T08:53:43.015 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.46069.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.117592.log 2026-02-18T08:53:43.019 INFO:teuthology.orchestra.run.trial163.stderr: 93.5% -- replaced with /var/log/ceph/cluster2-client.mirror.46069.log.gz 2026-02-18T08:53:43.020 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.79062.log: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.79062.log.gz 2026-02-18T08:53:43.020 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.122843.log 2026-02-18T08:53:43.020 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.117592.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.117592.log.gz 2026-02-18T08:53:43.021 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.79202.log 2026-02-18T08:53:43.021 INFO:teuthology.orchestra.run.trial163.stderr: 95.7% -- replaced with /var/log/ceph/cluster2-client.mirror.68446.log.gz 2026-02-18T08:53:43.021 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.122843.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.132127.log 2026-02-18T08:53:43.022 INFO:teuthology.orchestra.run.trial163.stderr: 88.3%/var/log/ceph/cluster2-client.mirror.79202.log: -- replaced with /var/log/ceph/cluster2-client.mirror.122843.log.gz 2026-02-18T08:53:43.022 INFO:teuthology.orchestra.run.trial163.stderr: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.79202.log.gzgzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.131795.log 2026-02-18T08:53:43.022 INFO:teuthology.orchestra.run.trial163.stderr: 2026-02-18T08:53:43.022 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.132127.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.34899.log 2026-02-18T08:53:43.023 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.131795.log: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.132127.log.gz 2026-02-18T08:53:43.023 INFO:teuthology.orchestra.run.trial163.stderr: 87.0% -- replaced with /var/log/ceph/cluster1-client.mirror.131795.log.gz 2026-02-18T08:53:43.023 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.130104.log 2026-02-18T08:53:43.024 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.112086.log 2026-02-18T08:53:43.024 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.130104.log: /var/log/ceph/cluster2-client.mirror.34899.log: 86.8% -- replaced with /var/log/ceph/cluster1-client.mirror.130104.log.gz 2026-02-18T08:53:43.024 INFO:teuthology.orchestra.run.trial163.stderr: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.34899.log.gz 2026-02-18T08:53:43.025 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.115991.log 2026-02-18T08:53:43.025 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.112086.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.35922.log 2026-02-18T08:53:43.025 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.115991.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.112086.log.gz 2026-02-18T08:53:43.025 INFO:teuthology.orchestra.run.trial163.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.115991.log.gz 2026-02-18T08:53:43.026 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.39816.log 2026-02-18T08:53:43.026 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.115332.log 2026-02-18T08:53:43.027 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.35922.log: /var/log/ceph/cluster2-client.mirror.39816.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.39816.log.gz 2026-02-18T08:53:43.027 INFO:teuthology.orchestra.run.trial163.stderr: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.35922.log.gz 2026-02-18T08:53:43.027 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.92261.log 2026-02-18T08:53:43.027 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.115332.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.116206.log 2026-02-18T08:53:43.028 INFO:teuthology.orchestra.run.trial163.stderr: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.115332.log.gz 2026-02-18T08:53:43.028 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.92261.log: 1.3% -- replaced with /var/log/ceph/cluster1-client.mirror.92261.log.gz 2026-02-18T08:53:43.028 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.116814.log 2026-02-18T08:53:43.028 INFO:teuthology.orchestra.run.trial163.stderr: 95.7% -- replaced with /var/log/ceph/cluster2-client.mirror.123996.log.gz 2026-02-18T08:53:43.028 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.116206.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.85617.log 2026-02-18T08:53:43.029 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.68874.log 2026-02-18T08:53:43.029 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.116814.log: /var/log/ceph/cluster1-client.mirror.85617.log: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.85617.log.gz 2026-02-18T08:53:43.030 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.83294.log 2026-02-18T08:53:43.030 INFO:teuthology.orchestra.run.trial163.stderr: 92.6% -- replaced with /var/log/ceph/cluster2-client.mirror.116814.log.gz 2026-02-18T08:53:43.030 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.68874.log: 88.9%gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.22745.log 2026-02-18T08:53:43.030 INFO:teuthology.orchestra.run.trial163.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.68874.log.gz 2026-02-18T08:53:43.031 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.83294.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.125522.log 2026-02-18T08:53:43.032 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.22745.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.22745.log.gz 2026-02-18T08:53:43.032 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.119758.log 2026-02-18T08:53:43.032 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.125522.log: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.125522.log.gz 2026-02-18T08:53:43.033 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.128229.log 2026-02-18T08:53:43.033 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.119758.log: 88.3% -- replaced with /var/log/ceph/cluster1-client.mirror.119758.log.gz 2026-02-18T08:53:43.034 INFO:teuthology.orchestra.run.trial163.stderr: 93.5%gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.82970.log 2026-02-18T08:53:43.034 INFO:teuthology.orchestra.run.trial163.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.83294.log.gz 2026-02-18T08:53:43.034 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.128229.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.99885.log 2026-02-18T08:53:43.034 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.82970.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.128229.log.gz 2026-02-18T08:53:43.035 INFO:teuthology.orchestra.run.trial163.stderr: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.82970.log.gz 2026-02-18T08:53:43.035 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.57409.log 2026-02-18T08:53:43.035 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.81306.log 2026-02-18T08:53:43.036 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.99885.log: /var/log/ceph/cluster2-client.mirror.57409.log: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.57409.log.gz 2026-02-18T08:53:43.036 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.39072.log 2026-02-18T08:53:43.036 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.81306.log: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.81306.log.gz 2026-02-18T08:53:43.037 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.70449.log 2026-02-18T08:53:43.037 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.39072.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.39072.log.gz 2026-02-18T08:53:43.038 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.134569.log 2026-02-18T08:53:43.038 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.70449.log: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.70449.log.gz 2026-02-18T08:53:43.039 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.39909.log 2026-02-18T08:53:43.039 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.134569.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.134569.log.gz 2026-02-18T08:53:43.040 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.68765.log 2026-02-18T08:53:43.040 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.39909.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.129857.log 2026-02-18T08:53:43.040 INFO:teuthology.orchestra.run.trial163.stderr: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.39909.log.gz 2026-02-18T08:53:43.041 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.68765.log: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.68765.log.gz 2026-02-18T08:53:43.041 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.49215.log 2026-02-18T08:53:43.042 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.129857.log: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.129857.log.gz 2026-02-18T08:53:43.042 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.131919.log 2026-02-18T08:53:43.043 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.49215.log: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.49215.log.gz 2026-02-18T08:53:43.043 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.34655.log 2026-02-18T08:53:43.043 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.131919.log: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.131919.log.gz 2026-02-18T08:53:43.044 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.68291.log 2026-02-18T08:53:43.044 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.34655.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.34655.log.gz 2026-02-18T08:53:43.045 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.69469.log 2026-02-18T08:53:43.047 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.68291.log: 94.9% -- replaced with /var/log/ceph/cluster2-client.mirror.68291.log.gz 2026-02-18T08:53:43.047 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.18052.log 2026-02-18T08:53:43.048 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.69469.log: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.69469.log.gz 2026-02-18T08:53:43.048 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.61940.log 2026-02-18T08:53:43.049 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.18052.log: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.18052.log.gz 2026-02-18T08:53:43.049 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.17029.log 2026-02-18T08:53:43.049 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.61940.log: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.61940.log.gz 2026-02-18T08:53:43.050 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.129249.log 2026-02-18T08:53:43.050 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.17029.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.32384.log 2026-02-18T08:53:43.050 INFO:teuthology.orchestra.run.trial163.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.17029.log.gz 2026-02-18T08:53:43.051 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.129249.log: 87.0% -- replaced with /var/log/ceph/cluster1-client.mirror.129249.log.gz 2026-02-18T08:53:43.051 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.115529.log 2026-02-18T08:53:43.052 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.32384.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.32384.log.gz 2026-02-18T08:53:43.052 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.61485.log 2026-02-18T08:53:43.053 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.115529.log: 91.7% -- replaced with /var/log/ceph/cluster1-client.mirror.115529.log.gz 2026-02-18T08:53:43.054 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.77067.log 2026-02-18T08:53:43.054 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.61485.log: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.61485.log.gz 2026-02-18T08:53:43.055 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.99398.log 2026-02-18T08:53:43.055 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.77067.log: 89.2% -- replaced with /var/log/ceph/cluster2-client.mirror.77067.log.gz 2026-02-18T08:53:43.056 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.35829.log 2026-02-18T08:53:43.057 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.99398.log: 92.0% -- replaced with /var/log/ceph/cluster1-client.mirror.99398.log.gz 2026-02-18T08:53:43.057 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.129272.log 2026-02-18T08:53:43.058 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.35829.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.114592.log 2026-02-18T08:53:43.058 INFO:teuthology.orchestra.run.trial163.stderr: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.35829.log.gz 2026-02-18T08:53:43.059 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.129272.log: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.129272.log.gz 2026-02-18T08:53:43.059 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.54962.log 2026-02-18T08:53:43.059 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.114592.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.114592.log.gz 2026-02-18T08:53:43.060 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.102800.log 2026-02-18T08:53:43.060 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.54962.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.83470.log 2026-02-18T08:53:43.060 INFO:teuthology.orchestra.run.trial163.stderr: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.54962.log.gz 2026-02-18T08:53:43.061 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.102800.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.102800.log.gz 2026-02-18T08:53:43.061 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.134527.log 2026-02-18T08:53:43.062 INFO:teuthology.orchestra.run.trial163.stderr: 95.7% -- replaced with /var/log/ceph/cluster2-client.mirror.93208.log.gz 2026-02-18T08:53:43.066 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.83470.log: 95.7% -- replaced with /var/log/ceph/cluster2-client.mirror.115045.log.gz 2026-02-18T08:53:43.067 INFO:teuthology.orchestra.run.trial163.stderr: 96.9% -- replaced with /var/log/ceph/cluster2-client.mirror.83470.log.gz 2026-02-18T08:53:43.067 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.84554.log 2026-02-18T08:53:43.067 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.134527.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.69574.log 2026-02-18T08:53:43.067 INFO:teuthology.orchestra.run.trial163.stderr: 86.9%/var/log/ceph/cluster1-client.mirror.84554.log: -- replaced with /var/log/ceph/cluster1-client.mirror.134527.log.gz 2026-02-18T08:53:43.067 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.14702.log 2026-02-18T08:53:43.068 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.69574.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.21594.log 2026-02-18T08:53:43.068 INFO:teuthology.orchestra.run.trial163.stderr: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.69574.log.gz 2026-02-18T08:53:43.068 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.admin.14702.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.14702.log.gz 2026-02-18T08:53:43.068 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.130465.log 2026-02-18T08:53:43.069 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.21594.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.134048.log 2026-02-18T08:53:43.069 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.133535.log 2026-02-18T08:53:43.069 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.130465.log: 91.7%/var/log/ceph/cluster1-client.mirror.134048.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.21594.log.gz 2026-02-18T08:53:43.070 INFO:teuthology.orchestra.run.trial163.stderr: -- replaced with /var/log/ceph/cluster1-client.mirror.130465.log.gz 87.7% 2026-02-18T08:53:43.070 INFO:teuthology.orchestra.run.trial163.stderr: -- replaced with /var/log/ceph/cluster1-client.mirror.134048.log.gz 2026-02-18T08:53:43.070 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.28158.log 2026-02-18T08:53:43.070 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.133535.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.68249.log 2026-02-18T08:53:43.070 INFO:teuthology.orchestra.run.trial163.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.133535.log.gz/var/log/ceph/cluster2-client.mirror.28158.log: 93.5% -- replaced with /var/log/ceph/cluster1-client.mirror.84554.log.gz 2026-02-18T08:53:43.070 INFO:teuthology.orchestra.run.trial163.stderr: 2026-02-18T08:53:43.071 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.14802.log 2026-02-18T08:53:43.071 INFO:teuthology.orchestra.run.trial163.stderr: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.28158.log.gz 2026-02-18T08:53:43.071 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.68418.log 2026-02-18T08:53:43.071 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.admin.14802.log: /var/log/ceph/cluster2-client.mirror.68249.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.14802.log.gz 2026-02-18T08:53:43.072 INFO:teuthology.orchestra.run.trial163.stderr: 95.6% -- replaced with /var/log/ceph/cluster2-client.mirror.112943.log.gz 2026-02-18T08:53:43.074 INFO:teuthology.orchestra.run.trial163.stderr: 93.6% -- replaced with /var/log/ceph/cluster2-client.mirror.68249.log.gz 2026-02-18T08:53:43.074 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.53944.log 2026-02-18T08:53:43.074 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.admin.68418.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.85395.log 2026-02-18T08:53:43.075 INFO:teuthology.orchestra.run.trial163.stderr: 88.2%/var/log/ceph/cluster2-client.mirror.53944.log: -- replaced with /var/log/ceph/cluster1-client.admin.68418.log.gz 2026-02-18T08:53:43.075 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.64320.log 2026-02-18T08:53:43.075 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.88584.log 2026-02-18T08:53:43.075 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.85395.log: /var/log/ceph/cluster2-client.mirror.64320.log: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.53944.log.gz 2026-02-18T08:53:43.075 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.113335.log 2026-02-18T08:53:43.076 INFO:teuthology.orchestra.run.trial163.stderr: 89.2% -- replaced with /var/log/ceph/cluster2-client.mirror.85395.log.gz 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.64320.log.gz 2026-02-18T08:53:43.076 INFO:teuthology.orchestra.run.trial163.stderr: 2026-02-18T08:53:43.076 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.88584.log: 84.8% -- replaced with /var/log/ceph/cluster2-client.mirror.88584.log.gz 2026-02-18T08:53:43.076 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.72339.log 2026-02-18T08:53:43.076 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.113335.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.90592.log 2026-02-18T08:53:43.076 INFO:teuthology.orchestra.run.trial163.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.113335.log.gz 2026-02-18T08:53:43.077 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.72339.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.106849.log 2026-02-18T08:53:43.077 INFO:teuthology.orchestra.run.trial163.stderr: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.72339.log.gz 2026-02-18T08:53:43.077 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.90592.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.75906.log 2026-02-18T08:53:43.077 INFO:teuthology.orchestra.run.trial163.stderr: 84.7%/var/log/ceph/cluster2-client.mirror.106849.log: -- replaced with /var/log/ceph/cluster2-client.mirror.90592.log.gzgzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.101916.log 2026-02-18T08:53:43.077 INFO:teuthology.orchestra.run.trial163.stderr: 2026-02-18T08:53:43.078 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.75906.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.106849.log.gzgzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.86778.log 2026-02-18T08:53:43.078 INFO:teuthology.orchestra.run.trial163.stderr: 2026-02-18T08:53:43.078 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.120193.log 2026-02-18T08:53:43.078 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.101916.log: /var/log/ceph/cluster2-client.mirror.86778.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.27077.log 2026-02-18T08:53:43.079 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.120193.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.120193.log.gz 2026-02-18T08:53:43.079 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.111661.log 2026-02-18T08:53:43.079 INFO:teuthology.orchestra.run.trial163.stderr: 91.9% -- replaced with /var/log/ceph/cluster1-client.mirror.101916.log.gz 2026-02-18T08:53:43.080 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.27077.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.31210.log 2026-02-18T08:53:43.080 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.111661.log: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.27077.log.gz 2026-02-18T08:53:43.081 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.125682.log 2026-02-18T08:53:43.081 INFO:teuthology.orchestra.run.trial163.stderr: 88.3% -- replaced with /var/log/ceph/cluster1-client.mirror.111661.log.gz 2026-02-18T08:53:43.081 INFO:teuthology.orchestra.run.trial163.stderr: 93.5% -- replaced with /var/log/ceph/cluster2-client.mirror.75906.log.gz 2026-02-18T08:53:43.081 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.31210.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.39537.log 2026-02-18T08:53:43.081 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.125682.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.108522.log 2026-02-18T08:53:43.081 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.39537.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.31210.log.gz 2026-02-18T08:53:43.081 INFO:teuthology.orchestra.run.trial163.stderr: 89.3% -- replaced with /var/log/ceph/cluster2-client.mirror.125682.log.gz 2026-02-18T08:53:43.082 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.68944.log 2026-02-18T08:53:43.082 INFO:teuthology.orchestra.run.trial163.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.39537.log.gz 2026-02-18T08:53:43.082 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.96060.log 2026-02-18T08:53:43.082 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.108522.log: /var/log/ceph/cluster2-client.mirror.68944.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.108522.log.gz 2026-02-18T08:53:43.082 INFO:teuthology.orchestra.run.trial163.stderr: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.68944.log.gz 2026-02-18T08:53:43.082 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.120353.log 2026-02-18T08:53:43.083 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.22338.log 2026-02-18T08:53:43.083 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.96060.log: /var/log/ceph/cluster2-client.mirror.120353.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.14089.log 2026-02-18T08:53:43.083 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.22338.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.96060.log.gz 2026-02-18T08:53:43.083 INFO:teuthology.orchestra.run.trial163.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.120353.log.gz 2026-02-18T08:53:43.083 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.43145.log 2026-02-18T08:53:43.084 INFO:teuthology.orchestra.run.trial163.stderr: 88.2%gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.127698.log 2026-02-18T08:53:43.084 INFO:teuthology.orchestra.run.trial163.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.22338.log.gz 2026-02-18T08:53:43.084 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.admin.14089.log: /var/log/ceph/cluster2-client.mirror.43145.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.14089.log.gz 2026-02-18T08:53:43.084 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.70834.log 2026-02-18T08:53:43.084 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.127698.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.43145.log.gz 2026-02-18T08:53:43.085 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.71079.log 2026-02-18T08:53:43.085 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.70834.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.55589.log 2026-02-18T08:53:43.085 INFO:teuthology.orchestra.run.trial163.stderr: 89.3% -- replaced with /var/log/ceph/cluster2-client.mirror.127698.log.gz/var/log/ceph/cluster2-client.mirror.71079.log: 2026-02-18T08:53:43.085 INFO:teuthology.orchestra.run.trial163.stderr: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.70834.log.gz 2026-02-18T08:53:43.085 INFO:teuthology.orchestra.run.trial163.stderr: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.71079.log.gzgzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.133756.log 2026-02-18T08:53:43.085 INFO:teuthology.orchestra.run.trial163.stderr: 2026-02-18T08:53:43.086 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.122797.log 2026-02-18T08:53:43.086 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.55589.log: /var/log/ceph/cluster1-client.mirror.133756.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.94650.log 2026-02-18T08:53:43.086 INFO:teuthology.orchestra.run.trial163.stderr: 86.4% -- replaced with /var/log/ceph/cluster1-client.mirror.133756.log.gz 2026-02-18T08:53:43.086 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.122797.log: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.55589.log.gz 2026-02-18T08:53:43.086 INFO:teuthology.orchestra.run.trial163.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.122797.log.gz 2026-02-18T08:53:43.087 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.132846.log 2026-02-18T08:53:43.087 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.99107.log 2026-02-18T08:53:43.087 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.94650.log: /var/log/ceph/cluster2-client.mirror.132846.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.126675.log 2026-02-18T08:53:43.087 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.99107.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.94650.log.gz 2026-02-18T08:53:43.087 INFO:teuthology.orchestra.run.trial163.stderr: 87.9% -- replaced with /var/log/ceph/cluster2-client.mirror.132846.log.gz 2026-02-18T08:53:43.087 INFO:teuthology.orchestra.run.trial163.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.99107.log.gz 2026-02-18T08:53:43.088 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.88465.log 2026-02-18T08:53:43.088 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.135445.log 2026-02-18T08:53:43.088 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.126675.log: /var/log/ceph/cluster2-client.mirror.88465.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.123058.log 2026-02-18T08:53:43.088 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.admin.135445.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.126675.log.gz 2026-02-18T08:53:43.088 INFO:teuthology.orchestra.run.trial163.stderr: 28.5% -- replaced with /var/log/ceph/cluster2-client.admin.135445.log.gz 2026-02-18T08:53:43.089 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.11455.log 2026-02-18T08:53:43.089 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.25277.log 2026-02-18T08:53:43.089 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.123058.log: /var/log/ceph/cluster2-client.admin.11455.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.11455.log.gz 2026-02-18T08:53:43.107 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.97545.log 2026-02-18T08:53:43.108 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.25277.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.25277.log.gz 2026-02-18T08:53:43.108 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.75353.log 2026-02-18T08:53:43.115 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.97545.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.111594.log 2026-02-18T08:53:43.131 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.75353.log: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.75353.log.gz 2026-02-18T08:53:43.131 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.29646.log 2026-02-18T08:53:43.132 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.111594.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.111594.log.gz 2026-02-18T08:53:43.139 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.2.92600.log 2026-02-18T08:53:43.148 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.29646.log: 95.7% -- replaced with /var/log/ceph/cluster2-client.mirror.90280.log.gz 2026-02-18T08:53:43.149 INFO:teuthology.orchestra.run.trial163.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.29646.log.gz 2026-02-18T08:53:43.149 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.126411.log 2026-02-18T08:53:43.163 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.2.92600.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.75969.log 2026-02-18T08:53:43.165 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.126411.log: 90.7% -- replaced with /var/log/ceph/cluster2-client.mirror.126411.log.gz 2026-02-18T08:53:43.171 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.132765.log 2026-02-18T08:53:43.187 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.75969.log: 93.5% -- replaced with /var/log/ceph/cluster1-client.mirror.75969.log.gz 2026-02-18T08:53:43.188 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.98154.log 2026-02-18T08:53:43.188 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.132765.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.132765.log.gz 2026-02-18T08:53:43.195 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.21164.log 2026-02-18T08:53:43.196 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.98154.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.98154.log.gz 2026-02-18T08:53:43.197 INFO:teuthology.orchestra.run.trial163.stderr: 95.6% -- replaced with /var/log/ceph/cluster2-client.mirror.55425.log.gz 2026-02-18T08:53:43.200 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.17157.log 2026-02-18T08:53:43.201 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.21164.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.21164.log.gz 2026-02-18T08:53:43.201 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.27449.log 2026-02-18T08:53:43.201 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.17157.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.17157.log.gz 2026-02-18T08:53:43.201 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.36784.log 2026-02-18T08:53:43.202 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.27449.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.27449.log.gz 2026-02-18T08:53:43.202 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.55624.log 2026-02-18T08:53:43.203 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.36784.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.36784.log.gz 2026-02-18T08:53:43.203 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.97209.log 2026-02-18T08:53:43.204 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.55624.log: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.55624.log.gz 2026-02-18T08:53:43.204 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.3.87596.log 2026-02-18T08:53:43.204 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.97209.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.97209.log.gz 2026-02-18T08:53:43.205 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.60689.log 2026-02-18T08:53:43.213 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.3.87596.log: 93.6%gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.119036.log 2026-02-18T08:53:43.214 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.60689.log: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.60689.log.gz 2026-02-18T08:53:43.214 INFO:teuthology.orchestra.run.trial163.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.3.87596.log.gz 2026-02-18T08:53:43.214 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.24384.log 2026-02-18T08:53:43.215 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.119036.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.119036.log.gz 2026-02-18T08:53:43.215 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.18517.log 2026-02-18T08:53:43.216 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.24384.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.24384.log.gz 2026-02-18T08:53:43.216 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.48795.log 2026-02-18T08:53:43.217 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.18517.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.18517.log.gz 2026-02-18T08:53:43.217 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.123355.log 2026-02-18T08:53:43.218 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.48795.log: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.48795.log.gz 2026-02-18T08:53:43.218 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.128663.log 2026-02-18T08:53:43.218 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.123355.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.123355.log.gz 2026-02-18T08:53:43.219 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.129038.log 2026-02-18T08:53:43.219 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.128663.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.128663.log.gz 2026-02-18T08:53:43.219 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.100523.log 2026-02-18T08:53:43.220 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.129038.log: 90.7% -- replaced with /var/log/ceph/cluster2-client.mirror.129038.log.gz 2026-02-18T08:53:43.221 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.95936.log 2026-02-18T08:53:43.222 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.100523.log: 92.5% -- replaced with /var/log/ceph/cluster2-client.mirror.100523.log.gz 2026-02-18T08:53:43.223 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.125049.log 2026-02-18T08:53:43.223 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.95936.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.95936.log.gz 2026-02-18T08:53:43.224 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.122267.log 2026-02-18T08:53:43.225 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.125049.log: 92.5% -- replaced with /var/log/ceph/cluster1-client.mirror.125049.log.gz 2026-02-18T08:53:43.225 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.133493.log 2026-02-18T08:53:43.226 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.122267.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.122267.log.gz 2026-02-18T08:53:43.226 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.112804.log 2026-02-18T08:53:43.226 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.133493.log: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.133493.log.gz 2026-02-18T08:53:43.227 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.43517.log 2026-02-18T08:53:43.227 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.112804.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.112804.log.gz 2026-02-18T08:53:43.227 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.124478.log 2026-02-18T08:53:43.228 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.43517.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.43517.log.gz 2026-02-18T08:53:43.228 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.54294.log 2026-02-18T08:53:43.229 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.124478.log: 88.2% -- replaced with /var/log/ceph/cluster1-client.mirror.124478.log.gz 2026-02-18T08:53:43.230 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.115180.log 2026-02-18T08:53:43.230 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.54294.log: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.54294.log.gz 2026-02-18T08:53:43.230 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.32698.log 2026-02-18T08:53:43.231 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.115180.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.115180.log.gz 2026-02-18T08:53:43.231 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.110891.log 2026-02-18T08:53:43.232 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.32698.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.82636.log 2026-02-18T08:53:43.233 INFO:teuthology.orchestra.run.trial163.stderr: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.32698.log.gz 2026-02-18T08:53:43.233 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.110891.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.110891.log.gz 2026-02-18T08:53:43.233 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.102006.log 2026-02-18T08:53:43.234 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.82636.log: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.82636.log.gz 2026-02-18T08:53:43.234 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.53664.log 2026-02-18T08:53:43.235 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.102006.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.102006.log.gz 2026-02-18T08:53:43.235 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.102166.log 2026-02-18T08:53:43.236 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.53664.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.53664.log.gz 2026-02-18T08:53:43.237 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.75283.log 2026-02-18T08:53:43.238 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.102166.log: 92.5% -- replaced with /var/log/ceph/cluster1-client.mirror.102166.log.gz 2026-02-18T08:53:43.238 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.88793.log 2026-02-18T08:53:43.239 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.75283.log: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.75283.log.gz 2026-02-18T08:53:43.239 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.132806.log 2026-02-18T08:53:43.240 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.88793.log: 84.8% -- replaced with /var/log/ceph/cluster2-client.mirror.88793.log.gz 2026-02-18T08:53:43.240 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.125753.log 2026-02-18T08:53:43.242 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.132806.log: 91.8% -- replaced with /var/log/ceph/cluster2-client.mirror.132806.log.gz 2026-02-18T08:53:43.242 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.33163.log 2026-02-18T08:53:43.243 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.125753.log: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.125753.log.gz 2026-02-18T08:53:43.243 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.38235.log 2026-02-18T08:53:43.244 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.33163.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.33163.log.gz 2026-02-18T08:53:43.244 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.41866.log 2026-02-18T08:53:43.245 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.38235.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.46482.log 2026-02-18T08:53:43.245 INFO:teuthology.orchestra.run.trial163.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.38235.log.gz 2026-02-18T08:53:43.246 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.41866.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.53769.log 2026-02-18T08:53:43.246 INFO:teuthology.orchestra.run.trial163.stderr: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.41866.log.gz 2026-02-18T08:53:43.247 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.46482.log: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.46482.log.gz 2026-02-18T08:53:43.247 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.78677.log 2026-02-18T08:53:43.248 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.53769.log: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.53769.log.gz 2026-02-18T08:53:43.248 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.52474.log 2026-02-18T08:53:43.249 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.78677.log: 89.2% -- replaced with /var/log/ceph/cluster2-client.mirror.78677.log.gz 2026-02-18T08:53:43.249 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.38700.log 2026-02-18T08:53:43.250 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.52474.log: 87.9% -- replaced with /var/log/ceph/cluster2-client.mirror.52474.log.gz 2026-02-18T08:53:43.250 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.127008.log 2026-02-18T08:53:43.250 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.38700.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.40374.log 2026-02-18T08:53:43.251 INFO:teuthology.orchestra.run.trial163.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.38700.log.gz 2026-02-18T08:53:43.252 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.127008.log: 89.3% -- replaced with /var/log/ceph/cluster2-client.mirror.127008.log.gz 2026-02-18T08:53:43.252 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.60234.log 2026-02-18T08:53:43.252 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.40374.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.101302.log 2026-02-18T08:53:43.253 INFO:teuthology.orchestra.run.trial163.stderr: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.40374.log.gz 2026-02-18T08:53:43.254 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.60234.log: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.60234.log.gz 2026-02-18T08:53:43.254 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.121703.log 2026-02-18T08:53:43.254 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.101302.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.101302.log.gz 2026-02-18T08:53:43.255 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.25370.log 2026-02-18T08:53:43.256 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.121703.log: 92.4% -- replaced with /var/log/ceph/cluster1-client.mirror.121703.log.gz 2026-02-18T08:53:43.257 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.53104.log 2026-02-18T08:53:43.257 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.25370.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.25370.log.gz 2026-02-18T08:53:43.257 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.41978.log 2026-02-18T08:53:43.258 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.53104.log: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.53104.log.gz 2026-02-18T08:53:43.258 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.71149.log 2026-02-18T08:53:43.259 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.41978.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.41978.log.gz 2026-02-18T08:53:43.259 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.45434.log 2026-02-18T08:53:43.260 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.71149.log: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.71149.log.gz 2026-02-18T08:53:43.260 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.128016.log 2026-02-18T08:53:43.261 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.45434.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.81936.log 2026-02-18T08:53:43.261 INFO:teuthology.orchestra.run.trial163.stderr: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.45434.log.gz 2026-02-18T08:53:43.262 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.128016.log: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.128016.log.gz 2026-02-18T08:53:43.262 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.73564.log 2026-02-18T08:53:43.263 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.81936.log: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.81936.log.gz 2026-02-18T08:53:43.263 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.118665.log 2026-02-18T08:53:43.264 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.73564.log: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.73564.log.gz 2026-02-18T08:53:43.264 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.20477.log 2026-02-18T08:53:43.265 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.118665.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.118665.log.gz 2026-02-18T08:53:43.265 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.30332.log 2026-02-18T08:53:43.265 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.20477.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.111505.log 2026-02-18T08:53:43.266 INFO:teuthology.orchestra.run.trial163.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.20477.log.gz 2026-02-18T08:53:43.266 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.30332.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.85730.log 2026-02-18T08:53:43.267 INFO:teuthology.orchestra.run.trial163.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.30332.log.gz 2026-02-18T08:53:43.267 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.111505.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.111505.log.gz 2026-02-18T08:53:43.268 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.40932.log 2026-02-18T08:53:43.268 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.85730.log: 84.7% -- replaced with /var/log/ceph/cluster2-client.mirror.85730.log.gz 2026-02-18T08:53:43.268 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-mon.a.log 2026-02-18T08:53:43.269 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.40932.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.40932.log.gz 2026-02-18T08:53:43.269 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.84686.log 2026-02-18T08:53:43.270 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-mon.a.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.37528.log 2026-02-18T08:53:43.284 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.84686.log: 93.4%gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.15025.log 2026-02-18T08:53:43.284 INFO:teuthology.orchestra.run.trial163.stderr: -- replaced with /var/log/ceph/cluster1-client.mirror.84686.log.gz 2026-02-18T08:53:43.285 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.37528.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.37528.log.gz 2026-02-18T08:53:43.292 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.67729.log 2026-02-18T08:53:43.300 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.15025.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.92526.log 2026-02-18T08:53:43.301 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.67729.log: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.67729.log.gz 2026-02-18T08:53:43.303 INFO:teuthology.orchestra.run.trial163.stderr: 90.7% -- replaced with /var/log/ceph/cluster1-client.mirror.15025.log.gz 2026-02-18T08:53:43.307 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.129654.log 2026-02-18T08:53:43.321 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.92526.log: 95.7% -- replaced with /var/log/ceph/cluster2-client.mirror.42504.log.gz 2026-02-18T08:53:43.323 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.28007.log 2026-02-18T08:53:43.324 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.129654.log: 87.0% -- replaced with /var/log/ceph/cluster1-client.mirror.129654.log.gz 2026-02-18T08:53:43.324 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.134707.log 2026-02-18T08:53:43.325 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.28007.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.28007.log.gz 2026-02-18T08:53:43.326 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.26153.log 2026-02-18T08:53:43.326 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.134707.log: 87.6% -- replaced with /var/log/ceph/cluster1-client.mirror.134707.log.gz 2026-02-18T08:53:43.326 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.43482.log 2026-02-18T08:53:43.340 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.26153.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.42445.log 2026-02-18T08:53:43.340 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.43482.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.43482.log.gz 2026-02-18T08:53:43.352 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.45969.log 2026-02-18T08:53:43.356 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.admin.42445.log: 92.3% 88.8% -- replaced with /var/log/ceph/cluster1-client.admin.42445.log.gz 2026-02-18T08:53:43.356 INFO:teuthology.orchestra.run.trial163.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.26153.log.gz 2026-02-18T08:53:43.356 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.83428.log 2026-02-18T08:53:43.357 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.45969.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.56849.log 2026-02-18T08:53:43.357 INFO:teuthology.orchestra.run.trial163.stderr: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.45969.log.gz 2026-02-18T08:53:43.360 INFO:teuthology.orchestra.run.trial034.stderr: 90.4% -- replaced with /var/log/ceph/cluster1-mon.a.log.gz 2026-02-18T08:53:43.361 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.83428.log: 93.5% -- replaced with /var/log/ceph/cluster2-client.mirror.83428.log.gz 2026-02-18T08:53:43.362 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.42174.log 2026-02-18T08:53:43.362 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.56849.log: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.56849.log.gz 2026-02-18T08:53:43.362 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.103862.log 2026-02-18T08:53:43.363 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.42174.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.42174.log.gz 2026-02-18T08:53:43.363 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.130419.log 2026-02-18T08:53:43.364 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.103862.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.103862.log.gz 2026-02-18T08:53:43.365 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.113030.log 2026-02-18T08:53:43.365 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.130419.log: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.130419.log.gz 2026-02-18T08:53:43.365 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.45154.log 2026-02-18T08:53:43.365 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.113030.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.113030.log.gz 2026-02-18T08:53:43.367 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.65732.log 2026-02-18T08:53:43.367 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.45154.log: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.45154.log.gz 2026-02-18T08:53:43.367 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.32791.log 2026-02-18T08:53:43.367 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.65732.log: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.65732.log.gz 2026-02-18T08:53:43.368 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.121725.log 2026-02-18T08:53:43.368 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.32791.log: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.32791.log.gz 2026-02-18T08:53:43.368 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.89393.log 2026-02-18T08:53:43.384 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.121725.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.26333.log 2026-02-18T08:53:43.392 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.89393.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.37621.log 2026-02-18T08:53:43.394 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.26333.log: 95.7% 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.26333.log.gz 2026-02-18T08:53:43.394 INFO:teuthology.orchestra.run.trial163.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.30847.log.gz 2026-02-18T08:53:43.408 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.18145.log 2026-02-18T08:53:43.408 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.37621.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.37621.log.gz 2026-02-18T08:53:43.409 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.45612.log 2026-02-18T08:53:43.411 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.18145.log: 95.5% -- replaced with /var/log/ceph/cluster2-client.mirror.86428.log.gz 2026-02-18T08:53:43.420 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.86487.log 2026-02-18T08:53:43.424 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.45612.log: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.18145.log.gz 2026-02-18T08:53:43.432 INFO:teuthology.orchestra.run.trial163.stderr: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.45612.log.gz 2026-02-18T08:53:43.435 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.75926.log 2026-02-18T08:53:43.447 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.86487.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.1.130465.log 2026-02-18T08:53:43.455 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.75926.log: 93.5%gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.127994.log 2026-02-18T08:53:43.463 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.1.130465.log: 92.8% -- replaced with /var/log/ceph/cluster1-client.mirror.75926.log.gz 2026-02-18T08:53:43.464 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.132691.log 2026-02-18T08:53:43.465 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.127994.log: 89.3% -- replaced with /var/log/ceph/cluster2-client.mirror.127994.log.gz 2026-02-18T08:53:43.471 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.76232.log 2026-02-18T08:53:43.473 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.132691.log: 91.4% -- replaced with /var/log/ceph/cluster2-client.mirror.132691.log.gz 2026-02-18T08:53:43.475 INFO:teuthology.orchestra.run.trial163.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.1.130465.log.gz 2026-02-18T08:53:43.484 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.93074.log 2026-02-18T08:53:43.484 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.admin.76232.log: 88.6% -- replaced with /var/log/ceph/cluster1-client.admin.76232.log.gz 2026-02-18T08:53:43.488 INFO:teuthology.orchestra.run.trial163.stderr: 94.7% -- replaced with /var/log/ceph/cluster1-client.mirror.2.92600.log.gz 2026-02-18T08:53:43.499 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.107823.log 2026-02-18T08:53:43.500 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.93074.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.93074.log.gz 2026-02-18T08:53:43.514 INFO:teuthology.orchestra.run.trial163.stderr: 95.7% -- replaced with /var/log/ceph/cluster2-client.mirror.90144.log.gz 2026-02-18T08:53:43.515 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.134286.log 2026-02-18T08:53:43.516 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.107823.log: 87.9% -- replaced with /var/log/ceph/cluster2-client.mirror.107823.log.gz 2026-02-18T08:53:43.516 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.76647.log 2026-02-18T08:53:43.517 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.134286.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.134286.log.gz 2026-02-18T08:53:43.517 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.44412.log 2026-02-18T08:53:43.517 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.76647.log: 89.2% -- replaced with /var/log/ceph/cluster2-client.mirror.76647.log.gz 2026-02-18T08:53:43.518 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.42773.log 2026-02-18T08:53:43.519 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.44412.log: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.44412.log.gz 2026-02-18T08:53:43.519 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.115945.log 2026-02-18T08:53:43.520 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.42773.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.42773.log.gz 2026-02-18T08:53:43.520 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.39293.log 2026-02-18T08:53:43.520 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.115945.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.115945.log.gz 2026-02-18T08:53:43.521 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.93109.log 2026-02-18T08:53:43.522 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.39293.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.39293.log.gz 2026-02-18T08:53:43.522 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.30518.log 2026-02-18T08:53:43.522 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.93109.log: 87.9% -- replaced with /var/log/ceph/cluster2-client.mirror.93109.log.gz 2026-02-18T08:53:43.522 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.112634.log 2026-02-18T08:53:43.523 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.30518.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.117061.log 2026-02-18T08:53:43.524 INFO:teuthology.orchestra.run.trial163.stderr: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.30518.log.gz 2026-02-18T08:53:43.524 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.112634.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.112634.log.gz 2026-02-18T08:53:43.524 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.40839.log 2026-02-18T08:53:43.525 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.117061.log: 92.3% -- replaced with /var/log/ceph/cluster1-client.mirror.117061.log.gz 2026-02-18T08:53:43.526 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.41773.log 2026-02-18T08:53:43.527 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.40839.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.40839.log.gz 2026-02-18T08:53:43.527 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.95882.log 2026-02-18T08:53:43.528 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.41773.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.41773.log.gz 2026-02-18T08:53:43.528 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.15514.log 2026-02-18T08:53:43.529 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.95882.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.95882.log.gz 2026-02-18T08:53:43.529 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.39351.log 2026-02-18T08:53:43.529 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.15514.log: 64.1% -- replaced with /var/log/ceph/cluster2-client.mirror.15514.log.gz 2026-02-18T08:53:43.530 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.125895.log 2026-02-18T08:53:43.530 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.39351.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.39351.log.gz 2026-02-18T08:53:43.531 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.26056.log 2026-02-18T08:53:43.531 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.125895.log: 89.2% -- replaced with /var/log/ceph/cluster2-client.mirror.125895.log.gz 2026-02-18T08:53:43.532 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.22373.log 2026-02-18T08:53:43.532 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.26056.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.26056.log.gz 2026-02-18T08:53:43.533 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.107070.log 2026-02-18T08:53:43.533 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.22373.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.22373.log.gz 2026-02-18T08:53:43.533 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.131389.log 2026-02-18T08:53:43.534 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.107070.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.107070.log.gz 2026-02-18T08:53:43.534 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.3.15077.log 2026-02-18T08:53:43.534 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.131389.log: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.131389.log.gz 2026-02-18T08:53:43.535 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.80637.log 2026-02-18T08:53:43.540 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.3.15077.log: 94.3% -- replaced with /var/log/ceph/cluster2-client.mirror.3.15077.log.gz 2026-02-18T08:53:43.540 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.101556.log 2026-02-18T08:53:43.541 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.80637.log: 89.2% -- replaced with /var/log/ceph/cluster2-client.mirror.80637.log.gz 2026-02-18T08:53:43.541 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.84640.log 2026-02-18T08:53:43.541 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.101556.log: 95.7% -- replaced with /var/log/ceph/cluster2-client.mirror.116206.log.gz 2026-02-18T08:53:43.541 INFO:teuthology.orchestra.run.trial163.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.101556.log.gz 2026-02-18T08:53:43.541 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.130239.log 2026-02-18T08:53:43.542 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.84640.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.133274.log 2026-02-18T08:53:43.543 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.130239.log: 87.0% -- replaced with /var/log/ceph/cluster1-client.mirror.130239.log.gz 2026-02-18T08:53:43.545 INFO:teuthology.orchestra.run.trial163.stderr: 93.5% -- replaced with /var/log/ceph/cluster1-client.mirror.84640.log.gz 2026-02-18T08:53:43.546 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.122991.log 2026-02-18T08:53:43.546 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.133274.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.84092.log 2026-02-18T08:53:43.547 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.122991.log: 88.1% -- replaced with /var/log/ceph/cluster1-client.mirror.122991.log.gz 2026-02-18T08:53:43.547 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.33035.log 2026-02-18T08:53:43.547 INFO:teuthology.orchestra.run.trial163.stderr: 91.8% -- replaced with /var/log/ceph/cluster2-client.mirror.133274.log.gz 2026-02-18T08:53:43.548 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.84092.log: 89.2% -- replaced with /var/log/ceph/cluster2-client.mirror.84092.log.gz 2026-02-18T08:53:43.548 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.55321.log 2026-02-18T08:53:43.548 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.68979.log 2026-02-18T08:53:43.549 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.33035.log: /var/log/ceph/cluster2-client.mirror.55321.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.119001.log 2026-02-18T08:53:43.549 INFO:teuthology.orchestra.run.trial163.stderr: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.33035.log.gz 2026-02-18T08:53:43.553 INFO:teuthology.orchestra.run.trial163.stderr: 96.9% -- replaced with /var/log/ceph/cluster2-client.mirror.55321.log.gz 2026-02-18T08:53:43.554 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.68979.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.108111.log 2026-02-18T08:53:43.554 INFO:teuthology.orchestra.run.trial163.stderr: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.68979.log.gz 2026-02-18T08:53:43.555 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.119001.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.119001.log.gz 2026-02-18T08:53:43.555 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.134265.log 2026-02-18T08:53:43.556 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.81796.log 2026-02-18T08:53:43.556 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.134265.log: /var/log/ceph/cluster2-client.mirror.108111.log: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.134265.log.gz 2026-02-18T08:53:43.557 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.133212.log 2026-02-18T08:53:43.558 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.81796.log: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.81796.log.gz 2026-02-18T08:53:43.558 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.111327.log 2026-02-18T08:53:43.559 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.133212.log: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.133212.log.gz 2026-02-18T08:53:43.559 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.132609.log 2026-02-18T08:53:43.559 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.111327.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.111327.log.gz 2026-02-18T08:53:43.559 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.39479.log 2026-02-18T08:53:43.568 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.132609.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.23733.log 2026-02-18T08:53:43.571 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.39479.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.39479.log.gz 2026-02-18T08:53:43.585 INFO:teuthology.orchestra.run.trial163.stderr: 95.7% -- replaced with /var/log/ceph/cluster2-client.mirror.90967.log.gz 2026-02-18T08:53:43.595 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.36970.log 2026-02-18T08:53:43.596 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.23733.log: 95.8% -- replaced with /var/log/ceph/cluster2-client.mirror.55344.log.gz 2026-02-18T08:53:43.596 INFO:teuthology.orchestra.run.trial163.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.23733.log.gz 2026-02-18T08:53:43.599 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.135048.log 2026-02-18T08:53:43.601 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.15002.log 2026-02-18T08:53:43.601 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.36970.log: /var/log/ceph/cluster1-client.admin.135048.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.135048.log.gz 2026-02-18T08:53:43.601 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.46517.log 2026-02-18T08:53:43.601 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.133233.log 2026-02-18T08:53:43.601 INFO:teuthology.orchestra.run.trial163.stderr: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.36970.log.gz 2026-02-18T08:53:43.604 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.46517.log: /var/log/ceph/cluster1-client.mirror.15002.log: 80.8% -- replaced with /var/log/ceph/cluster1-client.mirror.15002.log.gz 2026-02-18T08:53:43.604 INFO:teuthology.orchestra.run.trial163.stderr: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.46517.log.gz 2026-02-18T08:53:43.604 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.38549.log 2026-02-18T08:53:43.604 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.60869.log 2026-02-18T08:53:43.604 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.133233.log: /var/log/ceph/cluster2-client.mirror.38549.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.133233.log.gz 2026-02-18T08:53:43.605 INFO:teuthology.orchestra.run.trial163.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.38549.log.gz 2026-02-18T08:53:43.605 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.20419.log 2026-02-18T08:53:43.605 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.60869.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.82846.log 2026-02-18T08:53:43.605 INFO:teuthology.orchestra.run.trial163.stderr: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.60869.log.gz 2026-02-18T08:53:43.606 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.20419.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.126127.log 2026-02-18T08:53:43.607 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.82846.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.20419.log.gz 2026-02-18T08:53:43.607 INFO:teuthology.orchestra.run.trial163.stderr: 89.2% -- replaced with /var/log/ceph/cluster2-client.mirror.82846.log.gz 2026-02-18T08:53:43.607 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.23919.log 2026-02-18T08:53:43.607 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.122133.log 2026-02-18T08:53:43.607 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.126127.log: /var/log/ceph/cluster2-client.mirror.23919.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.23919.log.gz 2026-02-18T08:53:43.607 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.37671.log 2026-02-18T08:53:43.608 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.122133.log: 90.7% -- replaced with /var/log/ceph/cluster2-client.mirror.126127.log.gz 2026-02-18T08:53:43.608 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.102671.log 2026-02-18T08:53:43.610 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.37671.log: 93.1% -- replaced with /var/log/ceph/cluster2-client.mirror.122133.log.gz 2026-02-18T08:53:43.610 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.131583.log 2026-02-18T08:53:43.611 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.102671.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.102671.log.gz 2026-02-18T08:53:43.611 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.78537.log 2026-02-18T08:53:43.612 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.131583.log: 83.6% -- replaced with /var/log/ceph/cluster2-client.mirror.131583.log.gz 2026-02-18T08:53:43.612 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.14114.log 2026-02-18T08:53:43.614 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.78537.log: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.78537.log.gz 2026-02-18T08:53:43.614 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.78957.log 2026-02-18T08:53:43.614 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.admin.14114.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.14114.log.gz 2026-02-18T08:53:43.614 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.96092.log 2026-02-18T08:53:43.615 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.78957.log: 89.2% -- replaced with /var/log/ceph/cluster2-client.mirror.78957.log.gz 2026-02-18T08:53:43.615 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.89170.log 2026-02-18T08:53:43.616 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.96092.log: 88.2% -- replaced with /var/log/ceph/cluster1-client.mirror.96092.log.gz 2026-02-18T08:53:43.616 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.25591.log 2026-02-18T08:53:43.624 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.89170.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.32198.log 2026-02-18T08:53:43.628 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.25591.log: 95.7% -- replaced with /var/log/ceph/cluster2-client.mirror.58898.log.gz 2026-02-18T08:53:43.634 INFO:teuthology.orchestra.run.trial163.stderr: 92.1% -- replaced with /var/log/ceph/cluster2-client.mirror.37671.log.gz 2026-02-18T08:53:43.635 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.114826.log 2026-02-18T08:53:43.635 INFO:teuthology.orchestra.run.trial163.stderr: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.25591.log.gz 2026-02-18T08:53:43.635 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.32198.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.19075.log 2026-02-18T08:53:43.636 INFO:teuthology.orchestra.run.trial163.stderr: 88.5%/var/log/ceph/cluster2-client.mirror.114826.log: -- replaced with /var/log/ceph/cluster2-client.mirror.32198.log.gz 2026-02-18T08:53:43.637 INFO:teuthology.orchestra.run.trial163.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.114826.log.gz 2026-02-18T08:53:43.637 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.50619.log 2026-02-18T08:53:43.637 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.19075.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.26391.log 2026-02-18T08:53:43.638 INFO:teuthology.orchestra.run.trial163.stderr: 88.3%/var/log/ceph/cluster2-client.mirror.50619.log: -- replaced with /var/log/ceph/cluster2-client.mirror.19075.log.gz 2026-02-18T08:53:43.638 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.65164.log 2026-02-18T08:53:43.638 INFO:teuthology.orchestra.run.trial163.stderr: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.50619.log.gz 2026-02-18T08:53:43.638 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.26391.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.26391.log.gz 2026-02-18T08:53:43.639 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.107640.log 2026-02-18T08:53:43.639 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.65164.log: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.65164.log.gz 2026-02-18T08:53:43.640 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.93605.log 2026-02-18T08:53:43.640 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.107640.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.58202.log 2026-02-18T08:53:43.641 INFO:teuthology.orchestra.run.trial163.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.107640.log.gz 2026-02-18T08:53:43.641 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.93605.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.93605.log.gzgzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.82006.log 2026-02-18T08:53:43.641 INFO:teuthology.orchestra.run.trial163.stderr: 2026-02-18T08:53:43.641 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.58202.log: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.58202.log.gz 2026-02-18T08:53:43.642 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.55090.log 2026-02-18T08:53:43.642 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.82006.log: 89.2% -- replaced with /var/log/ceph/cluster2-client.mirror.82006.log.gz 2026-02-18T08:53:43.642 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.132315.log 2026-02-18T08:53:43.643 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.55090.log: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.55090.log.gz 2026-02-18T08:53:43.643 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.68314.log 2026-02-18T08:53:43.644 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.132315.log: 91.4% -- replaced with /var/log/ceph/cluster2-client.mirror.132315.log.gz 2026-02-18T08:53:43.644 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.132106.log 2026-02-18T08:53:43.659 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.68314.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.84399.log 2026-02-18T08:53:43.660 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.132106.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.132106.log.gz 2026-02-18T08:53:43.661 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.115686.log 2026-02-18T08:53:43.662 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.84399.log: 95.7% -- replaced with /var/log/ceph/cluster2-client.mirror.99885.log.gz 2026-02-18T08:53:43.662 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.129137.log 2026-02-18T08:53:43.663 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.115686.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.115686.log.gz 2026-02-18T08:53:43.663 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.93186.log 2026-02-18T08:53:43.664 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.129137.log: 87.0% -- replaced with /var/log/ceph/cluster1-client.mirror.129137.log.gz 2026-02-18T08:53:43.664 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.28530.log 2026-02-18T08:53:43.665 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.93186.log: 92.5% -- replaced with /var/log/ceph/cluster1-client.mirror.93186.log.gz 2026-02-18T08:53:43.666 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.89231.log 2026-02-18T08:53:43.666 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.28530.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.28530.log.gz 2026-02-18T08:53:43.668 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.24820.log 2026-02-18T08:53:43.668 INFO:teuthology.orchestra.run.trial163.stderr: 92.5% -- replaced with /var/log/ceph/cluster2-client.mirror.84399.log.gz 2026-02-18T08:53:43.668 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.89231.log: 84.8% -- replaced with /var/log/ceph/cluster2-client.mirror.89231.log.gz 2026-02-18T08:53:43.668 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.99786.log 2026-02-18T08:53:43.668 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.43796.log 2026-02-18T08:53:43.669 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.24820.log: /var/log/ceph/cluster2-client.mirror.99786.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.1.90855.log 2026-02-18T08:53:43.669 INFO:teuthology.orchestra.run.trial163.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.99786.log.gz 2026-02-18T08:53:43.669 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.117704.log 2026-02-18T08:53:43.679 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.1.90855.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.117515.log 2026-02-18T08:53:43.680 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.117704.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.117704.log.gz 2026-02-18T08:53:43.688 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.43796.log: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.43796.log.gz 2026-02-18T08:53:43.689 INFO:teuthology.orchestra.run.trial163.stderr: 95.7% -- replaced with /var/log/ceph/cluster2-client.mirror.97545.log.gz 2026-02-18T08:53:43.691 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.27042.log 2026-02-18T08:53:43.692 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.117515.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.117515.log.gz 2026-02-18T08:53:43.707 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.102969.log 2026-02-18T08:53:43.709 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.27042.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.27042.log.gz 2026-02-18T08:53:43.709 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.56219.log 2026-02-18T08:53:43.709 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.102969.log: 88.3% -- replaced with /var/log/ceph/cluster1-client.mirror.102969.log.gz 2026-02-18T08:53:43.710 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.96025.log 2026-02-18T08:53:43.711 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.56219.log: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.56219.log.gz 2026-02-18T08:53:43.711 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.43982.log 2026-02-18T08:53:43.711 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.96025.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.96025.log.gz 2026-02-18T08:53:43.712 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.99581.log 2026-02-18T08:53:43.713 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.43982.log: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.43982.log.gz 2026-02-18T08:53:43.713 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.135002.log 2026-02-18T08:53:43.714 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.99581.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.99581.log.gz 2026-02-18T08:53:43.714 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.118071.log 2026-02-18T08:53:43.715 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.135002.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.mirror.135002.log.gz 2026-02-18T08:53:43.715 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.19110.log 2026-02-18T08:53:43.715 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.118071.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.118071.log.gz 2026-02-18T08:53:43.716 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.43610.log 2026-02-18T08:53:43.717 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.19110.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.19110.log.gz 2026-02-18T08:53:43.717 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.65998.log 2026-02-18T08:53:43.718 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.43610.log: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.43610.log.gz 2026-02-18T08:53:43.718 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.52719.log 2026-02-18T08:53:43.719 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.admin.65998.log: 89.2% -- replaced with /var/log/ceph/cluster1-client.admin.65998.log.gz 2026-02-18T08:53:43.719 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.78467.log 2026-02-18T08:53:43.719 INFO:teuthology.orchestra.run.trial163.stderr: 94.4% -- replaced with /var/log/ceph/cluster2-client.mirror.1.90855.log.gz 2026-02-18T08:53:43.719 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.52719.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.131837.log 2026-02-18T08:53:43.722 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.78467.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.52719.log.gz 2026-02-18T08:53:43.722 INFO:teuthology.orchestra.run.trial163.stderr: 89.2% -- replaced with /var/log/ceph/cluster2-client.mirror.78467.log.gz 2026-02-18T08:53:43.722 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.134734.log 2026-02-18T08:53:43.722 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.40595.log 2026-02-18T08:53:43.723 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.131837.log: /var/log/ceph/cluster1-client.mirror.134734.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.131837.log.gz 2026-02-18T08:53:43.723 INFO:teuthology.orchestra.run.trial163.stderr: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.134734.log.gz 2026-02-18T08:53:43.723 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.101804.log 2026-02-18T08:53:43.723 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.108211.log 2026-02-18T08:53:43.723 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.40595.log: /var/log/ceph/cluster2-client.mirror.101804.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.101804.log.gz 2026-02-18T08:53:43.723 INFO:teuthology.orchestra.run.trial163.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.40595.log.gz 2026-02-18T08:53:43.723 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.75636.log 2026-02-18T08:53:43.723 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.64460.log 2026-02-18T08:53:43.727 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.108211.log: /var/log/ceph/cluster2-client.mirror.75636.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.108211.log.gz 2026-02-18T08:53:43.727 INFO:teuthology.orchestra.run.trial163.stderr: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.75636.log.gz 2026-02-18T08:53:43.727 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.24477.log 2026-02-18T08:53:43.728 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.133994.log 2026-02-18T08:53:43.728 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.64460.log: /var/log/ceph/cluster2-client.mirror.24477.log: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.64460.log.gz 2026-02-18T08:53:43.728 INFO:teuthology.orchestra.run.trial163.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.24477.log.gz 2026-02-18T08:53:43.728 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.44633.log 2026-02-18T08:53:43.728 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.133994.log: 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.133994.log.gz 2026-02-18T08:53:43.728 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.20139.log 2026-02-18T08:53:43.728 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.68839.log 2026-02-18T08:53:43.728 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.44633.log: /var/log/ceph/cluster2-client.mirror.20139.log: 95.7% -- replaced with /var/log/ceph/cluster2-client.mirror.123058.log.gz 2026-02-18T08:53:43.728 INFO:teuthology.orchestra.run.trial163.stderr: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.44633.log.gz 2026-02-18T08:53:43.728 INFO:teuthology.orchestra.run.trial163.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.20139.log.gzgzip 2026-02-18T08:53:43.728 INFO:teuthology.orchestra.run.trial163.stderr: -5 --verbose -- /var/log/ceph/cluster2-client.mirror.57689.log 2026-02-18T08:53:43.728 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose --/var/log/ceph/cluster2-client.mirror.68839.log: /var/log/ceph/cluster2-client.mirror.121492.log 2026-02-18T08:53:43.728 INFO:teuthology.orchestra.run.trial163.stderr: 89.1%gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.99353.log 2026-02-18T08:53:43.729 INFO:teuthology.orchestra.run.trial163.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.68839.log.gz 2026-02-18T08:53:43.729 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.57689.log: /var/log/ceph/cluster2-client.mirror.121492.log: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.57689.log.gz 2026-02-18T08:53:43.729 INFO:teuthology.orchestra.run.trial163.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.121492.log.gz 2026-02-18T08:53:43.729 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.72514.log 2026-02-18T08:53:43.733 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.74548.log 2026-02-18T08:53:43.733 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.99353.log: /var/log/ceph/cluster2-client.mirror.72514.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.24012.log 2026-02-18T08:53:43.733 INFO:teuthology.orchestra.run.trial163.stderr: 88.1% -- replaced with /var/log/ceph/cluster1-client.mirror.99353.log.gz 2026-02-18T08:53:43.733 INFO:teuthology.orchestra.run.trial163.stderr: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.72514.log.gz 2026-02-18T08:53:43.733 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.74548.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.75881.log 2026-02-18T08:53:43.734 INFO:teuthology.orchestra.run.trial163.stderr: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.74548.log.gz 2026-02-18T08:53:43.734 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.24012.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.1.15025.log 2026-02-18T08:53:43.734 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.75881.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.59639.log 2026-02-18T08:53:43.734 INFO:teuthology.orchestra.run.trial163.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.24012.log.gz 2026-02-18T08:53:43.734 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.1.15025.log: 88.9% -- replaced with /var/log/ceph/cluster1-client.mirror.75881.log.gz 2026-02-18T08:53:43.734 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.35550.log 2026-02-18T08:53:43.734 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.59639.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.126817.log 2026-02-18T08:53:43.734 INFO:teuthology.orchestra.run.trial163.stderr: 88.7%/var/log/ceph/cluster2-client.mirror.35550.log: -- replaced with /var/log/ceph/cluster2-client.mirror.59639.log.gz 2026-02-18T08:53:43.734 INFO:teuthology.orchestra.run.trial163.stderr: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.35550.log.gz 2026-02-18T08:53:43.734 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.99234.log 2026-02-18T08:53:43.734 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.0.83316.log 2026-02-18T08:53:43.734 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.126817.log: /var/log/ceph/cluster2-client.mirror.99234.log: 88.2% 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.126817.log.gz 2026-02-18T08:53:43.734 INFO:teuthology.orchestra.run.trial163.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.99234.log.gz 2026-02-18T08:53:43.735 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.124357.log 2026-02-18T08:53:43.735 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.14497.log 2026-02-18T08:53:43.735 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.0.83316.log: /var/log/ceph/cluster2-client.mirror.124357.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.124357.log.gz 2026-02-18T08:53:43.735 INFO:teuthology.orchestra.run.trial163.stderr: 92.4% -- replaced with /var/log/ceph/cluster2-client.mirror.1.15025.log.gz 2026-02-18T08:53:43.736 INFO:teuthology.orchestra.run.trial163.stderr: 92.6% -- replaced with /var/log/ceph/cluster2-client.mirror.0.83316.log.gz 2026-02-18T08:53:43.736 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.112001.log 2026-02-18T08:53:43.736 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.admin.14497.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.44226.log 2026-02-18T08:53:43.736 INFO:teuthology.orchestra.run.trial163.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.14497.log.gz 2026-02-18T08:53:43.737 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.112001.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.85688.log 2026-02-18T08:53:43.737 INFO:teuthology.orchestra.run.trial163.stderr: 88.0%/var/log/ceph/cluster2-client.mirror.44226.log: -- replaced with /var/log/ceph/cluster2-client.mirror.112001.log.gz 2026-02-18T08:53:43.737 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.96603.log 2026-02-18T08:53:43.737 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.107977.log 2026-02-18T08:53:43.739 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.85688.log: /var/log/ceph/cluster2-client.mirror.96603.log: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.44226.log.gz 2026-02-18T08:53:43.739 INFO:teuthology.orchestra.run.trial163.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.96603.log.gz 2026-02-18T08:53:43.739 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.68114.log 2026-02-18T08:53:43.739 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.43331.log 2026-02-18T08:53:43.739 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.107977.log: /var/log/ceph/cluster2-client.mirror.68114.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.107977.log.gz 2026-02-18T08:53:43.739 INFO:teuthology.orchestra.run.trial163.stderr: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.68114.log.gz 2026-02-18T08:53:43.739 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.11503.log 2026-02-18T08:53:43.740 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.117738.log 2026-02-18T08:53:43.740 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.43331.log: /var/log/ceph/cluster2-client.admin.11503.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.11503.log.gz 2026-02-18T08:53:43.740 INFO:teuthology.orchestra.run.trial163.stderr: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.43331.log.gz 2026-02-18T08:53:43.747 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.47570.log 2026-02-18T08:53:43.748 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.117738.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.105485.log 2026-02-18T08:53:43.749 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.47570.log: 94.3% -- replaced with /var/log/ceph/cluster1-client.mirror.117738.log.gz 2026-02-18T08:53:43.749 INFO:teuthology.orchestra.run.trial163.stderr: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.47570.log.gz 2026-02-18T08:53:43.749 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.69994.log 2026-02-18T08:53:43.750 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.132744.log 2026-02-18T08:53:43.750 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.105485.log: /var/log/ceph/cluster2-client.mirror.69994.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.105485.log.gz 2026-02-18T08:53:43.750 INFO:teuthology.orchestra.run.trial163.stderr: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.69994.log.gz 2026-02-18T08:53:43.751 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.42329.log 2026-02-18T08:53:43.751 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.131411.log 2026-02-18T08:53:43.751 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.132744.log: 86.8% -- replaced with /var/log/ceph/cluster1-client.mirror.132744.log.gz 2026-02-18T08:53:43.752 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.42329.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.14627.log 2026-02-18T08:53:43.752 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.131411.log: 86.8% -- replaced with /var/log/ceph/cluster1-client.mirror.131411.log.gz 2026-02-18T08:53:43.752 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.77802.log 2026-02-18T08:53:43.752 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.admin.14627.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.14627.log.gz 2026-02-18T08:53:43.752 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.133932.log 2026-02-18T08:53:43.753 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.77802.log: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.77802.log.gz 2026-02-18T08:53:43.753 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.95090.log 2026-02-18T08:53:43.754 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.133932.log: 93.4% -- replaced with /var/log/ceph/cluster1-client.mirror.42329.log.gz 2026-02-18T08:53:43.754 INFO:teuthology.orchestra.run.trial163.stderr: 91.8% -- replaced with /var/log/ceph/cluster2-client.mirror.133932.log.gz 2026-02-18T08:53:43.755 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.116757.log 2026-02-18T08:53:43.756 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.102706.log 2026-02-18T08:53:43.756 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.95090.log: /var/log/ceph/cluster2-client.mirror.116757.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.95090.log.gz 2026-02-18T08:53:43.756 INFO:teuthology.orchestra.run.trial163.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.116757.log.gz 2026-02-18T08:53:43.757 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.133870.log 2026-02-18T08:53:43.757 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.29402.log 2026-02-18T08:53:43.757 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.102706.log: /var/log/ceph/cluster1-client.mirror.133870.log: 87.0% 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.102706.log.gz 2026-02-18T08:53:43.757 INFO:teuthology.orchestra.run.trial163.stderr: -- replaced with /var/log/ceph/cluster1-client.mirror.133870.log.gz 2026-02-18T08:53:43.757 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.87704.log 2026-02-18T08:53:43.758 INFO:teuthology.orchestra.run.trial163.stderr: 95.7% -- replaced with /var/log/ceph/cluster2-client.mirror.132894.log.gz 2026-02-18T08:53:43.761 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.77277.log 2026-02-18T08:53:43.761 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.29402.log: /var/log/ceph/cluster2-client.mirror.87704.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.112533.log 2026-02-18T08:53:43.761 INFO:teuthology.orchestra.run.trial163.stderr: 84.7% -- replaced with /var/log/ceph/cluster2-client.mirror.87704.log.gz 2026-02-18T08:53:43.761 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.77277.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.29402.log.gz 2026-02-18T08:53:43.761 INFO:teuthology.orchestra.run.trial163.stderr: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.77277.log.gz 2026-02-18T08:53:43.761 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.19482.log 2026-02-18T08:53:43.761 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.24663.log 2026-02-18T08:53:43.761 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.112533.log: /var/log/ceph/cluster2-client.mirror.19482.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.35178.log 2026-02-18T08:53:43.761 INFO:teuthology.orchestra.run.trial163.stderr: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.112533.log.gz 2026-02-18T08:53:43.761 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.24663.log: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.19482.log.gz 2026-02-18T08:53:43.762 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.135341.log 2026-02-18T08:53:43.762 INFO:teuthology.orchestra.run.trial163.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.24663.log.gz 2026-02-18T08:53:43.762 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.35178.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.84620.log 2026-02-18T08:53:43.762 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.129902.log 2026-02-18T08:53:43.762 INFO:teuthology.orchestra.run.trial163.stderr: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.35178.log.gz 2026-02-18T08:53:43.762 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.admin.135341.log: /var/log/ceph/cluster2-client.mirror.84620.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.135341.log.gz 2026-02-18T08:53:43.763 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.26612.log 2026-02-18T08:53:43.763 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.129902.log: 87.0% -- replaced with /var/log/ceph/cluster1-client.mirror.129902.log.gz 2026-02-18T08:53:43.764 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.45577.log 2026-02-18T08:53:43.764 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.26612.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.26612.log.gz 2026-02-18T08:53:43.764 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.62360.log 2026-02-18T08:53:43.765 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.45577.log: 93.5% -- replaced with /var/log/ceph/cluster2-client.mirror.84620.log.gz 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.45577.log.gz 2026-02-18T08:53:43.765 INFO:teuthology.orchestra.run.trial163.stderr: 2026-02-18T08:53:43.765 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.129294.log 2026-02-18T08:53:43.765 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.129227.log 2026-02-18T08:53:43.765 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.62360.log: /var/log/ceph/cluster1-client.mirror.129294.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.122099.log 2026-02-18T08:53:43.766 INFO:teuthology.orchestra.run.trial163.stderr: 87.0% -- replaced with /var/log/ceph/cluster1-client.mirror.129294.log.gz 2026-02-18T08:53:43.766 INFO:teuthology.orchestra.run.trial163.stderr: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.62360.log.gz 2026-02-18T08:53:43.766 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.129227.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.129407.log 2026-02-18T08:53:43.766 INFO:teuthology.orchestra.run.trial163.stderr: 87.0% -- replaced with /var/log/ceph/cluster1-client.mirror.129227.log.gz 2026-02-18T08:53:43.766 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.122099.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.39630.log 2026-02-18T08:53:43.767 INFO:teuthology.orchestra.run.trial163.stderr: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.122099.log.gz 2026-02-18T08:53:43.767 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.32756.log 2026-02-18T08:53:43.767 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.129407.log: /var/log/ceph/cluster2-client.mirror.39630.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.85564.log 2026-02-18T08:53:43.768 INFO:teuthology.orchestra.run.trial163.stderr: 87.0% -- replaced with /var/log/ceph/cluster1-client.mirror.129407.log.gz 2026-02-18T08:53:43.768 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.32756.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.39630.log.gz 2026-02-18T08:53:43.768 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.77382.log 2026-02-18T08:53:43.768 INFO:teuthology.orchestra.run.trial163.stderr: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.32756.log.gz 2026-02-18T08:53:43.768 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.85564.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.93694.log 2026-02-18T08:53:43.768 INFO:teuthology.orchestra.run.trial163.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.85564.log.gz 2026-02-18T08:53:43.769 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.77382.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.55939.log 2026-02-18T08:53:43.769 INFO:teuthology.orchestra.run.trial163.stderr: 88.9%/var/log/ceph/cluster2-client.mirror.93694.log: -- replaced with /var/log/ceph/cluster2-client.mirror.77382.log.gz 2026-02-18T08:53:43.770 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.69784.log 2026-02-18T08:53:43.770 INFO:teuthology.orchestra.run.trial163.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.93694.log.gz/var/log/ceph/cluster2-client.mirror.55939.log: 2026-02-18T08:53:43.770 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.42866.log 2026-02-18T08:53:43.770 INFO:teuthology.orchestra.run.trial163.stderr: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.55939.log.gz 2026-02-18T08:53:43.770 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.44968.log 2026-02-18T08:53:43.770 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.69784.log: /var/log/ceph/cluster2-client.mirror.42866.log: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.69784.log.gz 2026-02-18T08:53:43.770 INFO:teuthology.orchestra.run.trial163.stderr: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.42866.log.gz 2026-02-18T08:53:43.770 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.25649.log 2026-02-18T08:53:43.771 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.44968.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.128641.log 2026-02-18T08:53:43.771 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.25649.log: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.44968.log.gz 2026-02-18T08:53:43.771 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.133388.log 2026-02-18T08:53:43.771 INFO:teuthology.orchestra.run.trial163.stderr: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.25649.log.gz 2026-02-18T08:53:43.771 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.128641.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.74723.log 2026-02-18T08:53:43.771 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.133388.log: 89.3% -- replaced with /var/log/ceph/cluster2-client.mirror.128641.log.gz 2026-02-18T08:53:43.772 INFO:teuthology.orchestra.run.trial163.stderr: 87.6% -- replaced with /var/log/ceph/cluster1-client.mirror.133388.log.gz 2026-02-18T08:53:43.772 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.97321.log 2026-02-18T08:53:43.772 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.74723.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.121181.log 2026-02-18T08:53:43.772 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.97321.log: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.74723.log.gz 2026-02-18T08:53:43.773 INFO:teuthology.orchestra.run.trial163.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.97321.log.gz 2026-02-18T08:53:43.773 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.40409.log 2026-02-18T08:53:43.773 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.121181.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.84815.log 2026-02-18T08:53:43.774 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.40409.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.93028.log 2026-02-18T08:53:43.774 INFO:teuthology.orchestra.run.trial163.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.121181.log.gz 2026-02-18T08:53:43.774 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.84815.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.40409.log.gz 2026-02-18T08:53:43.777 INFO:teuthology.orchestra.run.trial163.stderr: 93.5% -- replaced with /var/log/ceph/cluster1-client.mirror.84815.log.gz 2026-02-18T08:53:43.777 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.71394.log 2026-02-18T08:53:43.777 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.93028.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.63165.log 2026-02-18T08:53:43.779 INFO:teuthology.orchestra.run.trial163.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.93028.log.gz 2026-02-18T08:53:43.779 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.71394.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.86666.log 2026-02-18T08:53:43.779 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose 89.1% -- -- replaced with /var/log/ceph/cluster2-client.mirror.71394.log.gz /var/log/ceph/cluster1-client.mirror.129789.log 2026-02-18T08:53:43.779 INFO:teuthology.orchestra.run.trial163.stderr: 2026-02-18T08:53:43.779 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.63165.log: /var/log/ceph/cluster2-client.mirror.86666.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.134651.log 2026-02-18T08:53:43.780 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.129789.log: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.63165.log.gz 2026-02-18T08:53:43.780 INFO:teuthology.orchestra.run.trial163.stderr: 86.7% -- replaced with /var/log/ceph/cluster1-client.mirror.129789.log.gz 2026-02-18T08:53:43.780 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.118828.log 2026-02-18T08:53:43.780 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.134651.log: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.134651.log.gz 2026-02-18T08:53:43.781 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.60549.log 2026-02-18T08:53:43.791 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.118828.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.110357.log 2026-02-18T08:53:43.792 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.60549.log: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.60549.log.gz 2026-02-18T08:53:43.793 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.104901.log 2026-02-18T08:53:43.793 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.110357.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.110357.log.gz 2026-02-18T08:53:43.794 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.108922.log 2026-02-18T08:53:43.794 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.104901.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.104901.log.gz 2026-02-18T08:53:43.795 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.15052.log 2026-02-18T08:53:43.795 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.108922.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.108922.log.gz 2026-02-18T08:53:43.796 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.31547.log 2026-02-18T08:53:43.799 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.15052.log: 94.8% -- replaced with /var/log/ceph/cluster1-client.mirror.15052.log.gz 2026-02-18T08:53:43.800 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.89443.log 2026-02-18T08:53:43.800 INFO:teuthology.orchestra.run.trial163.stderr: 95.7%/var/log/ceph/cluster2-client.mirror.31547.log: -- replaced with /var/log/ceph/cluster2-client.mirror.121725.log.gz 2026-02-18T08:53:43.800 INFO:teuthology.orchestra.run.trial163.stderr: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.31547.log.gz 2026-02-18T08:53:43.801 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.48690.log 2026-02-18T08:53:43.801 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.89443.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.24948.log 2026-02-18T08:53:43.802 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.48690.log: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.48690.log.gz 2026-02-18T08:53:43.802 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.125471.log 2026-02-18T08:53:43.803 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.24948.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.97503.log 2026-02-18T08:53:43.809 INFO:teuthology.orchestra.run.trial163.stderr: 95.7% -- replaced with /var/log/ceph/cluster2-client.mirror.108111.log.gz 2026-02-18T08:53:43.811 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.125471.log: 90.8% -- replaced with /var/log/ceph/cluster2-client.mirror.125471.log.gz 2026-02-18T08:53:43.811 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.63690.log 2026-02-18T08:53:43.813 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.97503.log: 92.2% -- replaced with /var/log/ceph/cluster2-client.mirror.97503.log.gz 2026-02-18T08:53:43.813 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.51249.log 2026-02-18T08:53:43.814 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.63690.log: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.63690.log.gz 2026-02-18T08:53:43.814 INFO:teuthology.orchestra.run.trial163.stderr: 92.0% -- replaced with /var/log/ceph/cluster2-client.mirror.86666.log.gz 2026-02-18T08:53:43.814 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.107539.log 2026-02-18T08:53:43.815 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.51249.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.73389.log 2026-02-18T08:53:43.815 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.107539.log: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.51249.log.gz 2026-02-18T08:53:43.815 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.68387.log 2026-02-18T08:53:43.816 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.73389.log: 89.1% 92.4% -- replaced with /var/log/ceph/cluster1-client.mirror.107539.log.gz 2026-02-18T08:53:43.816 INFO:teuthology.orchestra.run.trial163.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.73389.log.gz 2026-02-18T08:53:43.817 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.53139.log 2026-02-18T08:53:43.817 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.admin.68387.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.132027.log 2026-02-18T08:53:43.817 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.53139.log: 89.1% -- replaced with /var/log/ceph/cluster1-client.admin.68387.log.gz 2026-02-18T08:53:43.817 INFO:teuthology.orchestra.run.trial163.stderr: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.53139.log.gz 2026-02-18T08:53:43.818 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.17401.log 2026-02-18T08:53:43.818 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.132027.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.27356.log 2026-02-18T08:53:43.819 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.17401.log: 88.0% 91.2% -- replaced with /var/log/ceph/cluster2-client.mirror.132027.log.gz 2026-02-18T08:53:43.819 INFO:teuthology.orchestra.run.trial163.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.17401.log.gz 2026-02-18T08:53:43.819 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.30930.log 2026-02-18T08:53:43.819 INFO:teuthology.orchestra.run.trial163.stderr: 95.7% -- replaced with /var/log/ceph/cluster2-client.mirror.87727.log.gz 2026-02-18T08:53:43.820 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.27356.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.86346.log 2026-02-18T08:53:43.820 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.30930.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.110139.log 2026-02-18T08:53:43.820 INFO:teuthology.orchestra.run.trial163.stderr: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.27356.log.gz 2026-02-18T08:53:43.820 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.86346.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.30930.log.gz 2026-02-18T08:53:43.821 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.45701.log 2026-02-18T08:53:43.822 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.110139.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.60479.log 2026-02-18T08:53:43.822 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.45701.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.110139.log.gz 2026-02-18T08:53:43.822 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.31954.log 2026-02-18T08:53:43.822 INFO:teuthology.orchestra.run.trial163.stderr: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.45701.log.gz 2026-02-18T08:53:43.822 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.60479.log: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.60479.log.gz 2026-02-18T08:53:43.823 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.120487.log 2026-02-18T08:53:43.823 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.31954.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.81726.log 2026-02-18T08:53:43.823 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.120487.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.31954.log.gz 2026-02-18T08:53:43.823 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.109333.log 2026-02-18T08:53:43.824 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.81726.log: 92.1% 89.2% -- replaced with /var/log/ceph/cluster2-client.mirror.81726.log.gz 2026-02-18T08:53:43.824 INFO:teuthology.orchestra.run.trial163.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.120487.log.gz 2026-02-18T08:53:43.835 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.96395.log 2026-02-18T08:53:43.836 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.19825.log 2026-02-18T08:53:43.837 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.109333.log: /var/log/ceph/cluster2-client.mirror.96395.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.109333.log.gz 2026-02-18T08:53:43.837 INFO:teuthology.orchestra.run.trial163.stderr: 87.4% -- replaced with /var/log/ceph/cluster2-client.mirror.96395.log.gz 2026-02-18T08:53:43.837 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.95448.log 2026-02-18T08:53:43.837 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.19825.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.84248.log 2026-02-18T08:53:43.837 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.95448.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.19825.log.gz 2026-02-18T08:53:43.838 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.31826.log 2026-02-18T08:53:43.841 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.84248.log: 93.0% -- replaced with /var/log/ceph/cluster1-client.mirror.84248.log.gz 2026-02-18T08:53:43.842 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.98244.log 2026-02-18T08:53:43.842 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.31826.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.31826.log.gz 2026-02-18T08:53:43.843 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.94170.log 2026-02-18T08:53:43.843 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.98244.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.98244.log.gz 2026-02-18T08:53:43.843 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.45523.log 2026-02-18T08:53:43.845 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.94170.log: 92.6% -- replaced with /var/log/ceph/cluster1-client.mirror.94170.log.gz 2026-02-18T08:53:43.845 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.70764.log 2026-02-18T08:53:43.846 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.45523.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.45523.log.gz 2026-02-18T08:53:43.846 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.35027.log 2026-02-18T08:53:43.847 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.70764.log: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.70764.log.gz 2026-02-18T08:53:43.847 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.118339.log 2026-02-18T08:53:43.847 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.35027.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.55279.log 2026-02-18T08:53:43.848 INFO:teuthology.orchestra.run.trial163.stderr: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.35027.log.gz 2026-02-18T08:53:43.848 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.118339.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.118339.log.gz 2026-02-18T08:53:43.849 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.26021.log 2026-02-18T08:53:43.849 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.55279.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.19767.log 2026-02-18T08:53:43.851 INFO:teuthology.orchestra.run.trial163.stderr: 95.7% -- replaced with /var/log/ceph/cluster2-client.mirror.87992.log.gz 2026-02-18T08:53:43.851 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.44261.log 2026-02-18T08:53:43.852 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.19767.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.19767.log.gz 2026-02-18T08:53:43.852 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.42309.log 2026-02-18T08:53:43.853 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.44261.log: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.44261.log.gz 2026-02-18T08:53:43.853 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.19296.log 2026-02-18T08:53:43.853 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.42309.log: 93.5% -- replaced with /var/log/ceph/cluster2-client.mirror.55279.log.gz 2026-02-18T08:53:43.854 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.26021.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.129339.log 2026-02-18T08:53:43.854 INFO:teuthology.orchestra.run.trial163.stderr: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.26021.log.gz 2026-02-18T08:53:43.855 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.80742.log 2026-02-18T08:53:43.855 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.129339.log: 86.8% -- replaced with /var/log/ceph/cluster1-client.mirror.129339.log.gz 2026-02-18T08:53:43.856 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.76437.log 2026-02-18T08:53:43.856 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.80742.log: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.80742.log.gz 2026-02-18T08:53:43.857 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.34992.log 2026-02-18T08:53:43.857 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.76437.log: 89.2% -- replaced with /var/log/ceph/cluster2-client.mirror.76437.log.gz 2026-02-18T08:53:43.858 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.17959.log 2026-02-18T08:53:43.858 INFO:teuthology.orchestra.run.trial163.stderr: 93.4% -- replaced with /var/log/ceph/cluster2-client.mirror.42309.log.gz 2026-02-18T08:53:43.859 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.34992.log: /var/log/ceph/cluster2-client.mirror.19296.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.81586.log 2026-02-18T08:53:43.859 INFO:teuthology.orchestra.run.trial163.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.19296.log.gz 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.34992.log.gz 2026-02-18T08:53:43.859 INFO:teuthology.orchestra.run.trial163.stderr: 2026-02-18T08:53:43.860 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.17959.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.84901.log 2026-02-18T08:53:43.860 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.81586.log: 89.2% -- replaced with /var/log/ceph/cluster2-client.mirror.81586.log.gz 2026-02-18T08:53:43.860 INFO:teuthology.orchestra.run.trial163.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.17959.log.gz 2026-02-18T08:53:43.860 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.131772.log 2026-02-18T08:53:43.861 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.95349.log 2026-02-18T08:53:43.861 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.84901.log: /var/log/ceph/cluster1-client.mirror.131772.log: 86.8% -- replaced with /var/log/ceph/cluster1-client.mirror.131772.log.gz 2026-02-18T08:53:43.861 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.109100.log 2026-02-18T08:53:43.861 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.95349.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.95349.log.gz 2026-02-18T08:53:43.862 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.94274.log 2026-02-18T08:53:43.862 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.109100.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.109100.log.gz 2026-02-18T08:53:43.863 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.48935.log 2026-02-18T08:53:43.863 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.94274.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.94274.log.gz 2026-02-18T08:53:43.863 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.15463.log 2026-02-18T08:53:43.864 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.48935.log: 93.5% -- replaced with /var/log/ceph/cluster1-client.mirror.84901.log.gz 2026-02-18T08:53:43.864 INFO:teuthology.orchestra.run.trial163.stderr: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.48935.log.gz 2026-02-18T08:53:43.865 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.67694.log 2026-02-18T08:53:43.865 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.admin.15463.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.85269.log 2026-02-18T08:53:43.865 INFO:teuthology.orchestra.run.trial163.stderr: 80.1% -- replaced with /var/log/ceph/cluster2-client.admin.15463.log.gz 2026-02-18T08:53:43.866 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.67694.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.89934.log 2026-02-18T08:53:43.866 INFO:teuthology.orchestra.run.trial163.stderr: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.67694.log.gz/var/log/ceph/cluster2-client.mirror.85269.log: 2026-02-18T08:53:43.866 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.20699.log 2026-02-18T08:53:43.866 INFO:teuthology.orchestra.run.trial163.stderr: 89.4% -- replaced with /var/log/ceph/cluster2-client.mirror.85269.log.gz 2026-02-18T08:53:43.866 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.89934.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.29681.log 2026-02-18T08:53:43.867 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.20699.log: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.20699.log.gz 2026-02-18T08:53:43.868 INFO:teuthology.orchestra.run.trial163.stderr: 95.7% -- replaced with /var/log/ceph/cluster2-client.mirror.134020.log.gz 2026-02-18T08:53:43.871 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.49880.log 2026-02-18T08:53:43.872 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.29681.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.32233.log 2026-02-18T08:53:43.872 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.49880.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.29681.log.gz 2026-02-18T08:53:43.873 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.83538.log 2026-02-18T08:53:43.873 INFO:teuthology.orchestra.run.trial163.stderr: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.49880.log.gz 2026-02-18T08:53:43.873 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.32233.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.14526.log 2026-02-18T08:53:43.874 INFO:teuthology.orchestra.run.trial163.stderr: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.32233.log.gz 2026-02-18T08:53:43.874 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.83538.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.75991.log 2026-02-18T08:53:43.875 INFO:teuthology.orchestra.run.trial163.stderr: 89.3% -- replaced with /var/log/ceph/cluster2-client.mirror.83538.log.gz/var/log/ceph/cluster2-client.admin.14526.log: 2026-02-18T08:53:43.875 INFO:teuthology.orchestra.run.trial163.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.14526.log.gz 2026-02-18T08:53:43.875 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.126440.log 2026-02-18T08:53:43.875 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.36480.log 2026-02-18T08:53:43.876 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.75991.log: /var/log/ceph/cluster2-client.mirror.126440.log: 85.2% -- replaced with /var/log/ceph/cluster1-client.mirror.75991.log.gz 2026-02-18T08:53:43.876 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.14575.log 2026-02-18T08:53:43.876 INFO:teuthology.orchestra.run.trial163.stderr: 89.2% -- replaced with /var/log/ceph/cluster2-client.mirror.126440.log.gz 2026-02-18T08:53:43.876 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.36480.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.36480.log.gz 2026-02-18T08:53:43.877 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.121769.log 2026-02-18T08:53:43.877 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.124169.log 2026-02-18T08:53:43.877 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.admin.14575.log: /var/log/ceph/cluster2-client.mirror.121769.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.14575.log.gz 2026-02-18T08:53:43.877 INFO:teuthology.orchestra.run.trial163.stderr: 87.3% -- replaced with /var/log/ceph/cluster2-client.mirror.121769.log.gz 2026-02-18T08:53:43.877 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.127504.log 2026-02-18T08:53:43.878 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.26149.log 2026-02-18T08:53:43.878 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.124169.log: /var/log/ceph/cluster2-client.mirror.127504.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.124169.log.gz 2026-02-18T08:53:43.878 INFO:teuthology.orchestra.run.trial163.stderr: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.127504.log.gz 2026-02-18T08:53:43.878 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.126269.log 2026-02-18T08:53:43.878 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.120529.log 2026-02-18T08:53:43.879 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.26149.log: /var/log/ceph/cluster2-client.mirror.126269.log: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.26149.log.gz 2026-02-18T08:53:43.879 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.15241.log 2026-02-18T08:53:43.880 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.120529.log: 90.8% -- replaced with /var/log/ceph/cluster2-client.mirror.126269.log.gz 2026-02-18T08:53:43.883 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.96211.log 2026-02-18T08:53:43.884 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.admin.15241.log: 73.2% -- replaced with /var/log/ceph/cluster1-client.admin.15241.log.gz 2026-02-18T08:53:43.884 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.75143.log 2026-02-18T08:53:43.900 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.96211.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.122344.log 2026-02-18T08:53:43.900 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.75143.log: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.75143.log.gz 2026-02-18T08:53:43.908 INFO:teuthology.orchestra.run.trial163.stderr: 95.7% -- replaced with /var/log/ceph/cluster2-client.mirror.92526.log.gz 2026-02-18T08:53:43.908 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.52404.log 2026-02-18T08:53:43.909 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.122344.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.122344.log.gz 2026-02-18T08:53:43.911 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.72969.log 2026-02-18T08:53:43.911 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.52404.log: 95.8% -- replaced with /var/log/ceph/cluster2-client.mirror.131650.log.gz 2026-02-18T08:53:43.911 INFO:teuthology.orchestra.run.trial163.stderr: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.52404.log.gz 2026-02-18T08:53:43.911 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.54908.log 2026-02-18T08:53:43.911 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.72969.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.48060.log 2026-02-18T08:53:43.911 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.54908.log: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.72969.log.gz 2026-02-18T08:53:43.915 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.43296.log 2026-02-18T08:53:43.916 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.48060.log: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.54908.log.gz 2026-02-18T08:53:43.916 INFO:teuthology.orchestra.run.trial163.stderr: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.48060.log.gz 2026-02-18T08:53:43.916 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.127862.log 2026-02-18T08:53:43.916 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.43296.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.20792.log 2026-02-18T08:53:43.916 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.127862.log: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.43296.log.gz 2026-02-18T08:53:43.916 INFO:teuthology.orchestra.run.trial163.stderr: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.127862.log.gz 2026-02-18T08:53:43.916 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.45790.log 2026-02-18T08:53:43.916 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.20792.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.66570.log 2026-02-18T08:53:43.916 INFO:teuthology.orchestra.run.trial163.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.20792.log.gzgzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.31582.log 2026-02-18T08:53:43.916 INFO:teuthology.orchestra.run.trial163.stderr: 2026-02-18T08:53:43.917 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.66570.log: /var/log/ceph/cluster2-client.mirror.45790.log: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.66570.log.gz 2026-02-18T08:53:43.917 INFO:teuthology.orchestra.run.trial163.stderr: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.45790.log.gz 2026-02-18T08:53:43.917 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.55236.log 2026-02-18T08:53:43.917 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.31582.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.129587.log 2026-02-18T08:53:43.917 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.55236.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.48480.log 2026-02-18T08:53:43.917 INFO:teuthology.orchestra.run.trial163.stderr: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.31582.log.gz 2026-02-18T08:53:43.917 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.117319.log 2026-02-18T08:53:43.920 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.129587.log: 86.7% -- replaced with /var/log/ceph/cluster1-client.mirror.129587.log.gz 2026-02-18T08:53:43.920 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.48480.log: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.48480.log.gz 2026-02-18T08:53:43.920 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.43947.log 2026-02-18T08:53:43.920 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.117319.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.117319.log.gz 2026-02-18T08:53:43.920 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.80672.log 2026-02-18T08:53:43.920 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.43947.log: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.43947.log.gz 2026-02-18T08:53:43.920 INFO:teuthology.orchestra.run.trial163.stderr: 93.5%gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.97523.log 2026-02-18T08:53:43.920 INFO:teuthology.orchestra.run.trial163.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.55236.log.gz 2026-02-18T08:53:43.920 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.80672.log: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.80672.log.gz 2026-02-18T08:53:43.920 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.119214.log 2026-02-18T08:53:43.921 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.97523.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.31489.log 2026-02-18T08:53:43.922 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.119214.log: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.119214.log.gz 2026-02-18T08:53:43.923 INFO:teuthology.orchestra.run.trial163.stderr: 92.1% -- replaced with /var/log/ceph/cluster1-client.mirror.97523.log.gz 2026-02-18T08:53:43.923 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.103662.log 2026-02-18T08:53:43.923 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.31489.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.37898.log 2026-02-18T08:53:43.923 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.103662.log: 88.2% 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.31489.log.gz 2026-02-18T08:53:43.923 INFO:teuthology.orchestra.run.trial163.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.103662.log.gz 2026-02-18T08:53:43.923 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.111706.log 2026-02-18T08:53:43.923 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.37898.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.98482.log 2026-02-18T08:53:43.924 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.111706.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.37898.log.gz 2026-02-18T08:53:43.924 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.81621.log 2026-02-18T08:53:43.924 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.98482.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.98482.log.gz 2026-02-18T08:53:43.924 INFO:teuthology.orchestra.run.trial163.stderr: 92.0% -- replaced with /var/log/ceph/cluster1-client.mirror.111706.log.gz 2026-02-18T08:53:43.925 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.38421.log 2026-02-18T08:53:43.925 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.81621.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.128683.log 2026-02-18T08:53:43.925 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.38421.log: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.81621.log.gz 2026-02-18T08:53:43.926 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.129677.log 2026-02-18T08:53:43.926 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.128683.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.38421.log.gz 2026-02-18T08:53:43.926 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.78747.log 2026-02-18T08:53:43.926 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.129677.log: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.129677.log.gz 2026-02-18T08:53:43.926 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.41025.log 2026-02-18T08:53:43.926 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.78747.log: 90.8% -- replaced with /var/log/ceph/cluster2-client.mirror.128683.log.gz 2026-02-18T08:53:43.927 INFO:teuthology.orchestra.run.trial163.stderr: 89.2%gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.119268.log 2026-02-18T08:53:43.927 INFO:teuthology.orchestra.run.trial163.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.78747.log.gz 2026-02-18T08:53:43.928 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.41025.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.41025.log.gz 2026-02-18T08:53:43.928 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.132501.log 2026-02-18T08:53:43.928 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.119268.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.56464.log 2026-02-18T08:53:43.928 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.132501.log: 87.9% -- replaced with /var/log/ceph/cluster2-client.mirror.119268.log.gz 2026-02-18T08:53:43.929 INFO:teuthology.orchestra.run.trial163.stderr: 88.4%gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.114273.log 2026-02-18T08:53:43.929 INFO:teuthology.orchestra.run.trial163.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.132501.log.gz 2026-02-18T08:53:43.929 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.56464.log: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.56464.log.gz 2026-02-18T08:53:43.930 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.42476.log 2026-02-18T08:53:43.930 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.114273.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.58148.log 2026-02-18T08:53:43.930 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.admin.42476.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.114273.log.gz 2026-02-18T08:53:43.930 INFO:teuthology.orchestra.run.trial163.stderr: 87.9% -- replaced with /var/log/ceph/cluster1-client.admin.42476.log.gz 2026-02-18T08:53:43.940 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.129542.log 2026-02-18T08:53:43.941 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.58148.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.43889.log 2026-02-18T08:53:43.941 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.129542.log: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.58148.log.gz 2026-02-18T08:53:43.942 INFO:teuthology.orchestra.run.trial163.stderr: 87.0% -- replaced with /var/log/ceph/cluster1-client.mirror.129542.log.gz 2026-02-18T08:53:43.942 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.48515.log 2026-02-18T08:53:43.942 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.43889.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.61310.log 2026-02-18T08:53:43.943 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.48515.log: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.43889.log.gz 2026-02-18T08:53:43.943 INFO:teuthology.orchestra.run.trial163.stderr: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.48515.log.gz 2026-02-18T08:53:43.943 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.119360.log 2026-02-18T08:53:43.943 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.61310.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.31919.log 2026-02-18T08:53:43.943 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.119360.log: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.61310.log.gz 2026-02-18T08:53:43.944 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.24605.log 2026-02-18T08:53:43.944 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.31919.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.88607.log 2026-02-18T08:53:43.944 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.24605.log: 92.4% -- replaced with /var/log/ceph/cluster2-client.mirror.119360.log.gz 2026-02-18T08:53:43.944 INFO:teuthology.orchestra.run.trial163.stderr: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.31919.log.gz 2026-02-18T08:53:43.945 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.106772.log 2026-02-18T08:53:43.945 INFO:teuthology.orchestra.run.trial163.stderr: 88.3%gzip -- replaced with /var/log/ceph/cluster2-client.mirror.24605.log.gz 2026-02-18T08:53:43.945 INFO:teuthology.orchestra.run.trial163.stderr: -5 --verbose -- /var/log/ceph/cluster2-client.mirror.134589.log 2026-02-18T08:53:43.946 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.106772.log: /var/log/ceph/cluster2-client.mirror.88607.log: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.106772.log.gz 2026-02-18T08:53:43.959 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.71779.log 2026-02-18T08:53:43.961 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.134589.log: 91.7% -- replaced with /var/log/ceph/cluster2-client.mirror.134589.log.gz 2026-02-18T08:53:43.962 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.127174.log 2026-02-18T08:53:43.962 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.71779.log: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.71779.log.gz 2026-02-18T08:53:43.962 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.89729.log 2026-02-18T08:53:43.963 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.127174.log: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.127174.log.gz 2026-02-18T08:53:43.964 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.135237.log 2026-02-18T08:53:43.976 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.89729.log: 95.8% -- replaced with /var/log/ceph/cluster2-client.mirror.132609.log.gz 2026-02-18T08:53:43.976 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.60024.log 2026-02-18T08:53:43.977 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.admin.135237.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.135237.log.gz 2026-02-18T08:53:43.978 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.30550.log 2026-02-18T08:53:43.978 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.60024.log: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.60024.log.gz 2026-02-18T08:53:43.978 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.34620.log 2026-02-18T08:53:43.979 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.30550.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.132976.log 2026-02-18T08:53:43.979 INFO:teuthology.orchestra.run.trial163.stderr: 88.2% -- replaced with /var/log/ceph/cluster1-client.mirror.30550.log.gz 2026-02-18T08:53:43.980 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.34620.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.34620.log.gz 2026-02-18T08:53:43.980 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.44354.log 2026-02-18T08:53:43.981 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.132976.log: 91.4% -- replaced with /var/log/ceph/cluster2-client.mirror.132976.log.gz 2026-02-18T08:53:43.982 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.60129.log 2026-02-18T08:53:43.982 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.44354.log: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.44354.log.gz 2026-02-18T08:53:43.983 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.41339.log 2026-02-18T08:53:43.983 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.60129.log: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.60129.log.gz 2026-02-18T08:53:43.984 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.132583.log 2026-02-18T08:53:43.984 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.41339.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.41339.log.gz 2026-02-18T08:53:43.985 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.115864.log 2026-02-18T08:53:43.985 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.132583.log: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.132583.log.gz 2026-02-18T08:53:43.985 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.17680.log 2026-02-18T08:53:43.986 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.115864.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.115864.log.gz 2026-02-18T08:53:43.986 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.33407.log 2026-02-18T08:53:43.986 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.17680.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.17680.log.gz 2026-02-18T08:53:43.987 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.92934.log 2026-02-18T08:53:43.988 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.33407.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.33407.log.gz 2026-02-18T08:53:43.988 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.24291.log 2026-02-18T08:53:43.989 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.92934.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.92934.log.gz 2026-02-18T08:53:43.989 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.105987.log 2026-02-18T08:53:43.990 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.24291.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.24291.log.gz 2026-02-18T08:53:43.990 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.0.116277.log 2026-02-18T08:53:43.990 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.105987.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.105987.log.gz 2026-02-18T08:53:43.991 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.58669.log 2026-02-18T08:53:44.004 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.0.116277.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.133514.log 2026-02-18T08:53:44.011 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.58669.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.76752.log 2026-02-18T08:53:44.012 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.133514.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.133514.log.gz 2026-02-18T08:53:44.039 INFO:teuthology.orchestra.run.trial163.stderr: 92.8%gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.114146.log 2026-02-18T08:53:44.040 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.76752.log: 89.2% -- replaced with /var/log/ceph/cluster2-client.mirror.76752.log.gz 2026-02-18T08:53:44.043 INFO:teuthology.orchestra.run.trial163.stderr: -- replaced with /var/log/ceph/cluster1-client.mirror.0.116277.log.gz 2026-02-18T08:53:44.047 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.61625.log 2026-02-18T08:53:44.048 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.114146.log: 88.2% -- replaced with /var/log/ceph/cluster1-client.mirror.114146.log.gz 2026-02-18T08:53:44.063 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.109772.log 2026-02-18T08:53:44.064 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.61625.log: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.61625.log.gz 2026-02-18T08:53:44.069 INFO:teuthology.orchestra.run.trial163.stderr: 95.7% -- replaced with /var/log/ceph/cluster2-client.mirror.96211.log.gz 2026-02-18T08:53:44.070 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.98660.log 2026-02-18T08:53:44.076 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.109772.log: 95.7% -- replaced with /var/log/ceph/cluster2-client.mirror.88465.log.gz 2026-02-18T08:53:44.079 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.18924.log 2026-02-18T08:53:44.080 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.98660.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.98660.log.gz 2026-02-18T08:53:44.081 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.25033.log 2026-02-18T08:53:44.082 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.18924.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.18924.log.gz 2026-02-18T08:53:44.083 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.112348.log 2026-02-18T08:53:44.084 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.25033.log: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.25033.log.gz 2026-02-18T08:53:44.085 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.102835.log 2026-02-18T08:53:44.086 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.112348.log: 92.6% -- replaced with /var/log/ceph/cluster2-client.mirror.112348.log.gz 2026-02-18T08:53:44.088 INFO:teuthology.orchestra.run.trial163.stderr: 95.8% -- replaced with /var/log/ceph/cluster2-client.mirror.86778.log.gz 2026-02-18T08:53:44.089 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.29367.log 2026-02-18T08:53:44.089 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.122959.log 2026-02-18T08:53:44.090 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.102835.log: /var/log/ceph/cluster2-client.mirror.29367.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.102835.log.gz 2026-02-18T08:53:44.090 INFO:teuthology.orchestra.run.trial163.stderr: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.29367.log.gz 2026-02-18T08:53:44.096 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.94367.log 2026-02-18T08:53:44.096 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.122959.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.81201.log 2026-02-18T08:53:44.096 INFO:teuthology.orchestra.run.trial163.stderr: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.122959.log.gz 2026-02-18T08:53:44.098 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.94367.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.111025.log 2026-02-18T08:53:44.098 INFO:teuthology.orchestra.run.trial163.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.94367.log.gz/var/log/ceph/cluster2-client.mirror.81201.log: 2026-02-18T08:53:44.098 INFO:teuthology.orchestra.run.trial163.stderr: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.81201.log.gz 2026-02-18T08:53:44.098 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.118293.log 2026-02-18T08:53:44.098 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.38735.log 2026-02-18T08:53:44.099 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.111025.log: /var/log/ceph/cluster2-client.mirror.118293.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.118293.log.gz 2026-02-18T08:53:44.099 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.91987.log 2026-02-18T08:53:44.100 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.38735.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.38735.log.gz 2026-02-18T08:53:44.100 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.35643.log 2026-02-18T08:53:44.105 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.91987.log: 95.7% -- replaced with /var/log/ceph/cluster2-client.mirror.120529.log.gz 2026-02-18T08:53:44.120 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.40223.log 2026-02-18T08:53:44.121 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.35643.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.35643.log.gz 2026-02-18T08:53:44.121 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.98408.log 2026-02-18T08:53:44.121 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.40223.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.40223.log.gz 2026-02-18T08:53:44.122 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.33314.log 2026-02-18T08:53:44.123 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.98408.log: 92.5% -- replaced with /var/log/ceph/cluster1-client.mirror.98408.log.gz 2026-02-18T08:53:44.123 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.119962.log 2026-02-18T08:53:44.124 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.33314.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.33314.log.gz 2026-02-18T08:53:44.125 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.132436.log 2026-02-18T08:53:44.125 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.119962.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.119962.log.gz 2026-02-18T08:53:44.125 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.133617.log 2026-02-18T08:53:44.126 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.132436.log: 87.0% -- replaced with /var/log/ceph/cluster1-client.mirror.132436.log.gz 2026-02-18T08:53:44.126 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.120973.log 2026-02-18T08:53:44.127 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.133617.log: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.133617.log.gz 2026-02-18T08:53:44.127 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.0.45258.log 2026-02-18T08:53:44.140 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.120973.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.34155.log 2026-02-18T08:53:44.142 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.0.45258.log: 93.4% -- replaced with /var/log/ceph/cluster1-client.mirror.0.45258.log.gz 2026-02-18T08:53:44.151 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.18331.log 2026-02-18T08:53:44.164 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.34155.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.34155.log.gz 2026-02-18T08:53:44.166 INFO:teuthology.orchestra.run.trial163.stderr: 95.7% -- replaced with /var/log/ceph/cluster2-client.mirror.118828.log.gz 2026-02-18T08:53:44.167 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.21536.log 2026-02-18T08:53:44.168 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.18331.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.18331.log.gz 2026-02-18T08:53:44.168 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.106807.log 2026-02-18T08:53:44.170 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.21536.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.21536.log.gz 2026-02-18T08:53:44.170 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.100431.log 2026-02-18T08:53:44.170 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.106807.log: 95.8% -- replaced with /var/log/ceph/cluster2-client.mirror.89393.log.gz 2026-02-18T08:53:44.170 INFO:teuthology.orchestra.run.trial163.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.106807.log.gz 2026-02-18T08:53:44.170 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.79797.log 2026-02-18T08:53:44.171 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.100431.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.100431.log.gz 2026-02-18T08:53:44.173 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.18645.log 2026-02-18T08:53:44.174 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.79797.log: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.79797.log.gz 2026-02-18T08:53:44.174 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.135393.log 2026-02-18T08:53:44.174 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.18645.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.47465.log 2026-02-18T08:53:44.174 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.admin.135393.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.135393.log.gz 2026-02-18T08:53:44.174 INFO:teuthology.orchestra.run.trial163.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.18645.log.gz 2026-02-18T08:53:44.174 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.128249.log 2026-02-18T08:53:44.174 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.47465.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.35585.log 2026-02-18T08:53:44.174 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.128249.log: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.47465.log.gz 2026-02-18T08:53:44.174 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.68206.log 2026-02-18T08:53:44.178 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.35585.log: 90.6% 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.35585.log.gz 2026-02-18T08:53:44.178 INFO:teuthology.orchestra.run.trial163.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.128249.log.gz 2026-02-18T08:53:44.179 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.52159.log 2026-02-18T08:53:44.179 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.68206.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.117837.log 2026-02-18T08:53:44.179 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.52159.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.52159.log.gz 2026-02-18T08:53:44.179 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.33500.log 2026-02-18T08:53:44.179 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.117837.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.117837.log.gz 2026-02-18T08:53:44.179 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.67589.log 2026-02-18T08:53:44.179 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.33500.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.33500.log.gz 2026-02-18T08:53:44.179 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.62045.log 2026-02-18T08:53:44.179 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.67589.log: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.67589.log.gz 2026-02-18T08:53:44.179 INFO:teuthology.orchestra.run.trial163.stderr: 93.5% -- replaced with /var/log/ceph/cluster2-client.mirror.68206.log.gz 2026-02-18T08:53:44.180 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.53874.log 2026-02-18T08:53:44.180 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.62045.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.14677.log 2026-02-18T08:53:44.180 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.53874.log: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.62045.log.gz 2026-02-18T08:53:44.189 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.106072.log 2026-02-18T08:53:44.189 INFO:teuthology.orchestra.run.trial163.stderr: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.53874.log.gz 2026-02-18T08:53:44.189 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.admin.14677.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.14677.log.gz 2026-02-18T08:53:44.189 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.75849.log 2026-02-18T08:53:44.189 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.106072.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.133336.log 2026-02-18T08:53:44.190 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.75849.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.106072.log.gz 2026-02-18T08:53:44.190 INFO:teuthology.orchestra.run.trial163.stderr: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.75849.log.gz 2026-02-18T08:53:44.190 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.85311.log 2026-02-18T08:53:44.190 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.133336.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.133314.log 2026-02-18T08:53:44.190 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.85311.log: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.133336.log.gz 2026-02-18T08:53:44.190 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.39851.log 2026-02-18T08:53:44.190 INFO:teuthology.orchestra.run.trial163.stderr: 89.5% -- replaced with /var/log/ceph/cluster2-client.mirror.85311.log.gz 2026-02-18T08:53:44.190 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.133314.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.133314.log.gz 2026-02-18T08:53:44.190 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.106170.log 2026-02-18T08:53:44.190 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.39851.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.124810.log 2026-02-18T08:53:44.191 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.106170.log: 88.0% 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.39851.log.gz 2026-02-18T08:53:44.191 INFO:teuthology.orchestra.run.trial163.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.106170.log.gz 2026-02-18T08:53:44.191 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.89911.log 2026-02-18T08:53:44.191 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.124810.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.96557.log 2026-02-18T08:53:44.191 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.89911.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.124810.log.gz 2026-02-18T08:53:44.191 INFO:teuthology.orchestra.run.trial163.stderr: 84.8% -- replaced with /var/log/ceph/cluster2-client.mirror.89911.log.gz 2026-02-18T08:53:44.191 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.44319.log 2026-02-18T08:53:44.191 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.96557.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.108012.log 2026-02-18T08:53:44.191 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.44319.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.96557.log.gz 2026-02-18T08:53:44.191 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.116983.log 2026-02-18T08:53:44.191 INFO:teuthology.orchestra.run.trial163.stderr: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.44319.log.gz 2026-02-18T08:53:44.191 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.108012.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.108012.log.gz 2026-02-18T08:53:44.192 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.134677.log 2026-02-18T08:53:44.192 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.116983.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.131624.log 2026-02-18T08:53:44.192 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.134677.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.116983.log.gz 2026-02-18T08:53:44.192 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.122064.log 2026-02-18T08:53:44.192 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.131624.log: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.131624.log.gz 2026-02-18T08:53:44.192 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.117872.log 2026-02-18T08:53:44.192 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.122064.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.122064.log.gz 2026-02-18T08:53:44.192 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.114457.log 2026-02-18T08:53:44.192 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.117872.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.117872.log.gz 2026-02-18T08:53:44.192 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.22245.log 2026-02-18T08:53:44.196 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.114457.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.114457.log.gz 2026-02-18T08:53:44.196 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.79412.log 2026-02-18T08:53:44.196 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.22245.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.22245.log.gz 2026-02-18T08:53:44.196 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.129767.log 2026-02-18T08:53:44.196 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.79412.log: 89.2% -- replaced with /var/log/ceph/cluster2-client.mirror.79412.log.gz 2026-02-18T08:53:44.196 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.97055.log 2026-02-18T08:53:44.196 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.129767.log: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.129767.log.gz 2026-02-18T08:53:44.197 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.29553.log 2026-02-18T08:53:44.197 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.97055.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.97055.log.gz 2026-02-18T08:53:44.197 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.133953.log 2026-02-18T08:53:44.197 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.29553.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.131005.log 2026-02-18T08:53:44.197 INFO:teuthology.orchestra.run.trial163.stderr: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.29553.log.gz 2026-02-18T08:53:44.197 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.133953.log: 83.6% -- replaced with /var/log/ceph/cluster2-client.mirror.133953.log.gz 2026-02-18T08:53:44.197 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.71884.log 2026-02-18T08:53:44.197 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.131005.log: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.131005.log.gz 2026-02-18T08:53:44.197 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.45666.log 2026-02-18T08:53:44.199 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.71884.log: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.71884.log.gz 2026-02-18T08:53:44.199 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.21408.log 2026-02-18T08:53:44.199 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.45666.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.45061.log 2026-02-18T08:53:44.199 INFO:teuthology.orchestra.run.trial163.stderr: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.45666.log.gz 2026-02-18T08:53:44.199 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.21408.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.134075.log 2026-02-18T08:53:44.199 INFO:teuthology.orchestra.run.trial163.stderr: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.21408.log.gz 2026-02-18T08:53:44.199 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.45061.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.116072.log 2026-02-18T08:53:44.200 INFO:teuthology.orchestra.run.trial163.stderr: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.45061.log.gz 2026-02-18T08:53:44.200 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.134075.log: 87.9% -- replaced with /var/log/ceph/cluster1-client.mirror.134075.log.gz 2026-02-18T08:53:44.200 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.69154.log 2026-02-18T08:53:44.201 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.116072.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.116072.log.gz 2026-02-18T08:53:44.201 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.95268.log 2026-02-18T08:53:44.202 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.69154.log: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.69154.log.gz 2026-02-18T08:53:44.202 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.62675.log 2026-02-18T08:53:44.203 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.95268.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.95268.log.gz 2026-02-18T08:53:44.203 INFO:teuthology.orchestra.run.trial163.stderr: 95.7% -- replaced with /var/log/ceph/cluster2-client.mirror.30687.log.gz 2026-02-18T08:53:44.203 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.118019.log 2026-02-18T08:53:44.204 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.62675.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.64005.log 2026-02-18T08:53:44.204 INFO:teuthology.orchestra.run.trial163.stderr: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.62675.log.gz 2026-02-18T08:53:44.204 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.118019.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.133254.log 2026-02-18T08:53:44.205 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.64005.log: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.64005.log.gz 2026-02-18T08:53:44.205 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.99666.log 2026-02-18T08:53:44.205 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.133254.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.133254.log.gz 2026-02-18T08:53:44.206 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.38642.log 2026-02-18T08:53:44.206 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.99666.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.99666.log.gz 2026-02-18T08:53:44.206 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.106205.log 2026-02-18T08:53:44.208 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.38642.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.38642.log.gz 2026-02-18T08:53:44.208 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.87814.log 2026-02-18T08:53:44.208 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.106205.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.106205.log.gz 2026-02-18T08:53:44.209 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.17250.log 2026-02-18T08:53:44.217 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.87814.log: 95.8% -- replaced with /var/log/ceph/cluster2-client.mirror.89170.log.gz 2026-02-18T08:53:44.220 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.68149.log 2026-02-18T08:53:44.221 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.17250.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.17250.log.gz 2026-02-18T08:53:44.221 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.39944.log 2026-02-18T08:53:44.221 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.68149.log: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.68149.log.gz 2026-02-18T08:53:44.227 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.114079.log 2026-02-18T08:53:44.229 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.39944.log: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.39944.log.gz 2026-02-18T08:53:44.229 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.128712.log 2026-02-18T08:53:44.229 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.114079.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.114079.log.gz 2026-02-18T08:53:44.229 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.52054.log 2026-02-18T08:53:44.230 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.128712.log: 89.2% -- replaced with /var/log/ceph/cluster2-client.mirror.128712.log.gz 2026-02-18T08:53:44.231 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.107300.log 2026-02-18T08:53:44.231 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.52054.log: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.52054.log.gz 2026-02-18T08:53:44.232 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.65552.log 2026-02-18T08:53:44.232 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.107300.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.107300.log.gz 2026-02-18T08:53:44.232 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.32419.log 2026-02-18T08:53:44.233 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.65552.log: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.65552.log.gz 2026-02-18T08:53:44.234 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.55144.log 2026-02-18T08:53:44.234 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.32419.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.32419.log.gz 2026-02-18T08:53:44.234 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.112995.log 2026-02-18T08:53:44.235 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.55144.log: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.55144.log.gz 2026-02-18T08:53:44.235 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.35399.log 2026-02-18T08:53:44.236 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.112995.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.112995.log.gz 2026-02-18T08:53:44.236 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.127553.log 2026-02-18T08:53:44.237 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.35399.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.35399.log.gz 2026-02-18T08:53:44.237 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.43110.log 2026-02-18T08:53:44.238 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.127553.log: 89.3% -- replaced with /var/log/ceph/cluster2-client.mirror.127553.log.gz 2026-02-18T08:53:44.239 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.63725.log 2026-02-18T08:53:44.239 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.43110.log: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.43110.log.gz 2026-02-18T08:53:44.240 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.71604.log 2026-02-18T08:53:44.240 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.63725.log: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.63725.log.gz 2026-02-18T08:53:44.241 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.129699.log 2026-02-18T08:53:44.241 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.71604.log: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.71604.log.gz 2026-02-18T08:53:44.242 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.120074.log 2026-02-18T08:53:44.242 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.129699.log: 87.0% -- replaced with /var/log/ceph/cluster1-client.mirror.129699.log.gz 2026-02-18T08:53:44.242 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.54504.log 2026-02-18T08:53:44.243 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.120074.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.120074.log.gz 2026-02-18T08:53:44.243 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.84334.log 2026-02-18T08:53:44.244 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.54504.log: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.54504.log.gz 2026-02-18T08:53:44.245 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.117473.log 2026-02-18T08:53:44.245 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.84334.log: 95.6% -- replaced with /var/log/ceph/cluster2-client.mirror.86346.log.gz 2026-02-18T08:53:44.245 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.25091.log 2026-02-18T08:53:44.246 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.117473.log: 87.9% -- replaced with /var/log/ceph/cluster2-client.mirror.117473.log.gz 2026-02-18T08:53:44.246 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.29088.log 2026-02-18T08:53:44.246 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.25091.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.124099.log 2026-02-18T08:53:44.246 INFO:teuthology.orchestra.run.trial163.stderr: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.25091.log.gz 2026-02-18T08:53:44.247 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.29088.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.57304.log 2026-02-18T08:53:44.248 INFO:teuthology.orchestra.run.trial163.stderr: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.29088.log.gz 2026-02-18T08:53:44.248 INFO:teuthology.orchestra.run.trial163.stderr: 93.2% -- replaced with /var/log/ceph/cluster1-client.mirror.84334.log.gz 2026-02-18T08:53:44.248 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.124099.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.29216.log 2026-02-18T08:53:44.249 INFO:teuthology.orchestra.run.trial163.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.124099.log.gz 2026-02-18T08:53:44.250 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.57304.log: gzip 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.57304.log.gz -5 2026-02-18T08:53:44.250 INFO:teuthology.orchestra.run.trial163.stderr: --verbose -- /var/log/ceph/cluster2-client.mirror.121076.log 2026-02-18T08:53:44.250 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.29216.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.16458.log 2026-02-18T08:53:44.250 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.121076.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.29216.log.gz 2026-02-18T08:53:44.250 INFO:teuthology.orchestra.run.trial163.stderr: 87.9% -- replaced with /var/log/ceph/cluster2-client.mirror.121076.log.gz 2026-02-18T08:53:44.250 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.37028.log 2026-02-18T08:53:44.251 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.120318.log 2026-02-18T08:53:44.251 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.16458.log: /var/log/ceph/cluster2-client.mirror.37028.log: 79.0% -- replaced with /var/log/ceph/cluster1-client.mirror.16458.log.gz 2026-02-18T08:53:44.251 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.128754.log 2026-02-18T08:53:44.251 INFO:teuthology.orchestra.run.trial163.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.37028.log.gz 2026-02-18T08:53:44.252 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.120318.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.86133.log 2026-02-18T08:53:44.252 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.128754.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.120318.log.gz 2026-02-18T08:53:44.252 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.53314.log 2026-02-18T08:53:44.253 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.86133.log: 90.6% -- replaced with /var/log/ceph/cluster2-client.mirror.128754.log.gz 2026-02-18T08:53:44.253 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.83113.log 2026-02-18T08:53:44.254 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.53314.log: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.53314.log.gz 2026-02-18T08:53:44.254 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.0.117738.log 2026-02-18T08:53:44.254 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.83113.log: 89.2% -- replaced with /var/log/ceph/cluster2-client.mirror.83113.log.gz 2026-02-18T08:53:44.255 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.66959.log 2026-02-18T08:53:44.267 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.0.117738.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.135263.log 2026-02-18T08:53:44.268 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.66959.log: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.66959.log.gz 2026-02-18T08:53:44.271 INFO:teuthology.orchestra.run.trial163.stderr: 94.0% -- replaced with /var/log/ceph/cluster2-client.mirror.0.117738.log.gz 2026-02-18T08:53:44.272 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.84291.log 2026-02-18T08:53:44.272 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.admin.135263.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.135263.log.gz 2026-02-18T08:53:44.273 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.115247.log 2026-02-18T08:53:44.276 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.84291.log: 93.3% -- replaced with /var/log/ceph/cluster1-client.mirror.84291.log.gz 2026-02-18T08:53:44.277 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.44598.log 2026-02-18T08:53:44.277 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.115247.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.115247.log.gz 2026-02-18T08:53:44.278 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.97478.log 2026-02-18T08:53:44.279 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.44598.log: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.44598.log.gz 2026-02-18T08:53:44.279 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.69924.log 2026-02-18T08:53:44.279 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.97478.log: 88.1% -- replaced with /var/log/ceph/cluster1-client.mirror.97478.log.gz 2026-02-18T08:53:44.280 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.3.89784.log 2026-02-18T08:53:44.280 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.69924.log: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.69924.log.gz 2026-02-18T08:53:44.281 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.43575.log 2026-02-18T08:53:44.299 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.3.89784.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.108751.log 2026-02-18T08:53:44.300 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.43575.log: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.43575.log.gz 2026-02-18T08:53:44.307 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.107427.log 2026-02-18T08:53:44.308 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.108751.log: 88.4% -- replaced with /var/log/ceph/cluster1-client.mirror.108751.log.gz 2026-02-18T08:53:44.315 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.26798.log 2026-02-18T08:53:44.316 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.107427.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.107427.log.gz 2026-02-18T08:53:44.325 INFO:teuthology.orchestra.run.trial163.stderr: 95.7% -- replaced with /var/log/ceph/cluster2-client.mirror.95448.log.gz 2026-02-18T08:53:44.326 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.90944.log 2026-02-18T08:53:44.327 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.26798.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.26798.log.gz 2026-02-18T08:53:44.327 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.124411.log 2026-02-18T08:53:44.328 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.90944.log: 84.7% -- replaced with /var/log/ceph/cluster2-client.mirror.90944.log.gz 2026-02-18T08:53:44.328 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.134789.log 2026-02-18T08:53:44.329 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.124411.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.124411.log.gz 2026-02-18T08:53:44.329 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.37214.log 2026-02-18T08:53:44.329 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.134789.log: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.134789.log.gz 2026-02-18T08:53:44.330 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.110958.log 2026-02-18T08:53:44.330 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.37214.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.37214.log.gz 2026-02-18T08:53:44.330 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.115748.log 2026-02-18T08:53:44.331 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.110958.log: 88.1% -- replaced with /var/log/ceph/cluster1-client.mirror.110958.log.gz 2026-02-18T08:53:44.331 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.84534.log 2026-02-18T08:53:44.332 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.115748.log: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.115748.log.gz 2026-02-18T08:53:44.332 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.44540.log 2026-02-18T08:53:44.336 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.84534.log: 93.6% -- replaced with /var/log/ceph/cluster2-client.mirror.84534.log.gz 2026-02-18T08:53:44.336 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.103408.log 2026-02-18T08:53:44.337 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.44540.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.126553.log 2026-02-18T08:53:44.337 INFO:teuthology.orchestra.run.trial163.stderr: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.44540.log.gz 2026-02-18T08:53:44.338 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.103408.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.103408.log.gz 2026-02-18T08:53:44.338 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.72199.log 2026-02-18T08:53:44.339 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.126553.log: 90.7% -- replaced with /var/log/ceph/cluster2-client.mirror.126553.log.gz 2026-02-18T08:53:44.340 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.85053.log 2026-02-18T08:53:44.340 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.72199.log: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.72199.log.gz 2026-02-18T08:53:44.341 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.71499.log 2026-02-18T08:53:44.344 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.85053.log: 93.5% -- replaced with /var/log/ceph/cluster2-client.mirror.85053.log.gz 2026-02-18T08:53:44.345 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.64530.log 2026-02-18T08:53:44.345 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.71499.log: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.71499.log.gz 2026-02-18T08:53:44.346 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.104577.log 2026-02-18T08:53:44.346 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.64530.log: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.64530.log.gz 2026-02-18T08:53:44.347 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.76082.log 2026-02-18T08:53:44.347 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.104577.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.104577.log.gz 2026-02-18T08:53:44.347 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.131163.log 2026-02-18T08:53:44.349 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.76082.log: 94.1% -- replaced with /var/log/ceph/cluster2-client.mirror.3.89784.log.gz 2026-02-18T08:53:44.350 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.132666.log 2026-02-18T08:53:44.351 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.131163.log: 87.0% -- replaced with /var/log/ceph/cluster1-client.mirror.131163.log.gz 2026-02-18T08:53:44.351 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.45934.log 2026-02-18T08:53:44.352 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.132666.log: 96.5% -- replaced with /var/log/ceph/cluster2-client.mirror.76082.log.gz 2026-02-18T08:53:44.352 INFO:teuthology.orchestra.run.trial163.stderr: 87.9% -- replaced with /var/log/ceph/cluster1-client.mirror.132666.log.gz 2026-02-18T08:53:44.352 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.84666.log 2026-02-18T08:53:44.352 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.45934.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.126653.log 2026-02-18T08:53:44.353 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.20757.log 2026-02-18T08:53:44.353 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.84666.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.45934.log.gz 2026-02-18T08:53:44.353 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.126653.log: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.126653.log.gz 2026-02-18T08:53:44.354 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.67309.log 2026-02-18T08:53:44.355 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.20757.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.20757.log.gz 2026-02-18T08:53:44.355 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.132147.log 2026-02-18T08:53:44.355 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.67309.log: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.67309.log.gz 2026-02-18T08:53:44.356 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.74933.log 2026-02-18T08:53:44.356 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.132147.log: 93.5% -- replaced with /var/log/ceph/cluster2-client.mirror.84666.log.gz 2026-02-18T08:53:44.357 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.2.109402.log 2026-02-18T08:53:44.357 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.74933.log: 91.8% -- replaced with /var/log/ceph/cluster2-client.mirror.132147.log.gz 2026-02-18T08:53:44.358 INFO:teuthology.orchestra.run.trial163.stderr: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.74933.log.gz 2026-02-18T08:53:44.358 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.55299.log 2026-02-18T08:53:44.358 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.131524.log 2026-02-18T08:53:44.358 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.2.109402.log: /var/log/ceph/cluster1-client.mirror.55299.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.102625.log 2026-02-18T08:53:44.361 INFO:teuthology.orchestra.run.trial163.stderr: 93.5% -- replaced with /var/log/ceph/cluster1-client.mirror.55299.log.gz 2026-02-18T08:53:44.362 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.131524.log: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.131524.log.gz 2026-02-18T08:53:44.363 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.125822.log 2026-02-18T08:53:44.363 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.102625.log: 95.7% -- replaced with /var/log/ceph/cluster2-client.mirror.24948.log.gz 2026-02-18T08:53:44.364 INFO:teuthology.orchestra.run.trial163.stderr: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.102625.log.gz 2026-02-18T08:53:44.364 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.47042.log 2026-02-18T08:53:44.364 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.125822.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.109540.log 2026-02-18T08:53:44.365 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.47042.log: 89.4% -- replaced with /var/log/ceph/cluster2-client.mirror.125822.log.gz 2026-02-18T08:53:44.365 INFO:teuthology.orchestra.run.trial163.stderr: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.47042.log.gz 2026-02-18T08:53:44.365 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.96866.log 2026-02-18T08:53:44.365 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.109540.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.109540.log.gz 2026-02-18T08:53:44.366 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.17529.log 2026-02-18T08:53:44.366 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.96866.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.85120.log 2026-02-18T08:53:44.367 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.17529.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.17529.log.gz 2026-02-18T08:53:44.367 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.98579.log 2026-02-18T08:53:44.371 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.85120.log: 93.5% -- replaced with /var/log/ceph/cluster1-client.mirror.85120.log.gz 2026-02-18T08:53:44.376 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.68625.log 2026-02-18T08:53:44.376 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.98579.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.98579.log.gz 2026-02-18T08:53:44.376 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.52299.log 2026-02-18T08:53:44.377 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.68625.log: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.68625.log.gz 2026-02-18T08:53:44.377 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.46377.log 2026-02-18T08:53:44.378 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.52299.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.52299.log.gz 2026-02-18T08:53:44.378 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.106961.log 2026-02-18T08:53:44.379 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.46377.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.51354.log 2026-02-18T08:53:44.379 INFO:teuthology.orchestra.run.trial163.stderr: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.46377.log.gz 2026-02-18T08:53:44.380 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.106961.log: 92.2% -- replaced with /var/log/ceph/cluster1-client.mirror.106961.log.gz 2026-02-18T08:53:44.381 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.85479.log 2026-02-18T08:53:44.382 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.51354.log: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.51354.log.gz 2026-02-18T08:53:44.382 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.96962.log 2026-02-18T08:53:44.382 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.85479.log: 89.3% -- replaced with /var/log/ceph/cluster2-client.mirror.85479.log.gz 2026-02-18T08:53:44.383 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.87942.log 2026-02-18T08:53:44.383 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.96962.log: 87.4% -- replaced with /var/log/ceph/cluster2-client.mirror.96962.log.gz 2026-02-18T08:53:44.384 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.86977.log 2026-02-18T08:53:44.390 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.87942.log: 94.2% -- replaced with /var/log/ceph/cluster2-client.mirror.2.109402.log.gz 2026-02-18T08:53:44.390 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.118931.log 2026-02-18T08:53:44.400 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.86977.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.19354.log 2026-02-18T08:53:44.400 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.118931.log: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.118931.log.gz 2026-02-18T08:53:44.402 INFO:teuthology.orchestra.run.trial163.stderr: 95.7% -- replaced with /var/log/ceph/cluster2-client.mirror.109772.log.gz 2026-02-18T08:53:44.402 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.86464.log 2026-02-18T08:53:44.402 INFO:teuthology.orchestra.run.trial163.stderr: 95.6% -- replaced with /var/log/ceph/cluster2-client.mirror.89729.log.gz 2026-02-18T08:53:44.404 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.19354.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.19354.log.gz 2026-02-18T08:53:44.404 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.129072.log 2026-02-18T08:53:44.404 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.87424.log 2026-02-18T08:53:44.404 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.86464.log: /var/log/ceph/cluster1-client.mirror.129072.log: 1.3% -- replaced with /var/log/ceph/cluster1-client.mirror.129072.log.gz 2026-02-18T08:53:44.404 INFO:teuthology.orchestra.run.trial163.stderr: 84.7% -- replaced with /var/log/ceph/cluster2-client.mirror.86464.log.gz 2026-02-18T08:53:44.404 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.119803.log 2026-02-18T08:53:44.405 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.127524.log 2026-02-18T08:53:44.405 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.87424.log: /var/log/ceph/cluster1-client.mirror.119803.log: 84.8% -- replaced with /var/log/ceph/cluster2-client.mirror.87424.log.gz 2026-02-18T08:53:44.405 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.95314.log 2026-02-18T08:53:44.406 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.127524.log: 92.4% -- replaced with /var/log/ceph/cluster1-client.mirror.119803.log.gz 2026-02-18T08:53:44.407 INFO:teuthology.orchestra.run.trial163.stderr: 90.7% -- replaced with /var/log/ceph/cluster2-client.mirror.127524.log.gz 2026-02-18T08:53:44.407 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.55799.log 2026-02-18T08:53:44.407 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.52264.log 2026-02-18T08:53:44.409 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.95314.log: /var/log/ceph/cluster2-client.mirror.55799.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.95314.log.gz 2026-02-18T08:53:44.409 INFO:teuthology.orchestra.run.trial163.stderr: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.55799.log.gz 2026-02-18T08:53:44.409 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.64565.log 2026-02-18T08:53:44.409 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.102275.log 2026-02-18T08:53:44.409 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.52264.log: /var/log/ceph/cluster2-client.mirror.64565.log: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.64565.log.gz 2026-02-18T08:53:44.409 INFO:teuthology.orchestra.run.trial163.stderr: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.52264.log.gz 2026-02-18T08:53:44.409 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.17122.log 2026-02-18T08:53:44.411 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.27728.log 2026-02-18T08:53:44.411 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.102275.log: /var/log/ceph/cluster2-client.mirror.17122.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.102275.log.gz 2026-02-18T08:53:44.411 INFO:teuthology.orchestra.run.trial163.stderr: 88.2%gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.135289.log 2026-02-18T08:53:44.411 INFO:teuthology.orchestra.run.trial163.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.17122.log.gz 2026-02-18T08:53:44.411 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.27728.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.27728.log.gz 2026-02-18T08:53:44.411 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.105801.log 2026-02-18T08:53:44.411 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.73669.log 2026-02-18T08:53:44.414 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.admin.135289.log: /var/log/ceph/cluster2-client.mirror.105801.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.135289.log.gz 2026-02-18T08:53:44.414 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.30053.log 2026-02-18T08:53:44.414 INFO:teuthology.orchestra.run.trial163.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.105801.log.gz 2026-02-18T08:53:44.414 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.73669.log: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.73669.log.gz 2026-02-18T08:53:44.414 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.14185.log 2026-02-18T08:53:44.414 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.90021.log 2026-02-18T08:53:44.414 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.30053.log: /var/log/ceph/cluster2-client.admin.14185.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.14185.log.gz 2026-02-18T08:53:44.415 INFO:teuthology.orchestra.run.trial163.stderr: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.30053.log.gz 2026-02-18T08:53:44.415 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.105313.log 2026-02-18T08:53:44.415 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.78257.log 2026-02-18T08:53:44.415 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.90021.log: /var/log/ceph/cluster2-client.mirror.105313.log: 84.8% -- replaced with /var/log/ceph/cluster2-client.mirror.90021.log.gz 2026-02-18T08:53:44.415 INFO:teuthology.orchestra.run.trial163.stderr: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.105313.log.gz 2026-02-18T08:53:44.415 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.69679.log 2026-02-18T08:53:44.416 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.84205.log 2026-02-18T08:53:44.416 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.78257.log: /var/log/ceph/cluster2-client.mirror.69679.log: 89.1% 89.2% -- replaced with /var/log/ceph/cluster2-client.mirror.78257.log.gz 2026-02-18T08:53:44.416 INFO:teuthology.orchestra.run.trial163.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.69679.log.gz 2026-02-18T08:53:44.417 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.105572.log 2026-02-18T08:53:44.417 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.128379.log 2026-02-18T08:53:44.417 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.84205.log: /var/log/ceph/cluster2-client.mirror.105572.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.105572.log.gz 2026-02-18T08:53:44.419 INFO:teuthology.orchestra.run.trial163.stderr: 93.0% -- replaced with /var/log/ceph/cluster1-client.mirror.84205.log.gz 2026-02-18T08:53:44.419 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.99188.log 2026-02-18T08:53:44.419 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.19203.log 2026-02-18T08:53:44.421 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.128379.log: /var/log/ceph/cluster2-client.mirror.99188.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.128379.log.gz 2026-02-18T08:53:44.421 INFO:teuthology.orchestra.run.trial163.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.99188.log.gz 2026-02-18T08:53:44.421 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.48900.log 2026-02-18T08:53:44.421 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.19203.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.62570.log 2026-02-18T08:53:44.422 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.48900.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.19203.log.gz 2026-02-18T08:53:44.422 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.46303.log 2026-02-18T08:53:44.422 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.62570.log: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.48900.log.gz 2026-02-18T08:53:44.422 INFO:teuthology.orchestra.run.trial163.stderr: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.62570.log.gz 2026-02-18T08:53:44.422 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.57619.log 2026-02-18T08:53:44.423 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.46303.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.49775.log 2026-02-18T08:53:44.423 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.22524.log 2026-02-18T08:53:44.423 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.57619.log: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.57619.log.gz 2026-02-18T08:53:44.431 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.49775.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.101092.log 2026-02-18T08:53:44.432 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.22524.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.22524.log.gz 2026-02-18T08:53:44.433 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.51144.log 2026-02-18T08:53:44.433 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.101092.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.101092.log.gz 2026-02-18T08:53:44.439 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.41587.log 2026-02-18T08:53:44.441 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.51144.log: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.51144.log.gz 2026-02-18T08:53:44.441 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.21071.log 2026-02-18T08:53:44.442 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.41587.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.41587.log.gz 2026-02-18T08:53:44.442 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.115484.log 2026-02-18T08:53:44.442 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.21071.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.21071.log.gz 2026-02-18T08:53:44.443 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.54609.log 2026-02-18T08:53:44.443 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.115484.log: 88.2% -- replaced with /var/log/ceph/cluster1-client.mirror.115484.log.gz 2026-02-18T08:53:44.444 INFO:teuthology.orchestra.run.trial163.stderr: 88.5%gzip -- replaced with /var/log/ceph/cluster2-client.mirror.49775.log.gz 2026-02-18T08:53:44.444 INFO:teuthology.orchestra.run.trial163.stderr: -5 --verbose -- /var/log/ceph/cluster2-client.mirror.69714.log 2026-02-18T08:53:44.444 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.54609.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.23210.log 2026-02-18T08:53:44.445 INFO:teuthology.orchestra.run.trial163.stderr: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.54609.log.gz 2026-02-18T08:53:44.445 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.69714.log: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.69714.log.gz 2026-02-18T08:53:44.445 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.84357.log 2026-02-18T08:53:44.446 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.23210.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.65676.log 2026-02-18T08:53:44.447 INFO:teuthology.orchestra.run.trial163.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.23210.log.gz 2026-02-18T08:53:44.450 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.84357.log: 93.4% -- replaced with /var/log/ceph/cluster2-client.mirror.84357.log.gz 2026-02-18T08:53:44.451 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.77312.log 2026-02-18T08:53:44.451 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.65676.log: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.65676.log.gz 2026-02-18T08:53:44.451 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.112390.log 2026-02-18T08:53:44.452 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.77312.log: 95.7% -- replaced with /var/log/ceph/cluster2-client.mirror.120973.log.gz 89.2% -- replaced with /var/log/ceph/cluster2-client.mirror.77312.log.gz 2026-02-18T08:53:44.452 INFO:teuthology.orchestra.run.trial163.stderr: 2026-02-18T08:53:44.453 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.51039.log 2026-02-18T08:53:44.453 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.69294.log 2026-02-18T08:53:44.454 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.112390.log: /var/log/ceph/cluster2-client.mirror.51039.log: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.51039.log.gz 2026-02-18T08:53:44.467 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.93748.log 2026-02-18T08:53:44.469 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.69294.log: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.69294.log.gz 2026-02-18T08:53:44.469 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.2.26153.log 2026-02-18T08:53:44.470 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.93748.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.93748.log.gz 2026-02-18T08:53:44.470 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.83316.log 2026-02-18T08:53:44.470 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.2.26153.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.70659.log 2026-02-18T08:53:44.475 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.107265.log 2026-02-18T08:53:44.483 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.70659.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.108818.log 2026-02-18T08:53:44.484 INFO:teuthology.orchestra.run.trial163.stderr: 89.1%/var/log/ceph/cluster2-client.mirror.107265.log: -- replaced with /var/log/ceph/cluster2-client.mirror.70659.log.gz 2026-02-18T08:53:44.484 INFO:teuthology.orchestra.run.trial163.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.107265.log.gz 2026-02-18T08:53:44.492 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.83316.log: 89.3% -- replaced with /var/log/ceph/cluster1-client.mirror.83316.log.gz 2026-02-18T08:53:44.499 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.65824.log 2026-02-18T08:53:44.511 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.108818.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.30618.log 2026-02-18T08:53:44.523 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.65824.log: 93.5%gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.132459.log 2026-02-18T08:53:44.527 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.30618.log: -- replaced with /var/log/ceph/cluster2-client.mirror.65824.log.gz 2026-02-18T08:53:44.531 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.70414.log 2026-02-18T08:53:44.532 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.132459.log: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.132459.log.gz 2026-02-18T08:53:44.533 INFO:teuthology.orchestra.run.trial163.stderr: 95.8% -- replaced with /var/log/ceph/cluster2-client.mirror.89443.log.gz 2026-02-18T08:53:44.539 INFO:teuthology.orchestra.run.trial163.stderr: 93.3% -- replaced with /var/log/ceph/cluster2-client.mirror.30618.log.gz 2026-02-18T08:53:44.547 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.63830.log 2026-02-18T08:53:44.548 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.70414.log: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.70414.log.gz 2026-02-18T08:53:44.564 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.31675.log 2026-02-18T08:53:44.564 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.63830.log: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.63830.log.gz 2026-02-18T08:53:44.580 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.29030.log 2026-02-18T08:53:44.580 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.31675.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.31675.log.gz 2026-02-18T08:53:44.592 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.15326.log 2026-02-18T08:53:44.596 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.29030.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.29030.log.gz 2026-02-18T08:53:44.610 INFO:teuthology.orchestra.run.trial163.stderr: 95.7%gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.28286.log 2026-02-18T08:53:44.611 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.15326.log: -- replaced with /var/log/ceph/cluster2-client.mirror.86487.log.gz 2026-02-18T08:53:44.611 INFO:teuthology.orchestra.run.trial163.stderr: 68.4% -- replaced with /var/log/ceph/cluster1-client.mirror.15326.log.gz 2026-02-18T08:53:44.611 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.117041.log 2026-02-18T08:53:44.612 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.28286.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.28286.log.gz 2026-02-18T08:53:44.612 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.110581.log 2026-02-18T08:53:44.613 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.117041.log: 92.5% -- replaced with /var/log/ceph/cluster2-client.mirror.117041.log.gz 2026-02-18T08:53:44.614 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.96117.log 2026-02-18T08:53:44.622 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.110581.log: 95.7% -- replaced with /var/log/ceph/cluster2-client.mirror.118019.log.gz 2026-02-18T08:53:44.622 INFO:teuthology.orchestra.run.trial163.stderr: 95.8% -- replaced with /var/log/ceph/cluster2-client.mirror.68314.log.gz 2026-02-18T08:53:44.623 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.67064.log 2026-02-18T08:53:44.623 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.96117.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.128521.log 2026-02-18T08:53:44.624 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.67064.log: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.67064.log.gz 2026-02-18T08:53:44.625 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.102455.log 2026-02-18T08:53:44.625 INFO:teuthology.orchestra.run.trial163.stderr: 92.6% -- replaced with /var/log/ceph/cluster2-client.mirror.96117.log.gz 2026-02-18T08:53:44.625 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.128521.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.128521.log.gz 2026-02-18T08:53:44.626 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.123778.log 2026-02-18T08:53:44.626 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.102455.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.123320.log 2026-02-18T08:53:44.627 INFO:teuthology.orchestra.run.trial163.stderr: 87.9% -- replaced with /var/log/ceph/cluster2-client.mirror.102455.log.gz 2026-02-18T08:53:44.627 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.123778.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.123778.log.gz 2026-02-18T08:53:44.627 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.49530.log 2026-02-18T08:53:44.627 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.123320.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.129744.log 2026-02-18T08:53:44.627 INFO:teuthology.orchestra.run.trial163.stderr: 95.7% -- replaced with /var/log/ceph/cluster2-client.mirror.134677.log.gz 2026-02-18T08:53:44.627 INFO:teuthology.orchestra.run.trial163.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.123320.log.gz 2026-02-18T08:53:44.631 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.49530.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.36015.log 2026-02-18T08:53:44.631 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.129744.log: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.49530.log.gz 2026-02-18T08:53:44.631 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.130059.log 2026-02-18T08:53:44.631 INFO:teuthology.orchestra.run.trial163.stderr: 87.0% -- replaced with /var/log/ceph/cluster1-client.mirror.129744.log.gz 2026-02-18T08:53:44.631 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.36015.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.35492.log 2026-02-18T08:53:44.631 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.130059.log: 86.8% -- replaced with /var/log/ceph/cluster1-client.mirror.130059.log.gz 2026-02-18T08:53:44.631 INFO:teuthology.orchestra.run.trial163.stderr: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.36015.log.gz 2026-02-18T08:53:44.631 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.133643.log 2026-02-18T08:53:44.631 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.35492.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.74408.log 2026-02-18T08:53:44.631 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.133643.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.41715.log 2026-02-18T08:53:44.631 INFO:teuthology.orchestra.run.trial163.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.35492.log.gz 2026-02-18T08:53:44.632 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.74408.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.126369.log 2026-02-18T08:53:44.632 INFO:teuthology.orchestra.run.trial163.stderr: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.74408.log.gz 2026-02-18T08:53:44.632 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.41715.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.41715.log.gz 2026-02-18T08:53:44.632 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.34000.log 2026-02-18T08:53:44.632 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.126369.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.134368.log 2026-02-18T08:53:44.632 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.34000.log: 89.4% -- replaced with /var/log/ceph/cluster2-client.mirror.126369.log.gz 2026-02-18T08:53:44.632 INFO:teuthology.orchestra.run.trial163.stderr: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.34000.log.gz 2026-02-18T08:53:44.634 INFO:teuthology.orchestra.run.trial163.stderr: 95.8% -- replaced with /var/log/ceph/cluster2-client.mirror.24820.log.gz 2026-02-18T08:53:44.639 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.34283.log 2026-02-18T08:53:44.640 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.134368.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.71184.log 2026-02-18T08:53:44.640 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.34283.log: 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.134368.log.gz 2026-02-18T08:53:44.642 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.102041.log 2026-02-18T08:53:44.642 INFO:teuthology.orchestra.run.trial163.stderr: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.34283.log.gz 2026-02-18T08:53:44.642 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.71184.log: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.71184.log.gz 2026-02-18T08:53:44.642 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.123906.log 2026-02-18T08:53:44.642 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.102041.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.74338.log 2026-02-18T08:53:44.642 INFO:teuthology.orchestra.run.trial163.stderr: 88.2%/var/log/ceph/cluster2-client.mirror.123906.log: -- replaced with /var/log/ceph/cluster2-client.mirror.102041.log.gz 2026-02-18T08:53:44.644 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.26298.log 2026-02-18T08:53:44.644 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.74338.log: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.74338.log.gz 2026-02-18T08:53:44.644 INFO:teuthology.orchestra.run.trial163.stderr: 92.3% -- replaced with /var/log/ceph/cluster2-client.mirror.123906.log.gz 2026-02-18T08:53:44.644 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.29832.log 2026-02-18T08:53:44.644 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.26298.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.46832.log 2026-02-18T08:53:44.644 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.29832.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.26298.log.gz 2026-02-18T08:53:44.648 INFO:teuthology.orchestra.run.trial163.stderr: 88.1%gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.21129.log 2026-02-18T08:53:44.648 INFO:teuthology.orchestra.run.trial163.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.29832.log.gz 2026-02-18T08:53:44.648 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.46832.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.83059.log 2026-02-18T08:53:44.648 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.21129.log: gzip -5 88.7% --verbose -- -- replaced with /var/log/ceph/cluster2-client.mirror.46832.log.gz /var/log/ceph/cluster2-client.mirror.21780.log 2026-02-18T08:53:44.648 INFO:teuthology.orchestra.run.trial163.stderr: 2026-02-18T08:53:44.648 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.83059.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.49040.log 2026-02-18T08:53:44.648 INFO:teuthology.orchestra.run.trial163.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.21129.log.gz 2026-02-18T08:53:44.648 INFO:teuthology.orchestra.run.trial163.stderr: 89.2%/var/log/ceph/cluster2-client.mirror.21780.log: -- replaced with /var/log/ceph/cluster2-client.mirror.83059.log.gz 2026-02-18T08:53:44.648 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.55211.log 2026-02-18T08:53:44.649 INFO:teuthology.orchestra.run.trial163.stderr: 88.3%/var/log/ceph/cluster2-client.mirror.49040.log: -- replaced with /var/log/ceph/cluster2-client.mirror.21780.log.gz 2026-02-18T08:53:44.649 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.120874.log 2026-02-18T08:53:44.649 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.55211.log: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.49040.log.gz 2026-02-18T08:53:44.649 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.35864.log 2026-02-18T08:53:44.649 INFO:teuthology.orchestra.run.trial163.stderr: 88.4% -- replaced with /var/log/ceph/cluster1-client.mirror.55211.log.gz 2026-02-18T08:53:44.649 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.120874.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.14447.log 2026-02-18T08:53:44.649 INFO:teuthology.orchestra.run.trial163.stderr: 88.3%/var/log/ceph/cluster2-client.mirror.35864.log: -- replaced with /var/log/ceph/cluster2-client.mirror.120874.log.gz 2026-02-18T08:53:44.649 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.130983.log 2026-02-18T08:53:44.649 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.admin.14447.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.14447.log.gz 2026-02-18T08:53:44.649 INFO:teuthology.orchestra.run.trial163.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.35864.log.gz 2026-02-18T08:53:44.649 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.46275.log 2026-02-18T08:53:44.651 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.130983.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.84271.log 2026-02-18T08:53:44.651 INFO:teuthology.orchestra.run.trial163.stderr: 87.0% -- replaced with /var/log/ceph/cluster1-client.mirror.130983.log.gz 2026-02-18T08:53:44.651 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.admin.46275.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.77837.log 2026-02-18T08:53:44.651 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.84271.log: 88.2% -- replaced with /var/log/ceph/cluster1-client.admin.46275.log.gz 2026-02-18T08:53:44.651 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.28065.log 2026-02-18T08:53:44.651 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.77837.log: 89.2% -- replaced with /var/log/ceph/cluster2-client.mirror.77837.log.gz 2026-02-18T08:53:44.651 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.27414.log 2026-02-18T08:53:44.651 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.28065.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.28065.log.gz 2026-02-18T08:53:44.653 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-osd.2.log 2026-02-18T08:53:44.653 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.27414.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.27414.log.gz 2026-02-18T08:53:44.653 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.103475.log 2026-02-18T08:53:44.653 INFO:teuthology.orchestra.run.trial163.stderr: 93.3%/var/log/ceph/cluster2-osd.2.log: -- replaced with /var/log/ceph/cluster2-client.mirror.84271.log.gz 2026-02-18T08:53:44.653 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.102240.log 2026-02-18T08:53:44.653 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.103475.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.40095.log 2026-02-18T08:53:44.655 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.102240.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.82041.log 2026-02-18T08:53:44.655 INFO:teuthology.orchestra.run.trial163.stderr: 88.2%/var/log/ceph/cluster2-client.mirror.40095.log: -- replaced with /var/log/ceph/cluster2-client.mirror.102240.log.gz 2026-02-18T08:53:44.655 INFO:teuthology.orchestra.run.trial163.stderr: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.40095.log.gz 2026-02-18T08:53:44.656 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.62990.log 2026-02-18T08:53:44.656 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.82041.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.41808.log 2026-02-18T08:53:44.656 INFO:teuthology.orchestra.run.trial163.stderr: 89.1%/var/log/ceph/cluster2-client.mirror.62990.log: -- replaced with /var/log/ceph/cluster2-client.mirror.82041.log.gz 2026-02-18T08:53:44.656 INFO:teuthology.orchestra.run.trial163.stderr: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.62990.log.gz 2026-02-18T08:53:44.656 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.98772.log 2026-02-18T08:53:44.657 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.41808.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.41808.log.gz 2026-02-18T08:53:44.657 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.70869.log 2026-02-18T08:53:44.657 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.98772.log: 92.1% -- replaced with /var/log/ceph/cluster1-client.mirror.98772.log.gz 2026-02-18T08:53:44.658 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.72724.log 2026-02-18T08:53:44.658 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.70869.log: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.70869.log.gz 2026-02-18T08:53:44.659 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.133470.log 2026-02-18T08:53:44.659 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.72724.log: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.72724.log.gz 2026-02-18T08:53:44.660 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.124204.log 2026-02-18T08:53:44.660 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.133470.log: 88.2% -- replaced with /var/log/ceph/cluster1-client.mirror.103475.log.gz 2026-02-18T08:53:44.660 INFO:teuthology.orchestra.run.trial163.stderr: 86.7% -- replaced with /var/log/ceph/cluster1-client.mirror.133470.log.gz 2026-02-18T08:53:44.661 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.89679.log 2026-02-18T08:53:44.662 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.124204.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.105867.log 2026-02-18T08:53:44.662 INFO:teuthology.orchestra.run.trial163.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.124204.log.gz 2026-02-18T08:53:44.662 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.89679.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.103520.log 2026-02-18T08:53:44.662 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.105867.log: 87.9% -- replaced with /var/log/ceph/cluster2-client.mirror.105867.log.gz 2026-02-18T08:53:44.662 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.29774.log 2026-02-18T08:53:44.663 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.103520.log: 92.2% -- replaced with /var/log/ceph/cluster1-client.mirror.103520.log.gz 2026-02-18T08:53:44.664 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.60339.log 2026-02-18T08:53:44.665 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.29774.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.29774.log.gz 2026-02-18T08:53:44.665 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.79307.log 2026-02-18T08:53:44.666 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.60339.log: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.60339.log.gz 2026-02-18T08:53:44.666 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-osd.1.log 2026-02-18T08:53:44.667 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.79307.log: 89.2% -- replaced with /var/log/ceph/cluster2-client.mirror.79307.log.gz 2026-02-18T08:53:44.667 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.21501.log 2026-02-18T08:53:44.667 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-osd.1.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.132002.log 2026-02-18T08:53:44.684 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.21501.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.33872.log 2026-02-18T08:53:44.684 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.132002.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.21501.log.gz 2026-02-18T08:53:44.684 INFO:teuthology.orchestra.run.trial163.stderr: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.132002.log.gz 2026-02-18T08:53:44.692 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.54574.log 2026-02-18T08:53:44.695 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.33872.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.33872.log.gz 2026-02-18T08:53:44.708 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.0.84399.log 2026-02-18T08:53:44.708 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.54574.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.54574.log.gz 2026-02-18T08:53:44.716 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.39572.log 2026-02-18T08:53:44.720 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.0.84399.log: 95.7% -- replaced with /var/log/ceph/cluster2-client.mirror.88607.log.gz 2026-02-18T08:53:44.724 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.104661.log 2026-02-18T08:53:44.736 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.39572.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.106282.log 2026-02-18T08:53:44.736 INFO:teuthology.orchestra.run.trial163.stderr: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.39572.log.gz 2026-02-18T08:53:44.736 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.104661.log: 88.2% -- replaced with /var/log/ceph/cluster1-client.mirror.104661.log.gz 2026-02-18T08:53:44.750 INFO:teuthology.orchestra.run.trial163.stderr: 95.7% -- replaced with /var/log/ceph/cluster2-client.mirror.46303.log.gz 2026-02-18T08:53:44.752 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.90802.log 2026-02-18T08:53:44.753 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.106282.log: 92.5% -- replaced with /var/log/ceph/cluster1-client.mirror.106282.log.gz 2026-02-18T08:53:44.753 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.26114.log 2026-02-18T08:53:44.754 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.90802.log: 95.7% -- replaced with /var/log/ceph/cluster2-client.mirror.112390.log.gz 2026-02-18T08:53:44.768 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.76577.log 2026-02-18T08:53:44.768 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.26114.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.26114.log.gz 2026-02-18T08:53:44.770 INFO:teuthology.orchestra.run.trial163.stderr: 95.8% -- replaced with /var/log/ceph/cluster2-client.mirror.85688.log.gz 2026-02-18T08:53:44.783 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.82321.log 2026-02-18T08:53:44.785 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.105766.log 2026-02-18T08:53:44.785 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.76577.log: /var/log/ceph/cluster2-client.mirror.82321.log: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.76577.log.gz 2026-02-18T08:53:44.785 INFO:teuthology.orchestra.run.trial163.stderr: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.82321.log.gz 2026-02-18T08:53:44.786 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.132786.log 2026-02-18T08:53:44.786 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.105766.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.105450.log 2026-02-18T08:53:44.786 INFO:teuthology.orchestra.run.trial163.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.105766.log.gz 2026-02-18T08:53:44.786 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.132786.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.132786.log.gz 2026-02-18T08:53:44.787 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.25963.log 2026-02-18T08:53:44.787 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.100930.log 2026-02-18T08:53:44.788 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.105450.log: /var/log/ceph/cluster2-client.mirror.25963.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.105450.log.gz 2026-02-18T08:53:44.788 INFO:teuthology.orchestra.run.trial163.stderr: 88.4%gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.50304.log 2026-02-18T08:53:44.788 INFO:teuthology.orchestra.run.trial163.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.25963.log.gz 2026-02-18T08:53:44.788 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.100930.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.100930.log.gz 2026-02-18T08:53:44.788 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.15633.log 2026-02-18T08:53:44.790 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.120688.log 2026-02-18T08:53:44.790 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.50304.log: /var/log/ceph/cluster2-client.mirror.15633.log: 56.0% -- replaced with /var/log/ceph/cluster2-client.mirror.15633.log.gz 2026-02-18T08:53:44.790 INFO:teuthology.orchestra.run.trial163.stderr: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.50304.log.gz 2026-02-18T08:53:44.790 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.126959.log 2026-02-18T08:53:44.790 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.132868.log 2026-02-18T08:53:44.791 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.120688.log: /var/log/ceph/cluster2-client.mirror.126959.log: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.120688.log.gz 2026-02-18T08:53:44.791 INFO:teuthology.orchestra.run.trial163.stderr: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.126959.log.gz 2026-02-18T08:53:44.791 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.128207.log 2026-02-18T08:53:44.791 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.29460.log 2026-02-18T08:53:44.791 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.132868.log: /var/log/ceph/cluster2-client.mirror.128207.log: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.132868.log.gz 2026-02-18T08:53:44.792 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.126482.log 2026-02-18T08:53:44.792 INFO:teuthology.orchestra.run.trial163.stderr: 89.2% -- replaced with /var/log/ceph/cluster2-client.mirror.128207.log.gz 2026-02-18T08:53:44.792 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.29460.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.30660.log 2026-02-18T08:53:44.792 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.130605.log 2026-02-18T08:53:44.792 INFO:teuthology.orchestra.run.trial163.stderr: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.29460.log.gz 2026-02-18T08:53:44.793 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.30660.log: /var/log/ceph/cluster2-client.mirror.126482.log: 90.7% -- replaced with /var/log/ceph/cluster2-client.mirror.126482.log.gz 2026-02-18T08:53:44.793 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.102590.log 2026-02-18T08:53:44.794 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.130605.log: 86.8% -- replaced with /var/log/ceph/cluster1-client.mirror.130605.log.gz 2026-02-18T08:53:44.794 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.35213.log 2026-02-18T08:53:44.795 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.102590.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.102590.log.gz 2026-02-18T08:53:44.795 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.125029.log 2026-02-18T08:53:44.796 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.35213.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.35213.log.gz 2026-02-18T08:53:44.796 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.90615.log 2026-02-18T08:53:44.797 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.125029.log: 96.8% -- replaced with /var/log/ceph/cluster2-client.mirror.30660.log.gz 2026-02-18T08:53:44.797 INFO:teuthology.orchestra.run.trial163.stderr: 92.7% -- replaced with /var/log/ceph/cluster2-client.mirror.125029.log.gz 2026-02-18T08:53:44.798 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.134977.log 2026-02-18T08:53:44.799 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.61009.log 2026-02-18T08:53:44.799 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.90615.log: /var/log/ceph/cluster2-client.mirror.134977.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.mirror.134977.log.gz 2026-02-18T08:53:44.812 INFO:teuthology.orchestra.run.trial163.stderr: 95.7% -- replaced with /var/log/ceph/cluster2-client.mirror.111025.log.gz 2026-02-18T08:53:44.819 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.132480.log 2026-02-18T08:53:44.820 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.61009.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.58581.log 2026-02-18T08:53:44.822 INFO:teuthology.orchestra.run.trial163.stderr: 88.7%/var/log/ceph/cluster2-client.mirror.132480.log: -- replaced with /var/log/ceph/cluster2-client.mirror.61009.log.gz 2026-02-18T08:53:44.822 INFO:teuthology.orchestra.run.trial163.stderr: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.132480.log.gz 2026-02-18T08:53:44.822 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.85206.log 2026-02-18T08:53:44.822 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.58581.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.110802.log 2026-02-18T08:53:44.823 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.85206.log: 89.5% -- replaced with /var/log/ceph/cluster2-client.mirror.85206.log.gz 2026-02-18T08:53:44.823 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.94461.log 2026-02-18T08:53:44.825 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.110802.log: 95.5% -- replaced with /var/log/ceph/cluster2-client.mirror.86977.log.gz 2026-02-18T08:53:44.825 INFO:teuthology.orchestra.run.trial163.stderr: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.110802.log.gz 2026-02-18T08:53:44.825 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.83678.log 2026-02-18T08:53:44.825 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.94461.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.85827.log 2026-02-18T08:53:44.826 INFO:teuthology.orchestra.run.trial163.stderr: 88.4%/var/log/ceph/cluster2-client.mirror.83678.log: -- replaced with /var/log/ceph/cluster2-client.mirror.94461.log.gz 93.5% -- replaced with /var/log/ceph/cluster1-client.mirror.58581.log.gz 2026-02-18T08:53:44.826 INFO:teuthology.orchestra.run.trial163.stderr: 2026-02-18T08:53:44.826 INFO:teuthology.orchestra.run.trial163.stderr: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.83678.log.gz 2026-02-18T08:53:44.826 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.53559.log 2026-02-18T08:53:44.826 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.85827.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.2.14953.log 2026-02-18T08:53:44.826 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.53559.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.62780.log 2026-02-18T08:53:44.827 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.2.14953.log: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.53559.log.gz 2026-02-18T08:53:44.827 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.127769.log 2026-02-18T08:53:44.827 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.62780.log: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.62780.log.gz 2026-02-18T08:53:44.827 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.32663.log 2026-02-18T08:53:44.829 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.127769.log: 89.4% -- replaced with /var/log/ceph/cluster2-client.mirror.127769.log.gz 2026-02-18T08:53:44.829 INFO:teuthology.orchestra.run.trial163.stderr: 93.2% -- replaced with /var/log/ceph/cluster1-client.mirror.2.14953.log.gz 2026-02-18T08:53:44.829 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.41153.log 2026-02-18T08:53:44.829 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.32663.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.66784.log 2026-02-18T08:53:44.830 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.41153.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.32663.log.gz 2026-02-18T08:53:44.830 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.125724.log 2026-02-18T08:53:44.830 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.66784.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.41153.log.gz 2026-02-18T08:53:44.830 INFO:teuthology.orchestra.run.trial163.stderr: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.66784.log.gz 2026-02-18T08:53:44.831 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.123444.log 2026-02-18T08:53:44.831 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.75493.log 2026-02-18T08:53:44.832 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.125724.log: /var/log/ceph/cluster2-client.mirror.123444.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.123444.log.gz 2026-02-18T08:53:44.832 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.131878.log 2026-02-18T08:53:44.833 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.75493.log: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.75493.log.gz 2026-02-18T08:53:44.833 INFO:teuthology.orchestra.run.trial163.stderr: 90.7% -- replaced with /var/log/ceph/cluster2-client.mirror.125724.log.gz 2026-02-18T08:53:44.833 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.128734.log 2026-02-18T08:53:44.833 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.131878.log: 83.6% -- replaced with /var/log/ceph/cluster2-client.mirror.131878.log.gz 2026-02-18T08:53:44.833 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.42351.log 2026-02-18T08:53:44.834 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.128734.log: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.128734.log.gz 2026-02-18T08:53:44.834 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.52089.log 2026-02-18T08:53:44.835 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.42351.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.109054.log 2026-02-18T08:53:44.835 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.111238.log 2026-02-18T08:53:44.836 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.109054.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.109054.log.gz 2026-02-18T08:53:44.836 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.0.61174.log 2026-02-18T08:53:44.836 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.111238.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.111238.log.gz 2026-02-18T08:53:44.837 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.120931.log 2026-02-18T08:53:44.838 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.0.61174.log: 92.5% -- replaced with /var/log/ceph/cluster2-client.mirror.0.61174.log.gz 2026-02-18T08:53:44.839 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.112121.log 2026-02-18T08:53:44.840 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.120931.log: 96.8% -- replaced with /var/log/ceph/cluster2-client.mirror.42351.log.gz 2026-02-18T08:53:44.840 INFO:teuthology.orchestra.run.trial163.stderr: 92.0%/var/log/ceph/cluster2-client.mirror.52089.log: -- replaced with /var/log/ceph/cluster2-client.mirror.120931.log.gz 2026-02-18T08:53:44.841 INFO:teuthology.orchestra.run.trial163.stderr: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.52089.log.gz 2026-02-18T08:53:44.841 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.128541.log 2026-02-18T08:53:44.842 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.112121.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.112121.log.gz 2026-02-18T08:53:44.842 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.15650.log 2026-02-18T08:53:44.843 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.128541.log: 90.6% -- replaced with /var/log/ceph/cluster2-client.mirror.128541.log.gz 2026-02-18T08:53:44.844 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.14419.log 2026-02-18T08:53:44.845 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.22803.log 2026-02-18T08:53:44.845 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.15650.log: /var/log/ceph/cluster2-client.admin.14419.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.14419.log.gz 54.2% -- replaced with /var/log/ceph/cluster2-client.mirror.15650.log.gz 2026-02-18T08:53:44.845 INFO:teuthology.orchestra.run.trial163.stderr: 2026-02-18T08:53:44.845 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.112921.log 2026-02-18T08:53:44.846 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.88933.log 2026-02-18T08:53:44.846 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.22803.log: /var/log/ceph/cluster1-client.mirror.112921.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.22803.log.gz 2026-02-18T08:53:44.846 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.31116.log 2026-02-18T08:53:44.847 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.88933.log: 92.5% -- replaced with /var/log/ceph/cluster1-client.mirror.112921.log.gz 2026-02-18T08:53:44.847 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.30018.log 2026-02-18T08:53:44.848 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.31116.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.31116.log.gz 2026-02-18T08:53:44.849 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.73879.log 2026-02-18T08:53:44.849 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.30018.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.30018.log.gz 2026-02-18T08:53:44.850 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.63480.log 2026-02-18T08:53:44.850 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.73879.log: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.73879.log.gz 2026-02-18T08:53:44.850 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.95187.log 2026-02-18T08:53:44.851 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.63480.log: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.63480.log.gz 2026-02-18T08:53:44.851 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.0.58733.log 2026-02-18T08:53:44.852 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.95187.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.95187.log.gz 2026-02-18T08:53:44.852 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.80252.log 2026-02-18T08:53:44.854 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.0.58733.log: 95.7% -- replaced with /var/log/ceph/cluster2-client.mirror.86133.log.gz 2026-02-18T08:53:44.854 INFO:teuthology.orchestra.run.trial163.stderr: 91.2% -- replaced with /var/log/ceph/cluster2-mon.a.log.gz 2026-02-18T08:53:44.854 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.125451.log 2026-02-18T08:53:44.855 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.80252.log: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.80252.log.gz 2026-02-18T08:53:44.855 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.84967.log 2026-02-18T08:53:44.855 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.125451.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.91763.log 2026-02-18T08:53:44.855 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.84967.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.125451.log.gz 2026-02-18T08:53:44.856 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.75547.log 2026-02-18T08:53:44.856 INFO:teuthology.orchestra.run.trial163.stderr: 93.9% -- replaced with /var/log/ceph/cluster1-client.mirror.0.58733.log.gz 2026-02-18T08:53:44.857 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.91763.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.129632.log 2026-02-18T08:53:44.857 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.75547.log: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.75547.log.gz 2026-02-18T08:53:44.857 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.26577.log 2026-02-18T08:53:44.857 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.129632.log: 87.0% -- replaced with /var/log/ceph/cluster1-client.mirror.129632.log.gz 2026-02-18T08:53:44.859 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.33779.log 2026-02-18T08:53:44.859 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.26577.log: 93.5% -- replaced with /var/log/ceph/cluster2-client.mirror.84967.log.gz 2026-02-18T08:53:44.859 INFO:teuthology.orchestra.run.trial163.stderr: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.26577.log.gz 2026-02-18T08:53:44.859 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.28937.log 2026-02-18T08:53:44.859 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.33779.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.88208.log 2026-02-18T08:53:44.860 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.28937.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.33779.log.gz 2026-02-18T08:53:44.860 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.87969.log 2026-02-18T08:53:44.860 INFO:teuthology.orchestra.run.trial163.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.28937.log.gz 2026-02-18T08:53:44.860 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.88208.log: 84.7% -- replaced with /var/log/ceph/cluster2-client.mirror.88208.log.gz 2026-02-18T08:53:44.861 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.100466.log 2026-02-18T08:53:44.862 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.87969.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.51109.log 2026-02-18T08:53:44.862 INFO:teuthology.orchestra.run.trial163.stderr: 84.8% -- replaced with /var/log/ceph/cluster2-client.mirror.87969.log.gz 2026-02-18T08:53:44.862 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.100466.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.100466.log.gzgzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.41246.log 2026-02-18T08:53:44.862 INFO:teuthology.orchestra.run.trial163.stderr: 2026-02-18T08:53:44.863 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.51109.log: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.51109.log.gz 2026-02-18T08:53:44.863 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.95793.log 2026-02-18T08:53:44.863 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.41246.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.63935.log 2026-02-18T08:53:44.863 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.95793.log: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.41246.log.gz 2026-02-18T08:53:44.864 INFO:teuthology.orchestra.run.trial163.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.95793.log.gz 2026-02-18T08:53:44.864 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.35120.log 2026-02-18T08:53:44.864 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.63935.log: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.63935.log.gz 2026-02-18T08:53:44.864 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.15183.log 2026-02-18T08:53:44.864 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.35120.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.28344.log 2026-02-18T08:53:44.866 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.admin.15183.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.35120.log.gz 2026-02-18T08:53:44.866 INFO:teuthology.orchestra.run.trial163.stderr: 95.7% 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.15183.log.gz 2026-02-18T08:53:44.866 INFO:teuthology.orchestra.run.trial163.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.96866.log.gz 2026-02-18T08:53:44.866 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.14777.log 2026-02-18T08:53:44.866 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.28344.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.74194.log 2026-02-18T08:53:44.866 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.admin.14777.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.83856.log 2026-02-18T08:53:44.866 INFO:teuthology.orchestra.run.trial163.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.14777.log.gz 2026-02-18T08:53:44.866 INFO:teuthology.orchestra.run.trial163.stderr: 88.4%/var/log/ceph/cluster2-client.mirror.74194.log: -- replaced with /var/log/ceph/cluster2-client.mirror.28344.log.gz 2026-02-18T08:53:44.867 INFO:teuthology.orchestra.run.trial163.stderr: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.74194.log.gz 2026-02-18T08:53:44.879 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.31454.log 2026-02-18T08:53:44.880 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.85753.log 2026-02-18T08:53:44.881 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.83856.log: 89.2%/var/log/ceph/cluster2-client.mirror.31454.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.16971.log 2026-02-18T08:53:44.881 INFO:teuthology.orchestra.run.trial163.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.83856.log.gz 2026-02-18T08:53:44.881 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.68730.log 2026-02-18T08:53:44.881 INFO:teuthology.orchestra.run.trial163.stderr: 88.5%/var/log/ceph/cluster2-client.mirror.85753.log: -- replaced with /var/log/ceph/cluster2-client.mirror.31454.log.gz 2026-02-18T08:53:44.882 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.16971.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.101359.log 2026-02-18T08:53:44.882 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.68730.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.16971.log.gz 2026-02-18T08:53:44.882 INFO:teuthology.orchestra.run.trial163.stderr: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.68730.log.gzgzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.34934.log 2026-02-18T08:53:44.883 INFO:teuthology.orchestra.run.trial163.stderr: 2026-02-18T08:53:44.883 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.101359.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.15437.log 2026-02-18T08:53:44.884 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.34934.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.34934.log.gz 2026-02-18T08:53:44.884 INFO:teuthology.orchestra.run.trial163.stderr: 92.7% -- replaced with /var/log/ceph/cluster2-client.mirror.101359.log.gz 2026-02-18T08:53:44.884 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.40002.log 2026-02-18T08:53:44.884 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.admin.15437.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.19732.log 2026-02-18T08:53:44.884 INFO:teuthology.orchestra.run.trial163.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.15437.log.gz 2026-02-18T08:53:44.886 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.40002.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.130217.log 2026-02-18T08:53:44.886 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.19732.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.64744.log 2026-02-18T08:53:44.886 INFO:teuthology.orchestra.run.trial163.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.19732.log.gz 2026-02-18T08:53:44.886 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.130217.log: 87.0% -- replaced with /var/log/ceph/cluster1-client.mirror.130217.log.gz 2026-02-18T08:53:44.886 INFO:teuthology.orchestra.run.trial163.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.40002.log.gz 2026-02-18T08:53:44.886 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.115910.log 2026-02-18T08:53:44.887 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.64744.log: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.64744.log.gz 2026-02-18T08:53:44.887 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.93783.log 2026-02-18T08:53:44.887 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.115910.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.70239.log 2026-02-18T08:53:44.889 INFO:teuthology.orchestra.run.trial163.stderr: 87.4% -- replaced with /var/log/ceph/cluster2-client.mirror.115910.log.gz 2026-02-18T08:53:44.889 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.93783.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.109730.log 2026-02-18T08:53:44.889 INFO:teuthology.orchestra.run.trial163.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.93783.log.gz 2026-02-18T08:53:44.889 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.70239.log: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.70239.log.gz 2026-02-18T08:53:44.889 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.104967.log 2026-02-18T08:53:44.889 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.39107.log 2026-02-18T08:53:44.890 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.109730.log: /var/log/ceph/cluster2-client.mirror.104967.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.104967.log.gz 2026-02-18T08:53:44.890 INFO:teuthology.orchestra.run.trial163.stderr: 92.5% -- replaced with /var/log/ceph/cluster2-client.mirror.109730.log.gz 2026-02-18T08:53:44.891 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.90121.log 2026-02-18T08:53:44.891 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.39107.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.64989.log 2026-02-18T08:53:44.891 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.90121.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.78047.log 2026-02-18T08:53:44.891 INFO:teuthology.orchestra.run.trial163.stderr: 84.8% -- replaced with /var/log/ceph/cluster2-client.mirror.90121.log.gz 2026-02-18T08:53:44.891 INFO:teuthology.orchestra.run.trial163.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.39107.log.gz 2026-02-18T08:53:44.892 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.64989.log: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.64989.log.gz 2026-02-18T08:53:44.892 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.91084.log 2026-02-18T08:53:44.893 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.42097.log 2026-02-18T08:53:44.893 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.78047.log: /var/log/ceph/cluster2-client.mirror.91084.log: 89.2% -- replaced with /var/log/ceph/cluster2-client.mirror.78047.log.gz 2026-02-18T08:53:44.893 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.58624.log 2026-02-18T08:53:44.894 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.42097.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.42097.log.gz 2026-02-18T08:53:44.894 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.96776.log 2026-02-18T08:53:44.898 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.58624.log: 93.5% -- replaced with /var/log/ceph/cluster1-client.mirror.58624.log.gz 2026-02-18T08:53:44.898 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.135027.log 2026-02-18T08:53:44.899 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.96776.log: 92.2% -- replaced with /var/log/ceph/cluster2-client.mirror.96776.log.gz 2026-02-18T08:53:44.900 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.106678.log 2026-02-18T08:53:44.900 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.135027.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.mirror.135027.log.gz 2026-02-18T08:53:44.901 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.24198.log 2026-02-18T08:53:44.901 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.106678.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.106678.log.gz 2026-02-18T08:53:44.901 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.104866.log 2026-02-18T08:53:44.902 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.24198.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.24198.log.gz 2026-02-18T08:53:44.902 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.82531.log 2026-02-18T08:53:44.903 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.104866.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.104866.log.gz 2026-02-18T08:53:44.904 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.31303.log 2026-02-18T08:53:44.904 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.82531.log: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.82531.log.gz 2026-02-18T08:53:44.904 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.101839.log 2026-02-18T08:53:44.905 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.31303.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.92818.log 2026-02-18T08:53:44.905 INFO:teuthology.orchestra.run.trial163.stderr: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.31303.log.gz 2026-02-18T08:53:44.906 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.101839.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.101839.log.gz 2026-02-18T08:53:44.906 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.128783.log 2026-02-18T08:53:44.907 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.92818.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.92818.log.gz 2026-02-18T08:53:44.907 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.82111.log 2026-02-18T08:53:44.908 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.128783.log: 89.2% -- replaced with /var/log/ceph/cluster2-client.mirror.128783.log.gz 2026-02-18T08:53:44.908 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.133555.log 2026-02-18T08:53:44.909 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.82111.log: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.82111.log.gz 2026-02-18T08:53:44.909 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.126766.log 2026-02-18T08:53:44.911 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.133555.log: 91.9% -- replaced with /var/log/ceph/cluster2-client.mirror.133555.log.gz 2026-02-18T08:53:44.911 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.79832.log 2026-02-18T08:53:44.913 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.126766.log: 90.7% -- replaced with /var/log/ceph/cluster2-client.mirror.126766.log.gz 2026-02-18T08:53:44.913 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.39258.log 2026-02-18T08:53:44.914 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.79832.log: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.79832.log.gz 2026-02-18T08:53:44.914 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.61695.log 2026-02-18T08:53:44.915 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.39258.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.39258.log.gz 2026-02-18T08:53:44.915 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.94804.log 2026-02-18T08:53:44.916 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.61695.log: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.61695.log.gz 2026-02-18T08:53:44.916 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.121683.log 2026-02-18T08:53:44.917 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.94804.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.94804.log.gz 2026-02-18T08:53:44.917 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.129384.log 2026-02-18T08:53:44.918 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.121683.log: 92.6% -- replaced with /var/log/ceph/cluster2-client.mirror.121683.log.gz 2026-02-18T08:53:44.919 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.101626.log 2026-02-18T08:53:44.919 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.129384.log: 86.7% -- replaced with /var/log/ceph/cluster1-client.mirror.129384.log.gz 2026-02-18T08:53:44.919 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.89254.log 2026-02-18T08:53:44.920 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.101626.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.101626.log.gz 2026-02-18T08:53:44.920 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.3.55454.log 2026-02-18T08:53:44.931 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.89254.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.118630.log 2026-02-18T08:53:44.944 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.3.55454.log: 93.6% 95.8% -- replaced with /var/log/ceph/cluster2-client.mirror.89934.log.gz 2026-02-18T08:53:44.947 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.131501.log 2026-02-18T08:53:44.948 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.118630.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.118630.log.gz 2026-02-18T08:53:44.949 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.130284.log 2026-02-18T08:53:44.949 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.131501.log: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.131501.log.gz 2026-02-18T08:53:44.950 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.85185.log 2026-02-18T08:53:44.950 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.130284.log: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.130284.log.gz 2026-02-18T08:53:44.950 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.67939.log 2026-02-18T08:53:44.951 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.85185.log: 89.6% -- replaced with /var/log/ceph/cluster2-client.mirror.85185.log.gz 2026-02-18T08:53:44.952 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.83767.log 2026-02-18T08:53:44.952 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.67939.log: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.67939.log.gz 2026-02-18T08:53:44.953 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.63095.log 2026-02-18T08:53:44.953 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.83767.log: 89.2% -- replaced with /var/log/ceph/cluster2-client.mirror.83767.log.gz 2026-02-18T08:53:44.953 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.121270.log 2026-02-18T08:53:44.954 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.63095.log: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.63095.log.gz 2026-02-18T08:53:44.955 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.42266.log 2026-02-18T08:53:44.955 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.121270.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.121270.log.gz 2026-02-18T08:53:44.956 INFO:teuthology.orchestra.run.trial163.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.3.55454.log.gz 2026-02-18T08:53:44.956 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.112323.log 2026-02-18T08:53:44.956 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.42266.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.119869.log 2026-02-18T08:53:44.960 INFO:teuthology.orchestra.run.trial163.stderr: 93.3% -- replaced with /var/log/ceph/cluster2-client.mirror.42266.log.gz 2026-02-18T08:53:44.960 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.112323.log: 88.4% -- replaced with /var/log/ceph/cluster1-client.mirror.112323.log.gz 2026-02-18T08:53:44.961 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.54714.log 2026-02-18T08:53:44.961 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.119869.log: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.119869.log.gz 2026-02-18T08:53:44.962 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.128065.log 2026-02-18T08:53:44.962 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.54714.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.130560.log 2026-02-18T08:53:44.963 INFO:teuthology.orchestra.run.trial163.stderr: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.54714.log.gz 2026-02-18T08:53:44.964 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.128065.log: 89.2% -- replaced with /var/log/ceph/cluster2-client.mirror.128065.log.gz 2026-02-18T08:53:44.964 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.124545.log 2026-02-18T08:53:44.964 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.130560.log: 86.8% -- replaced with /var/log/ceph/cluster1-client.mirror.130560.log.gz 2026-02-18T08:53:44.965 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.31861.log 2026-02-18T08:53:44.971 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.124545.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.119125.log 2026-02-18T08:53:44.979 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.31861.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.119335.log 2026-02-18T08:53:44.980 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.119125.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.31861.log.gz 2026-02-18T08:53:44.980 INFO:teuthology.orchestra.run.trial163.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.119125.log.gz 2026-02-18T08:53:44.983 INFO:teuthology.orchestra.run.trial163.stderr: 95.7% -- replaced with /var/log/ceph/cluster2-client.mirror.58669.log.gz 2026-02-18T08:53:44.987 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.70344.log 2026-02-18T08:53:44.988 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.119335.log: 88.1% -- replaced with /var/log/ceph/cluster1-client.mirror.119335.log.gz 2026-02-18T08:53:44.989 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.87146.log 2026-02-18T08:53:44.989 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.70344.log: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.70344.log.gz 2026-02-18T08:53:44.990 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.122232.log 2026-02-18T08:53:44.990 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.87146.log: 84.7% -- replaced with /var/log/ceph/cluster2-client.mirror.87146.log.gz 2026-02-18T08:53:44.990 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.127453.log 2026-02-18T08:53:44.991 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.122232.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.122232.log.gz 2026-02-18T08:53:44.991 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.88259.log 2026-02-18T08:53:44.993 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.127453.log: 90.6% -- replaced with /var/log/ceph/cluster2-client.mirror.127453.log.gz 2026-02-18T08:53:44.994 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.84987.log 2026-02-18T08:53:44.994 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.88259.log: 84.8% -- replaced with /var/log/ceph/cluster2-client.mirror.88259.log.gz 2026-02-18T08:53:44.994 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.61114.log 2026-02-18T08:53:44.998 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.84987.log: 93.4% -- replaced with /var/log/ceph/cluster1-client.mirror.84987.log.gz 2026-02-18T08:53:44.998 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.126979.log 2026-02-18T08:53:44.999 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.61114.log: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.61114.log.gz 2026-02-18T08:53:44.999 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.59429.log 2026-02-18T08:53:45.001 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.126979.log: 90.6% -- replaced with /var/log/ceph/cluster2-client.mirror.126979.log.gz 2026-02-18T08:53:45.001 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.59534.log 2026-02-18T08:53:45.002 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.59429.log: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.59429.log.gz 2026-02-18T08:53:45.002 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.72409.log 2026-02-18T08:53:45.003 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.59534.log: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.59534.log.gz 2026-02-18T08:53:45.003 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.27600.log 2026-02-18T08:53:45.003 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.72409.log: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.72409.log.gz 2026-02-18T08:53:45.004 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.132561.log 2026-02-18T08:53:45.005 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.27600.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.27600.log.gz 2026-02-18T08:53:45.005 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.110482.log 2026-02-18T08:53:45.006 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.132561.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.132561.log.gz 2026-02-18T08:53:45.006 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.34527.log 2026-02-18T08:53:45.007 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.110482.log: 95.7% -- replaced with /var/log/ceph/cluster2-client.mirror.87942.log.gz 2026-02-18T08:53:45.007 INFO:teuthology.orchestra.run.trial163.stderr: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.110482.log.gz 2026-02-18T08:53:45.007 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.129834.log 2026-02-18T08:53:45.008 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.34527.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.95599.log 2026-02-18T08:53:45.008 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.129834.log: 87.0% -- replaced with /var/log/ceph/cluster1-client.mirror.129834.log.gz 2026-02-18T08:53:45.008 INFO:teuthology.orchestra.run.trial163.stderr: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.34527.log.gz 2026-02-18T08:53:45.009 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.15275.log 2026-02-18T08:53:45.009 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.95599.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.52999.log 2026-02-18T08:53:45.010 INFO:teuthology.orchestra.run.trial163.stderr: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.95599.log.gz 2026-02-18T08:53:45.010 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.15275.log: 4.7% -- replaced with /var/log/ceph/cluster1-client.mirror.15275.log.gz 2026-02-18T08:53:45.010 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.84511.log 2026-02-18T08:53:45.010 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.52999.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.110104.log 2026-02-18T08:53:45.010 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.84511.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.52999.log.gz 2026-02-18T08:53:45.011 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.118584.log 2026-02-18T08:53:45.011 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.110104.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.110104.log.gz 2026-02-18T08:53:45.011 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.93166.log 2026-02-18T08:53:45.012 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.118584.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.118584.log.gz 2026-02-18T08:53:45.012 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.17436.log 2026-02-18T08:53:45.013 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.93166.log: 92.6% -- replaced with /var/log/ceph/cluster2-client.mirror.93166.log.gz 2026-02-18T08:53:45.014 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.114308.log 2026-02-18T08:53:45.014 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.17436.log: 93.4% -- replaced with /var/log/ceph/cluster1-client.mirror.84511.log.gz 2026-02-18T08:53:45.015 INFO:teuthology.orchestra.run.trial163.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.17436.log.gz 2026-02-18T08:53:45.015 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.127882.log 2026-02-18T08:53:45.015 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.114308.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.114308.log.gz 2026-02-18T08:53:45.015 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.65199.log 2026-02-18T08:53:45.017 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.127882.log: 90.7% -- replaced with /var/log/ceph/cluster2-client.mirror.127882.log.gz 2026-02-18T08:53:45.017 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.54084.log 2026-02-18T08:53:45.018 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.65199.log: 95.7% -- replaced with /var/log/ceph/cluster2-client.mirror.108818.log.gz 2026-02-18T08:53:45.018 INFO:teuthology.orchestra.run.trial163.stderr: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.65199.log.gz 2026-02-18T08:53:45.019 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.71814.log 2026-02-18T08:53:45.019 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.54084.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.73459.log 2026-02-18T08:53:45.019 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.71814.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.54084.log.gz 2026-02-18T08:53:45.019 INFO:teuthology.orchestra.run.trial163.stderr: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.71814.log.gz 2026-02-18T08:53:45.019 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.19544.log 2026-02-18T08:53:45.020 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.73459.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.107733.log 2026-02-18T08:53:45.021 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.19544.log: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.73459.log.gz 2026-02-18T08:53:45.021 INFO:teuthology.orchestra.run.trial163.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.19544.log.gz 2026-02-18T08:53:45.021 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.135419.log 2026-02-18T08:53:45.021 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.107733.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.107733.log.gz 2026-02-18T08:53:45.022 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.108089.log 2026-02-18T08:53:45.022 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.admin.135419.log: gzip 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.135419.log.gz -5 2026-02-18T08:53:45.022 INFO:teuthology.orchestra.run.trial163.stderr: --verbose -- /var/log/ceph/cluster2-client.mirror.83643.log 2026-02-18T08:53:45.022 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.108089.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.57794.log 2026-02-18T08:53:45.022 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.83643.log: 89.2% -- replaced with /var/log/ceph/cluster2-client.mirror.83643.log.gz 2026-02-18T08:53:45.023 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.127575.log 2026-02-18T08:53:45.023 INFO:teuthology.orchestra.run.trial163.stderr: 92.7%/var/log/ceph/cluster2-client.mirror.57794.log: -- replaced with /var/log/ceph/cluster1-client.mirror.108089.log.gz 2026-02-18T08:53:45.023 INFO:teuthology.orchestra.run.trial163.stderr: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.57794.log.gz 2026-02-18T08:53:45.023 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.67624.log 2026-02-18T08:53:45.023 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.127575.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.106983.log 2026-02-18T08:53:45.025 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.67624.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.127575.log.gz 2026-02-18T08:53:45.025 INFO:teuthology.orchestra.run.trial163.stderr: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.67624.log.gz 2026-02-18T08:53:45.025 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.73494.log 2026-02-18T08:53:45.025 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.106983.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.131816.log 2026-02-18T08:53:45.025 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.73494.log: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.73494.log.gz 2026-02-18T08:53:45.031 INFO:teuthology.orchestra.run.trial163.stderr: 95.7% -- replaced with /var/log/ceph/cluster2-client.mirror.91987.log.gz 2026-02-18T08:53:45.031 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.118503.log 2026-02-18T08:53:45.031 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.131816.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.75248.log 2026-02-18T08:53:45.032 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.118503.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.131816.log.gz 2026-02-18T08:53:45.035 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.46412.log 2026-02-18T08:53:45.035 INFO:teuthology.orchestra.run.trial163.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.118503.log.gz 2026-02-18T08:53:45.035 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.75248.log: 95.8%gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.121457.log 2026-02-18T08:53:45.035 INFO:teuthology.orchestra.run.trial163.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.133643.log.gz 2026-02-18T08:53:45.035 INFO:teuthology.orchestra.run.trial163.stderr: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.75248.log.gz 2026-02-18T08:53:45.035 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.46412.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.60974.log 2026-02-18T08:53:45.035 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.121457.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.122421.log 2026-02-18T08:53:45.035 INFO:teuthology.orchestra.run.trial163.stderr: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.46412.log.gz 2026-02-18T08:53:45.035 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.60974.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.121457.log.gz 2026-02-18T08:53:45.035 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.116277.log 2026-02-18T08:53:45.035 INFO:teuthology.orchestra.run.trial163.stderr: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.60974.log.gz 2026-02-18T08:53:45.035 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.25684.log 2026-02-18T08:53:45.035 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.122421.log: /var/log/ceph/cluster2-client.mirror.116277.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.69084.log 2026-02-18T08:53:45.036 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.25684.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.134244.log 2026-02-18T08:53:45.036 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.69084.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.25684.log.gz 2026-02-18T08:53:45.038 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.32605.log 2026-02-18T08:53:45.038 INFO:teuthology.orchestra.run.trial163.stderr: 92.5% -- replaced with /var/log/ceph/cluster1-client.mirror.122421.log.gz 2026-02-18T08:53:45.038 INFO:teuthology.orchestra.run.trial163.stderr: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.69084.log.gz 2026-02-18T08:53:45.038 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.134244.log: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.134244.log.gz 2026-02-18T08:53:45.038 INFO:teuthology.orchestra.run.trial163.stderr: 92.5% -- replaced with /var/log/ceph/cluster2-client.mirror.116277.log.gz 2026-02-18T08:53:45.038 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.34093.log 2026-02-18T08:53:45.038 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.84159.log 2026-02-18T08:53:45.038 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.32605.log: /var/log/ceph/cluster2-client.mirror.34093.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.59324.log 2026-02-18T08:53:45.038 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.84159.log: 87.7% 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.34093.log.gz -- replaced with /var/log/ceph/cluster2-client.mirror.32605.log.gz 2026-02-18T08:53:45.038 INFO:teuthology.orchestra.run.trial163.stderr: 2026-02-18T08:53:45.039 INFO:teuthology.orchestra.run.trial163.stderr: 88.9% -- replaced with /var/log/ceph/cluster1-client.mirror.84159.log.gz 2026-02-18T08:53:45.039 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.28902.log 2026-02-18T08:53:45.039 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.53419.log 2026-02-18T08:53:45.043 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.59324.log: /var/log/ceph/cluster2-client.mirror.28902.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.65799.log 2026-02-18T08:53:45.044 INFO:teuthology.orchestra.run.trial163.stderr: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.59324.log.gz 2026-02-18T08:53:45.044 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.53419.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.28902.log.gz 2026-02-18T08:53:45.044 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.53454.log 2026-02-18T08:53:45.044 INFO:teuthology.orchestra.run.trial163.stderr: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.53419.log.gz 2026-02-18T08:53:45.044 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.124597.log 2026-02-18T08:53:45.044 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.65799.log: /var/log/ceph/cluster2-client.mirror.53454.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.79937.log 2026-02-18T08:53:45.044 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.124597.log: 88.6% -- replaced with /var/log/ceph/cluster1-client.mirror.65799.log.gzgzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.27879.log 2026-02-18T08:53:45.044 INFO:teuthology.orchestra.run.trial163.stderr: 2026-02-18T08:53:45.044 INFO:teuthology.orchestra.run.trial163.stderr: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.53454.log.gz 2026-02-18T08:53:45.044 INFO:teuthology.orchestra.run.trial163.stderr: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.124597.log.gz 2026-02-18T08:53:45.044 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.79937.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.128592.log 2026-02-18T08:53:45.044 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.27879.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.128876.log 2026-02-18T08:53:45.044 INFO:teuthology.orchestra.run.trial163.stderr: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.79937.log.gz 2026-02-18T08:53:45.044 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.128592.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.27879.log.gz 2026-02-18T08:53:45.045 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.20570.log 2026-02-18T08:53:45.045 INFO:teuthology.orchestra.run.trial163.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.128592.log.gz 2026-02-18T08:53:45.045 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.128876.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.82216.log 2026-02-18T08:53:45.045 INFO:teuthology.orchestra.run.trial163.stderr: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.128876.log.gz 2026-02-18T08:53:45.045 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.20570.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.23117.log 2026-02-18T08:53:45.045 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.82216.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.87008.log 2026-02-18T08:53:45.045 INFO:teuthology.orchestra.run.trial163.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.20570.log.gz/var/log/ceph/cluster2-client.mirror.23117.log: 2026-02-18T08:53:45.045 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.47675.log 2026-02-18T08:53:45.045 INFO:teuthology.orchestra.run.trial163.stderr: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.82216.log.gz 2026-02-18T08:53:45.045 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.87008.log: 87.9%gzip -- replaced with /var/log/ceph/cluster2-client.mirror.23117.log.gz 2026-02-18T08:53:45.045 INFO:teuthology.orchestra.run.trial163.stderr: -5 --verbose -- /var/log/ceph/cluster2-client.mirror.106643.log 2026-02-18T08:53:45.045 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.47675.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.73809.log 2026-02-18T08:53:45.045 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.106643.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.35736.log 2026-02-18T08:53:45.045 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.73809.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.47675.log.gz 2026-02-18T08:53:45.045 INFO:teuthology.orchestra.run.trial163.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.106643.log.gz 2026-02-18T08:53:45.045 INFO:teuthology.orchestra.run.trial163.stderr: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.73809.log.gz 2026-02-18T08:53:45.045 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.110539.log 2026-02-18T08:53:45.048 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.17773.log 2026-02-18T08:53:45.048 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.35736.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.125795.log 2026-02-18T08:53:45.048 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.110539.log: 95.1% -- replaced with /var/log/ceph/cluster1-client.mirror.87008.log.gz 2026-02-18T08:53:45.048 INFO:teuthology.orchestra.run.trial163.stderr: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.35736.log.gz 2026-02-18T08:53:45.048 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.17773.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.47147.log 2026-02-18T08:53:45.048 INFO:teuthology.orchestra.run.trial163.stderr: 92.4% -- replaced with /var/log/ceph/cluster2-client.mirror.110539.log.gz 2026-02-18T08:53:45.048 INFO:teuthology.orchestra.run.trial163.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.17773.log.gz 2026-02-18T08:53:45.048 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.110280.log 2026-02-18T08:53:45.048 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.95758.log 2026-02-18T08:53:45.048 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.125795.log: /var/log/ceph/cluster2-client.mirror.47147.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.92503.log 2026-02-18T08:53:45.048 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.110280.log: /var/log/ceph/cluster2-client.mirror.95758.log: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.47147.log.gz 2026-02-18T08:53:45.049 INFO:teuthology.orchestra.run.trial163.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.110280.log.gz 2026-02-18T08:53:45.049 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.19017.log 2026-02-18T08:53:45.049 INFO:teuthology.orchestra.run.trial163.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.95758.log.gz 2026-02-18T08:53:45.049 INFO:teuthology.orchestra.run.trial163.stderr: 90.6% -- replaced with /var/log/ceph/cluster2-client.mirror.125795.log.gz 2026-02-18T08:53:45.049 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.92503.log: 3.9% -- replaced with /var/log/ceph/cluster1-client.mirror.92503.log.gz 2026-02-18T08:53:45.049 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.102457.log 2026-02-18T08:53:45.049 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.19017.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.64635.log 2026-02-18T08:53:45.049 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.102457.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.99420.log 2026-02-18T08:53:45.049 INFO:teuthology.orchestra.run.trial163.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.19017.log.gz 2026-02-18T08:53:45.050 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.64635.log: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.64635.log.gz 2026-02-18T08:53:45.050 INFO:teuthology.orchestra.run.trial163.stderr: 95.2% -- replaced with /var/log/ceph/cluster1-client.mirror.102457.log.gz 2026-02-18T08:53:45.050 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.77907.log 2026-02-18T08:53:45.050 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.99420.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.120039.log 2026-02-18T08:53:45.050 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.77907.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.122619.log 2026-02-18T08:53:45.050 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.120039.log: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.77907.log.gz 2026-02-18T08:53:45.050 INFO:teuthology.orchestra.run.trial163.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.120039.log.gz 2026-02-18T08:53:45.051 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.109638.log 2026-02-18T08:53:45.052 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.122619.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.122584.log 2026-02-18T08:53:45.052 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.109638.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.122619.log.gz 2026-02-18T08:53:45.052 INFO:teuthology.orchestra.run.trial163.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.109638.log.gz 2026-02-18T08:53:45.052 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.44910.log 2026-02-18T08:53:45.053 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.122584.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.122584.log.gz 2026-02-18T08:53:45.053 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.92282.log 2026-02-18T08:53:45.053 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.44910.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.21908.log 2026-02-18T08:53:45.054 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.92282.log: 88.6% 1.3% -- replaced with /var/log/ceph/cluster2-client.mirror.44910.log.gz 2026-02-18T08:53:45.054 INFO:teuthology.orchestra.run.trial163.stderr: -- replaced with /var/log/ceph/cluster1-client.mirror.92282.log.gz 2026-02-18T08:53:45.054 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.130713.log 2026-02-18T08:53:45.054 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.21908.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.47112.log 2026-02-18T08:53:45.055 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.130713.log: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.21908.log.gz 2026-02-18T08:53:45.055 INFO:teuthology.orchestra.run.trial163.stderr: 87.0% -- replaced with /var/log/ceph/cluster1-client.mirror.130713.log.gz 2026-02-18T08:53:45.055 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.82461.log 2026-02-18T08:53:45.055 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.47112.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.80952.log 2026-02-18T08:53:45.056 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.82461.log: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.47112.log.gz 2026-02-18T08:53:45.056 INFO:teuthology.orchestra.run.trial163.stderr: 89.2% -- replaced with /var/log/ceph/cluster2-client.mirror.82461.log.gz 2026-02-18T08:53:45.056 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.48725.log 2026-02-18T08:53:45.057 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.80952.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.50164.log 2026-02-18T08:53:45.057 INFO:teuthology.orchestra.run.trial163.stderr: 89.2%/var/log/ceph/cluster2-client.mirror.48725.log: -- replaced with /var/log/ceph/cluster2-client.mirror.80952.log.gz 2026-02-18T08:53:45.057 INFO:teuthology.orchestra.run.trial163.stderr: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.48725.log.gz 2026-02-18T08:53:45.057 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.15207.log 2026-02-18T08:53:45.058 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.50164.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.129362.log 2026-02-18T08:53:45.058 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.admin.15207.log: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.50164.log.gz 2026-02-18T08:53:45.058 INFO:teuthology.orchestra.run.trial163.stderr: 80.1% -- replaced with /var/log/ceph/cluster1-client.admin.15207.log.gz 2026-02-18T08:53:45.058 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.131857.log 2026-02-18T08:53:45.059 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.129362.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.80147.log 2026-02-18T08:53:45.059 INFO:teuthology.orchestra.run.trial163.stderr: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.129362.log.gz 2026-02-18T08:53:45.060 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.131857.log: 91.8% -- replaced with /var/log/ceph/cluster2-client.mirror.131857.log.gz 2026-02-18T08:53:45.060 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.125287.log 2026-02-18T08:53:45.060 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.80147.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.27914.log 2026-02-18T08:53:45.061 INFO:teuthology.orchestra.run.trial163.stderr: 88.9%/var/log/ceph/cluster1-client.mirror.125287.log: -- replaced with /var/log/ceph/cluster2-client.mirror.80147.log.gz 2026-02-18T08:53:45.061 INFO:teuthology.orchestra.run.trial163.stderr: 88.3% -- replaced with /var/log/ceph/cluster1-client.mirror.125287.log.gz 2026-02-18T08:53:45.063 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.50934.log 2026-02-18T08:53:45.064 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.27914.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.98894.log 2026-02-18T08:53:45.064 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.50934.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.27914.log.gz 2026-02-18T08:53:45.064 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.53524.log 2026-02-18T08:53:45.064 INFO:teuthology.orchestra.run.trial163.stderr: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.50934.log.gz 2026-02-18T08:53:45.065 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.98894.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.37586.log 2026-02-18T08:53:45.065 INFO:teuthology.orchestra.run.trial163.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.98894.log.gz 2026-02-18T08:53:45.065 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.114911.log 2026-02-18T08:53:45.066 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.53524.log: /var/log/ceph/cluster2-client.mirror.37586.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.131434.log 2026-02-18T08:53:45.066 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.114911.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.31023.log 2026-02-18T08:53:45.066 INFO:teuthology.orchestra.run.trial163.stderr: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.53524.log.gz 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.114911.log.gz 2026-02-18T08:53:45.066 INFO:teuthology.orchestra.run.trial163.stderr: 2026-02-18T08:53:45.066 INFO:teuthology.orchestra.run.trial163.stderr: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.37586.log.gz 2026-02-18T08:53:45.067 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.37156.log 2026-02-18T08:53:45.067 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.131434.log: /var/log/ceph/cluster2-client.mirror.31023.log: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.131434.log.gz 2026-02-18T08:53:45.067 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.56674.log 2026-02-18T08:53:45.067 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.37156.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.31023.log.gz 2026-02-18T08:53:45.068 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.118896.log 2026-02-18T08:53:45.068 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.56674.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.15411.log 2026-02-18T08:53:45.068 INFO:teuthology.orchestra.run.trial163.stderr: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.37156.log.gz 2026-02-18T08:53:45.068 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.118896.log: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.56674.log.gz 2026-02-18T08:53:45.068 INFO:teuthology.orchestra.run.trial163.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.118896.log.gz 2026-02-18T08:53:45.071 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.65932.log 2026-02-18T08:53:45.072 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.admin.15411.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.30595.log 2026-02-18T08:53:45.072 INFO:teuthology.orchestra.run.trial163.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.15411.log.gz 2026-02-18T08:53:45.072 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.65932.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.107462.log 2026-02-18T08:53:45.072 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.30595.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.78012.log 2026-02-18T08:53:45.073 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.107462.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.107462.log.gz 2026-02-18T08:53:45.073 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.36877.log 2026-02-18T08:53:45.074 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.78012.log: 89.2% -- replaced with /var/log/ceph/cluster2-client.mirror.78012.log.gz 2026-02-18T08:53:45.074 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.61380.log 2026-02-18T08:53:45.075 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.36877.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.36877.log.gz 2026-02-18T08:53:45.075 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.77207.log 2026-02-18T08:53:45.076 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.61380.log: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.61380.log.gz 2026-02-18T08:53:45.076 INFO:teuthology.orchestra.run.trial163.stderr: 93.0% -- replaced with /var/log/ceph/cluster1-client.mirror.30595.log.gz 2026-02-18T08:53:45.076 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.57938.log 2026-02-18T08:53:45.077 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.46244.log 2026-02-18T08:53:45.078 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.77207.log: /var/log/ceph/cluster2-client.mirror.57938.log: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.77207.log.gz 2026-02-18T08:53:45.078 INFO:teuthology.orchestra.run.trial163.stderr: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.57938.log.gz 2026-02-18T08:53:45.078 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.43017.log 2026-02-18T08:53:45.078 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.11538.log 2026-02-18T08:53:45.079 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.admin.46244.log: /var/log/ceph/cluster2-client.mirror.43017.log: 88.9% -- replaced with /var/log/ceph/cluster1-client.admin.46244.log.gz 2026-02-18T08:53:45.079 INFO:teuthology.orchestra.run.trial163.stderr: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.43017.log.gz 2026-02-18T08:53:45.079 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.25463.log 2026-02-18T08:53:45.080 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.admin.11538.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.111133.log 2026-02-18T08:53:45.080 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.25463.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.11538.log.gz 2026-02-18T08:53:45.080 INFO:teuthology.orchestra.run.trial163.stderr: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.25463.log.gz 2026-02-18T08:53:45.083 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.58008.log 2026-02-18T08:53:45.084 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.111133.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.119581.log 2026-02-18T08:53:45.084 INFO:teuthology.orchestra.run.trial163.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.111133.log.gz 2026-02-18T08:53:45.084 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.58008.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.65304.log 2026-02-18T08:53:45.085 INFO:teuthology.orchestra.run.trial163.stderr: 88.8%/var/log/ceph/cluster2-client.mirror.119581.log: -- replaced with /var/log/ceph/cluster2-client.mirror.58008.log.gz 2026-02-18T08:53:45.085 INFO:teuthology.orchestra.run.trial163.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.119581.log.gz 2026-02-18T08:53:45.085 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.100613.log 2026-02-18T08:53:45.085 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.72234.log 2026-02-18T08:53:45.086 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.65304.log: /var/log/ceph/cluster2-client.mirror.100613.log: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.65304.log.gz 2026-02-18T08:53:45.086 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.90472.log 2026-02-18T08:53:45.087 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.72234.log: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.72234.log.gz 2026-02-18T08:53:45.087 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.108603.log 2026-02-18T08:53:45.091 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.90472.log: 95.7% -- replaced with /var/log/ceph/cluster2-client.mirror.110581.log.gz 2026-02-18T08:53:45.091 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.79377.log 2026-02-18T08:53:45.092 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.108603.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.108603.log.gz 2026-02-18T08:53:45.092 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.132290.log 2026-02-18T08:53:45.093 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.79377.log: 89.2% -- replaced with /var/log/ceph/cluster2-client.mirror.79377.log.gz 2026-02-18T08:53:45.093 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.60444.log 2026-02-18T08:53:45.094 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.132290.log: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.132290.log.gz 2026-02-18T08:53:45.094 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.119691.log 2026-02-18T08:53:45.094 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.60444.log: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.60444.log.gz 2026-02-18T08:53:45.095 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.115551.log 2026-02-18T08:53:45.095 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.119691.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.119691.log.gz 2026-02-18T08:53:45.096 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.121897.log 2026-02-18T08:53:45.115 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.115551.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.121626.log 2026-02-18T08:53:45.116 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.121897.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.121897.log.gz 2026-02-18T08:53:45.139 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.125332.log 2026-02-18T08:53:45.140 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.121626.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.121626.log.gz 2026-02-18T08:53:45.156 INFO:teuthology.orchestra.run.trial163.stderr: 94.8% -- replaced with /var/log/ceph/cluster1-client.mirror.0.84399.log.gz 2026-02-18T08:53:45.159 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.66465.log 2026-02-18T08:53:45.160 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.125332.log: gzip -5 --verbose -- /var/log/ceph/cluster2.log 2026-02-18T08:53:45.160 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.66465.log: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.66465.log.gz 2026-02-18T08:53:45.161 INFO:teuthology.orchestra.run.trial163.stderr: 92.6% -- replaced with /var/log/ceph/cluster1-client.mirror.125332.log.gz 2026-02-18T08:53:45.161 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.127411.log 2026-02-18T08:53:45.165 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2.log: 87.4% -- replaced with /var/log/ceph/cluster2.log.gz 2026-02-18T08:53:45.166 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.129114.log 2026-02-18T08:53:45.166 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.127411.log: 89.3% -- replaced with /var/log/ceph/cluster2-client.mirror.127411.log.gz 2026-02-18T08:53:45.167 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.17587.log 2026-02-18T08:53:45.167 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.129114.log: 87.0% -- replaced with /var/log/ceph/cluster1-client.mirror.129114.log.gz 2026-02-18T08:53:45.168 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.78852.log 2026-02-18T08:53:45.168 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.17587.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.17587.log.gz 2026-02-18T08:53:45.169 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.62255.log 2026-02-18T08:53:45.169 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.78852.log: 89.2% -- replaced with /var/log/ceph/cluster2-client.mirror.78852.log.gz 2026-02-18T08:53:45.169 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.15548.log 2026-02-18T08:53:45.170 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.62255.log: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.62255.log.gz 2026-02-18T08:53:45.170 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.15309.log 2026-02-18T08:53:45.171 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.15548.log: 4.7% -- replaced with /var/log/ceph/cluster2-client.mirror.15548.log.gz 2026-02-18T08:53:45.171 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.82671.log 2026-02-18T08:53:45.173 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.15309.log: 4.7% -- replaced with /var/log/ceph/cluster1-client.mirror.15309.log.gz 2026-02-18T08:53:45.174 INFO:teuthology.orchestra.run.trial163.stderr: 96.0% -- replaced with /var/log/ceph/cluster2-client.mirror.90802.log.gz 2026-02-18T08:53:45.174 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.94192.log 2026-02-18T08:53:45.174 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.82671.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.37307.log 2026-02-18T08:53:45.174 INFO:teuthology.orchestra.run.trial163.stderr: 88.9%/var/log/ceph/cluster2-client.mirror.94192.log: -- replaced with /var/log/ceph/cluster2-client.mirror.82671.log.gz 2026-02-18T08:53:45.174 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.109892.log 2026-02-18T08:53:45.174 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.37307.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.37307.log.gz 2026-02-18T08:53:45.175 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.109019.log 2026-02-18T08:53:45.175 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.109892.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.109892.log.gz 2026-02-18T08:53:45.175 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.24889.log 2026-02-18T08:53:45.175 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.109019.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.109019.log.gz 2026-02-18T08:53:45.176 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.107561.log 2026-02-18T08:53:45.177 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.admin.24889.log: 88.8% -- replaced with /var/log/ceph/cluster1-client.admin.24889.log.gz 2026-02-18T08:53:45.177 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.131095.log 2026-02-18T08:53:45.188 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.107561.log: 95.7% -- replaced with /var/log/ceph/cluster2-client.mirror.115551.log.gz 2026-02-18T08:53:45.188 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.90044.log 2026-02-18T08:53:45.189 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.131095.log: 86.8% -- replaced with /var/log/ceph/cluster1-client.mirror.131095.log.gz 2026-02-18T08:53:45.189 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.102937.log 2026-02-18T08:53:45.200 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.90044.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.104728.log 2026-02-18T08:53:45.200 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.102937.log: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.102937.log.gz 2026-02-18T08:53:45.208 INFO:teuthology.orchestra.run.trial163.stderr: 95.6% -- replaced with /var/log/ceph/cluster2-client.mirror.87814.log.gz 2026-02-18T08:53:45.209 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.73984.log 2026-02-18T08:53:45.216 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.104728.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.98042.log 2026-02-18T08:53:45.228 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.73984.log: 89.1%gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.30788.log 2026-02-18T08:53:45.228 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.98042.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.98042.log.gz 2026-02-18T08:53:45.231 INFO:teuthology.orchestra.run.trial163.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.73984.log.gz 2026-02-18T08:53:45.241 INFO:teuthology.orchestra.run.trial163.stderr: 95.7% -- replaced with /var/log/ceph/cluster2-client.mirror.104728.log.gz 2026-02-18T08:53:45.242 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.20384.log 2026-02-18T08:53:45.242 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.admin.30788.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.131565.log 2026-02-18T08:53:45.243 INFO:teuthology.orchestra.run.trial163.stderr: 88.8% -- replaced with /var/log/ceph/cluster1-client.admin.30788.log.gz 2026-02-18T08:53:45.243 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.20384.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.20384.log.gz 2026-02-18T08:53:45.243 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.99378.log 2026-02-18T08:53:45.244 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.131565.log: 62.9% -- replaced with /var/log/ceph/cluster2-client.mirror.131565.log.gz 2026-02-18T08:53:45.244 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.40967.log 2026-02-18T08:53:45.245 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.99378.log: 92.2% -- replaced with /var/log/ceph/cluster2-client.mirror.99378.log.gz 2026-02-18T08:53:45.246 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.110027.log 2026-02-18T08:53:45.247 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.40967.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.32884.log 2026-02-18T08:53:45.247 INFO:teuthology.orchestra.run.trial163.stderr: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.40967.log.gz 2026-02-18T08:53:45.248 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.110027.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.110027.log.gz 2026-02-18T08:53:45.248 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.83269.log 2026-02-18T08:53:45.249 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.32884.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.32884.log.gz 2026-02-18T08:53:45.249 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.113289.log 2026-02-18T08:53:45.249 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.83269.log: 88.9% -- replaced with /var/log/ceph/cluster1-client.mirror.83269.log.gz 2026-02-18T08:53:45.250 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.132827.log 2026-02-18T08:53:45.251 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.113289.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.113289.log.gz 2026-02-18T08:53:45.251 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.118106.log 2026-02-18T08:53:45.251 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.132827.log: 83.6% -- replaced with /var/log/ceph/cluster2-client.mirror.132827.log.gz 2026-02-18T08:53:45.252 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.86405.log 2026-02-18T08:53:45.252 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.118106.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.118106.log.gz 2026-02-18T08:53:45.252 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.123533.log 2026-02-18T08:53:45.253 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.86405.log: 84.7% -- replaced with /var/log/ceph/cluster2-client.mirror.86405.log.gz 2026-02-18T08:53:45.253 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.44505.log 2026-02-18T08:53:45.254 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.123533.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.123533.log.gz 2026-02-18T08:53:45.254 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.68009.log 2026-02-18T08:53:45.255 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.44505.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.44505.log.gz 2026-02-18T08:53:45.255 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.123215.log 2026-02-18T08:53:45.255 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.68009.log: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.68009.log.gz 2026-02-18T08:53:45.256 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.89706.log 2026-02-18T08:53:45.256 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.123215.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.123215.log.gz 2026-02-18T08:53:45.257 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.107494.log 2026-02-18T08:53:45.257 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.89706.log: 84.8% -- replaced with /var/log/ceph/cluster2-client.mirror.89706.log.gz 2026-02-18T08:53:45.257 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.26984.log 2026-02-18T08:53:45.258 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.107494.log: 88.1% -- replaced with /var/log/ceph/cluster1-client.mirror.107494.log.gz 2026-02-18T08:53:45.258 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.55179.log 2026-02-18T08:53:45.259 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.26984.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.26984.log.gz 2026-02-18T08:53:45.259 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.115023.log 2026-02-18T08:53:45.260 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.55179.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.81057.log 2026-02-18T08:53:45.260 INFO:teuthology.orchestra.run.trial163.stderr: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.55179.log.gz 2026-02-18T08:53:45.261 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.115023.log: 91.9% -- replaced with /var/log/ceph/cluster1-client.mirror.115023.log.gz 2026-02-18T08:53:45.262 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.69049.log 2026-02-18T08:53:45.262 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.81057.log: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.81057.log.gz 2026-02-18T08:53:45.263 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.15599.log 2026-02-18T08:53:45.263 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.69049.log: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.69049.log.gz 2026-02-18T08:53:45.264 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.71989.log 2026-02-18T08:53:45.264 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.15599.log: 65.1% -- replaced with /var/log/ceph/cluster2-client.mirror.15599.log.gz 2026-02-18T08:53:45.264 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.128967.log 2026-02-18T08:53:45.265 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.71989.log: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.71989.log.gz 2026-02-18T08:53:45.265 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.35457.log 2026-02-18T08:53:45.266 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.128967.log: 90.6% -- replaced with /var/log/ceph/cluster2-client.mirror.128967.log.gz 2026-02-18T08:53:45.267 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.79587.log 2026-02-18T08:53:45.267 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.35457.log: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.35457.log.gz 2026-02-18T08:53:45.268 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.17494.log 2026-02-18T08:53:45.268 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.79587.log: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.79587.log.gz 2026-02-18T08:53:45.268 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.48375.log 2026-02-18T08:53:45.269 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.17494.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.116107.log 2026-02-18T08:53:45.269 INFO:teuthology.orchestra.run.trial163.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.17494.log.gz 2026-02-18T08:53:45.270 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.48375.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.48375.log.gz 2026-02-18T08:53:45.270 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.126604.log 2026-02-18T08:53:45.271 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.116107.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.116107.log.gz 2026-02-18T08:53:45.271 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.117361.log 2026-02-18T08:53:45.272 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.126604.log: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.126604.log.gz 2026-02-18T08:53:45.272 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.79902.log 2026-02-18T08:53:45.273 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.117361.log: 87.3% -- replaced with /var/log/ceph/cluster2-client.mirror.117361.log.gz 2026-02-18T08:53:45.273 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.59394.log 2026-02-18T08:53:45.273 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.79902.log: 89.2% -- replaced with /var/log/ceph/cluster2-client.mirror.79902.log.gz 2026-02-18T08:53:45.274 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.20104.log 2026-02-18T08:53:45.274 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.59394.log: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.59394.log.gz 2026-02-18T08:53:45.274 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.130780.log 2026-02-18T08:53:45.275 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.20104.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.20104.log.gz 2026-02-18T08:53:45.275 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.40130.log 2026-02-18T08:53:45.276 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.130780.log: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.130780.log.gz 2026-02-18T08:53:45.276 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.40281.log 2026-02-18T08:53:45.277 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.40130.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.40130.log.gz 2026-02-18T08:53:45.277 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.43203.log 2026-02-18T08:53:45.278 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.40281.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.40281.log.gz 2026-02-18T08:53:45.278 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.101896.log 2026-02-18T08:53:45.278 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.43203.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.43203.log.gz 2026-02-18T08:53:45.279 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.106262.log 2026-02-18T08:53:45.280 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.101896.log: 92.1% -- replaced with /var/log/ceph/cluster2-client.mirror.101896.log.gz 2026-02-18T08:53:45.280 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.61800.log 2026-02-18T08:53:45.282 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.106262.log: 92.7% -- replaced with /var/log/ceph/cluster2-client.mirror.106262.log.gz 2026-02-18T08:53:45.282 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.54259.log 2026-02-18T08:53:45.282 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.61800.log: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.61800.log.gz 2026-02-18T08:53:45.283 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.109705.log 2026-02-18T08:53:45.283 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.54259.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.22559.log 2026-02-18T08:53:45.285 INFO:teuthology.orchestra.run.trial163.stderr: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.54259.log.gz 2026-02-18T08:53:45.285 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.109705.log: 88.2% -- replaced with /var/log/ceph/cluster1-client.mirror.109705.log.gz 2026-02-18T08:53:45.285 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.112036.log 2026-02-18T08:53:45.286 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.22559.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.22559.log.gz 2026-02-18T08:53:45.286 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.38456.log 2026-02-18T08:53:45.286 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.112036.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.112036.log.gz 2026-02-18T08:53:45.287 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.30204.log 2026-02-18T08:53:45.287 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.38456.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.125704.log 2026-02-18T08:53:45.287 INFO:teuthology.orchestra.run.trial163.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.38456.log.gz 2026-02-18T08:53:45.288 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.30204.log: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.30204.log.gz 2026-02-18T08:53:45.288 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.24730.log 2026-02-18T08:53:45.289 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.125704.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.125704.log.gz 2026-02-18T08:53:45.289 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.99153.log 2026-02-18T08:53:45.290 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.24730.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.125358.log 2026-02-18T08:53:45.290 INFO:teuthology.orchestra.run.trial163.stderr: 88.1% -- replaced with /var/log/ceph/cluster1-client.mirror.24730.log.gz 2026-02-18T08:53:45.291 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.99153.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.99153.log.gz 2026-02-18T08:53:45.291 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.107346.log 2026-02-18T08:53:45.291 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.125358.log: 89.4% -- replaced with /var/log/ceph/cluster2-client.mirror.125358.log.gz 2026-02-18T08:53:45.292 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.20011.log 2026-02-18T08:53:45.293 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.107346.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.107346.log.gz 2026-02-18T08:53:45.293 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.85248.log 2026-02-18T08:53:45.294 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.20011.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.20011.log.gz 2026-02-18T08:53:45.294 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.37435.log 2026-02-18T08:53:45.294 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.85248.log: 89.4% -- replaced with /var/log/ceph/cluster2-client.mirror.85248.log.gz 2026-02-18T08:53:45.295 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.98279.log 2026-02-18T08:53:45.296 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.37435.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.122309.log 2026-02-18T08:53:45.296 INFO:teuthology.orchestra.run.trial163.stderr: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.37435.log.gz 2026-02-18T08:53:45.297 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.98279.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.98279.log.gz 2026-02-18T08:53:45.297 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.127291.log 2026-02-18T08:53:45.297 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.122309.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.122309.log.gz 2026-02-18T08:53:45.298 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.56884.log 2026-02-18T08:53:45.298 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.127291.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.127291.log.gz 2026-02-18T08:53:45.298 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.82741.log 2026-02-18T08:53:45.299 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.56884.log: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.56884.log.gz 2026-02-18T08:53:45.299 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.124937.log 2026-02-18T08:53:45.300 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.82741.log: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.82741.log.gz 2026-02-18T08:53:45.300 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.80882.log 2026-02-18T08:53:45.301 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.124937.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.124937.log.gz 2026-02-18T08:53:45.301 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.132235.log 2026-02-18T08:53:45.301 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.80882.log: 89.2% -- replaced with /var/log/ceph/cluster2-client.mirror.80882.log.gz 2026-02-18T08:53:45.302 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.128854.log 2026-02-18T08:53:45.311 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.132235.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.63200.log 2026-02-18T08:53:45.313 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.128854.log: 89.4% -- replaced with /var/log/ceph/cluster2-client.mirror.128854.log.gz 2026-02-18T08:53:45.323 INFO:teuthology.orchestra.run.trial163.stderr: 95.8% -- replaced with /var/log/ceph/cluster2-client.mirror.90615.log.gz 2026-02-18T08:53:45.327 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.133724.log 2026-02-18T08:53:45.328 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.63200.log: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.63200.log.gz 2026-02-18T08:53:45.329 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.85437.log 2026-02-18T08:53:45.329 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.133724.log: 91.3% -- replaced with /var/log/ceph/cluster2-client.mirror.133724.log.gz 2026-02-18T08:53:45.330 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.128328.log 2026-02-18T08:53:45.331 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.85437.log: 89.4% -- replaced with /var/log/ceph/cluster2-client.mirror.85437.log.gz 2026-02-18T08:53:45.331 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.19860.log 2026-02-18T08:53:45.333 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.128328.log: 90.7% -- replaced with /var/log/ceph/cluster2-client.mirror.128328.log.gz 2026-02-18T08:53:45.333 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.117952.log 2026-02-18T08:53:45.335 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.19860.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.19860.log.gz 2026-02-18T08:53:45.335 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.21873.log 2026-02-18T08:53:45.335 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.117952.log: 88.2% -- replaced with /var/log/ceph/cluster1-client.mirror.117952.log.gz 2026-02-18T08:53:45.336 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.129812.log 2026-02-18T08:53:45.336 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.21873.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.85521.log 2026-02-18T08:53:45.336 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.129812.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.21873.log.gz 2026-02-18T08:53:45.336 INFO:teuthology.orchestra.run.trial163.stderr: 87.0% -- replaced with /var/log/ceph/cluster1-client.mirror.129812.log.gz 2026-02-18T08:53:45.337 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.64884.log 2026-02-18T08:53:45.337 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.85521.log: 89.4% -- replaced with /var/log/ceph/cluster2-client.mirror.85521.log.gz 2026-02-18T08:53:45.338 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.33593.log 2026-02-18T08:53:45.339 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.64884.log: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.64884.log.gz 2026-02-18T08:53:45.339 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.56429.log 2026-02-18T08:53:45.339 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.33593.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.33593.log.gz 2026-02-18T08:53:45.340 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.74863.log 2026-02-18T08:53:45.341 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.56429.log: 95.7% 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.56429.log.gz 2026-02-18T08:53:45.341 INFO:teuthology.orchestra.run.trial163.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.88933.log.gz 2026-02-18T08:53:45.341 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.0.105121.log 2026-02-18T08:53:45.341 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.74863.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.0.75991.log 2026-02-18T08:53:45.341 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.0.105121.log: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.74863.log.gz 2026-02-18T08:53:45.342 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.42959.log 2026-02-18T08:53:45.343 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.0.75991.log: 92.6% -- replaced with /var/log/ceph/cluster2-client.mirror.0.75991.log.gz 2026-02-18T08:53:45.344 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.94727.log 2026-02-18T08:53:45.344 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.42959.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.42959.log.gz 2026-02-18T08:53:45.344 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.123689.log 2026-02-18T08:53:45.345 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.94727.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.94727.log.gz 2026-02-18T08:53:45.345 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.37805.log 2026-02-18T08:53:45.346 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.123689.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.123689.log.gz 2026-02-18T08:53:45.346 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.91490.log 2026-02-18T08:53:45.347 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.37805.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.37805.log.gz 2026-02-18T08:53:45.347 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.105607.log 2026-02-18T08:53:45.349 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.91490.log: 93.6% -- replaced with /var/log/ceph/cluster1-client.mirror.0.105121.log.gz 2026-02-18T08:53:45.352 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.36201.log 2026-02-18T08:53:45.352 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.105607.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.105607.log.gz 2026-02-18T08:53:45.353 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.43238.log 2026-02-18T08:53:45.353 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.36201.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.112368.log 2026-02-18T08:53:45.353 INFO:teuthology.orchestra.run.trial163.stderr: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.36201.log.gz 2026-02-18T08:53:45.354 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.43238.log: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.43238.log.gz 2026-02-18T08:53:45.354 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.1.37671.log 2026-02-18T08:53:45.355 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.112368.log: 92.4% -- replaced with /var/log/ceph/cluster1-client.mirror.112368.log.gz 2026-02-18T08:53:45.356 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.116184.log 2026-02-18T08:53:45.356 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.1.37671.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.131257.log 2026-02-18T08:53:45.362 INFO:teuthology.orchestra.run.trial163.stderr: 95.9% -- replaced with /var/log/ceph/cluster2-client.mirror.89679.log.gz 2026-02-18T08:53:45.363 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.95055.log 2026-02-18T08:53:45.364 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.85353.log 2026-02-18T08:53:45.368 INFO:teuthology.orchestra.run.trial163.stderr: 95.8% -- replaced with /var/log/ceph/cluster2-client.mirror.91763.log.gz 2026-02-18T08:53:45.370 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.116184.log: /var/log/ceph/cluster2-client.mirror.131257.log: 82.9% -- replaced with /var/log/ceph/cluster2-client.mirror.131257.log.gz 2026-02-18T08:53:45.371 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.95055.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.95055.log.gz 2026-02-18T08:53:45.371 INFO:teuthology.orchestra.run.trial163.stderr: 92.3% -- replaced with /var/log/ceph/cluster1-client.mirror.116184.log.gz 2026-02-18T08:53:45.371 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.64425.log 2026-02-18T08:53:45.371 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.85353.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.20885.log 2026-02-18T08:53:45.371 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.64425.log: 89.5% -- replaced with /var/log/ceph/cluster2-client.mirror.85353.log.gz 2026-02-18T08:53:45.371 INFO:teuthology.orchestra.run.trial163.stderr: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.64425.log.gz 2026-02-18T08:53:45.371 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.47217.log 2026-02-18T08:53:45.373 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.20885.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.134952.log 2026-02-18T08:53:45.373 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.47217.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.20885.log.gz 2026-02-18T08:53:45.374 INFO:teuthology.orchestra.run.trial163.stderr: 88.7%gzip -- replaced with /var/log/ceph/cluster2-client.mirror.47217.log.gz -5 --verbose -- /var/log/ceph/cluster2-client.mirror.25312.log 2026-02-18T08:53:45.374 INFO:teuthology.orchestra.run.trial163.stderr: 2026-02-18T08:53:45.374 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.134952.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.mirror.134952.log.gz 2026-02-18T08:53:45.374 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.15394.log 2026-02-18T08:53:45.374 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.25312.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.25312.log.gz 2026-02-18T08:53:45.376 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.105348.log 2026-02-18T08:53:45.376 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.15394.log: 54.2% -- replaced with /var/log/ceph/cluster1-client.mirror.15394.log.gz 2026-02-18T08:53:45.376 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.17064.log 2026-02-18T08:53:45.376 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.105348.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.105348.log.gz 2026-02-18T08:53:45.376 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.102372.log 2026-02-18T08:53:45.376 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.17064.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.15292.log 2026-02-18T08:53:45.376 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.102372.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.17064.log.gz 2026-02-18T08:53:45.378 INFO:teuthology.orchestra.run.trial163.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.102372.log.gz 2026-02-18T08:53:45.378 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.0.16458.log 2026-02-18T08:53:45.378 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.15292.log: 4.7%gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.126837.log 2026-02-18T08:53:45.378 INFO:teuthology.orchestra.run.trial163.stderr: -- replaced with /var/log/ceph/cluster1-client.mirror.15292.log.gz 2026-02-18T08:53:45.379 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.0.16458.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.54189.log 2026-02-18T08:53:45.379 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.126837.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.49985.log 2026-02-18T08:53:45.379 INFO:teuthology.orchestra.run.trial163.stderr: 90.7% -- replaced with /var/log/ceph/cluster2-client.mirror.126837.log.gz 2026-02-18T08:53:45.380 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.44168.log 2026-02-18T08:53:45.380 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.49985.log: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.49985.log.gz 2026-02-18T08:53:45.380 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.18180.log 2026-02-18T08:53:45.381 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.44168.log: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.44168.log.gz 2026-02-18T08:53:45.381 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.62220.log 2026-02-18T08:53:45.382 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.18180.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.79482.log 2026-02-18T08:53:45.382 INFO:teuthology.orchestra.run.trial163.stderr: 94.2% -- replaced with /var/log/ceph/cluster2-client.mirror.0.16458.log.gz 2026-02-18T08:53:45.382 INFO:teuthology.orchestra.run.trial163.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.18180.log.gz 2026-02-18T08:53:45.383 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.54189.log: /var/log/ceph/cluster2-client.mirror.62220.log: 88.6% 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.62220.log.gz 2026-02-18T08:53:45.383 INFO:teuthology.orchestra.run.trial163.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.54189.log.gz 2026-02-18T08:53:45.383 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.59709.log 2026-02-18T08:53:45.384 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.79482.log: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.79482.log.gz 2026-02-18T08:53:45.384 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.75814.log 2026-02-18T08:53:45.385 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.59709.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.59709.log.gz 2026-02-18T08:53:45.385 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.46139.log 2026-02-18T08:53:45.385 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.75814.log: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.75814.log.gz 2026-02-18T08:53:45.386 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.131230.log 2026-02-18T08:53:45.386 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.46139.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.130442.log 2026-02-18T08:53:45.388 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.131230.log: 86.8% -- replaced with /var/log/ceph/cluster1-client.mirror.131230.log.gz 2026-02-18T08:53:45.388 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.54997.log 2026-02-18T08:53:45.389 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.130442.log: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.130442.log.gz 2026-02-18T08:53:45.395 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.105121.log 2026-02-18T08:53:45.396 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.54997.log: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.54997.log.gz 2026-02-18T08:53:45.397 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.61415.log 2026-02-18T08:53:45.398 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.105121.log: 92.5% -- replaced with /var/log/ceph/cluster2-client.mirror.105121.log.gz 2026-02-18T08:53:45.398 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.135315.log 2026-02-18T08:53:45.399 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.61415.log: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.61415.log.gz 2026-02-18T08:53:45.399 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.110983.log 2026-02-18T08:53:45.400 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.admin.135315.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.135315.log.gz 2026-02-18T08:53:45.400 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.77592.log 2026-02-18T08:53:45.401 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.110983.log: 92.2% -- replaced with /var/log/ceph/cluster2-client.mirror.110983.log.gz 2026-02-18T08:53:45.402 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.34748.log 2026-02-18T08:53:45.402 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.77592.log: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.77592.log.gz 2026-02-18T08:53:45.403 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.54873.log 2026-02-18T08:53:45.403 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.34748.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.107035.log 2026-02-18T08:53:45.404 INFO:teuthology.orchestra.run.trial163.stderr: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.34748.log.gz 2026-02-18T08:53:45.404 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.54873.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.45399.log 2026-02-18T08:53:45.405 INFO:teuthology.orchestra.run.trial163.stderr: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.54873.log.gz 2026-02-18T08:53:45.405 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.107035.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.107035.log.gz 2026-02-18T08:53:45.406 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.55397.log 2026-02-18T08:53:45.407 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.45399.log: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.45399.log.gz 2026-02-18T08:53:45.407 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.48585.log 2026-02-18T08:53:45.407 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.admin.55397.log: 88.0% -- replaced with /var/log/ceph/cluster1-client.admin.55397.log.gz 2026-02-18T08:53:45.408 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.122878.log 2026-02-18T08:53:45.409 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.48585.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.48585.log.gz 2026-02-18T08:53:45.409 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.24698.log 2026-02-18T08:53:45.409 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.122878.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.122878.log.gz 2026-02-18T08:53:45.410 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.80462.log 2026-02-18T08:53:45.411 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.24698.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.24698.log.gz 2026-02-18T08:53:45.411 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.91898.log 2026-02-18T08:53:45.411 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.80462.log: 89.2% -- replaced with /var/log/ceph/cluster2-client.mirror.80462.log.gz 2026-02-18T08:53:45.412 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.33349.log 2026-02-18T08:53:45.412 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.91898.log: 84.9% -- replaced with /var/log/ceph/cluster2-client.mirror.91898.log.gz 2026-02-18T08:53:45.413 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.120430.log 2026-02-18T08:53:45.414 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.33349.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.33349.log.gz 2026-02-18T08:53:45.414 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.84795.log 2026-02-18T08:53:45.414 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.120430.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.120430.log.gz 2026-02-18T08:53:45.415 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.124446.log 2026-02-18T08:53:45.418 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.84795.log: 93.6% -- replaced with /var/log/ceph/cluster2-client.mirror.84795.log.gz 2026-02-18T08:53:45.419 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.76787.log 2026-02-18T08:53:45.419 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.124446.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.124446.log.gz 2026-02-18T08:53:45.419 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.20326.log 2026-02-18T08:53:45.420 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.76787.log: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.76787.log.gz 2026-02-18T08:53:45.420 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.3.14969.log 2026-02-18T08:53:45.421 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.20326.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.101267.log 2026-02-18T08:53:45.421 INFO:teuthology.orchestra.run.trial163.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.20326.log.gz 2026-02-18T08:53:45.422 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.3.14969.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.57059.log 2026-02-18T08:53:45.431 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.27170.log 2026-02-18T08:53:45.433 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.101267.log: /var/log/ceph/cluster2-client.mirror.57059.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.101267.log.gz 2026-02-18T08:53:45.433 INFO:teuthology.orchestra.run.trial163.stderr: 94.6% -- replaced with /var/log/ceph/cluster1-client.mirror.1.91378.log.gz 2026-02-18T08:53:45.433 INFO:teuthology.orchestra.run.trial163.stderr: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.57059.log.gz 2026-02-18T08:53:45.433 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.126198.log 2026-02-18T08:53:45.434 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.27170.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.27170.log.gz 2026-02-18T08:53:45.434 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.31268.log 2026-02-18T08:53:45.435 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.126198.log: 90.7% -- replaced with /var/log/ceph/cluster2-client.mirror.126198.log.gz 2026-02-18T08:53:45.435 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.87363.log 2026-02-18T08:53:45.436 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.31268.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.31268.log.gz 2026-02-18T08:53:45.437 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.79097.log 2026-02-18T08:53:45.448 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.87363.log: 95.7% -- replaced with /var/log/ceph/cluster2-client.mirror.124545.log.gz 2026-02-18T08:53:45.448 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.36842.log 2026-02-18T08:53:45.449 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.79097.log: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.79097.log.gz 2026-02-18T08:53:45.449 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.133295.log 2026-02-18T08:53:45.450 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.36842.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.36842.log.gz 2026-02-18T08:53:45.450 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.38921.log 2026-02-18T08:53:45.451 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.133295.log: 83.6% -- replaced with /var/log/ceph/cluster2-client.mirror.133295.log.gz 2026-02-18T08:53:45.451 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.125917.log 2026-02-18T08:53:45.452 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.38921.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.38921.log.gz 2026-02-18T08:53:45.452 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.41304.log 2026-02-18T08:53:45.452 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.125917.log: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.125917.log.gz 2026-02-18T08:53:45.452 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.74229.log 2026-02-18T08:53:45.453 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.41304.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.41304.log.gz 2026-02-18T08:53:45.454 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.98430.log 2026-02-18T08:53:45.454 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.74229.log: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.74229.log.gz 2026-02-18T08:53:45.454 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.113173.log 2026-02-18T08:53:45.464 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.98430.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.92160.log 2026-02-18T08:53:45.464 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.113173.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.113173.log.gz 2026-02-18T08:53:45.479 INFO:teuthology.orchestra.run.trial163.stderr: 95.8% -- replaced with /var/log/ceph/cluster2-client.mirror.89254.log.gz 2026-02-18T08:53:45.480 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.71289.log 2026-02-18T08:53:45.481 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.92160.log: 84.8% -- replaced with /var/log/ceph/cluster2-client.mirror.92160.log.gz 2026-02-18T08:53:45.481 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.64110.log 2026-02-18T08:53:45.482 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.71289.log: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.71289.log.gz 2026-02-18T08:53:45.482 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.120116.log 2026-02-18T08:53:45.483 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.64110.log: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.64110.log.gz 2026-02-18T08:53:45.483 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.53349.log 2026-02-18T08:53:45.483 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.120116.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.120116.log.gz 2026-02-18T08:53:45.484 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.132639.log 2026-02-18T08:53:45.485 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.53349.log: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.53349.log.gz 2026-02-18T08:53:45.485 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.112256.log 2026-02-18T08:53:45.485 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.132639.log: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.132639.log.gz 2026-02-18T08:53:45.486 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.59849.log 2026-02-18T08:53:45.486 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.112256.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.112256.log.gz 2026-02-18T08:53:45.486 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.99026.log 2026-02-18T08:53:45.487 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.59849.log: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.59849.log.gz 2026-02-18T08:53:45.487 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.23768.log 2026-02-18T08:53:45.488 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.99026.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.99026.log.gz 2026-02-18T08:53:45.488 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.17808.log 2026-02-18T08:53:45.489 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.23768.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.23768.log.gz 2026-02-18T08:53:45.489 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.34434.log 2026-02-18T08:53:45.490 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.17808.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.17808.log.gz 2026-02-18T08:53:45.490 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.115367.log 2026-02-18T08:53:45.491 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.34434.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.34434.log.gz 2026-02-18T08:53:45.491 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.85458.log 2026-02-18T08:53:45.491 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.115367.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.115367.log.gz 2026-02-18T08:53:45.491 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.90399.log 2026-02-18T08:53:45.492 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.85458.log: 89.3% -- replaced with /var/log/ceph/cluster2-client.mirror.85458.log.gz 2026-02-18T08:53:45.492 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.101938.log 2026-02-18T08:53:45.493 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.90399.log: 84.7% -- replaced with /var/log/ceph/cluster2-client.mirror.90399.log.gz 2026-02-18T08:53:45.493 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.47780.log 2026-02-18T08:53:45.500 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.101938.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.33221.log 2026-02-18T08:53:45.510 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.47780.log: 95.7% -- replaced with /var/log/ceph/cluster2-client.mirror.94192.log.gz 2026-02-18T08:53:45.510 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.73914.log 2026-02-18T08:53:45.511 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.33221.log: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.47780.log.gz 2026-02-18T08:53:45.512 INFO:teuthology.orchestra.run.trial163.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.33221.log.gz 2026-02-18T08:53:45.512 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.22896.log 2026-02-18T08:53:45.513 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.73914.log: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.73914.log.gz 2026-02-18T08:53:45.513 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.51844.log 2026-02-18T08:53:45.514 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.22896.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.22896.log.gz 2026-02-18T08:53:45.514 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.80427.log 2026-02-18T08:53:45.515 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.51844.log: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.51844.log.gz 2026-02-18T08:53:45.515 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.125775.log 2026-02-18T08:53:45.515 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.80427.log: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.80427.log.gz 2026-02-18T08:53:45.516 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.121658.log 2026-02-18T08:53:45.517 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.125775.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.125775.log.gz 2026-02-18T08:53:45.517 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.14139.log 2026-02-18T08:53:45.517 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.121658.log: 88.2% -- replaced with /var/log/ceph/cluster1-client.mirror.121658.log.gz 2026-02-18T08:53:45.518 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.113710.log 2026-02-18T08:53:45.518 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.admin.14139.log: 80.3% -- replaced with /var/log/ceph/cluster2-client.admin.14139.log.gz 2026-02-18T08:53:45.519 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.18424.log 2026-02-18T08:53:45.519 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.113710.log: 88.4% -- replaced with /var/log/ceph/cluster1-client.mirror.113710.log.gz 2026-02-18T08:53:45.520 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.92899.log 2026-02-18T08:53:45.521 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.18424.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.18424.log.gz 2026-02-18T08:53:45.521 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.131548.log 2026-02-18T08:53:45.521 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.92899.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.92899.log.gz 2026-02-18T08:53:45.522 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.130149.log 2026-02-18T08:53:45.522 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.131548.log: 62.9% -- replaced with /var/log/ceph/cluster1-client.mirror.131548.log.gz 2026-02-18T08:53:45.522 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.134548.log 2026-02-18T08:53:45.523 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.130149.log: 87.0% -- replaced with /var/log/ceph/cluster1-client.mirror.130149.log.gz 2026-02-18T08:53:45.523 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.99751.log 2026-02-18T08:53:45.524 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.134548.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.134548.log.gz 2026-02-18T08:53:45.524 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.52789.log 2026-02-18T08:53:45.525 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.99751.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.99751.log.gz 2026-02-18T08:53:45.525 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.92600.log 2026-02-18T08:53:45.526 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.52789.log: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.52789.log.gz 2026-02-18T08:53:45.526 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.65374.log 2026-02-18T08:53:45.529 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.92600.log: 95.7% -- replaced with /var/log/ceph/cluster2-client.mirror.99420.log.gz 2026-02-18T08:53:45.531 INFO:teuthology.orchestra.run.trial163.stderr: 95.9% -- replaced with /var/log/ceph/cluster2-client.mirror.90472.log.gz 2026-02-18T08:53:45.532 INFO:teuthology.orchestra.run.trial163.stderr: 92.7% -- replaced with /var/log/ceph/cluster2-client.mirror.92600.log.gz 2026-02-18T08:53:45.534 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.67204.log 2026-02-18T08:53:45.534 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.65374.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.49635.log 2026-02-18T08:53:45.534 INFO:teuthology.orchestra.run.trial163.stderr: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.65374.log.gz 2026-02-18T08:53:45.535 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.67204.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.110245.log 2026-02-18T08:53:45.535 INFO:teuthology.orchestra.run.trial163.stderr: 88.9%gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.21257.log 2026-02-18T08:53:45.535 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.49635.log: -- replaced with /var/log/ceph/cluster2-client.mirror.67204.log.gz 2026-02-18T08:53:45.535 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.110245.log: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.49635.log.gz 2026-02-18T08:53:45.535 INFO:teuthology.orchestra.run.trial163.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.110245.log.gz 2026-02-18T08:53:45.535 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.130938.log 2026-02-18T08:53:45.535 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.21257.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.118786.log 2026-02-18T08:53:45.542 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.130938.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.21257.log.gz 2026-02-18T08:53:45.542 INFO:teuthology.orchestra.run.trial163.stderr: 87.0% -- replaced with /var/log/ceph/cluster1-client.mirror.130938.log.gz 2026-02-18T08:53:45.542 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.130915.log 2026-02-18T08:53:45.542 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.118786.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.127649.log 2026-02-18T08:53:45.542 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.130915.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.51984.log 2026-02-18T08:53:45.542 INFO:teuthology.orchestra.run.trial163.stderr: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.130915.log.gz 2026-02-18T08:53:45.542 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.127649.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.127649.log.gz 2026-02-18T08:53:45.542 INFO:teuthology.orchestra.run.trial163.stderr: 92.7% -- replaced with /var/log/ceph/cluster2-client.mirror.118786.log.gz 2026-02-18T08:53:45.542 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.128612.log 2026-02-18T08:53:45.542 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.116948.log 2026-02-18T08:53:45.542 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.51984.log: /var/log/ceph/cluster2-client.mirror.128612.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.34190.log 2026-02-18T08:53:45.543 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.116948.log: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.51984.log.gz 2026-02-18T08:53:45.543 INFO:teuthology.orchestra.run.trial163.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.116948.log.gz 2026-02-18T08:53:45.543 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.128925.log 2026-02-18T08:53:45.543 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.120228.log 2026-02-18T08:53:45.543 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.34190.log: 90.7% -- replaced with /var/log/ceph/cluster2-client.mirror.128612.log.gz 2026-02-18T08:53:45.543 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.128925.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.34190.log.gz 2026-02-18T08:53:45.543 INFO:teuthology.orchestra.run.trial163.stderr: 89.4% -- replaced with /var/log/ceph/cluster2-client.mirror.128925.log.gz 2026-02-18T08:53:45.543 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.71464.log 2026-02-18T08:53:45.543 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.113941.log 2026-02-18T08:53:45.543 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.120228.log: /var/log/ceph/cluster2-client.mirror.71464.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.91964.log 2026-02-18T08:53:45.543 INFO:teuthology.orchestra.run.trial163.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.120228.log.gz 2026-02-18T08:53:45.543 INFO:teuthology.orchestra.run.trial163.stderr: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.71464.log.gz 2026-02-18T08:53:45.544 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.113941.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.113941.log.gz 2026-02-18T08:53:45.544 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.127340.log 2026-02-18T08:53:45.544 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.91964.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.89527.log 2026-02-18T08:53:45.544 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.127340.log: 84.7% -- replaced with /var/log/ceph/cluster2-client.mirror.91964.log.gz 2026-02-18T08:53:45.544 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.124503.log 2026-02-18T08:53:45.544 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.89527.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.27972.log 2026-02-18T08:53:45.544 INFO:teuthology.orchestra.run.trial163.stderr: 89.2% -- replaced with /var/log/ceph/cluster2-client.mirror.127340.log.gz 2026-02-18T08:53:45.547 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.124503.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.85542.log 2026-02-18T08:53:45.547 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.27972.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.27972.log.gz 2026-02-18T08:53:45.547 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.15480.log 2026-02-18T08:53:45.547 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.85542.log: 92.7% -- replaced with /var/log/ceph/cluster2-client.mirror.124503.log.gz 2026-02-18T08:53:45.547 INFO:teuthology.orchestra.run.trial163.stderr: 89.4%gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.125400.log 2026-02-18T08:53:45.547 INFO:teuthology.orchestra.run.trial163.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.85542.log.gz 2026-02-18T08:53:45.547 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.admin.15480.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.25498.log 2026-02-18T08:53:45.547 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.125400.log: 80.3% -- replaced with /var/log/ceph/cluster2-client.admin.15480.log.gz 2026-02-18T08:53:45.547 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.35678.log 2026-02-18T08:53:45.547 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.25498.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.117016.log 2026-02-18T08:53:45.547 INFO:teuthology.orchestra.run.trial163.stderr: 90.7% 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.25498.log.gz 2026-02-18T08:53:45.548 INFO:teuthology.orchestra.run.trial163.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.125400.log.gz 2026-02-18T08:53:45.548 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.35678.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.14211.log 2026-02-18T08:53:45.548 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.117016.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.35678.log.gz 2026-02-18T08:53:45.548 INFO:teuthology.orchestra.run.trial163.stderr: 88.2% -- replaced with /var/log/ceph/cluster1-client.mirror.117016.log.gz 2026-02-18T08:53:45.548 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.50199.log 2026-02-18T08:53:45.549 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.97132.log 2026-02-18T08:53:45.549 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.admin.14211.log: /var/log/ceph/cluster2-client.mirror.50199.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.14211.log.gz 2026-02-18T08:53:45.549 INFO:teuthology.orchestra.run.trial163.stderr: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.50199.log.gz 2026-02-18T08:53:45.549 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.14852.log 2026-02-18T08:53:45.549 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.97132.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.97132.log.gz 2026-02-18T08:53:45.549 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.27635.log 2026-02-18T08:53:45.549 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.admin.14852.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.14852.log.gz 2026-02-18T08:53:45.550 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.73179.log 2026-02-18T08:53:45.550 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.27635.log: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.27635.log.gz 2026-02-18T08:53:45.550 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.54154.log 2026-02-18T08:53:45.551 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.73179.log: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.73179.log.gz 2026-02-18T08:53:45.551 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.65767.log 2026-02-18T08:53:45.552 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.54154.log: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.54154.log.gz 2026-02-18T08:53:45.552 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.127811.log 2026-02-18T08:53:45.552 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.65767.log: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.65767.log.gz 2026-02-18T08:53:45.553 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.76997.log 2026-02-18T08:53:45.554 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.127811.log: 90.7% -- replaced with /var/log/ceph/cluster2-client.mirror.127811.log.gz 2026-02-18T08:53:45.554 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.124632.log 2026-02-18T08:53:45.555 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.76997.log: 89.2% -- replaced with /var/log/ceph/cluster2-client.mirror.76997.log.gz 2026-02-18T08:53:45.555 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.88442.log 2026-02-18T08:53:45.556 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.124632.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.124632.log.gz 2026-02-18T08:53:45.556 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.14969.log 2026-02-18T08:53:45.556 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.88442.log: 84.9% -- replaced with /var/log/ceph/cluster2-client.mirror.88442.log.gz 2026-02-18T08:53:45.557 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.67484.log 2026-02-18T08:53:45.557 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.14969.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.28100.log 2026-02-18T08:53:45.557 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.130803.log 2026-02-18T08:53:45.558 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.28100.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.28100.log.gz 2026-02-18T08:53:45.559 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.122401.log 2026-02-18T08:53:45.559 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.130803.log: 87.0% -- replaced with /var/log/ceph/cluster1-client.mirror.130803.log.gz 2026-02-18T08:53:45.559 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.80042.log 2026-02-18T08:53:45.561 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.122401.log: 92.7% -- replaced with /var/log/ceph/cluster2-client.mirror.122401.log.gz 2026-02-18T08:53:45.561 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.128570.log 2026-02-18T08:53:45.562 INFO:teuthology.orchestra.run.trial163.stderr: 95.5%/var/log/ceph/cluster2-client.mirror.80042.log: -- replaced with /var/log/ceph/cluster2-client.mirror.85753.log.gz 2026-02-18T08:53:45.562 INFO:teuthology.orchestra.run.trial163.stderr: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.80042.log.gz 2026-02-18T08:53:45.563 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.119442.log 2026-02-18T08:53:45.563 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.67484.log: /var/log/ceph/cluster2-client.mirror.128570.log: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.67484.log.gz 2026-02-18T08:53:45.564 INFO:teuthology.orchestra.run.trial163.stderr: 89.3% -- replaced with /var/log/ceph/cluster2-client.mirror.128570.log.gz 2026-02-18T08:53:45.564 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.23826.log 2026-02-18T08:53:45.564 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.119442.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.20291.log 2026-02-18T08:53:45.565 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.23826.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.23826.log.gz 2026-02-18T08:53:45.567 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.134854.log 2026-02-18T08:53:45.568 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.20291.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.130870.log 2026-02-18T08:53:45.569 INFO:teuthology.orchestra.run.trial163.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.20291.log.gz 2026-02-18T08:53:45.569 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.134854.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.134854.log.gz 2026-02-18T08:53:45.570 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.133972.log 2026-02-18T08:53:45.570 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.130870.log: 86.8% -- replaced with /var/log/ceph/cluster1-client.mirror.130870.log.gz 2026-02-18T08:53:45.571 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.42645.log 2026-02-18T08:53:45.571 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.133972.log: 91.9% -- replaced with /var/log/ceph/cluster2-client.mirror.14969.log.gz 2026-02-18T08:53:45.571 INFO:teuthology.orchestra.run.trial163.stderr: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.133972.log.gz 2026-02-18T08:53:45.571 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.67099.log 2026-02-18T08:53:45.572 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.60374.log 2026-02-18T08:53:45.573 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.42645.log: /var/log/ceph/cluster2-client.mirror.67099.log: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.67099.log.gz 2026-02-18T08:53:45.573 INFO:teuthology.orchestra.run.trial163.stderr: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.42645.log.gz 2026-02-18T08:53:45.573 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.42209.log 2026-02-18T08:53:45.574 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.49355.log 2026-02-18T08:53:45.574 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.60374.log: /var/log/ceph/cluster2-client.mirror.42209.log: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.60374.log.gz 2026-02-18T08:53:45.574 INFO:teuthology.orchestra.run.trial163.stderr: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.42209.log.gz 2026-02-18T08:53:45.574 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.126340.log 2026-02-18T08:53:45.574 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.49355.log: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.49355.log.gz 2026-02-18T08:53:45.583 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.133189.log 2026-02-18T08:53:45.584 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.126340.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.129879.log 2026-02-18T08:53:45.584 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.133189.log: 87.0% -- replaced with /var/log/ceph/cluster1-client.mirror.133189.log.gz 2026-02-18T08:53:45.585 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.134346.log 2026-02-18T08:53:45.585 INFO:teuthology.orchestra.run.trial163.stderr: 90.6% -- replaced with /var/log/ceph/cluster2-client.mirror.126340.log.gz 2026-02-18T08:53:45.586 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.129879.log: 87.0% -- replaced with /var/log/ceph/cluster1-client.mirror.129879.log.gz 2026-02-18T08:53:45.586 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.113735.log 2026-02-18T08:53:45.586 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.134346.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.77942.log 2026-02-18T08:53:45.586 INFO:teuthology.orchestra.run.trial163.stderr: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.134346.log.gz 2026-02-18T08:53:45.586 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.113735.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.103542.log 2026-02-18T08:53:45.587 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.77942.log: 89.2% -- replaced with /var/log/ceph/cluster2-client.mirror.77942.log.gz 2026-02-18T08:53:45.587 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.55904.log 2026-02-18T08:53:45.588 INFO:teuthology.orchestra.run.trial163.stderr: 92.8% -- replaced with /var/log/ceph/cluster2-client.mirror.113735.log.gz 2026-02-18T08:53:45.588 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.103542.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.80532.log 2026-02-18T08:53:45.588 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.55904.log: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.55904.log.gz 2026-02-18T08:53:45.589 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.105228.log 2026-02-18T08:53:45.589 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.80532.log: 89.2% -- replaced with /var/log/ceph/cluster2-client.mirror.80532.log.gz 2026-02-18T08:53:45.590 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.0.102457.log 2026-02-18T08:53:45.590 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.105228.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.105228.log.gz 2026-02-18T08:53:45.591 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.84597.log 2026-02-18T08:53:45.600 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.0.102457.log: 95.7% -- replaced with /var/log/ceph/cluster2-client.mirror.106983.log.gz 2026-02-18T08:53:45.601 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.70099.log 2026-02-18T08:53:45.604 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.84597.log: 93.5% -- replaced with /var/log/ceph/cluster1-client.mirror.84597.log.gz 2026-02-18T08:53:45.605 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.22431.log 2026-02-18T08:53:45.605 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.70099.log: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.70099.log.gz 2026-02-18T08:53:45.606 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.115282.log 2026-02-18T08:53:45.606 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.22431.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.70729.log 2026-02-18T08:53:45.606 INFO:teuthology.orchestra.run.trial163.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.22431.log.gz 2026-02-18T08:53:45.607 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.115282.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.115282.log.gz 2026-02-18T08:53:45.607 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.45845.log 2026-02-18T08:53:45.608 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.70729.log: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.70729.log.gz 2026-02-18T08:53:45.608 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.132263.log 2026-02-18T08:53:45.609 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.45845.log: 94.2% -- replaced with /var/log/ceph/cluster2-client.mirror.0.102457.log.gz 2026-02-18T08:53:45.609 INFO:teuthology.orchestra.run.trial163.stderr: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.45845.log.gz 2026-02-18T08:53:45.609 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.95152.log 2026-02-18T08:53:45.611 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.132263.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.134759.log 2026-02-18T08:53:45.611 INFO:teuthology.orchestra.run.trial163.stderr: 87.9% -- replaced with /var/log/ceph/cluster1-client.mirror.132263.log.gz/var/log/ceph/cluster2-client.mirror.95152.log: 2026-02-18T08:53:45.611 INFO:teuthology.orchestra.run.trial163.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.95152.log.gz 2026-02-18T08:53:45.611 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.76892.log 2026-02-18T08:53:45.611 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.134759.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.46094.log 2026-02-18T08:53:45.612 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.76892.log: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.76892.log.gz 2026-02-18T08:53:45.612 INFO:teuthology.orchestra.run.trial163.stderr: 91.5% -- replaced with /var/log/ceph/cluster2-client.mirror.134759.log.gz 2026-02-18T08:53:45.612 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.78327.log 2026-02-18T08:53:45.613 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.27263.log 2026-02-18T08:53:45.613 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.46094.log: /var/log/ceph/cluster2-client.mirror.78327.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.85030.log 2026-02-18T08:53:45.613 INFO:teuthology.orchestra.run.trial163.stderr: 89.2% -- replaced with /var/log/ceph/cluster2-client.mirror.78327.log.gz 2026-02-18T08:53:45.615 INFO:teuthology.orchestra.run.trial163.stderr: 93.4% -- replaced with /var/log/ceph/cluster1-client.mirror.46094.log.gz 2026-02-18T08:53:45.615 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.78362.log 2026-02-18T08:53:45.616 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.27263.log: /var/log/ceph/cluster1-client.mirror.85030.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.27263.log.gz 2026-02-18T08:53:45.616 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.124729.log 2026-02-18T08:53:45.617 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.78362.log: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.78362.log.gz 2026-02-18T08:53:45.617 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.25556.log 2026-02-18T08:53:45.618 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.124729.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.124729.log.gz 2026-02-18T08:53:45.618 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.30483.log 2026-02-18T08:53:45.619 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.25556.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.19168.log 2026-02-18T08:53:45.620 INFO:teuthology.orchestra.run.trial163.stderr: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.25556.log.gz 2026-02-18T08:53:45.620 INFO:teuthology.orchestra.run.trial163.stderr: 93.5% -- replaced with /var/log/ceph/cluster1-client.mirror.85030.log.gz 2026-02-18T08:53:45.620 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.30483.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.109673.log 2026-02-18T08:53:45.620 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.19168.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.30483.log.gz 2026-02-18T08:53:45.621 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.17622.log 2026-02-18T08:53:45.621 INFO:teuthology.orchestra.run.trial163.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.19168.log.gz 2026-02-18T08:53:45.621 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.109673.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.51319.log 2026-02-18T08:53:45.621 INFO:teuthology.orchestra.run.trial163.stderr: 88.0%/var/log/ceph/cluster2-client.mirror.17622.log: -- replaced with /var/log/ceph/cluster2-client.mirror.109673.log.gz 2026-02-18T08:53:45.621 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.120754.log 2026-02-18T08:53:45.622 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.51319.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.17622.log.gz 2026-02-18T08:53:45.623 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.135469.log 2026-02-18T08:53:45.623 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.120754.log: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.51319.log.gz 2026-02-18T08:53:45.623 INFO:teuthology.orchestra.run.trial163.stderr: 88.2%gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.30425.log 2026-02-18T08:53:45.624 INFO:teuthology.orchestra.run.trial163.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.120754.log.gz 2026-02-18T08:53:45.624 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.admin.135469.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.91467.log 2026-02-18T08:53:45.624 INFO:teuthology.orchestra.run.trial163.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.135469.log.gz 2026-02-18T08:53:45.624 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.30425.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.129159.log 2026-02-18T08:53:45.624 INFO:teuthology.orchestra.run.trial163.stderr: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.30425.log.gz 2026-02-18T08:53:45.624 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.91467.log: 84.8% -- replaced with /var/log/ceph/cluster2-client.mirror.91467.log.gz 2026-02-18T08:53:45.625 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.57829.log 2026-02-18T08:53:45.625 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.112498.log 2026-02-18T08:53:45.625 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.129159.log: /var/log/ceph/cluster2-client.mirror.57829.log: 86.8% -- replaced with /var/log/ceph/cluster1-client.mirror.129159.log.gz 2026-02-18T08:53:45.625 INFO:teuthology.orchestra.run.trial163.stderr: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.57829.log.gz 2026-02-18T08:53:45.625 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.84314.log 2026-02-18T08:53:45.626 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.112498.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.112498.log.gz 2026-02-18T08:53:45.626 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.62150.log 2026-02-18T08:53:45.626 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.22989.log 2026-02-18T08:53:45.626 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.84314.log: /var/log/ceph/cluster2-client.mirror.62150.log: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.62150.log.gz 2026-02-18T08:53:45.629 INFO:teuthology.orchestra.run.trial163.stderr: 93.3% -- replaced with /var/log/ceph/cluster2-client.mirror.84314.log.gz 2026-02-18T08:53:45.629 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.104686.log 2026-02-18T08:53:45.630 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.22989.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.125571.log 2026-02-18T08:53:45.630 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.104686.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.22989.log.gz 2026-02-18T08:53:45.630 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.34376.log 2026-02-18T08:53:45.631 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.125571.log: 92.4% -- replaced with /var/log/ceph/cluster2-client.mirror.104686.log.gz 2026-02-18T08:53:45.631 INFO:teuthology.orchestra.run.trial163.stderr: 89.2% -- replaced with /var/log/ceph/cluster2-client.mirror.125571.log.gz 2026-02-18T08:53:45.632 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.47427.log 2026-02-18T08:53:45.632 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.34376.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.88047.log 2026-02-18T08:53:45.632 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.47427.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.34376.log.gz 2026-02-18T08:53:45.633 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.67834.log 2026-02-18T08:53:45.633 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.88047.log: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.47427.log.gz 2026-02-18T08:53:45.633 INFO:teuthology.orchestra.run.trial163.stderr: 84.8% -- replaced with /var/log/ceph/cluster2-client.mirror.88047.log.gz 2026-02-18T08:53:45.635 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.52824.log 2026-02-18T08:53:45.636 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.0.113404.log 2026-02-18T08:53:45.636 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.67834.log: /var/log/ceph/cluster2-client.mirror.52824.log: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.67834.log.gz 2026-02-18T08:53:45.636 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.134874.log 2026-02-18T08:53:45.636 INFO:teuthology.orchestra.run.trial163.stderr: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.52824.log.gz 2026-02-18T08:53:45.637 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.0.113404.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.92219.log 2026-02-18T08:53:45.638 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.134874.log: 91.8% -- replaced with /var/log/ceph/cluster2-client.mirror.134874.log.gz 2026-02-18T08:53:45.639 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.90645.log 2026-02-18T08:53:45.639 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.92219.log: 1.3% -- replaced with /var/log/ceph/cluster2-client.mirror.92219.log.gz 2026-02-18T08:53:45.640 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2.audit.log 2026-02-18T08:53:45.640 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.90645.log: 84.8% -- replaced with /var/log/ceph/cluster2-client.mirror.90645.log.gz 2026-02-18T08:53:45.640 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.112171.log 2026-02-18T08:53:45.641 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2.audit.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.113678.log 2026-02-18T08:53:45.642 INFO:teuthology.orchestra.run.trial163.stderr: 94.1% -- replaced with /var/log/ceph/cluster2-client.mirror.0.113404.log.gz 2026-02-18T08:53:45.642 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.38177.log 2026-02-18T08:53:45.643 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.113678.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.113678.log.gz 2026-02-18T08:53:45.643 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.26949.log 2026-02-18T08:53:45.643 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.38177.log: 92.9% -- replaced with /var/log/ceph/cluster2.audit.log.gz 2026-02-18T08:53:45.644 INFO:teuthology.orchestra.run.trial163.stderr: 88.5%/var/log/ceph/cluster2-client.mirror.112171.log: -- replaced with /var/log/ceph/cluster2-client.mirror.38177.log.gz 2026-02-18T08:53:45.644 INFO:teuthology.orchestra.run.trial163.stderr: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.112171.log.gz 2026-02-18T08:53:45.644 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.94692.log 2026-02-18T08:53:45.645 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.26949.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.131118.log 2026-02-18T08:53:45.645 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.94692.log: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.26949.log.gz 2026-02-18T08:53:45.645 INFO:teuthology.orchestra.run.trial163.stderr: 87.9% -- replaced with /var/log/ceph/cluster2-client.mirror.94692.log.gz 2026-02-18T08:53:45.646 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.45003.log 2026-02-18T08:53:45.646 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.131118.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.48410.log 2026-02-18T08:53:45.646 INFO:teuthology.orchestra.run.trial163.stderr: 86.7% -- replaced with /var/log/ceph/cluster1-client.mirror.131118.log.gz 2026-02-18T08:53:45.647 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.45003.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.51739.log 2026-02-18T08:53:45.647 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.48410.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.45003.log.gz 2026-02-18T08:53:45.647 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.130082.log 2026-02-18T08:53:45.647 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.51739.log: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.48410.log.gz 2026-02-18T08:53:45.648 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.120507.log 2026-02-18T08:53:45.648 INFO:teuthology.orchestra.run.trial163.stderr: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.51739.log.gz 2026-02-18T08:53:45.648 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.130082.log: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.130082.log.gz 2026-02-18T08:53:45.655 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.84491.log 2026-02-18T08:53:45.656 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.84772.log 2026-02-18T08:53:45.657 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.120507.log: /var/log/ceph/cluster2-client.mirror.84491.log: 91.9% -- replaced with /var/log/ceph/cluster1-client.mirror.120507.log.gz 2026-02-18T08:53:45.657 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.2.131257.log 2026-02-18T08:53:45.660 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.84772.log: 93.4% -- replaced with /var/log/ceph/cluster2-client.mirror.84491.log.gz 2026-02-18T08:53:45.660 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.130690.log 2026-02-18T08:53:45.661 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.2.131257.log: 93.5% -- replaced with /var/log/ceph/cluster1-client.mirror.84772.log.gz 2026-02-18T08:53:45.661 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.134449.log 2026-02-18T08:53:45.662 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.130690.log: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.130690.log.gz 2026-02-18T08:53:45.662 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.29495.log 2026-02-18T08:53:45.663 INFO:teuthology.orchestra.run.trial163.stderr: 93.4% -- replaced with /var/log/ceph/cluster1-client.mirror.2.131257.log.gz 2026-02-18T08:53:45.663 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.134449.log: 87.9% -- replaced with /var/log/ceph/cluster1-client.mirror.134449.log.gz 2026-02-18T08:53:45.663 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.132187.log 2026-02-18T08:53:45.664 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.118549.log 2026-02-18T08:53:45.664 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.29495.log: /var/log/ceph/cluster2-client.mirror.132187.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.132187.log.gz 2026-02-18T08:53:45.664 INFO:teuthology.orchestra.run.trial163.stderr: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.29495.log.gz 2026-02-18T08:53:45.664 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.29867.log 2026-02-18T08:53:45.666 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.118549.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.14501.log 2026-02-18T08:53:45.666 INFO:teuthology.orchestra.run.trial163.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.118549.log.gz 2026-02-18T08:53:45.666 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.29867.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.35085.log 2026-02-18T08:53:45.666 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.admin.14501.log: 88.3% 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.14501.log.gz 2026-02-18T08:53:45.666 INFO:teuthology.orchestra.run.trial163.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.29867.log.gz 2026-02-18T08:53:45.666 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.134422.log 2026-02-18T08:53:45.667 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.35085.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.96360.log 2026-02-18T08:53:45.667 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.134422.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.35085.log.gz 2026-02-18T08:53:45.667 INFO:teuthology.orchestra.run.trial163.stderr: 87.6% -- replaced with /var/log/ceph/cluster1-client.mirror.134422.log.gz 2026-02-18T08:53:45.667 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.16343.log 2026-02-18T08:53:45.667 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.96360.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.104706.log 2026-02-18T08:53:45.668 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.16343.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.96360.log.gz 2026-02-18T08:53:45.668 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.126462.log 2026-02-18T08:53:45.668 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.29274.log 2026-02-18T08:53:45.668 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.126462.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.126462.log.gz 2026-02-18T08:53:45.675 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.64145.log 2026-02-18T08:53:45.676 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.17901.log/var/log/ceph/cluster2-client.mirror.29274.log: 2026-02-18T08:53:45.676 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.64145.log: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.64145.log.gz 2026-02-18T08:53:45.677 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.129018.log 2026-02-18T08:53:45.678 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.17901.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.17901.log.gz 2026-02-18T08:53:45.678 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.122924.log 2026-02-18T08:53:45.678 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.129018.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.129018.log.gz 2026-02-18T08:53:45.678 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.102188.log 2026-02-18T08:53:45.679 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.122924.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.122924.log.gz 2026-02-18T08:53:45.680 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.104706.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.51529.log 2026-02-18T08:53:45.680 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.102188.log: 92.2% -- replaced with /var/log/ceph/cluster1-client.mirror.104706.log.gz 2026-02-18T08:53:45.691 INFO:teuthology.orchestra.run.trial163.stderr: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.29274.log.gz 2026-02-18T08:53:45.695 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.24419.log 2026-02-18T08:53:45.696 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.51529.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.51529.log.gz 2026-02-18T08:53:45.703 INFO:teuthology.orchestra.run.trial163.stderr: 95.8% -- replaced with /var/log/ceph/cluster2-client.mirror.65932.log.gz 2026-02-18T08:53:45.703 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.59919.log 2026-02-18T08:53:45.705 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.24419.log: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.24419.log.gz 2026-02-18T08:53:45.705 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.107858.log 2026-02-18T08:53:45.705 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.59919.log: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.59919.log.gz 2026-02-18T08:53:45.706 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.87169.log 2026-02-18T08:53:45.706 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.107858.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.107858.log.gz 2026-02-18T08:53:45.707 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.98929.log 2026-02-18T08:53:45.718 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.87169.log: 95.7% -- replaced with /var/log/ceph/cluster2-client.mirror.100613.log.gz 2026-02-18T08:53:45.719 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.32570.log 2026-02-18T08:53:45.720 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.98929.log: 87.4% -- replaced with /var/log/ceph/cluster2-client.mirror.98929.log.gz 2026-02-18T08:53:45.721 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.88070.log 2026-02-18T08:53:45.722 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.32570.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.32570.log.gz 2026-02-18T08:53:45.722 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.43389.log 2026-02-18T08:53:45.731 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.88070.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.35306.log 2026-02-18T08:53:45.734 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.43389.log: 95.7% -- replaced with /var/log/ceph/cluster2-client.mirror.107561.log.gz 2026-02-18T08:53:45.735 INFO:teuthology.orchestra.run.trial163.stderr: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.43389.log.gz 2026-02-18T08:53:45.744 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.46046.log 2026-02-18T08:53:45.745 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.35306.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.35306.log.gz 2026-02-18T08:53:45.745 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.69364.log 2026-02-18T08:53:45.745 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.46046.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.15159.log 2026-02-18T08:53:45.749 INFO:teuthology.orchestra.run.trial163.stderr: 93.3% -- replaced with /var/log/ceph/cluster1-client.mirror.46046.log.gz 2026-02-18T08:53:45.750 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.69364.log: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.69364.log.gz 2026-02-18T08:53:45.751 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.115783.log 2026-02-18T08:53:45.751 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.admin.15159.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.15159.log.gz 2026-02-18T08:53:45.751 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.111003.log 2026-02-18T08:53:45.752 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.115783.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.115783.log.gz 2026-02-18T08:53:45.752 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.104129.log 2026-02-18T08:53:45.754 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.111003.log: 91.9% -- replaced with /var/log/ceph/cluster1-client.mirror.111003.log.gz 2026-02-18T08:53:45.754 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.21629.log 2026-02-18T08:53:45.755 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.104129.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.104129.log.gz 2026-02-18T08:53:45.755 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.34248.log 2026-02-18T08:53:45.755 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.21629.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.92980.log 2026-02-18T08:53:45.756 INFO:teuthology.orchestra.run.trial163.stderr: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.21629.log.gz 2026-02-18T08:53:45.757 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.34248.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.34248.log.gz 2026-02-18T08:53:45.757 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.37342.log 2026-02-18T08:53:45.758 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.92980.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.92980.log.gz 2026-02-18T08:53:45.758 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.64670.log 2026-02-18T08:53:45.759 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.37342.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.37342.log.gz 2026-02-18T08:53:45.759 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.2.133098.log 2026-02-18T08:53:45.760 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.64670.log: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.64670.log.gz 2026-02-18T08:53:45.760 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.121041.log 2026-02-18T08:53:45.762 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.2.133098.log: 92.4% -- replaced with /var/log/ceph/cluster2-client.mirror.2.133098.log.gz 2026-02-18T08:53:45.762 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.132951.log 2026-02-18T08:53:45.763 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.121041.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.121041.log.gz 2026-02-18T08:53:45.763 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.73774.log 2026-02-18T08:53:45.764 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.132951.log: 87.6% -- replaced with /var/log/ceph/cluster1-client.mirror.132951.log.gz 2026-02-18T08:53:45.764 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.46937.log 2026-02-18T08:53:45.765 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.73774.log: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.73774.log.gz 2026-02-18T08:53:45.765 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.94538.log 2026-02-18T08:53:45.766 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.46937.log: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.46937.log.gz 2026-02-18T08:53:45.766 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.15497.log 2026-02-18T08:53:45.767 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.94538.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.94538.log.gz 2026-02-18T08:53:45.767 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.101057.log 2026-02-18T08:53:45.768 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.admin.15497.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.115145.log 2026-02-18T08:53:45.768 INFO:teuthology.orchestra.run.trial163.stderr: 73.4% -- replaced with /var/log/ceph/cluster2-client.admin.15497.log.gz 2026-02-18T08:53:45.769 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.101057.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.101057.log.gz 2026-02-18T08:53:45.769 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.48200.log 2026-02-18T08:53:45.770 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.115145.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.115145.log.gz 2026-02-18T08:53:45.770 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.126511.log 2026-02-18T08:53:45.771 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.48200.log: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.48200.log.gz 2026-02-18T08:53:45.771 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.52194.log 2026-02-18T08:53:45.772 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.126511.log: 89.4% -- replaced with /var/log/ceph/cluster2-client.mirror.126511.log.gz 2026-02-18T08:53:45.772 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.111629.log 2026-02-18T08:53:45.773 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.52194.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.52194.log.gz 2026-02-18T08:53:45.773 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.97759.log 2026-02-18T08:53:45.776 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.111629.log: 95.8% -- replaced with /var/log/ceph/cluster2-client.mirror.91490.log.gz 2026-02-18T08:53:45.776 INFO:teuthology.orchestra.run.trial163.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.111629.log.gz 2026-02-18T08:53:45.776 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.60654.log 2026-02-18T08:53:45.776 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.97759.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.94125.log 2026-02-18T08:53:45.776 INFO:teuthology.orchestra.run.trial163.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.97759.log.gz 2026-02-18T08:53:45.776 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.60654.log: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.60654.log.gzgzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.65909.log 2026-02-18T08:53:45.776 INFO:teuthology.orchestra.run.trial163.stderr: 2026-02-18T08:53:45.776 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.94125.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.45189.log 2026-02-18T08:53:45.776 INFO:teuthology.orchestra.run.trial163.stderr: 88.4% -- replaced with /var/log/ceph/cluster1-client.mirror.94125.log.gz 2026-02-18T08:53:45.777 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.65909.log: 94.4% -- replaced with /var/log/ceph/cluster2-client.mirror.65909.log.gz 2026-02-18T08:53:45.778 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.52894.log 2026-02-18T08:53:45.779 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.83999.log 2026-02-18T08:53:45.779 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.45189.log: /var/log/ceph/cluster2-client.mirror.52894.log: 95.7% 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.45189.log.gz 2026-02-18T08:53:45.779 INFO:teuthology.orchestra.run.trial163.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.85827.log.gz 2026-02-18T08:53:45.780 INFO:teuthology.orchestra.run.trial163.stderr: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.52894.log.gzgzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.112719.log 2026-02-18T08:53:45.780 INFO:teuthology.orchestra.run.trial163.stderr: 2026-02-18T08:53:45.780 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.83999.log: 89.2% -- replaced with /var/log/ceph/cluster2-client.mirror.83999.log.gz 2026-02-18T08:53:45.782 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.110649.log 2026-02-18T08:53:45.782 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.74828.log 2026-02-18T08:53:45.782 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.112719.log: /var/log/ceph/cluster2-client.mirror.110649.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.98388.log 2026-02-18T08:53:45.783 INFO:teuthology.orchestra.run.trial163.stderr: 88.3%/var/log/ceph/cluster2-client.mirror.74828.log: -- replaced with /var/log/ceph/cluster2-client.mirror.112719.log.gz 2026-02-18T08:53:45.783 INFO:teuthology.orchestra.run.trial163.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.110649.log.gz 2026-02-18T08:53:45.783 INFO:teuthology.orchestra.run.trial163.stderr: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.74828.log.gz 2026-02-18T08:53:45.783 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.117191.log 2026-02-18T08:53:45.783 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.119090.log 2026-02-18T08:53:45.783 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.98388.log: /var/log/ceph/cluster2-client.mirror.117191.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.29123.log 2026-02-18T08:53:45.783 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.119090.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.117191.log.gz 2026-02-18T08:53:45.783 INFO:teuthology.orchestra.run.trial163.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.119090.log.gz 2026-02-18T08:53:45.783 INFO:teuthology.orchestra.run.trial163.stderr: 92.6% -- replaced with /var/log/ceph/cluster2-client.mirror.98388.log.gz 2026-02-18T08:53:45.787 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.108246.log 2026-02-18T08:53:45.787 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.72654.log 2026-02-18T08:53:45.787 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.29123.log: /var/log/ceph/cluster2-client.mirror.108246.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.59184.log 2026-02-18T08:53:45.787 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.72654.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.108246.log.gz 2026-02-18T08:53:45.787 INFO:teuthology.orchestra.run.trial163.stderr: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.29123.log.gz 2026-02-18T08:53:45.787 INFO:teuthology.orchestra.run.trial163.stderr: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.72654.log.gz 2026-02-18T08:53:45.788 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.83945.log 2026-02-18T08:53:45.788 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.39723.log 2026-02-18T08:53:45.788 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.59184.log: /var/log/ceph/cluster2-client.mirror.83945.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.62850.log 2026-02-18T08:53:45.788 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.39723.log: 88.9% 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.59184.log.gz -- replaced with /var/log/ceph/cluster2-client.mirror.83945.log.gz 2026-02-18T08:53:45.788 INFO:teuthology.orchestra.run.trial163.stderr: 2026-02-18T08:53:45.788 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.68269.log 2026-02-18T08:53:45.788 INFO:teuthology.orchestra.run.trial163.stderr: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.39723.log.gz 2026-02-18T08:53:45.788 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.30988.log 2026-02-18T08:53:45.788 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.62850.log: /var/log/ceph/cluster1-client.mirror.68269.log: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.62850.log.gz 2026-02-18T08:53:45.788 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.97446.log 2026-02-18T08:53:45.788 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.96638.log 2026-02-18T08:53:45.788 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.30988.log: /var/log/ceph/cluster2-client.mirror.97446.log: 95.8% -- replaced with /var/log/ceph/cluster2-client.mirror.91084.log.gz 2026-02-18T08:53:45.788 INFO:teuthology.orchestra.run.trial163.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.97446.log.gz 2026-02-18T08:53:45.789 INFO:teuthology.orchestra.run.trial163.stderr: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.30988.log.gz 2026-02-18T08:53:45.789 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.87447.log 2026-02-18T08:53:45.791 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.40037.log 2026-02-18T08:53:45.791 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.96638.log: /var/log/ceph/cluster2-client.mirror.87447.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.89059.log 2026-02-18T08:53:45.791 INFO:teuthology.orchestra.run.trial163.stderr: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.96638.log.gz 2026-02-18T08:53:45.791 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.40037.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.62955.log 2026-02-18T08:53:45.791 INFO:teuthology.orchestra.run.trial163.stderr: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.40037.log.gz 2026-02-18T08:53:45.791 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.89059.log: 93.5% -- replaced with /var/log/ceph/cluster1-client.mirror.68269.log.gz 2026-02-18T08:53:45.791 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.101379.log 2026-02-18T08:53:45.791 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.64355.log 2026-02-18T08:53:45.792 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.62955.log: 88.6%/var/log/ceph/cluster1-client.mirror.101379.log: -- replaced with /var/log/ceph/cluster2-client.mirror.62955.log.gz 2026-02-18T08:53:45.792 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.47640.log 2026-02-18T08:53:45.792 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.64355.log: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.64355.log.gz 2026-02-18T08:53:45.792 INFO:teuthology.orchestra.run.trial163.stderr: 92.4% -- replaced with /var/log/ceph/cluster1-client.mirror.101379.log.gz 2026-02-18T08:53:45.792 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.131185.log 2026-02-18T08:53:45.792 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.47640.log: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.47640.log.gz 2026-02-18T08:53:45.792 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.28716.log 2026-02-18T08:53:45.793 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.131185.log: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.131185.log.gz 2026-02-18T08:53:45.793 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.134221.log 2026-02-18T08:53:45.794 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.28716.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.28716.log.gz 2026-02-18T08:53:45.794 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.98695.log 2026-02-18T08:53:45.795 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.134221.log: 87.0% -- replaced with /var/log/ceph/cluster1-client.mirror.134221.log.gz 2026-02-18T08:53:45.795 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.21315.log 2026-02-18T08:53:45.795 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.98695.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.98695.log.gz 2026-02-18T08:53:45.796 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.19672.log 2026-02-18T08:53:45.796 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.21315.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.21315.log.gz 2026-02-18T08:53:45.797 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.122507.log 2026-02-18T08:53:45.797 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.19672.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.19672.log.gz 2026-02-18T08:53:45.798 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.70029.log 2026-02-18T08:53:45.799 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.122507.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.130627.log 2026-02-18T08:53:45.800 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.70029.log: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.70029.log.gz 2026-02-18T08:53:45.800 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.107132.log 2026-02-18T08:53:45.801 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.130627.log: 87.0% -- replaced with /var/log/ceph/cluster1-client.mirror.130627.log.gz 2026-02-18T08:53:45.801 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.38793.log 2026-02-18T08:53:45.802 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.107132.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.107132.log.gz 2026-02-18T08:53:45.802 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.118212.log 2026-02-18T08:53:45.803 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.38793.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.38793.log.gz 2026-02-18T08:53:45.803 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.111935.log 2026-02-18T08:53:45.804 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.118212.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.118212.log.gz 2026-02-18T08:53:45.804 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.30111.log 2026-02-18T08:53:45.805 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.111935.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.111935.log.gz 2026-02-18T08:53:45.805 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.26670.log 2026-02-18T08:53:45.806 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.30111.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.30111.log.gz 2026-02-18T08:53:45.806 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.70939.log 2026-02-18T08:53:45.807 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.26670.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.26670.log.gz 2026-02-18T08:53:45.807 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.122681.log 2026-02-18T08:53:45.808 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.70939.log: 95.6% -- replaced with /var/log/ceph/cluster2-client.mirror.90044.log.gz 2026-02-18T08:53:45.808 INFO:teuthology.orchestra.run.trial163.stderr: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.70939.log.gz 2026-02-18T08:53:45.808 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.58415.log 2026-02-18T08:53:45.809 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.122681.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.36935.log 2026-02-18T08:53:45.809 INFO:teuthology.orchestra.run.trial163.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.122681.log.gz 2026-02-18T08:53:45.809 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.58415.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.58415.log.gzgzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.96844.log 2026-02-18T08:53:45.809 INFO:teuthology.orchestra.run.trial163.stderr: 2026-02-18T08:53:45.810 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.36935.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.86954.log 2026-02-18T08:53:45.810 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.96844.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.36935.log.gz 2026-02-18T08:53:45.810 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.135211.log 2026-02-18T08:53:45.811 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.86954.log: 84.7% -- replaced with /var/log/ceph/cluster2-client.mirror.86954.log.gz 2026-02-18T08:53:45.811 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.131602.log 2026-02-18T08:53:45.811 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.admin.135211.log: 92.1% -- replaced with /var/log/ceph/cluster1-client.mirror.96844.log.gz 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.135211.log.gz 2026-02-18T08:53:45.811 INFO:teuthology.orchestra.run.trial163.stderr: 2026-02-18T08:53:45.812 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.84577.log 2026-02-18T08:53:45.812 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.131602.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.93516.log 2026-02-18T08:53:45.812 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.84577.log: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.131602.log.gz 2026-02-18T08:53:45.813 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.51774.log 2026-02-18T08:53:45.813 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.93516.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.93516.log.gz 2026-02-18T08:53:45.813 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.131975.log 2026-02-18T08:53:45.814 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.51774.log: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.51774.log.gz 2026-02-18T08:53:45.814 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.135185.log 2026-02-18T08:53:45.815 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.131975.log: 87.9% -- replaced with /var/log/ceph/cluster1-client.mirror.131975.log.gz 2026-02-18T08:53:45.815 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.42587.log 2026-02-18T08:53:45.815 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.admin.135185.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.135185.log.gz 2026-02-18T08:53:45.816 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.109402.log 2026-02-18T08:53:45.816 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.42587.log: 93.5% -- replaced with /var/log/ceph/cluster2-client.mirror.84577.log.gz 2026-02-18T08:53:45.816 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.32977.log 2026-02-18T08:53:45.816 INFO:teuthology.orchestra.run.trial163.stderr: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.42587.log.gz 2026-02-18T08:53:45.818 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.109402.log: 95.0% -- replaced with /var/log/ceph/cluster1-client.mirror.109402.log.gz 2026-02-18T08:53:45.818 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.124972.log 2026-02-18T08:53:45.819 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.32977.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.32977.log.gz 2026-02-18T08:53:45.820 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.129947.log 2026-02-18T08:53:45.820 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.11425.log 2026-02-18T08:53:45.821 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.124972.log: /var/log/ceph/cluster1-client.mirror.129947.log: 86.9% 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.124972.log.gz 2026-02-18T08:53:45.821 INFO:teuthology.orchestra.run.trial163.stderr: -- replaced with /var/log/ceph/cluster1-client.mirror.129947.log.gz 2026-02-18T08:53:45.821 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.15377.log 2026-02-18T08:53:45.821 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.1.99956.log 2026-02-18T08:53:45.821 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.admin.11425.log: /var/log/ceph/cluster1-client.mirror.15377.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.11425.log.gz 2026-02-18T08:53:45.822 INFO:teuthology.orchestra.run.trial163.stderr: 56.3% -- replaced with /var/log/ceph/cluster1-client.mirror.15377.log.gz 2026-02-18T08:53:45.822 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.85804.log 2026-02-18T08:53:45.831 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.1.99956.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.83448.log 2026-02-18T08:53:45.832 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.85804.log: 84.7% -- replaced with /var/log/ceph/cluster2-client.mirror.85804.log.gz 2026-02-18T08:53:45.832 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.106601.log 2026-02-18T08:53:45.836 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.83448.log: 93.4% -- replaced with /var/log/ceph/cluster1-client.mirror.83448.log.gz 2026-02-18T08:53:45.836 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.112291.log 2026-02-18T08:53:45.837 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.106601.log: 86.9% -- replaced with /var/log/ceph/cluster2-client.mirror.106601.log.gz 2026-02-18T08:53:45.837 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.120653.log 2026-02-18T08:53:45.837 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.112291.log: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.112291.log.gz 2026-02-18T08:53:45.838 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.71254.log 2026-02-18T08:53:45.838 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.120653.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.120653.log.gz 2026-02-18T08:53:45.838 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.50374.log 2026-02-18T08:53:45.839 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.71254.log: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.71254.log.gz 2026-02-18T08:53:45.839 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.36108.log 2026-02-18T08:53:45.840 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.50374.log: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.50374.log.gz 2026-02-18T08:53:45.840 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.108343.log 2026-02-18T08:53:45.841 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.36108.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.36108.log.gz 2026-02-18T08:53:45.841 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.127669.log 2026-02-18T08:53:45.842 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.108343.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.108343.log.gz 2026-02-18T08:53:45.842 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.94769.log 2026-02-18T08:53:45.843 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.127669.log: 90.7% -- replaced with /var/log/ceph/cluster2-client.mirror.127669.log.gz 2026-02-18T08:53:45.844 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.18552.log 2026-02-18T08:53:45.844 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.94769.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.94769.log.gz 2026-02-18T08:53:45.845 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.27542.log 2026-02-18T08:53:45.845 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.18552.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.18552.log.gz 2026-02-18T08:53:45.845 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.38142.log 2026-02-18T08:53:45.846 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.27542.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.27542.log.gz 2026-02-18T08:53:45.846 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.130262.log 2026-02-18T08:53:45.847 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.38142.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.38142.log.gz 2026-02-18T08:53:45.847 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.52509.log 2026-02-18T08:53:45.848 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.130262.log: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.130262.log.gz 2026-02-18T08:53:45.848 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.119303.log 2026-02-18T08:53:45.849 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.52509.log: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.52509.log.gz 2026-02-18T08:53:45.849 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.99863.log 2026-02-18T08:53:45.849 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.119303.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.119303.log.gz 2026-02-18T08:53:45.850 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.123813.log 2026-02-18T08:53:45.851 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.99863.log: 92.4% -- replaced with /var/log/ceph/cluster1-client.mirror.99863.log.gz 2026-02-18T08:53:45.851 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.49845.log 2026-02-18T08:53:45.852 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.123813.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.123813.log.gz 2026-02-18T08:53:45.852 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.87596.log 2026-02-18T08:53:45.853 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.49845.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.85332.log 2026-02-18T08:53:45.853 INFO:teuthology.orchestra.run.trial163.stderr: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.49845.log.gz 2026-02-18T08:53:45.854 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.87596.log: 94.5% -- replaced with /var/log/ceph/cluster1-client.mirror.87596.log.gz 2026-02-18T08:53:45.854 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.22931.log 2026-02-18T08:53:45.855 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.85332.log: 89.4% -- replaced with /var/log/ceph/cluster2-client.mirror.85332.log.gz 2026-02-18T08:53:45.855 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.97244.log 2026-02-18T08:53:45.856 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.22931.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.133912.log 2026-02-18T08:53:45.856 INFO:teuthology.orchestra.run.trial163.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.22931.log.gz 2026-02-18T08:53:45.856 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.97244.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.97244.log.gz 2026-02-18T08:53:45.857 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.107381.log 2026-02-18T08:53:45.857 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.133912.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.133912.log.gz 2026-02-18T08:53:45.858 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.62325.log 2026-02-18T08:53:45.858 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.107381.log: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.107381.log.gz 2026-02-18T08:53:45.858 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.88738.log 2026-02-18T08:53:45.859 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.62325.log: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.62325.log.gz 2026-02-18T08:53:45.860 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.18831.log 2026-02-18T08:53:45.860 INFO:teuthology.orchestra.run.trial163.stderr: 93.7% -- replaced with /var/log/ceph/cluster1-client.mirror.1.99956.log.gz 2026-02-18T08:53:45.860 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.88738.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.51669.log 2026-02-18T08:53:45.860 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.18831.log: 94.8% -- replaced with /var/log/ceph/cluster1-client.mirror.88738.log.gz 2026-02-18T08:53:45.861 INFO:teuthology.orchestra.run.trial163.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.18831.log.gz 2026-02-18T08:53:45.861 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.131255.log 2026-02-18T08:53:45.861 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.90668.log 2026-02-18T08:53:45.862 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.51669.log: /var/log/ceph/cluster1-client.mirror.131255.log: 87.0% -- replaced with /var/log/ceph/cluster1-client.mirror.131255.log.gz 2026-02-18T08:53:45.862 INFO:teuthology.orchestra.run.trial163.stderr: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.51669.log.gz 2026-02-18T08:53:45.862 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.131897.log 2026-02-18T08:53:45.862 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.54399.log 2026-02-18T08:53:45.863 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.90668.log: /var/log/ceph/cluster2-client.mirror.131897.log: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.131897.log.gz 2026-02-18T08:53:45.863 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.84127.log 2026-02-18T08:53:45.864 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.54399.log: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.54399.log.gz 2026-02-18T08:53:45.875 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.87919.log 2026-02-18T08:53:45.876 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.84127.log: 89.2% -- replaced with /var/log/ceph/cluster2-client.mirror.84127.log.gz 2026-02-18T08:53:45.876 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.25219.log 2026-02-18T08:53:45.877 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.87919.log: 84.8% -- replaced with /var/log/ceph/cluster2-client.mirror.87919.log.gz 2026-02-18T08:53:45.877 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.34562.log 2026-02-18T08:53:45.878 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.25219.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.25219.log.gz 2026-02-18T08:53:45.878 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.110837.log 2026-02-18T08:53:45.879 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.34562.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.34562.log.gz 2026-02-18T08:53:45.879 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.24775.log 2026-02-18T08:53:45.879 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.110837.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.110837.log.gz 2026-02-18T08:53:45.880 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.46026.log 2026-02-18T08:53:45.880 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.24775.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.125542.log 2026-02-18T08:53:45.883 INFO:teuthology.orchestra.run.trial163.stderr: 93.0% -- replaced with /var/log/ceph/cluster1-client.mirror.24775.log.gz 2026-02-18T08:53:45.884 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.46026.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.126795.log 2026-02-18T08:53:45.885 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.125542.log: 90.7% -- replaced with /var/log/ceph/cluster2-client.mirror.125542.log.gz 2026-02-18T08:53:45.889 INFO:teuthology.orchestra.run.trial163.stderr: 93.4% -- replaced with /var/log/ceph/cluster2-client.mirror.46026.log.gz 2026-02-18T08:53:45.889 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.127382.log 2026-02-18T08:53:45.890 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.126795.log: 89.4% -- replaced with /var/log/ceph/cluster2-client.mirror.126795.log.gz 2026-02-18T08:53:45.890 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.49460.log 2026-02-18T08:53:45.892 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.127382.log: 90.7% -- replaced with /var/log/ceph/cluster2-client.mirror.127382.log.gz 2026-02-18T08:53:45.892 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.81691.log 2026-02-18T08:53:45.893 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.49460.log: 95.8% -- replaced with /var/log/ceph/cluster2-client.mirror.132235.log.gz 2026-02-18T08:53:45.893 INFO:teuthology.orchestra.run.trial163.stderr: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.49460.log.gz 2026-02-18T08:53:45.893 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.117669.log 2026-02-18T08:53:45.894 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.81691.log: 89.2% -- replaced with /var/log/ceph/cluster2-client.mirror.81691.log.gz 2026-02-18T08:53:45.894 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.107698.log 2026-02-18T08:53:45.894 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.117669.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.117669.log.gz 2026-02-18T08:53:45.895 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.77487.log 2026-02-18T08:53:45.895 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.107698.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.107698.log.gz 2026-02-18T08:53:45.895 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.31396.log 2026-02-18T08:53:45.896 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.77487.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.111728.log 2026-02-18T08:53:45.896 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.31396.log: 89.2% -- replaced with /var/log/ceph/cluster2-client.mirror.77487.log.gz 2026-02-18T08:53:45.896 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.79167.log 2026-02-18T08:53:45.897 INFO:teuthology.orchestra.run.trial163.stderr: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.31396.log.gz 2026-02-18T08:53:45.897 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.111728.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.97624.log 2026-02-18T08:53:45.898 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.79167.log: 89.2% -- replaced with /var/log/ceph/cluster2-client.mirror.79167.log.gz 2026-02-18T08:53:45.898 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.79692.log 2026-02-18T08:53:45.898 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.97624.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.97624.log.gz 2026-02-18T08:53:45.899 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.117627.log 2026-02-18T08:53:45.899 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.79692.log: 89.2% -- replaced with /var/log/ceph/cluster2-client.mirror.79692.log.gz 2026-02-18T08:53:45.900 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.50584.log 2026-02-18T08:53:45.900 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.117627.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.117627.log.gz 2026-02-18T08:53:45.901 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.82251.log 2026-02-18T08:53:45.901 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.50584.log: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.50584.log.gz 2026-02-18T08:53:45.902 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.65641.log 2026-02-18T08:53:45.902 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.82251.log: 89.2% -- replaced with /var/log/ceph/cluster2-client.mirror.82251.log.gz 2026-02-18T08:53:45.903 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.40502.log 2026-02-18T08:53:45.903 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.65641.log: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.65641.log.gz 2026-02-18T08:53:45.904 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.66675.log 2026-02-18T08:53:45.904 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.40502.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.40502.log.gz 2026-02-18T08:53:45.905 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.65587.log 2026-02-18T08:53:45.905 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.66675.log: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.66675.log.gz 2026-02-18T08:53:45.906 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.50794.log 2026-02-18T08:53:45.906 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.65587.log: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.65587.log.gz 2026-02-18T08:53:45.906 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.66234.log 2026-02-18T08:53:45.907 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.50794.log: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.50794.log.gz 2026-02-18T08:53:45.908 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.47885.log 2026-02-18T08:53:45.908 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.66234.log: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.66234.log.gz 2026-02-18T08:53:45.908 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.52684.log 2026-02-18T08:53:45.909 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.47885.log: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.47885.log.gz 2026-02-18T08:53:45.909 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.62885.log 2026-02-18T08:53:45.910 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.52684.log: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.52684.log.gz 2026-02-18T08:53:45.911 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.125500.log 2026-02-18T08:53:45.911 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.62885.log: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.62885.log.gz 2026-02-18T08:53:45.911 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.134474.log 2026-02-18T08:53:45.912 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.125500.log: 89.3% -- replaced with /var/log/ceph/cluster2-client.mirror.125500.log.gz 2026-02-18T08:53:45.913 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.121862.log 2026-02-18T08:53:45.914 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.134474.log: 91.3% -- replaced with /var/log/ceph/cluster2-client.mirror.134474.log.gz 2026-02-18T08:53:45.914 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.56569.log 2026-02-18T08:53:45.915 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.121862.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.121862.log.gz 2026-02-18T08:53:45.915 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.32512.log 2026-02-18T08:53:45.916 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.56569.log: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.56569.log.gz 2026-02-18T08:53:45.916 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.59814.log 2026-02-18T08:53:45.917 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.32512.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.112901.log 2026-02-18T08:53:45.917 INFO:teuthology.orchestra.run.trial163.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.32512.log.gz 2026-02-18T08:53:45.918 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.59814.log: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.59814.log.gz 2026-02-18T08:53:45.918 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.15343.log 2026-02-18T08:53:45.920 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.112901.log: 92.6% -- replaced with /var/log/ceph/cluster2-client.mirror.112901.log.gz 2026-02-18T08:53:45.920 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.126937.log 2026-02-18T08:53:45.920 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.15343.log: 65.1% -- replaced with /var/log/ceph/cluster1-client.mirror.15343.log.gz 2026-02-18T08:53:45.921 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.14349.log 2026-02-18T08:53:45.922 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.126937.log: 89.4% -- replaced with /var/log/ceph/cluster2-client.mirror.126937.log.gz 2026-02-18T08:53:45.922 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.49320.log 2026-02-18T08:53:45.922 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.admin.14349.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.14349.log.gz 2026-02-18T08:53:45.923 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.28193.log 2026-02-18T08:53:45.923 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.49320.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.106476.log 2026-02-18T08:53:45.924 INFO:teuthology.orchestra.run.trial163.stderr: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.49320.log.gz 2026-02-18T08:53:45.924 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.28193.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.28193.log.gz 2026-02-18T08:53:45.925 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.17215.log 2026-02-18T08:53:45.925 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.106476.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.106476.log.gz 2026-02-18T08:53:45.925 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.33721.log 2026-02-18T08:53:45.926 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.17215.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.134504.log 2026-02-18T08:53:45.926 INFO:teuthology.orchestra.run.trial163.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.17215.log.gz 2026-02-18T08:53:45.927 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.33721.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.33721.log.gz 2026-02-18T08:53:45.928 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.74513.log 2026-02-18T08:53:45.928 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.134504.log: 86.8% -- replaced with /var/log/ceph/cluster1-client.mirror.134504.log.gz 2026-02-18T08:53:45.928 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.109750.log 2026-02-18T08:53:45.929 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.74513.log: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.74513.log.gz 2026-02-18T08:53:45.929 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.89656.log 2026-02-18T08:53:45.931 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.109750.log: 92.3% -- replaced with /var/log/ceph/cluster1-client.mirror.109750.log.gz 2026-02-18T08:53:45.931 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.23024.log 2026-02-18T08:53:45.932 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.89656.log: 84.8% -- replaced with /var/log/ceph/cluster2-client.mirror.89656.log.gz 2026-02-18T08:53:45.932 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.115417.log 2026-02-18T08:53:45.932 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.23024.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.66319.log 2026-02-18T08:53:45.933 INFO:teuthology.orchestra.run.trial163.stderr: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.23024.log.gz 2026-02-18T08:53:45.933 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.115417.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.115417.log.gz 2026-02-18T08:53:45.934 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.41622.log 2026-02-18T08:53:45.934 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.66319.log: 84.9% -- replaced with /var/log/ceph/cluster1-client.mirror.66319.log.gz 2026-02-18T08:53:45.935 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.128036.log 2026-02-18T08:53:45.935 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.41622.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.41622.log.gz 2026-02-18T08:53:45.936 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.44817.log 2026-02-18T08:53:45.937 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.128036.log: 90.6% -- replaced with /var/log/ceph/cluster2-client.mirror.128036.log.gz 2026-02-18T08:53:45.937 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.65269.log 2026-02-18T08:53:45.938 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.44817.log: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.44817.log.gz 2026-02-18T08:53:45.938 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.111686.log 2026-02-18T08:53:45.939 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.65269.log: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.65269.log.gz 2026-02-18T08:53:45.939 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.123926.log 2026-02-18T08:53:45.941 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.111686.log: 92.0% -- replaced with /var/log/ceph/cluster2-client.mirror.111686.log.gz 2026-02-18T08:53:45.941 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.110062.log 2026-02-18T08:53:45.942 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.123926.log: 92.2% -- replaced with /var/log/ceph/cluster1-client.mirror.123926.log.gz 2026-02-18T08:53:45.943 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.62115.log 2026-02-18T08:53:45.943 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.110062.log: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.110062.log.gz 2026-02-18T08:53:45.944 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.84944.log 2026-02-18T08:53:45.944 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.62115.log: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.62115.log.gz 2026-02-18T08:53:45.945 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.74968.log 2026-02-18T08:53:45.948 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.84944.log: 93.5% -- replaced with /var/log/ceph/cluster1-client.mirror.84944.log.gz 2026-02-18T08:53:45.949 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.88231.log 2026-02-18T08:53:45.949 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.74968.log: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.74968.log.gz 2026-02-18T08:53:45.950 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.92183.log 2026-02-18T08:53:45.953 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.88231.log: 95.7% -- replaced with /var/log/ceph/cluster2-client.mirror.119442.log.gz 2026-02-18T08:53:45.953 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.27821.log 2026-02-18T08:53:45.967 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.92183.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.127240.log 2026-02-18T08:53:45.968 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.27821.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.27821.log.gz 2026-02-18T08:53:45.983 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.62430.log 2026-02-18T08:53:45.985 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.127240.log: 90.7% -- replaced with /var/log/ceph/cluster2-client.mirror.127240.log.gz 2026-02-18T08:53:45.991 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.111362.log 2026-02-18T08:53:45.992 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.62430.log: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.62430.log.gz 2026-02-18T08:53:45.999 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.69504.log 2026-02-18T08:53:46.000 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.111362.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.111362.log.gz 2026-02-18T08:53:46.006 INFO:teuthology.orchestra.run.trial163.stderr: 95.7% -- replaced with /var/log/ceph/cluster2-client.mirror.102188.log.gz 2026-02-18T08:53:46.006 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.115651.log 2026-02-18T08:53:46.007 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.69504.log: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.69504.log.gz 2026-02-18T08:53:46.007 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.46902.log 2026-02-18T08:53:46.008 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.115651.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.115651.log.gz 2026-02-18T08:53:46.008 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.29181.log 2026-02-18T08:53:46.009 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.46902.log: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.46902.log.gz 2026-02-18T08:53:46.009 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.121235.log 2026-02-18T08:53:46.010 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.29181.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.29181.log.gz 2026-02-18T08:53:46.010 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.129317.log 2026-02-18T08:53:46.011 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.121235.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.121235.log.gz 2026-02-18T08:53:46.011 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.64040.log 2026-02-18T08:53:46.012 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.129317.log: 87.0% -- replaced with /var/log/ceph/cluster1-client.mirror.129317.log.gz 2026-02-18T08:53:46.012 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.51004.log 2026-02-18T08:53:46.013 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.64040.log: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.64040.log.gz 2026-02-18T08:53:46.013 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.90449.log 2026-02-18T08:53:46.014 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.51004.log: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.51004.log.gz 2026-02-18T08:53:46.014 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.84858.log 2026-02-18T08:53:46.015 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.90449.log: 84.7% -- replaced with /var/log/ceph/cluster2-client.mirror.90449.log.gz 2026-02-18T08:53:46.015 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.103443.log 2026-02-18T08:53:46.019 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.84858.log: 93.5% -- replaced with /var/log/ceph/cluster1-client.mirror.84858.log.gz 2026-02-18T08:53:46.019 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.128357.log 2026-02-18T08:53:46.020 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.103443.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.103443.log.gz 2026-02-18T08:53:46.020 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.94150.log 2026-02-18T08:53:46.021 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.128357.log: 89.2% -- replaced with /var/log/ceph/cluster2-client.mirror.128357.log.gz 2026-02-18T08:53:46.022 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.18366.log 2026-02-18T08:53:46.023 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.94150.log: 92.7% -- replaced with /var/log/ceph/cluster2-client.mirror.94150.log.gz 2026-02-18T08:53:46.023 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.58536.log 2026-02-18T08:53:46.024 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.18366.log: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.18366.log.gz 2026-02-18T08:53:46.024 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.117083.log 2026-02-18T08:53:46.025 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.58536.log: 88.6% -- replaced with /var/log/ceph/cluster1-client.mirror.58536.log.gz 2026-02-18T08:53:46.025 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.80217.log 2026-02-18T08:53:46.035 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.117083.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.121146.log 2026-02-18T08:53:46.036 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.80217.log: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.80217.log.gz 2026-02-18T08:53:46.039 INFO:teuthology.orchestra.run.trial163.stderr: 95.7% -- replaced with /var/log/ceph/cluster2-client.mirror.101938.log.gz 2026-02-18T08:53:46.043 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.46727.log 2026-02-18T08:53:46.044 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.121146.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.121146.log.gz 2026-02-18T08:53:46.045 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.66029.log 2026-02-18T08:53:46.046 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.46727.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.46727.log.gz 2026-02-18T08:53:46.046 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.130352.log 2026-02-18T08:53:46.046 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.admin.66029.log: 88.4% -- replaced with /var/log/ceph/cluster1-client.admin.66029.log.gz 2026-02-18T08:53:46.047 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.97411.log 2026-02-18T08:53:46.047 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.130352.log: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.130352.log.gz 2026-02-18T08:53:46.048 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.123409.log 2026-02-18T08:53:46.048 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.97411.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.97411.log.gz 2026-02-18T08:53:46.048 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.49425.log 2026-02-18T08:53:46.049 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.123409.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.123409.log.gz 2026-02-18T08:53:46.049 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.99269.log 2026-02-18T08:53:46.050 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.49425.log: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.49425.log.gz 2026-02-18T08:53:46.050 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.121939.log 2026-02-18T08:53:46.051 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.99269.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.99269.log.gz 2026-02-18T08:53:46.051 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.93141.log 2026-02-18T08:53:46.052 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.121939.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.121939.log.gz 2026-02-18T08:53:46.052 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.72094.log 2026-02-18T08:53:46.053 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.93141.log: 88.1% -- replaced with /var/log/ceph/cluster1-client.mirror.93141.log.gz 2026-02-18T08:53:46.053 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.32012.log 2026-02-18T08:53:46.054 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.72094.log: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.72094.log.gz 2026-02-18T08:53:46.054 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.38607.log 2026-02-18T08:53:46.055 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.32012.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.41432.log 2026-02-18T08:53:46.055 INFO:teuthology.orchestra.run.trial163.stderr: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.32012.log.gz 2026-02-18T08:53:46.056 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.38607.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.65887.log 2026-02-18T08:53:46.056 INFO:teuthology.orchestra.run.trial163.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.38607.log.gz 2026-02-18T08:53:46.057 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.41432.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.66269.log 2026-02-18T08:53:46.057 INFO:teuthology.orchestra.run.trial163.stderr: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.41432.log.gz 2026-02-18T08:53:46.061 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.65887.log: 93.5% -- replaced with /var/log/ceph/cluster1-client.mirror.65887.log.gz 2026-02-18T08:53:46.061 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.81481.log 2026-02-18T08:53:46.062 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.66269.log: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.66269.log.gz 2026-02-18T08:53:46.062 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.23303.log 2026-02-18T08:53:46.063 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.81481.log: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.81481.log.gz 2026-02-18T08:53:46.063 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.117550.log 2026-02-18T08:53:46.064 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.23303.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.23303.log.gz 2026-02-18T08:53:46.064 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.24755.log 2026-02-18T08:53:46.065 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.117550.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.117550.log.gz 2026-02-18T08:53:46.065 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.113254.log 2026-02-18T08:53:46.065 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.24755.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.73354.log 2026-02-18T08:53:46.069 INFO:teuthology.orchestra.run.trial163.stderr: 93.0% -- replaced with /var/log/ceph/cluster2-client.mirror.24755.log.gz 2026-02-18T08:53:46.070 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.113254.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.113254.log.gz 2026-02-18T08:53:46.070 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.81516.log 2026-02-18T08:53:46.071 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.73354.log: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.73354.log.gz 2026-02-18T08:53:46.071 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.80567.log 2026-02-18T08:53:46.072 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.81516.log: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.81516.log.gz 2026-02-18T08:53:46.072 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.119904.log 2026-02-18T08:53:46.073 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.80567.log: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.80567.log.gz 2026-02-18T08:53:46.073 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.108557.log 2026-02-18T08:53:46.074 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.119904.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.119904.log.gz 2026-02-18T08:53:46.074 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.100849.log 2026-02-18T08:53:46.075 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.108557.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.108557.log.gz 2026-02-18T08:53:46.075 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.63585.log 2026-02-18T08:53:46.076 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.100849.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.100849.log.gz 2026-02-18T08:53:46.076 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.25777.log 2026-02-18T08:53:46.076 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.63585.log: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.63585.log.gz 2026-02-18T08:53:46.077 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.98991.log 2026-02-18T08:53:46.078 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.25777.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.25777.log.gz 2026-02-18T08:53:46.078 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.68226.log 2026-02-18T08:53:46.079 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.98991.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.98991.log.gz 2026-02-18T08:53:46.079 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.60059.log 2026-02-18T08:53:46.082 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.68226.log: 93.5% -- replaced with /var/log/ceph/cluster1-client.mirror.68226.log.gz 2026-02-18T08:53:46.083 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.66164.log 2026-02-18T08:53:46.084 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.60059.log: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.60059.log.gz 2026-02-18T08:53:46.084 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.42680.log 2026-02-18T08:53:46.084 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.66164.log: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.66164.log.gz 2026-02-18T08:53:46.084 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.128947.log 2026-02-18T08:53:46.085 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.42680.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.103036.log 2026-02-18T08:53:46.086 INFO:teuthology.orchestra.run.trial163.stderr: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.42680.log.gz 2026-02-18T08:53:46.086 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.128947.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.128947.log.gz 2026-02-18T08:53:46.087 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.91921.log 2026-02-18T08:53:46.095 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.103036.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.29960.log 2026-02-18T08:53:46.097 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.91921.log: 95.7% -- replaced with /var/log/ceph/cluster2-client.mirror.98430.log.gz 2026-02-18T08:53:46.104 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.133098.log 2026-02-18T08:53:46.105 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.29960.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.29960.log.gz 2026-02-18T08:53:46.105 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.22466.log 2026-02-18T08:53:46.106 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.133098.log: 85.4% -- replaced with /var/log/ceph/cluster1-client.mirror.133098.log.gz 2026-02-18T08:53:46.106 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.63375.log 2026-02-18T08:53:46.106 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.22466.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.22466.log.gz 2026-02-18T08:53:46.107 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.127362.log 2026-02-18T08:53:46.107 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.63375.log: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.63375.log.gz 2026-02-18T08:53:46.107 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.31768.log 2026-02-18T08:53:46.108 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.127362.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.127362.log.gz 2026-02-18T08:53:46.109 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.69399.log 2026-02-18T08:53:46.109 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.31768.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.31768.log.gz 2026-02-18T08:53:46.110 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.133576.log 2026-02-18T08:53:46.110 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.69399.log: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.69399.log.gz 2026-02-18T08:53:46.110 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.123036.log 2026-02-18T08:53:46.111 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.133576.log: 83.6% -- replaced with /var/log/ceph/cluster2-client.mirror.133576.log.gz 2026-02-18T08:53:46.111 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.102146.log 2026-02-18T08:53:46.113 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.123036.log: 92.1% -- replaced with /var/log/ceph/cluster1-client.mirror.123036.log.gz 2026-02-18T08:53:46.113 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.120462.log 2026-02-18T08:53:46.114 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.102146.log: 92.8% -- replaced with /var/log/ceph/cluster2-client.mirror.102146.log.gz 2026-02-18T08:53:46.115 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.28379.log 2026-02-18T08:53:46.116 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.120462.log: 88.4% -- replaced with /var/log/ceph/cluster1-client.mirror.120462.log.gz 2026-02-18T08:53:46.116 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.41060.log 2026-02-18T08:53:46.117 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.28379.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.28379.log.gz 2026-02-18T08:53:46.117 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.70309.log 2026-02-18T08:53:46.117 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.41060.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.78572.log 2026-02-18T08:53:46.118 INFO:teuthology.orchestra.run.trial163.stderr: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.41060.log.gz 2026-02-18T08:53:46.119 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.70309.log: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.70309.log.gz 2026-02-18T08:53:46.119 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.127482.log 2026-02-18T08:53:46.119 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.78572.log: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.78572.log.gz 2026-02-18T08:53:46.120 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.97888.log 2026-02-18T08:53:46.121 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.127482.log: 89.3% -- replaced with /var/log/ceph/cluster2-client.mirror.127482.log.gz 2026-02-18T08:53:46.121 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.85010.log 2026-02-18T08:53:46.122 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.97888.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.97888.log.gz 2026-02-18T08:53:46.122 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.126908.log 2026-02-18T08:53:46.126 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.85010.log: 93.5% -- replaced with /var/log/ceph/cluster2-client.mirror.85010.log.gz 2026-02-18T08:53:46.126 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.45755.log 2026-02-18T08:53:46.128 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.126908.log: 90.8% -- replaced with /var/log/ceph/cluster2-client.mirror.126908.log.gz 2026-02-18T08:53:46.128 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.127311.log 2026-02-18T08:53:46.129 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.45755.log: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.45755.log.gz 2026-02-18T08:53:46.129 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.26891.log 2026-02-18T08:53:46.131 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.127311.log: 95.6% -- replaced with /var/log/ceph/cluster2-client.mirror.122507.log.gz 2026-02-18T08:53:46.131 INFO:teuthology.orchestra.run.trial163.stderr: 90.7% -- replaced with /var/log/ceph/cluster2-client.mirror.127311.log.gz 2026-02-18T08:53:46.131 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.106237.log 2026-02-18T08:53:46.132 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.26891.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.26891.log.gz 2026-02-18T08:53:46.132 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.128178.log 2026-02-18T08:53:46.133 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.106237.log: 88.1% -- replaced with /var/log/ceph/cluster1-client.mirror.106237.log.gz 2026-02-18T08:53:46.133 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.120395.log 2026-02-18T08:53:46.135 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.128178.log: 90.7% -- replaced with /var/log/ceph/cluster2-client.mirror.128178.log.gz 2026-02-18T08:53:46.135 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.29588.log 2026-02-18T08:53:46.136 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.120395.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.120395.log.gz 2026-02-18T08:53:46.136 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.57479.log 2026-02-18T08:53:46.136 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.29588.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.124856.log 2026-02-18T08:53:46.137 INFO:teuthology.orchestra.run.trial163.stderr: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.29588.log.gz 2026-02-18T08:53:46.138 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.57479.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.57479.log.gz 2026-02-18T08:53:46.138 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.33686.log 2026-02-18T08:53:46.139 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.124856.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.124856.log.gz 2026-02-18T08:53:46.139 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.58561.log 2026-02-18T08:53:46.140 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.33686.log: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.33686.log.gz 2026-02-18T08:53:46.140 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.85290.log 2026-02-18T08:53:46.143 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.58561.log: 93.5% -- replaced with /var/log/ceph/cluster2-client.mirror.58561.log.gz 2026-02-18T08:53:46.144 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.14652.log 2026-02-18T08:53:46.145 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.85290.log: 89.4% -- replaced with /var/log/ceph/cluster2-client.mirror.85290.log.gz 2026-02-18T08:53:46.145 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.111168.log 2026-02-18T08:53:46.146 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.admin.14652.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.14652.log.gz 2026-02-18T08:53:46.146 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.103273.log 2026-02-18T08:53:46.147 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.111168.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.111168.log.gz 2026-02-18T08:53:46.147 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.125846.log 2026-02-18T08:53:46.148 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.103273.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.103273.log.gz 2026-02-18T08:53:46.148 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.22710.log 2026-02-18T08:53:46.149 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.125846.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.125846.log.gz 2026-02-18T08:53:46.149 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.90422.log 2026-02-18T08:53:46.149 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.22710.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.41397.log 2026-02-18T08:53:46.150 INFO:teuthology.orchestra.run.trial163.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.22710.log.gz 2026-02-18T08:53:46.155 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.90422.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.33965.log 2026-02-18T08:53:46.171 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.41397.log: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.41397.log.gz 2026-02-18T08:53:46.175 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.108638.log 2026-02-18T08:53:46.176 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.33965.log: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.33965.log.gz 2026-02-18T08:53:46.178 INFO:teuthology.orchestra.run.trial163.stderr: 95.7% -- replaced with /var/log/ceph/cluster2-client.mirror.103542.log.gz 2026-02-18T08:53:46.178 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.131050.log 2026-02-18T08:53:46.179 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.108638.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.108638.log.gz 2026-02-18T08:53:46.180 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.101221.log 2026-02-18T08:53:46.180 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.131050.log: 86.8% -- replaced with /var/log/ceph/cluster1-client.mirror.131050.log.gz 2026-02-18T08:53:46.181 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.80987.log 2026-02-18T08:53:46.181 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.101221.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.101221.log.gz 2026-02-18T08:53:46.182 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.32477.log 2026-02-18T08:53:46.182 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.80987.log: 89.2% -- replaced with /var/log/ceph/cluster2-client.mirror.80987.log.gz 2026-02-18T08:53:46.182 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.3.88636.log 2026-02-18T08:53:46.183 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.32477.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.32477.log.gz 2026-02-18T08:53:46.184 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.60584.log 2026-02-18T08:53:46.191 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.3.88636.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.26705.log 2026-02-18T08:53:46.192 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.60584.log: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.60584.log.gz 2026-02-18T08:53:46.211 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.113777.log 2026-02-18T08:53:46.212 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.26705.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.26705.log.gz 2026-02-18T08:53:46.227 INFO:teuthology.orchestra.run.trial163.stderr: 95.6% -- replaced with /var/log/ceph/cluster2-client.mirror.90668.log.gz 2026-02-18T08:53:46.227 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.19261.log 2026-02-18T08:53:46.236 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.113777.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.78642.log 2026-02-18T08:53:46.240 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.19261.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.19261.log.gz 2026-02-18T08:53:46.255 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.76201.log 2026-02-18T08:53:46.256 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.78642.log: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.78642.log.gz 2026-02-18T08:53:46.268 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.94881.log 2026-02-18T08:53:46.268 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.admin.76201.log: 89.2% -- replaced with /var/log/ceph/cluster1-client.admin.76201.log.gz 2026-02-18T08:53:46.287 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.26426.log 2026-02-18T08:53:46.289 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.94881.log: 92.2% -- replaced with /var/log/ceph/cluster1-client.mirror.94881.log.gz 2026-02-18T08:53:46.299 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.64849.log 2026-02-18T08:53:46.312 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.26426.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.26426.log.gzgzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.68181.log 2026-02-18T08:53:46.312 INFO:teuthology.orchestra.run.trial163.stderr: 2026-02-18T08:53:46.312 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.64849.log: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.64849.log.gz 2026-02-18T08:53:46.313 INFO:teuthology.orchestra.run.trial163.stderr: 95.6% -- replaced with /var/log/ceph/cluster2-client.mirror.88231.log.gz 2026-02-18T08:53:46.314 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.19447.log 2026-02-18T08:53:46.314 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.68181.log: 88.6% -- replaced with /var/log/ceph/cluster1-client.mirror.68181.log.gz 2026-02-18T08:53:46.315 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.81092.log 2026-02-18T08:53:46.316 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.19447.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.19447.log.gz 2026-02-18T08:53:46.316 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.58113.log 2026-02-18T08:53:46.317 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.81092.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.81092.log.gz 2026-02-18T08:53:46.317 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.58291.log 2026-02-18T08:53:46.319 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.58113.log: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.58113.log.gz 2026-02-18T08:53:46.319 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.75949.log 2026-02-18T08:53:46.319 INFO:teuthology.orchestra.run.trial163.stderr: 95.7% -- replaced with /var/log/ceph/cluster2-client.mirror.111728.log.gz 2026-02-18T08:53:46.319 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.58291.log: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.58291.log.gz 2026-02-18T08:53:46.319 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.97682.log 2026-02-18T08:53:46.319 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.75949.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.88282.log 2026-02-18T08:53:46.319 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.97682.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.97682.log.gz 2026-02-18T08:53:46.320 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.84034.log 2026-02-18T08:53:46.322 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.88282.log: 93.5% -- replaced with /var/log/ceph/cluster2-client.mirror.75949.log.gz 2026-02-18T08:53:46.323 INFO:teuthology.orchestra.run.trial163.stderr: 95.8% -- replaced with /var/log/ceph/cluster2-client.mirror.87447.log.gz 2026-02-18T08:53:46.335 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.38886.log 2026-02-18T08:53:46.336 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.84034.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.131709.log 2026-02-18T08:53:46.337 INFO:teuthology.orchestra.run.trial163.stderr: 89.2%/var/log/ceph/cluster2-client.mirror.38886.log: -- replaced with /var/log/ceph/cluster2-client.mirror.84034.log.gz 2026-02-18T08:53:46.338 INFO:teuthology.orchestra.run.trial163.stderr: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.38886.log.gz 2026-02-18T08:53:46.338 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.72024.log 2026-02-18T08:53:46.338 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.22280.log 2026-02-18T08:53:46.338 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.131709.log: /var/log/ceph/cluster2-client.mirror.72024.log: 87.6% 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.72024.log.gz 2026-02-18T08:53:46.338 INFO:teuthology.orchestra.run.trial163.stderr: -- replaced with /var/log/ceph/cluster1-client.mirror.131709.log.gz 2026-02-18T08:53:46.340 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.93992.log 2026-02-18T08:53:46.341 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.22280.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.34469.log 2026-02-18T08:53:46.341 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.93992.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.22280.log.gz 2026-02-18T08:53:46.341 INFO:teuthology.orchestra.run.trial163.stderr: 88.3%gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.75582.log 2026-02-18T08:53:46.341 INFO:teuthology.orchestra.run.trial163.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.93992.log.gz 2026-02-18T08:53:46.341 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.34469.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.67379.log 2026-02-18T08:53:46.341 INFO:teuthology.orchestra.run.trial163.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.34469.log.gz 2026-02-18T08:53:46.341 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.75582.log: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.75582.log.gz 2026-02-18T08:53:46.352 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.93376.log 2026-02-18T08:53:46.352 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.67379.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.23582.log 2026-02-18T08:53:46.354 INFO:teuthology.orchestra.run.trial163.stderr: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.67379.log.gz 2026-02-18T08:53:46.354 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.93376.log: 88.3%gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.129992.log 2026-02-18T08:53:46.354 INFO:teuthology.orchestra.run.trial163.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.93376.log.gz 2026-02-18T08:53:46.354 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.23582.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.131028.log 2026-02-18T08:53:46.354 INFO:teuthology.orchestra.run.trial163.stderr: 88.3%/var/log/ceph/cluster1-client.mirror.129992.log: -- replaced with /var/log/ceph/cluster2-client.mirror.23582.log.gz 2026-02-18T08:53:46.354 INFO:teuthology.orchestra.run.trial163.stderr: 87.0% -- replaced with /var/log/ceph/cluster1-client.mirror.129992.log.gz 2026-02-18T08:53:46.354 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.132209.log 2026-02-18T08:53:46.356 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.131028.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.44782.log 2026-02-18T08:53:46.356 INFO:teuthology.orchestra.run.trial163.stderr: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.131028.log.gz 2026-02-18T08:53:46.356 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.132209.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.39200.log 2026-02-18T08:53:46.356 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.44782.log: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.132209.log.gz 2026-02-18T08:53:46.356 INFO:teuthology.orchestra.run.trial163.stderr: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.44782.log.gz 2026-02-18T08:53:46.356 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.39665.log 2026-02-18T08:53:46.356 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.39200.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.59110.log 2026-02-18T08:53:46.358 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.39665.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.39200.log.gz 2026-02-18T08:53:46.358 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.40560.log 2026-02-18T08:53:46.358 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.59110.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.39665.log.gz 2026-02-18T08:53:46.358 INFO:teuthology.orchestra.run.trial163.stderr: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.59110.log.gz 2026-02-18T08:53:46.358 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.110514.log 2026-02-18T08:53:46.360 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.45096.log 2026-02-18T08:53:46.360 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.40560.log: /var/log/ceph/cluster1-client.mirror.110514.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.127965.log 2026-02-18T08:53:46.360 INFO:teuthology.orchestra.run.trial163.stderr: 88.1% -- replaced with /var/log/ceph/cluster1-client.mirror.110514.log.gz 2026-02-18T08:53:46.360 INFO:teuthology.orchestra.run.trial163.stderr: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.40560.log.gz 2026-02-18T08:53:46.360 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.15582.log 2026-02-18T08:53:46.360 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.45096.log: /var/log/ceph/cluster2-client.mirror.127965.log: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.45096.log.gz 2026-02-18T08:53:46.362 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.33535.log 2026-02-18T08:53:46.362 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.15582.log: 68.6% -- replaced with /var/log/ceph/cluster2-client.mirror.15582.log.gz 2026-02-18T08:53:46.362 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.24047.log 2026-02-18T08:53:46.362 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.33535.log: 90.7% -- replaced with /var/log/ceph/cluster2-client.mirror.127965.log.gz 2026-02-18T08:53:46.362 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.25870.log 2026-02-18T08:53:46.362 INFO:teuthology.orchestra.run.trial163.stderr: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.33535.log.gz 2026-02-18T08:53:46.362 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.121804.log 2026-02-18T08:53:46.363 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.24047.log: /var/log/ceph/cluster2-client.mirror.25870.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.24047.log.gz 2026-02-18T08:53:46.364 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.118374.log 2026-02-18T08:53:46.364 INFO:teuthology.orchestra.run.trial163.stderr: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.25870.log.gz 2026-02-18T08:53:46.364 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.121804.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.121804.log.gz 2026-02-18T08:53:46.364 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.42020.log 2026-02-18T08:53:46.364 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.118374.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.118374.log.gz 2026-02-18T08:53:46.365 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.66854.log 2026-02-18T08:53:46.365 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.42020.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.42020.log.gz 2026-02-18T08:53:46.365 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.27507.log 2026-02-18T08:53:46.366 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.66854.log: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.66854.log.gz 2026-02-18T08:53:46.366 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.125866.log 2026-02-18T08:53:46.366 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.27507.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.27507.log.gz 2026-02-18T08:53:46.367 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.111273.log 2026-02-18T08:53:46.367 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.125866.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.14919.log 2026-02-18T08:53:46.369 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.111273.log: 90.7% -- replaced with /var/log/ceph/cluster2-client.mirror.125866.log.gz 2026-02-18T08:53:46.369 INFO:teuthology.orchestra.run.trial163.stderr: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.111273.log.gz 2026-02-18T08:53:46.369 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.81831.log 2026-02-18T08:53:46.370 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.14919.log: 85.8% -- replaced with /var/log/ceph/cluster2-client.mirror.14919.log.gz 2026-02-18T08:53:46.370 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.17994.log 2026-02-18T08:53:46.370 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.81831.log: 89.2% -- replaced with /var/log/ceph/cluster2-client.mirror.81831.log.gz 2026-02-18T08:53:46.370 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.56359.log 2026-02-18T08:53:46.371 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.17994.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.17994.log.gz 2026-02-18T08:53:46.371 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.114213.log 2026-02-18T08:53:46.371 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.56359.log: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.56359.log.gz 2026-02-18T08:53:46.372 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.69889.log 2026-02-18T08:53:46.372 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.114213.log: 95.7% -- replaced with /var/log/ceph/cluster2-client.mirror.117083.log.gz 2026-02-18T08:53:46.373 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.14270.log 2026-02-18T08:53:46.373 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.69889.log: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.69889.log.gz 2026-02-18T08:53:46.374 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.82776.log 2026-02-18T08:53:46.374 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.admin.14270.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.14270.log.gz 2026-02-18T08:53:46.376 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.49145.log 2026-02-18T08:53:46.376 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.82776.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.58604.log 2026-02-18T08:53:46.376 INFO:teuthology.orchestra.run.trial163.stderr: 89.2% -- replaced with /var/log/ceph/cluster2-client.mirror.82776.log.gz 2026-02-18T08:53:46.376 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.49145.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.90257.log 2026-02-18T08:53:46.376 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.58604.log: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.49145.log.gz 2026-02-18T08:53:46.378 INFO:teuthology.orchestra.run.trial163.stderr: 93.5% -- replaced with /var/log/ceph/cluster2-client.mirror.58604.log.gz 2026-02-18T08:53:46.379 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.70204.log 2026-02-18T08:53:46.382 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.90257.log: 84.8%gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.21350.log 2026-02-18T08:53:46.382 INFO:teuthology.orchestra.run.trial163.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.90257.log.gz 2026-02-18T08:53:46.382 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.70204.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.63515.log 2026-02-18T08:53:46.382 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.21350.log: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.70204.log.gz 2026-02-18T08:53:46.383 INFO:teuthology.orchestra.run.trial163.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.21350.log.gz 2026-02-18T08:53:46.383 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.77627.log 2026-02-18T08:53:46.383 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.63515.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.105664.log 2026-02-18T08:53:46.383 INFO:teuthology.orchestra.run.trial163.stderr: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.63515.log.gz/var/log/ceph/cluster2-client.mirror.77627.log: 2026-02-18T08:53:46.383 INFO:teuthology.orchestra.run.trial163.stderr: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.77627.log.gzgzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.24512.log 2026-02-18T08:53:46.383 INFO:teuthology.orchestra.run.trial163.stderr: 2026-02-18T08:53:46.383 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.105664.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.85374.log 2026-02-18T08:53:46.389 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.24512.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.24512.log.gz 2026-02-18T08:53:46.389 INFO:teuthology.orchestra.run.trial163.stderr: 92.7% -- replaced with /var/log/ceph/cluster2-client.mirror.105664.log.gz 2026-02-18T08:53:46.389 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.109505.log 2026-02-18T08:53:46.389 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.85374.log: 89.5% -- replaced with /var/log/ceph/cluster2-client.mirror.85374.log.gz 2026-02-18T08:53:46.389 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.81271.log 2026-02-18T08:53:46.389 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.56324.log 2026-02-18T08:53:46.389 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.109505.log: /var/log/ceph/cluster2-client.mirror.81271.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.109505.log.gz 2026-02-18T08:53:46.389 INFO:teuthology.orchestra.run.trial163.stderr: 89.2% -- replaced with /var/log/ceph/cluster2-client.mirror.81271.log.gz 2026-02-18T08:53:46.389 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.129182.log 2026-02-18T08:53:46.390 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.56324.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.25405.log 2026-02-18T08:53:46.390 INFO:teuthology.orchestra.run.trial163.stderr: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.56324.log.gz 2026-02-18T08:53:46.390 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.129182.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.121324.log 2026-02-18T08:53:46.390 INFO:teuthology.orchestra.run.trial163.stderr: 86.7% -- replaced with /var/log/ceph/cluster1-client.mirror.129182.log.gz 2026-02-18T08:53:46.390 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.25405.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.111900.log 2026-02-18T08:53:46.390 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.121324.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.25405.log.gz 2026-02-18T08:53:46.390 INFO:teuthology.orchestra.run.trial163.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.121324.log.gz 2026-02-18T08:53:46.390 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.100717.log 2026-02-18T08:53:46.390 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.62535.log 2026-02-18T08:53:46.390 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.111900.log: /var/log/ceph/cluster2-client.mirror.100717.log: 88.3% 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.111900.log.gz 2026-02-18T08:53:46.390 INFO:teuthology.orchestra.run.trial163.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.100717.log.gz 2026-02-18T08:53:46.391 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.40653.log 2026-02-18T08:53:46.391 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.25928.log 2026-02-18T08:53:46.391 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.62535.log: /var/log/ceph/cluster2-client.mirror.40653.log: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.62535.log.gz 2026-02-18T08:53:46.395 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.102121.log 2026-02-18T08:53:46.395 INFO:teuthology.orchestra.run.trial163.stderr: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.40653.log.gz 2026-02-18T08:53:46.395 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.25928.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.25928.log.gz 2026-02-18T08:53:46.395 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.119546.log 2026-02-18T08:53:46.395 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.106941.log 2026-02-18T08:53:46.395 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.102121.log: /var/log/ceph/cluster2-client.mirror.119546.log: 88.2% 88.1% -- replaced with /var/log/ceph/cluster1-client.mirror.102121.log.gz 2026-02-18T08:53:46.395 INFO:teuthology.orchestra.run.trial163.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.119546.log.gz 2026-02-18T08:53:46.395 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.14953.log 2026-02-18T08:53:46.395 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.41211.log 2026-02-18T08:53:46.395 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.106941.log: /var/log/ceph/cluster2-client.mirror.14953.log: 78.8% -- replaced with /var/log/ceph/cluster2-client.mirror.14953.log.gz 2026-02-18T08:53:46.395 INFO:teuthology.orchestra.run.trial163.stderr: 92.3% -- replaced with /var/log/ceph/cluster2-client.mirror.106941.log.gz 2026-02-18T08:53:46.395 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.105952.log 2026-02-18T08:53:46.395 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.72759.log 2026-02-18T08:53:46.396 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.41211.log: /var/log/ceph/cluster2-client.mirror.105952.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.105952.log.gz 2026-02-18T08:53:46.396 INFO:teuthology.orchestra.run.trial163.stderr: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.41211.log.gz 2026-02-18T08:53:46.396 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.65409.log 2026-02-18T08:53:46.396 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.64954.log 2026-02-18T08:53:46.396 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.72759.log: /var/log/ceph/cluster2-client.mirror.65409.log: 88.8% 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.65409.log.gz 2026-02-18T08:53:46.396 INFO:teuthology.orchestra.run.trial163.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.72759.log.gz 2026-02-18T08:53:46.398 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.129452.log 2026-02-18T08:53:46.398 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.64954.log: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.64954.log.gz 2026-02-18T08:53:46.398 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.99701.log 2026-02-18T08:53:46.398 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.126156.log 2026-02-18T08:53:46.398 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.129452.log: /var/log/ceph/cluster2-client.mirror.99701.log: 86.8% -- replaced with /var/log/ceph/cluster1-client.mirror.129452.log.gz 2026-02-18T08:53:46.398 INFO:teuthology.orchestra.run.trial163.stderr: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.99701.log.gz 2026-02-18T08:53:46.398 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.76260.log 2026-02-18T08:53:46.398 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.79272.log 2026-02-18T08:53:46.398 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.126156.log: /var/log/ceph/cluster2-client.mirror.76260.log: 89.3% -- replaced with /var/log/ceph/cluster2-client.mirror.126156.log.gz 2026-02-18T08:53:46.399 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.46692.log 2026-02-18T08:53:46.399 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.79272.log: 89.2% -- replaced with /var/log/ceph/cluster2-client.mirror.79272.log.gz 2026-02-18T08:53:46.399 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.97286.log 2026-02-18T08:53:46.400 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.46692.log: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.46692.log.gz 2026-02-18T08:53:46.400 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.18796.log 2026-02-18T08:53:46.401 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.97286.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.97286.log.gz 2026-02-18T08:53:46.401 INFO:teuthology.orchestra.run.trial163.stderr: 95.7% -- replaced with /var/log/ceph/cluster2-client.mirror.87363.log.gz 2026-02-18T08:53:46.401 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.35364.log 2026-02-18T08:53:46.401 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.103951.log 2026-02-18T08:53:46.402 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.18796.log: /var/log/ceph/cluster2-client.mirror.35364.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.18796.log.gz 2026-02-18T08:53:46.402 INFO:teuthology.orchestra.run.trial163.stderr:gzip 88.5% -5 --verbose -- /var/log/ceph/cluster1-client.mirror.133672.log 2026-02-18T08:53:46.402 INFO:teuthology.orchestra.run.trial163.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.35364.log.gz 2026-02-18T08:53:46.402 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.103951.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.103951.log.gz 2026-02-18T08:53:46.403 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.117997.log 2026-02-18T08:53:46.403 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.59289.log 2026-02-18T08:53:46.403 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.133672.log: /var/log/ceph/cluster1-client.mirror.117997.log: 87.9% -- replaced with /var/log/ceph/cluster1-client.mirror.133672.log.gz 2026-02-18T08:53:46.404 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.69259.log 2026-02-18T08:53:46.404 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.59289.log: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.59289.log.gz 2026-02-18T08:53:46.404 INFO:teuthology.orchestra.run.trial163.stderr: 92.4% -- replaced with /var/log/ceph/cluster1-client.mirror.117997.log.gz 2026-02-18T08:53:46.404 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.106383.log 2026-02-18T08:53:46.405 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.26763.log 2026-02-18T08:53:46.405 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.69259.log: /var/log/ceph/cluster2-client.mirror.106383.log: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.69259.log.gz 2026-02-18T08:53:46.405 INFO:teuthology.orchestra.run.trial163.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.106383.log.gz 2026-02-18T08:53:46.406 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.129519.log 2026-02-18T08:53:46.406 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.103172.log 2026-02-18T08:53:46.406 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.26763.log: /var/log/ceph/cluster1-client.mirror.129519.log: 87.0% -- replaced with /var/log/ceph/cluster1-client.mirror.129519.log.gz 2026-02-18T08:53:46.406 INFO:teuthology.orchestra.run.trial163.stderr: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.26763.log.gz 2026-02-18T08:53:46.406 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.134130.log 2026-02-18T08:53:46.407 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.80322.log 2026-02-18T08:53:46.407 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.103172.log: /var/log/ceph/cluster2-client.mirror.134130.log: 79.5% -- replaced with /var/log/ceph/cluster2-client.mirror.134130.log.gz 2026-02-18T08:53:46.407 INFO:teuthology.orchestra.run.trial163.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.103172.log.gz 2026-02-18T08:53:46.408 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.44133.log 2026-02-18T08:53:46.408 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.80322.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.131140.log 2026-02-18T08:53:46.408 INFO:teuthology.orchestra.run.trial163.stderr: 89.1%/var/log/ceph/cluster2-client.mirror.44133.log: -- replaced with /var/log/ceph/cluster2-client.mirror.80322.log.gz 2026-02-18T08:53:46.408 INFO:teuthology.orchestra.run.trial163.stderr: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.44133.log.gz 2026-02-18T08:53:46.409 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.76857.log 2026-02-18T08:53:46.409 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.35271.log 2026-02-18T08:53:46.409 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.131140.log: /var/log/ceph/cluster2-client.mirror.76857.log: 87.0% -- replaced with /var/log/ceph/cluster1-client.mirror.131140.log.gz 2026-02-18T08:53:46.409 INFO:teuthology.orchestra.run.trial163.stderr: 89.2% -- replaced with /var/log/ceph/cluster2-client.mirror.76857.log.gz 2026-02-18T08:53:46.409 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.48305.log 2026-02-18T08:53:46.410 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.131945.log 2026-02-18T08:53:46.410 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.35271.log: /var/log/ceph/cluster2-client.mirror.48305.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.50055.log 2026-02-18T08:53:46.410 INFO:teuthology.orchestra.run.trial163.stderr: 88.8% 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.35271.log.gz 2026-02-18T08:53:46.411 INFO:teuthology.orchestra.run.trial163.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.48305.log.gz 2026-02-18T08:53:46.419 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.131945.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.37400.log 2026-02-18T08:53:46.420 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.50055.log: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.50055.log.gz 2026-02-18T08:53:46.420 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.14601.log 2026-02-18T08:53:46.421 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.37400.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.37400.log.gz 2026-02-18T08:53:46.422 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.86755.log 2026-02-18T08:53:46.422 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.admin.14601.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.14601.log.gz 2026-02-18T08:53:46.422 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.131208.log 2026-02-18T08:53:46.422 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.86755.log: 84.9% -- replaced with /var/log/ceph/cluster2-client.mirror.86755.log.gz 2026-02-18T08:53:46.423 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.135120.log 2026-02-18T08:53:46.423 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.131208.log: 86.8% -- replaced with /var/log/ceph/cluster1-client.mirror.131208.log.gz 2026-02-18T08:53:46.424 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.95669.log 2026-02-18T08:53:46.424 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.admin.135120.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.135120.log.gz 2026-02-18T08:53:46.424 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.101871.log 2026-02-18T08:53:46.425 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.95669.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.95669.log.gz 2026-02-18T08:53:46.425 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.80777.log 2026-02-18T08:53:46.426 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.101871.log: 88.2% -- replaced with /var/log/ceph/cluster1-client.mirror.101871.log.gz 2026-02-18T08:53:46.426 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.124322.log 2026-02-18T08:53:46.427 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.80777.log: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.80777.log.gz 2026-02-18T08:53:46.427 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.95426.log 2026-02-18T08:53:46.428 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.124322.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.124322.log.gz 2026-02-18T08:53:46.428 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.121359.log 2026-02-18T08:53:46.429 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.95426.log: 92.4% -- replaced with /var/log/ceph/cluster1-client.mirror.95426.log.gz 2026-02-18T08:53:46.430 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.90855.log 2026-02-18T08:53:46.430 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.121359.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.121359.log.gz 2026-02-18T08:53:46.431 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.74443.log 2026-02-18T08:53:46.433 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.90855.log: 95.8% -- replaced with /var/log/ceph/cluster2-client.mirror.92183.log.gz 2026-02-18T08:53:46.433 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.98614.log 2026-02-18T08:53:46.434 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.74443.log: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.74443.log.gz 2026-02-18T08:53:46.434 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.59604.log 2026-02-18T08:53:46.435 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.98614.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.98614.log.gz 2026-02-18T08:53:46.435 INFO:teuthology.orchestra.run.trial163.stderr: 95.3% -- replaced with /var/log/ceph/cluster1-client.mirror.90855.log.gz 2026-02-18T08:53:46.435 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.36143.log 2026-02-18T08:53:46.436 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.59604.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.39014.log 2026-02-18T08:53:46.436 INFO:teuthology.orchestra.run.trial163.stderr: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.59604.log.gz 2026-02-18T08:53:46.436 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.36143.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.32326.log 2026-02-18T08:53:46.437 INFO:teuthology.orchestra.run.trial163.stderr: 88.5%/var/log/ceph/cluster2-client.mirror.39014.log: -- replaced with /var/log/ceph/cluster2-client.mirror.36143.log.gz 2026-02-18T08:53:46.437 INFO:teuthology.orchestra.run.trial163.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.39014.log.gz 2026-02-18T08:53:46.437 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.95847.log 2026-02-18T08:53:46.437 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.32326.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.73039.log 2026-02-18T08:53:46.438 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.95847.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.32326.log.gz 2026-02-18T08:53:46.438 INFO:teuthology.orchestra.run.trial163.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.95847.log.gz 2026-02-18T08:53:46.438 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.32047.log 2026-02-18T08:53:46.439 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.73039.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.130374.log 2026-02-18T08:53:46.439 INFO:teuthology.orchestra.run.trial163.stderr: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.73039.log.gz 2026-02-18T08:53:46.439 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.32047.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.126582.log 2026-02-18T08:53:46.440 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.130374.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.32047.log.gz 2026-02-18T08:53:46.440 INFO:teuthology.orchestra.run.trial163.stderr: 86.8% -- replaced with /var/log/ceph/cluster1-client.mirror.130374.log.gz 2026-02-18T08:53:46.440 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.123498.log 2026-02-18T08:53:46.440 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.59075.log 2026-02-18T08:53:46.441 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.126582.log: /var/log/ceph/cluster2-client.mirror.123498.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.123498.log.gz 2026-02-18T08:53:46.441 INFO:teuthology.orchestra.run.trial163.stderr: 89.4% -- replaced with /var/log/ceph/cluster2-client.mirror.126582.log.gz 2026-02-18T08:53:46.441 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.100752.log 2026-02-18T08:53:46.441 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.59075.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.129722.log 2026-02-18T08:53:46.442 INFO:teuthology.orchestra.run.trial163.stderr: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.59075.log.gz 2026-02-18T08:53:46.442 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.100752.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.116834.log 2026-02-18T08:53:46.443 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.129722.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.100752.log.gz 2026-02-18T08:53:46.443 INFO:teuthology.orchestra.run.trial163.stderr: 87.0% -- replaced with /var/log/ceph/cluster1-client.mirror.129722.log.gz 2026-02-18T08:53:46.443 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.100543.log 2026-02-18T08:53:46.443 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.105639.log 2026-02-18T08:53:46.444 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.116834.log: /var/log/ceph/cluster1-client.mirror.100543.log: 92.3% -- replaced with /var/log/ceph/cluster1-client.mirror.100543.log.gz 2026-02-18T08:53:46.444 INFO:teuthology.orchestra.run.trial163.stderr: 92.6% -- replaced with /var/log/ceph/cluster1-client.mirror.116834.log.gz 2026-02-18T08:53:46.445 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.129564.log 2026-02-18T08:53:46.445 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.105639.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.51424.log 2026-02-18T08:53:46.445 INFO:teuthology.orchestra.run.trial163.stderr: 88.4% -- replaced with /var/log/ceph/cluster1-client.mirror.105639.log.gz/var/log/ceph/cluster1-client.mirror.129564.log: 2026-02-18T08:53:46.446 INFO:teuthology.orchestra.run.trial163.stderr: 87.0% -- replaced with /var/log/ceph/cluster1-client.mirror.129564.log.gz 2026-02-18T08:53:46.446 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.51879.log 2026-02-18T08:53:46.446 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.51424.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.29925.log 2026-02-18T08:53:46.447 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.51879.log: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.51424.log.gz 2026-02-18T08:53:46.447 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.36422.log 2026-02-18T08:53:46.447 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.29925.log: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.51879.log.gz 2026-02-18T08:53:46.447 INFO:teuthology.orchestra.run.trial163.stderr: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.29925.log.gz 2026-02-18T08:53:46.448 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.40316.log 2026-02-18T08:53:46.448 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.36422.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.36422.log.gz 2026-02-18T08:53:46.449 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.61275.log 2026-02-18T08:53:46.449 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.40316.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.40316.log.gz 2026-02-18T08:53:46.449 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.132721.log 2026-02-18T08:53:46.450 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.61275.log: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.61275.log.gz 2026-02-18T08:53:46.450 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.96298.log 2026-02-18T08:53:46.451 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.132721.log: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.132721.log.gz 2026-02-18T08:53:46.451 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.23361.log 2026-02-18T08:53:46.451 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.96298.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.16888.log 2026-02-18T08:53:46.452 INFO:teuthology.orchestra.run.trial163.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.96298.log.gz 2026-02-18T08:53:46.452 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.23361.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.103137.log 2026-02-18T08:53:46.452 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.16888.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.23361.log.gz 2026-02-18T08:53:46.453 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.130467.log 2026-02-18T08:53:46.453 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.83821.log 2026-02-18T08:53:46.453 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.130467.log: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.130467.log.gz 2026-02-18T08:53:46.454 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.87242.log 2026-02-18T08:53:46.454 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.83821.log: 89.3% -- replaced with /var/log/ceph/cluster2-client.mirror.83821.log.gz 2026-02-18T08:53:46.455 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.125071.log 2026-02-18T08:53:46.458 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.87242.log: 95.7% -- replaced with /var/log/ceph/cluster2-client.mirror.89527.log.gz 2026-02-18T08:53:46.463 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.99956.log 2026-02-18T08:53:46.468 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.125071.log: /var/log/ceph/cluster2-client.mirror.103137.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.103137.log.gz 2026-02-18T08:53:46.471 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.57724.log 2026-02-18T08:53:46.476 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.99956.log: 92.9% -- replaced with /var/log/ceph/cluster2-client.mirror.99956.log.gz 2026-02-18T08:53:46.487 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.107900.log 2026-02-18T08:53:46.488 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.57724.log: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.57724.log.gz 2026-02-18T08:53:46.501 INFO:teuthology.orchestra.run.trial163.stderr: 95.8% -- replaced with /var/log/ceph/cluster2-client.mirror.88070.log.gz 2026-02-18T08:53:46.501 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.101334.log 2026-02-18T08:53:46.502 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.107900.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.107900.log.gz 2026-02-18T08:53:46.502 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.127720.log 2026-02-18T08:53:46.503 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.101334.log: 88.1% -- replaced with /var/log/ceph/cluster1-client.mirror.101334.log.gz 2026-02-18T08:53:46.503 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.127791.log 2026-02-18T08:53:46.504 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.127720.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.127720.log.gz 2026-02-18T08:53:46.504 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.129429.log 2026-02-18T08:53:46.505 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.127791.log: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.127791.log.gz 2026-02-18T08:53:46.506 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.125613.log 2026-02-18T08:53:46.506 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.129429.log: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.129429.log.gz 2026-02-18T08:53:46.506 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.60759.log 2026-02-18T08:53:46.508 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.125613.log: 90.6% -- replaced with /var/log/ceph/cluster2-client.mirror.125613.log.gz 2026-02-18T08:53:46.508 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.38049.log 2026-02-18T08:53:46.509 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.60759.log: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.60759.log.gz 2026-02-18T08:53:46.509 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.77172.log 2026-02-18T08:53:46.510 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.38049.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.38049.log.gz 2026-02-18T08:53:46.510 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.107935.log 2026-02-18T08:53:46.511 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.77172.log: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.77172.log.gz 2026-02-18T08:53:46.511 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.135072.log 2026-02-18T08:53:46.512 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.107935.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.107935.log.gz 2026-02-18T08:53:46.512 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.61520.log 2026-02-18T08:53:46.513 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.admin.135072.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.135072.log.gz 2026-02-18T08:53:46.513 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.108719.log 2026-02-18T08:53:46.513 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.61520.log: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.61520.log.gz 2026-02-18T08:53:46.514 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.34806.log 2026-02-18T08:53:46.515 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.108719.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.108719.log.gz 2026-02-18T08:53:46.515 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.103238.log 2026-02-18T08:53:46.515 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.34806.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.34806.log.gz 2026-02-18T08:53:46.516 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.2.133007.log 2026-02-18T08:53:46.516 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.103238.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.103238.log.gz 2026-02-18T08:53:46.516 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.120951.log 2026-02-18T08:53:46.518 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.2.133007.log: 92.8% -- replaced with /var/log/ceph/cluster1-client.mirror.2.133007.log.gz 2026-02-18T08:53:46.518 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.42831.log 2026-02-18T08:53:46.519 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.120951.log: 91.8% -- replaced with /var/log/ceph/cluster1-client.mirror.120951.log.gz 2026-02-18T08:53:46.520 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.89147.log 2026-02-18T08:53:46.521 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.42831.log: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.42831.log.gz 2026-02-18T08:53:46.521 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.106566.log 2026-02-18T08:53:46.521 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.89147.log: 84.7% -- replaced with /var/log/ceph/cluster2-client.mirror.89147.log.gz 2026-02-18T08:53:46.522 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.65867.log 2026-02-18T08:53:46.522 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.106566.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.106566.log.gz 2026-02-18T08:53:46.523 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.127145.log 2026-02-18T08:53:46.526 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.65867.log: 93.5% -- replaced with /var/log/ceph/cluster2-client.mirror.65867.log.gz 2026-02-18T08:53:46.526 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.53839.log 2026-02-18T08:53:46.528 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.127145.log: 90.8% -- replaced with /var/log/ceph/cluster2-client.mirror.127145.log.gz 2026-02-18T08:53:46.529 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.16829.log 2026-02-18T08:53:46.529 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.53839.log: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.53839.log.gz 2026-02-18T08:53:46.530 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.53034.log 2026-02-18T08:53:46.530 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.admin.16829.log: 88.3% -- replaced with /var/log/ceph/cluster1-client.admin.16829.log.gz 2026-02-18T08:53:46.531 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.105052.log 2026-02-18T08:53:46.532 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.53034.log: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.53034.log.gz 2026-02-18T08:53:46.532 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.95564.log 2026-02-18T08:53:46.532 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.105052.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.105052.log.gz 2026-02-18T08:53:46.532 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.97923.log 2026-02-18T08:53:46.533 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.95564.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.95564.log.gz 2026-02-18T08:53:46.534 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.60794.log 2026-02-18T08:53:46.534 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.97923.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.97923.log.gz 2026-02-18T08:53:46.534 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.23489.log 2026-02-18T08:53:46.535 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.60794.log: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.60794.log.gz 2026-02-18T08:53:46.535 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.58326.log 2026-02-18T08:53:46.536 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.23489.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.68555.log 2026-02-18T08:53:46.536 INFO:teuthology.orchestra.run.trial163.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.23489.log.gz 2026-02-18T08:53:46.536 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.58326.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.58326.log.gz 2026-02-18T08:53:46.537 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.3.132345.log 2026-02-18T08:53:46.537 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.68555.log: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.68555.log.gz 2026-02-18T08:53:46.537 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.74758.log 2026-02-18T08:53:46.540 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.3.132345.log: 95.7% -- replaced with /var/log/ceph/cluster2-client.mirror.103036.log.gz 2026-02-18T08:53:46.540 INFO:teuthology.orchestra.run.trial163.stderr: 93.6% -- replaced with /var/log/ceph/cluster2-client.mirror.3.132345.log.gz 2026-02-18T08:53:46.541 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.16936.log 2026-02-18T08:53:46.541 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.74758.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.73144.log 2026-02-18T08:53:46.541 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.16936.log: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.74758.log.gz 2026-02-18T08:53:46.542 INFO:teuthology.orchestra.run.trial163.stderr: 88.3%gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.54469.log 2026-02-18T08:53:46.542 INFO:teuthology.orchestra.run.trial163.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.16936.log.gz 2026-02-18T08:53:46.542 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.73144.log: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.73144.log.gz 2026-02-18T08:53:46.543 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.98727.log 2026-02-18T08:53:46.543 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.54469.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.77417.log 2026-02-18T08:53:46.544 INFO:teuthology.orchestra.run.trial163.stderr: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.54469.log.gz 2026-02-18T08:53:46.544 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.133891.log 2026-02-18T08:53:46.544 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.98727.log: /var/log/ceph/cluster2-client.mirror.77417.log: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.77417.log.gz 2026-02-18T08:53:46.545 INFO:teuthology.orchestra.run.trial163.stderr: 88.3% -- replaced with /var/log/ceph/cluster1-client.mirror.98727.log.gz 2026-02-18T08:53:46.545 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.101715.log 2026-02-18T08:53:46.545 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.31361.log 2026-02-18T08:53:46.546 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.133891.log: /var/log/ceph/cluster2-client.mirror.101715.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.133891.log.gz 2026-02-18T08:53:46.546 INFO:teuthology.orchestra.run.trial163.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.101715.log.gz 2026-02-18T08:53:46.546 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.19637.log 2026-02-18T08:53:46.546 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.31361.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.114677.log 2026-02-18T08:53:46.547 INFO:teuthology.orchestra.run.trial163.stderr: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.31361.log.gz 2026-02-18T08:53:46.548 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.31733.log 2026-02-18T08:53:46.548 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.114677.log: /var/log/ceph/cluster2-client.mirror.19637.log: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.114677.log.gz 2026-02-18T08:53:46.548 INFO:teuthology.orchestra.run.trial163.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.19637.log.gz 2026-02-18T08:53:46.548 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.131286.log 2026-02-18T08:53:46.549 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.31733.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.31733.log.gz 2026-02-18T08:53:46.549 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.48620.log 2026-02-18T08:53:46.549 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.131286.log: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.131286.log.gz 2026-02-18T08:53:46.549 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.85584.log 2026-02-18T08:53:46.550 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.48620.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.48620.log.gz 2026-02-18T08:53:46.550 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.61590.log 2026-02-18T08:53:46.551 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.85584.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.18087.log 2026-02-18T08:53:46.552 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.61590.log: 91.3% -- replaced with /var/log/ceph/cluster2-client.mirror.85584.log.gz 2026-02-18T08:53:46.552 INFO:teuthology.orchestra.run.trial163.stderr: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.61590.log.gz 2026-02-18T08:53:46.553 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.50829.log 2026-02-18T08:53:46.553 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.18087.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.18087.log.gz 2026-02-18T08:53:46.553 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.15616.log 2026-02-18T08:53:46.554 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.50829.log: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.50829.log.gz 2026-02-18T08:53:46.554 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.56114.log 2026-02-18T08:53:46.554 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.15616.log: 68.8% -- replaced with /var/log/ceph/cluster2-client.mirror.15616.log.gz 2026-02-18T08:53:46.555 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.53979.log 2026-02-18T08:53:46.555 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.56114.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.56114.log.gz 2026-02-18T08:53:46.555 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.80007.log 2026-02-18T08:53:46.556 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.53979.log: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.53979.log.gz 2026-02-18T08:53:46.556 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.14345.log 2026-02-18T08:53:46.557 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.80007.log: 89.2% -- replaced with /var/log/ceph/cluster2-client.mirror.80007.log.gz 2026-02-18T08:53:46.557 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.94903.log 2026-02-18T08:53:46.557 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.admin.14345.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.14345.log.gz 2026-02-18T08:53:46.558 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.98842.log 2026-02-18T08:53:46.567 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.94903.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.24570.log 2026-02-18T08:53:46.568 INFO:teuthology.orchestra.run.trial163.stderr: 95.8% -- replaced with /var/log/ceph/cluster2-client.mirror.46139.log.gz 2026-02-18T08:53:46.568 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.98842.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.58237.log 2026-02-18T08:53:46.569 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.24570.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.24570.log.gz 2026-02-18T08:53:46.569 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.126107.log 2026-02-18T08:53:46.570 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.58237.log: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.58237.log.gz 2026-02-18T08:53:46.570 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.134327.log 2026-02-18T08:53:46.570 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.126107.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.126107.log.gz 2026-02-18T08:53:46.571 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.46587.log 2026-02-18T08:53:46.571 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.134327.log: 83.5% -- replaced with /var/log/ceph/cluster2-client.mirror.134327.log.gz 2026-02-18T08:53:46.572 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.70519.log 2026-02-18T08:53:46.572 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.46587.log: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.46587.log.gz 2026-02-18T08:53:46.572 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.25742.log 2026-02-18T08:53:46.573 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.70519.log: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.70519.log.gz 2026-02-18T08:53:46.573 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.39444.log 2026-02-18T08:53:46.574 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.25742.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.25742.log.gz 2026-02-18T08:53:46.574 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.88910.log 2026-02-18T08:53:46.574 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.39444.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.39444.log.gz 2026-02-18T08:53:46.575 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.40746.log 2026-02-18T08:53:46.575 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.88910.log: 84.8% -- replaced with /var/log/ceph/cluster2-client.mirror.88910.log.gz 2026-02-18T08:53:46.575 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.54679.log 2026-02-18T08:53:46.576 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.40746.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.21815.log 2026-02-18T08:53:46.576 INFO:teuthology.orchestra.run.trial163.stderr: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.40746.log.gz 2026-02-18T08:53:46.577 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.54679.log: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.54679.log.gz 2026-02-18T08:53:46.577 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.36749.log 2026-02-18T08:53:46.578 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.21815.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.21815.log.gz 2026-02-18T08:53:46.578 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.30390.log 2026-02-18T08:53:46.579 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.36749.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.36749.log.gz 2026-02-18T08:53:46.579 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.127923.log 2026-02-18T08:53:46.579 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.30390.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.30390.log.gz 2026-02-18T08:53:46.580 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.59005.log 2026-02-18T08:53:46.580 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.127923.log: 89.3% -- replaced with /var/log/ceph/cluster2-client.mirror.127923.log.gz 2026-02-18T08:53:46.581 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.15077.log 2026-02-18T08:53:46.581 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.59005.log: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.59005.log.gz 2026-02-18T08:53:46.582 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.30575.log 2026-02-18T08:53:46.582 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.15077.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.128158.log 2026-02-18T08:53:46.582 INFO:teuthology.orchestra.run.trial163.stderr: 77.8% -- replaced with /var/log/ceph/cluster1-client.mirror.15077.log.gz 2026-02-18T08:53:46.583 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.30575.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.114978.log 2026-02-18T08:53:46.587 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.128158.log: 93.0% -- replaced with /var/log/ceph/cluster2-client.mirror.30575.log.gz 2026-02-18T08:53:46.587 INFO:teuthology.orchestra.run.trial163.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.128158.log.gz 2026-02-18T08:53:46.588 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.36579.log 2026-02-18T08:53:46.588 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.114978.log: 88.2% -- replaced with /var/log/ceph/cluster1-client.mirror.114978.log.gz 2026-02-18T08:53:46.589 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.57269.log 2026-02-18T08:53:46.589 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.36579.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.36579.log.gz 2026-02-18T08:53:46.589 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.33256.log 2026-02-18T08:53:46.590 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.57269.log: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.57269.log.gz 2026-02-18T08:53:46.590 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.71709.log 2026-02-18T08:53:46.591 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.33256.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.54364.log 2026-02-18T08:53:46.591 INFO:teuthology.orchestra.run.trial163.stderr: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.33256.log.gz 2026-02-18T08:53:46.592 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.71709.log: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.71709.log.gz 2026-02-18T08:53:46.592 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.77697.log 2026-02-18T08:53:46.593 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.54364.log: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.54364.log.gz 2026-02-18T08:53:46.593 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.33442.log 2026-02-18T08:53:46.594 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.77697.log: 89.2% -- replaced with /var/log/ceph/cluster2-client.mirror.77697.log.gz 2026-02-18T08:53:46.594 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.82566.log 2026-02-18T08:53:46.595 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.33442.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.28658.log 2026-02-18T08:53:46.595 INFO:teuthology.orchestra.run.trial163.stderr: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.33442.log.gz 2026-02-18T08:53:46.596 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.82566.log: 89.2% -- replaced with /var/log/ceph/cluster2-client.mirror.82566.log.gz 2026-02-18T08:53:46.596 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.27693.log 2026-02-18T08:53:46.597 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.28658.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.28658.log.gz 2026-02-18T08:53:46.597 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.81376.log 2026-02-18T08:53:46.598 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.27693.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.27693.log.gz 2026-02-18T08:53:46.598 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.19918.log 2026-02-18T08:53:46.599 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.81376.log: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.81376.log.gz 2026-02-18T08:53:46.599 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.96997.log 2026-02-18T08:53:46.600 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.19918.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.76105.log 2026-02-18T08:53:46.600 INFO:teuthology.orchestra.run.trial163.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.19918.log.gz 2026-02-18T08:53:46.601 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.96997.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.96997.log.gz 2026-02-18T08:53:46.601 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.47007.log 2026-02-18T08:53:46.609 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.76105.log: 95.7% -- replaced with /var/log/ceph/cluster2-client.mirror.76260.log.gz 2026-02-18T08:53:46.610 INFO:teuthology.orchestra.run.trial163.stderr: 95.7% -- replaced with /var/log/ceph/cluster2-client.mirror.113777.log.gz 2026-02-18T08:53:46.611 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.134100.log 2026-02-18T08:53:46.611 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.47007.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.42374.log 2026-02-18T08:53:46.612 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.134100.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.44075.log 2026-02-18T08:53:46.612 INFO:teuthology.orchestra.run.trial163.stderr: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.47007.log.gz 2026-02-18T08:53:46.612 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.42374.log: 91.4% -- replaced with /var/log/ceph/cluster2-client.mirror.134100.log.gz 2026-02-18T08:53:46.613 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.134927.log 2026-02-18T08:53:46.615 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.134629.log 2026-02-18T08:53:46.616 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.134927.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.mirror.134927.log.gz 2026-02-18T08:53:46.616 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.33814.log 2026-02-18T08:53:46.617 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.134629.log: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.134629.log.gz 2026-02-18T08:53:46.617 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.1.122133.log 2026-02-18T08:53:46.618 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.33814.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.66640.log 2026-02-18T08:53:46.618 INFO:teuthology.orchestra.run.trial163.stderr: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.33814.log.gz 2026-02-18T08:53:46.627 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.1.122133.log: /var/log/ceph/cluster2-client.mirror.44075.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.55256.log 2026-02-18T08:53:46.628 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.66640.log: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.44075.log.gz 2026-02-18T08:53:46.628 INFO:teuthology.orchestra.run.trial163.stderr: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.66640.log.gz 2026-02-18T08:53:46.639 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.119825.log 2026-02-18T08:53:46.644 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.55256.log: 93.5% -- replaced with /var/log/ceph/cluster1-client.mirror.55256.log.gz 2026-02-18T08:53:46.648 INFO:teuthology.orchestra.run.trial163.stderr: 92.6% -- replaced with /var/log/ceph/cluster1-client.mirror.1.122133.log.gz 2026-02-18T08:53:46.648 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.46622.log 2026-02-18T08:53:46.655 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.119825.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.37493.log 2026-02-18T08:53:46.671 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.46622.log: 95.8% -- replaced with /var/log/ceph/cluster2-client.mirror.87169.log.gz 2026-02-18T08:53:46.672 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.27321.log 2026-02-18T08:53:46.672 INFO:teuthology.orchestra.run.trial163.stderr: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.46622.log.gz 2026-02-18T08:53:46.672 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.37493.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.95971.log 2026-02-18T08:53:46.673 INFO:teuthology.orchestra.run.trial163.stderr: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.37493.log.gz 2026-02-18T08:53:46.673 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.27321.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.27321.log.gz 2026-02-18T08:53:46.673 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.83202.log 2026-02-18T08:53:46.674 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.95971.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.95971.log.gz 2026-02-18T08:53:46.675 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.26484.log 2026-02-18T08:53:46.675 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.83202.log: 89.2% -- replaced with /var/log/ceph/cluster2-client.mirror.83202.log.gz 2026-02-18T08:53:46.676 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.44040.log 2026-02-18T08:53:46.677 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.26484.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.26484.log.gz 2026-02-18T08:53:46.677 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.132521.log 2026-02-18T08:53:46.678 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.44040.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.36672.log 2026-02-18T08:53:46.678 INFO:teuthology.orchestra.run.trial163.stderr: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.44040.log.gz 2026-02-18T08:53:46.679 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.132521.log: 91.7% -- replaced with /var/log/ceph/cluster2-client.mirror.132521.log.gz 2026-02-18T08:53:46.680 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.28844.log 2026-02-18T08:53:46.681 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.36672.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.36672.log.gz 2026-02-18T08:53:46.681 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.26519.log 2026-02-18T08:53:46.682 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.28844.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.28844.log.gz 2026-02-18T08:53:46.682 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.49670.log 2026-02-18T08:53:46.683 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.26519.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.20232.log 2026-02-18T08:53:46.683 INFO:teuthology.orchestra.run.trial163.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.26519.log.gz 2026-02-18T08:53:46.684 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.49670.log: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.49670.log.gz 2026-02-18T08:53:46.684 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.62640.log 2026-02-18T08:53:46.685 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.20232.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.20232.log.gz 2026-02-18T08:53:46.685 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.58043.log 2026-02-18T08:53:46.686 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.62640.log: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.62640.log.gz 2026-02-18T08:53:46.686 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.43854.log 2026-02-18T08:53:46.687 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.58043.log: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.58043.log.gz 2026-02-18T08:53:46.688 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.62010.log 2026-02-18T08:53:46.688 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.43854.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.43854.log.gz 2026-02-18T08:53:46.688 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.99843.log 2026-02-18T08:53:46.689 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.62010.log: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.62010.log.gz 2026-02-18T08:53:46.689 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.125312.log 2026-02-18T08:53:46.690 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.99843.log: 92.6% -- replaced with /var/log/ceph/cluster2-client.mirror.99843.log.gz 2026-02-18T08:53:46.690 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.42286.log 2026-02-18T08:53:46.691 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.125312.log: 92.7% -- replaced with /var/log/ceph/cluster2-client.mirror.125312.log.gz 2026-02-18T08:53:46.692 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.128428.log 2026-02-18T08:53:46.692 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.42286.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.2.15052.log 2026-02-18T08:53:46.696 INFO:teuthology.orchestra.run.trial163.stderr: 93.3% -- replaced with /var/log/ceph/cluster1-client.mirror.42286.log.gz 2026-02-18T08:53:46.697 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.128428.log: 89.4% -- replaced with /var/log/ceph/cluster2-client.mirror.128428.log.gz 2026-02-18T08:53:46.698 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.127269.log 2026-02-18T08:53:46.698 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.2.15052.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.24105.log 2026-02-18T08:53:46.708 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.47535.log 2026-02-18T08:53:46.708 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.127269.log: /var/log/ceph/cluster2-client.mirror.24105.log: 89.4% -- replaced with /var/log/ceph/cluster2-client.mirror.127269.log.gz 2026-02-18T08:53:46.720 INFO:teuthology.orchestra.run.trial163.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.24105.log.gz 2026-02-18T08:53:46.728 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.101401.log 2026-02-18T08:53:46.728 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.47535.log: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.47535.log.gz 2026-02-18T08:53:46.735 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.47252.log 2026-02-18T08:53:46.760 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.101401.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.111451.log 2026-02-18T08:53:46.760 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.47252.log: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.47252.log.gz 2026-02-18T08:53:46.763 INFO:teuthology.orchestra.run.trial163.stderr: 95.7% -- replaced with /var/log/ceph/cluster2-client.mirror.89059.log.gz 2026-02-18T08:53:46.763 INFO:teuthology.orchestra.run.trial163.stderr: 93.5% -- replaced with /var/log/ceph/cluster2-client.mirror.2.15052.log.gz 2026-02-18T08:53:46.763 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.79727.log 2026-02-18T08:53:46.765 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.111451.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.111451.log.gz 2026-02-18T08:53:46.765 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.99480.log 2026-02-18T08:53:46.765 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.79727.log: 89.2% -- replaced with /var/log/ceph/cluster2-client.mirror.79727.log.gz 2026-02-18T08:53:46.772 INFO:teuthology.orchestra.run.trial163.stderr: 95.7% -- replaced with /var/log/ceph/cluster2-client.mirror.119825.log.gz 2026-02-18T08:53:46.784 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.116139.log 2026-02-18T08:53:46.784 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.99480.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.14877.log 2026-02-18T08:53:46.784 INFO:teuthology.orchestra.run.trial163.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.99480.log.gz 2026-02-18T08:53:46.786 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.116139.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.51564.log 2026-02-18T08:53:46.786 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.admin.14877.log: 88.3% -- replaced with /var/log/ceph/cluster1-client.mirror.116139.log.gz 2026-02-18T08:53:46.786 INFO:teuthology.orchestra.run.trial163.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.14877.log.gz 2026-02-18T08:53:46.786 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.38084.log 2026-02-18T08:53:46.786 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.51564.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.107605.log 2026-02-18T08:53:46.786 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.38084.log: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.51564.log.gz 2026-02-18T08:53:46.789 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.128996.log 2026-02-18T08:53:46.789 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.107605.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.38084.log.gz 2026-02-18T08:53:46.789 INFO:teuthology.orchestra.run.trial163.stderr: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.107605.log.gz 2026-02-18T08:53:46.790 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.23675.log 2026-02-18T08:53:46.790 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.24326.log 2026-02-18T08:53:46.790 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.128996.log: /var/log/ceph/cluster2-client.mirror.23675.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.23675.log.gz 2026-02-18T08:53:46.790 INFO:teuthology.orchestra.run.trial163.stderr: 89.4% -- replaced with /var/log/ceph/cluster2-client.mirror.128996.log.gz 2026-02-18T08:53:46.790 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.65844.log 2026-02-18T08:53:46.790 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.24326.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.50409.log 2026-02-18T08:53:46.790 INFO:teuthology.orchestra.run.trial163.stderr: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.24326.log.gz 2026-02-18T08:53:46.790 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.65844.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.115829.log 2026-02-18T08:53:46.791 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.50409.log: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.50409.log.gz 2026-02-18T08:53:46.792 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.115003.log 2026-02-18T08:53:46.792 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.115829.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.115829.log.gz 2026-02-18T08:53:46.792 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.57903.log 2026-02-18T08:53:46.793 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.115003.log: 92.3% -- replaced with /var/log/ceph/cluster2-client.mirror.115003.log.gz 2026-02-18T08:53:46.793 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.91061.log 2026-02-18T08:53:46.794 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.57903.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.57903.log.gz 2026-02-18T08:53:46.794 INFO:teuthology.orchestra.run.trial163.stderr: 93.5% -- replaced with /var/log/ceph/cluster1-client.mirror.65844.log.gz 2026-02-18T08:53:46.794 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.72549.log 2026-02-18T08:53:46.794 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.91061.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.99515.log 2026-02-18T08:53:46.795 INFO:teuthology.orchestra.run.trial163.stderr: 84.8% -- replaced with /var/log/ceph/cluster2-client.mirror.91061.log.gz 2026-02-18T08:53:46.797 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.72549.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.70134.log 2026-02-18T08:53:46.797 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.99515.log: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.72549.log.gz 2026-02-18T08:53:46.797 INFO:teuthology.orchestra.run.trial163.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.99515.log.gz 2026-02-18T08:53:46.797 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.48165.log 2026-02-18T08:53:46.797 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.113127.log 2026-02-18T08:53:46.797 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.70134.log: /var/log/ceph/cluster2-client.mirror.48165.log: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.70134.log.gz 2026-02-18T08:53:46.797 INFO:teuthology.orchestra.run.trial163.stderr: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.48165.log.gz 2026-02-18T08:53:46.797 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.25184.log 2026-02-18T08:53:46.798 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.113127.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.14902.log 2026-02-18T08:53:46.798 INFO:teuthology.orchestra.run.trial163.stderr: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.113127.log.gz 2026-02-18T08:53:46.799 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.25184.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.42241.log 2026-02-18T08:53:46.800 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.14902.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.25184.log.gz 2026-02-18T08:53:46.800 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.124064.log 2026-02-18T08:53:46.800 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.68520.log 2026-02-18T08:53:46.800 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.124064.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.124064.log.gz 2026-02-18T08:53:46.800 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.134394.log 2026-02-18T08:53:46.801 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.68520.log: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.68520.log.gz 2026-02-18T08:53:46.801 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.85164.log 2026-02-18T08:53:46.802 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.134394.log: 93.7% -- replaced with /var/log/ceph/cluster2-client.mirror.14902.log.gz 2026-02-18T08:53:46.803 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.42241.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.43761.log 2026-02-18T08:53:46.803 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.85164.log: 88.4% -- replaced with /var/log/ceph/cluster1-client.mirror.42241.log.gz 2026-02-18T08:53:46.804 INFO:teuthology.orchestra.run.trial163.stderr: 89.4% -- replaced with /var/log/ceph/cluster2-client.mirror.85164.log.gz 2026-02-18T08:53:46.805 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.29739.log 2026-02-18T08:53:46.805 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.43761.log: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.43761.log.gz 2026-02-18T08:53:46.805 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.78782.log 2026-02-18T08:53:46.806 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.29739.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.29739.log.gz 2026-02-18T08:53:46.806 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.73704.log 2026-02-18T08:53:46.807 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.78782.log: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.78782.log.gz 2026-02-18T08:53:46.807 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.72129.log 2026-02-18T08:53:46.808 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.73704.log: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.73704.log.gz 2026-02-18T08:53:46.808 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.18889.log 2026-02-18T08:53:46.808 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.72129.log: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.72129.log.gz 2026-02-18T08:53:46.809 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.104542.log 2026-02-18T08:53:46.809 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.18889.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.18982.log 2026-02-18T08:53:46.810 INFO:teuthology.orchestra.run.trial163.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.18889.log.gz 2026-02-18T08:53:46.810 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.104542.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.104542.log.gz 2026-02-18T08:53:46.810 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.105002.log 2026-02-18T08:53:46.811 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.18982.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.18982.log.gz 2026-02-18T08:53:46.812 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.56989.log 2026-02-18T08:53:46.813 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.105002.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.105002.log.gz 2026-02-18T08:53:46.813 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.76472.log 2026-02-18T08:53:46.813 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.56989.log: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.56989.log.gz 2026-02-18T08:53:46.814 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.92853.log 2026-02-18T08:53:46.815 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.76472.log: 89.2% -- replaced with /var/log/ceph/cluster2-client.mirror.76472.log.gz 2026-02-18T08:53:46.815 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.105902.log 2026-02-18T08:53:46.815 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.92853.log: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.92853.log.gz 2026-02-18T08:53:46.816 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.110926.log 2026-02-18T08:53:46.816 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.105902.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.105902.log.gz 2026-02-18T08:53:46.831 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.28995.log 2026-02-18T08:53:46.832 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.110926.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.110926.log.gz 2026-02-18T08:53:46.833 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.105706.log 2026-02-18T08:53:46.833 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.28995.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.114171.log 2026-02-18T08:53:46.834 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.105706.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.28995.log.gz 2026-02-18T08:53:46.840 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.135096.log 2026-02-18T08:53:46.853 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.114171.log: 92.8% -- replaced with /var/log/ceph/cluster2-client.mirror.114171.log.gz 2026-02-18T08:53:46.855 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.123016.log 2026-02-18T08:53:46.856 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.admin.135096.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.135096.log.gz 2026-02-18T08:53:46.863 INFO:teuthology.orchestra.run.trial163.stderr: 95.7% -- replaced with /var/log/ceph/cluster2-client.mirror.98842.log.gz 2026-02-18T08:53:46.863 INFO:teuthology.orchestra.run.trial163.stderr: 95.7% -- replaced with /var/log/ceph/cluster2-client.mirror.125071.log.gz 2026-02-18T08:53:46.864 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.103500.log 2026-02-18T08:53:46.864 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.1.14919.log 2026-02-18T08:53:46.865 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.123016.log: /var/log/ceph/cluster2-client.mirror.103500.log: 92.2% -- replaced with /var/log/ceph/cluster2-client.mirror.123016.log.gz 2026-02-18T08:53:46.865 INFO:teuthology.orchestra.run.trial163.stderr: 92.3% -- replaced with /var/log/ceph/cluster2-client.mirror.103500.log.gz 2026-02-18T08:53:46.865 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.71919.log 2026-02-18T08:53:46.867 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.128499.log 2026-02-18T08:53:46.867 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.1.14919.log: /var/log/ceph/cluster2-client.mirror.71919.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.116026.log 2026-02-18T08:53:46.867 INFO:teuthology.orchestra.run.trial163.stderr: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.71919.log.gz 2026-02-18T08:53:46.867 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.20664.log 2026-02-18T08:53:46.868 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.116026.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.116026.log.gz 2026-02-18T08:53:46.868 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.30638.log 2026-02-18T08:53:46.869 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.20664.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.20664.log.gz 2026-02-18T08:53:46.870 INFO:teuthology.orchestra.run.trial163.stderr: 93.1% -- replaced with /var/log/ceph/cluster1-client.mirror.1.14919.log.gz 2026-02-18T08:53:46.870 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.0.15002.log 2026-02-18T08:53:46.871 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.128499.log: /var/log/ceph/cluster1-client.mirror.30638.log: 89.4%gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.85100.log 2026-02-18T08:53:46.871 INFO:teuthology.orchestra.run.trial163.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.128499.log.gz 2026-02-18T08:53:46.871 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.125593.log 2026-02-18T08:53:46.874 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.85100.log: 93.3% -- replaced with /var/log/ceph/cluster1-client.mirror.30638.log.gz 2026-02-18T08:53:46.875 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.0.15002.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.36387.log 2026-02-18T08:53:46.875 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.125593.log: 93.5% -- replaced with /var/log/ceph/cluster2-client.mirror.85100.log.gz 2026-02-18T08:53:46.875 INFO:teuthology.orchestra.run.trial163.stderr: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.125593.log.gz 2026-02-18T08:53:46.876 INFO:teuthology.orchestra.run.trial163.stderr: 91.2% -- replaced with /var/log/ceph/cluster2-client.mirror.0.15002.log.gz 2026-02-18T08:53:46.876 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.94573.log 2026-02-18T08:53:46.877 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.36387.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.36387.log.gz 2026-02-18T08:53:46.878 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.30239.log 2026-02-18T08:53:46.878 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.94573.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.94573.log.gz 2026-02-18T08:53:46.879 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.133362.log 2026-02-18T08:53:46.879 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.30239.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.30239.log.gz 2026-02-18T08:53:46.880 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.113092.log 2026-02-18T08:53:46.881 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.113404.log 2026-02-18T08:53:46.881 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.133362.log: /var/log/ceph/cluster2-client.mirror.113092.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.113092.log.gz 2026-02-18T08:53:46.882 INFO:teuthology.orchestra.run.trial163.stderr: 95.7% -- replaced with /var/log/ceph/cluster2-client.mirror.16888.log.gz 2026-02-18T08:53:46.885 INFO:teuthology.orchestra.run.trial163.stderr: 95.8% -- replaced with /var/log/ceph/cluster2-client.mirror.91921.log.gz 2026-02-18T08:53:46.895 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.66057.log 2026-02-18T08:53:46.896 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.113404.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.66889.log 2026-02-18T08:53:46.896 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.130307.log 2026-02-18T08:53:46.898 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.66057.log: /var/log/ceph/cluster2-client.mirror.66889.log: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.66889.log.gz 2026-02-18T08:53:46.898 INFO:teuthology.orchestra.run.trial163.stderr: 94.0% -- replaced with /var/log/ceph/cluster1-client.mirror.113404.log.gz 2026-02-18T08:53:46.898 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.56009.log 2026-02-18T08:53:46.898 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.130307.log: 87.0%gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.132168.log 2026-02-18T08:53:46.898 INFO:teuthology.orchestra.run.trial163.stderr: -- replaced with /var/log/ceph/cluster1-client.mirror.130307.log.gz 2026-02-18T08:53:46.899 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.56009.log: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.56009.log.gz 2026-02-18T08:53:46.901 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.61730.log 2026-02-18T08:53:46.902 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.81901.log 2026-02-18T08:53:46.902 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.132168.log: /var/log/ceph/cluster2-client.mirror.61730.log: 83.6% -- replaced with /var/log/ceph/cluster2-client.mirror.132168.log.gz 2026-02-18T08:53:46.902 INFO:teuthology.orchestra.run.trial163.stderr: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.61730.log.gz 2026-02-18T08:53:46.902 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.36515.log 2026-02-18T08:53:46.902 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.81901.log: 89.2% -- replaced with /var/log/ceph/cluster2-client.mirror.81901.log.gz 2026-02-18T08:53:46.902 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.125380.log 2026-02-18T08:53:46.902 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.58504.log 2026-02-18T08:53:46.902 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.36515.log: /var/log/ceph/cluster2-client.mirror.125380.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.18238.log 2026-02-18T08:53:46.902 INFO:teuthology.orchestra.run.trial163.stderr: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.125380.log.gz 2026-02-18T08:53:46.902 INFO:teuthology.orchestra.run.trial163.stderr: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.36515.log.gz 2026-02-18T08:53:46.903 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.58504.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.58504.log.gz 2026-02-18T08:53:46.904 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.40188.log 2026-02-18T08:53:46.905 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.35957.log 2026-02-18T08:53:46.905 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.18238.log: /var/log/ceph/cluster2-client.mirror.40188.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.88848.log 2026-02-18T08:53:46.905 INFO:teuthology.orchestra.run.trial163.stderr: 88.3% 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.40188.log.gz 2026-02-18T08:53:46.905 INFO:teuthology.orchestra.run.trial163.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.18238.log.gz 2026-02-18T08:53:46.905 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.66749.log 2026-02-18T08:53:46.905 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.35957.log: /var/log/ceph/cluster2-client.mirror.88848.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.35957.log.gz 2026-02-18T08:53:46.919 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.75725.log 2026-02-18T08:53:46.920 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.66749.log: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.66749.log.gz 2026-02-18T08:53:46.921 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.28623.log 2026-02-18T08:53:46.922 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.75725.log: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.75725.log.gz 2026-02-18T08:53:46.922 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.85416.log 2026-02-18T08:53:46.923 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.28623.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.28623.log.gz 2026-02-18T08:53:46.923 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.106441.log 2026-02-18T08:53:46.924 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.85416.log: 89.5% -- replaced with /var/log/ceph/cluster2-client.mirror.85416.log.gz 2026-02-18T08:53:46.924 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.0.14902.log 2026-02-18T08:53:46.924 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.106441.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.106441.log.gz 2026-02-18T08:53:46.925 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.43424.log 2026-02-18T08:53:46.925 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.0.14902.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.70624.log 2026-02-18T08:53:46.937 INFO:teuthology.orchestra.run.trial163.stderr: 95.8% -- replaced with /var/log/ceph/cluster2-client.mirror.16343.log.gz 2026-02-18T08:53:46.937 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.82426.log 2026-02-18T08:53:46.938 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.70624.log: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.70624.log.gz 2026-02-18T08:53:46.938 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.130893.log 2026-02-18T08:53:46.939 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.82426.log: 89.2% -- replaced with /var/log/ceph/cluster2-client.mirror.82426.log.gz 2026-02-18T08:53:46.939 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.118806.log 2026-02-18T08:53:46.940 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.130893.log: /var/log/ceph/cluster2-client.mirror.43424.log: 87.0% -- replaced with /var/log/ceph/cluster1-client.mirror.130893.log.gz 2026-02-18T08:53:46.941 INFO:teuthology.orchestra.run.trial163.stderr: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.43424.log.gz 2026-02-18T08:53:46.941 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.103697.log 2026-02-18T08:53:46.942 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.118806.log: 92.6% -- replaced with /var/log/ceph/cluster1-client.mirror.118806.log.gz 2026-02-18T08:53:46.942 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.97589.log 2026-02-18T08:53:46.943 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.103697.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.103697.log.gz 2026-02-18T08:53:46.943 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.113976.log 2026-02-18T08:53:46.944 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.97589.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.97589.log.gz 2026-02-18T08:53:46.944 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.117226.log 2026-02-18T08:53:46.945 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.113976.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.113976.log.gz 2026-02-18T08:53:46.945 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.28565.log 2026-02-18T08:53:46.946 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.117226.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.117226.log.gz 2026-02-18T08:53:46.946 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.1.88738.log 2026-02-18T08:53:46.947 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.28565.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.28565.log.gz 2026-02-18T08:53:46.947 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.86209.log 2026-02-18T08:53:46.956 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.1.88738.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.108389.log 2026-02-18T08:53:46.956 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.86209.log: 93.9% -- replaced with /var/log/ceph/cluster2-client.mirror.1.88738.log.gz 2026-02-18T08:53:46.957 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.127079.log 2026-02-18T08:53:46.957 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.108389.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.108389.log.gz 2026-02-18T08:53:46.957 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.67904.log 2026-02-18T08:53:46.958 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.127079.log: 89.4% -- replaced with /var/log/ceph/cluster2-client.mirror.127079.log.gz 2026-02-18T08:53:46.958 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.46116.log 2026-02-18T08:53:46.959 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.67904.log: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.67904.log.gz 2026-02-18T08:53:46.959 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.57094.log 2026-02-18T08:53:46.960 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.46116.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.37121.log 2026-02-18T08:53:46.968 INFO:teuthology.orchestra.run.trial163.stderr: 97.2%/var/log/ceph/cluster2-client.mirror.57094.log: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.57094.log.gz 2026-02-18T08:53:46.968 INFO:teuthology.orchestra.run.trial163.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.46116.log.gz 2026-02-18T08:53:46.968 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.81411.log 2026-02-18T08:53:46.969 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.37121.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.37121.log.gz 2026-02-18T08:53:46.970 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.96684.log 2026-02-18T08:53:46.970 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.81411.log: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.81411.log.gz 2026-02-18T08:53:46.971 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.129924.log 2026-02-18T08:53:46.971 INFO:teuthology.orchestra.run.trial163.stderr: 95.7% -- replaced with /var/log/ceph/cluster2-client.mirror.94903.log.gz 2026-02-18T08:53:46.972 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.96684.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.96684.log.gz 2026-02-18T08:53:46.973 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.64779.log 2026-02-18T08:53:46.973 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.64215.log 2026-02-18T08:53:46.973 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.129924.log: /var/log/ceph/cluster2-client.mirror.64779.log: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.129924.log.gz 2026-02-18T08:53:46.973 INFO:teuthology.orchestra.run.trial163.stderr: 88.8%gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.102994.log 2026-02-18T08:53:46.973 INFO:teuthology.orchestra.run.trial163.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.64779.log.gz 2026-02-18T08:53:46.973 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.64215.log: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.64215.log.gz 2026-02-18T08:53:46.975 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.44683.log 2026-02-18T08:53:46.975 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.108796.log 2026-02-18T08:53:46.975 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.102994.log: /var/log/ceph/cluster1-client.mirror.44683.log: 93.2% -- replaced with /var/log/ceph/cluster1-client.mirror.44683.log.gz 2026-02-18T08:53:46.975 INFO:teuthology.orchestra.run.trial163.stderr: 92.6% -- replaced with /var/log/ceph/cluster2-client.mirror.102994.log.gz 2026-02-18T08:53:46.975 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.33907.log 2026-02-18T08:53:46.976 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.127595.log 2026-02-18T08:53:46.976 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.108796.log: /var/log/ceph/cluster2-client.mirror.33907.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.33907.log.gz 2026-02-18T08:53:46.976 INFO:teuthology.orchestra.run.trial163.stderr: 91.9% -- replaced with /var/log/ceph/cluster1-client.mirror.108796.log.gz 2026-02-18T08:53:46.977 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.63620.log 2026-02-18T08:53:46.978 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.48270.log 2026-02-18T08:53:46.978 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.127595.log: /var/log/ceph/cluster2-client.mirror.63620.log: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.63620.log.gz 2026-02-18T08:53:46.978 INFO:teuthology.orchestra.run.trial163.stderr: 90.7% -- replaced with /var/log/ceph/cluster2-client.mirror.127595.log.gz 2026-02-18T08:53:46.979 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.91740.log 2026-02-18T08:53:46.980 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.94861.log 2026-02-18T08:53:46.980 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.48270.log: /var/log/ceph/cluster2-client.mirror.91740.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.57164.log 2026-02-18T08:53:46.980 INFO:teuthology.orchestra.run.trial163.stderr: 84.7% -- replaced with /var/log/ceph/cluster2-client.mirror.91740.log.gz 2026-02-18T08:53:46.980 INFO:teuthology.orchestra.run.trial163.stderr: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.48270.log.gz 2026-02-18T08:53:46.980 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.94861.log: 92.4% -- replaced with /var/log/ceph/cluster2-client.mirror.94861.log.gz 2026-02-18T08:53:46.982 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.94615.log 2026-02-18T08:53:46.982 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.51459.log 2026-02-18T08:53:46.982 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.57164.log: /var/log/ceph/cluster2-client.mirror.94615.log: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.57164.log.gz 2026-02-18T08:53:46.983 INFO:teuthology.orchestra.run.trial163.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.94615.log.gz 2026-02-18T08:53:46.983 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.104266.log 2026-02-18T08:53:46.983 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.131073.log 2026-02-18T08:53:46.988 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.51459.log: /var/log/ceph/cluster2-client.mirror.104266.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.104266.log.gz 2026-02-18T08:53:46.988 INFO:teuthology.orchestra.run.trial163.stderr: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.51459.log.gz 2026-02-18T08:53:46.988 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.113208.log 2026-02-18T08:53:46.988 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.92756.log 2026-02-18T08:53:46.988 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.131073.log: /var/log/ceph/cluster2-client.mirror.113208.log: 86.8% -- replaced with /var/log/ceph/cluster1-client.mirror.131073.log.gz 2026-02-18T08:53:46.988 INFO:teuthology.orchestra.run.trial163.stderr: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.113208.log.gz 2026-02-18T08:53:46.988 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.27786.log 2026-02-18T08:53:46.988 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.92756.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.92756.log.gz 2026-02-18T08:53:46.988 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.96263.log 2026-02-18T08:53:46.989 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.28437.log 2026-02-18T08:53:46.989 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.27786.log: /var/log/ceph/cluster2-client.mirror.96263.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.96263.log.gz 2026-02-18T08:53:46.989 INFO:teuthology.orchestra.run.trial163.stderr: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.27786.log.gz 2026-02-18T08:53:46.989 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.71359.log 2026-02-18T08:53:46.989 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.41943.log 2026-02-18T08:53:46.989 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.28437.log: /var/log/ceph/cluster2-client.mirror.71359.log: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.71359.log.gz 2026-02-18T08:53:46.989 INFO:teuthology.orchestra.run.trial163.stderr: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.28437.log.gz 2026-02-18T08:53:46.989 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.67169.log 2026-02-18T08:53:46.989 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.133595.log 2026-02-18T08:53:46.989 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.41943.log: /var/log/ceph/cluster2-client.mirror.67169.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.119380.log 2026-02-18T08:53:46.989 INFO:teuthology.orchestra.run.trial163.stderr: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.67169.log.gz 2026-02-18T08:53:46.990 INFO:teuthology.orchestra.run.trial163.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.41943.log.gz 2026-02-18T08:53:46.993 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.133595.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.133595.log.gz 2026-02-18T08:53:46.993 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.63270.log 2026-02-18T08:53:46.994 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.126746.log 2026-02-18T08:53:46.994 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.119380.log: /var/log/ceph/cluster2-client.mirror.63270.log: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.63270.log.gz 2026-02-18T08:53:46.994 INFO:teuthology.orchestra.run.trial163.stderr: 92.3% -- replaced with /var/log/ceph/cluster1-client.mirror.119380.log.gz 2026-02-18T08:53:46.994 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.38828.log 2026-02-18T08:53:46.994 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.30146.log 2026-02-18T08:53:46.994 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.126746.log: /var/log/ceph/cluster2-client.mirror.38828.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.126746.log.gz 2026-02-18T08:53:46.994 INFO:teuthology.orchestra.run.trial163.stderr: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.38828.log.gzgzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.114542.log 2026-02-18T08:53:46.994 INFO:teuthology.orchestra.run.trial163.stderr: 2026-02-18T08:53:46.994 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.30146.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.30146.log.gz 2026-02-18T08:53:46.994 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.132542.log 2026-02-18T08:53:46.994 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.47357.log 2026-02-18T08:53:46.994 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.114542.log: /var/log/ceph/cluster2-client.mirror.132542.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.114542.log.gz 2026-02-18T08:53:46.995 INFO:teuthology.orchestra.run.trial163.stderr: 83.6% -- replaced with /var/log/ceph/cluster2-client.mirror.132542.log.gz 2026-02-18T08:53:46.995 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.37991.log 2026-02-18T08:53:46.995 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.38328.log 2026-02-18T08:53:46.995 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.47357.log: /var/log/ceph/cluster2-client.mirror.37991.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.30819.log 2026-02-18T08:53:46.996 INFO:teuthology.orchestra.run.trial163.stderr: 88.4% 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.37991.log.gz 2026-02-18T08:53:46.996 INFO:teuthology.orchestra.run.trial163.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.47357.log.gz 2026-02-18T08:53:46.996 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.106304.log 2026-02-18T08:53:46.996 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.38328.log: /var/log/ceph/cluster1-client.admin.30819.log: 88.5%gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.97090.log 2026-02-18T08:53:46.996 INFO:teuthology.orchestra.run.trial163.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.38328.log.gz 2026-02-18T08:53:46.997 INFO:teuthology.orchestra.run.trial163.stderr: 87.7% -- replaced with /var/log/ceph/cluster1-client.admin.30819.log.gz 2026-02-18T08:53:47.015 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.106304.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.87510.log 2026-02-18T08:53:47.016 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.97090.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.97090.log.gz 2026-02-18T08:53:47.016 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.69189.log 2026-02-18T08:53:47.027 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.87510.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.109857.log 2026-02-18T08:53:47.028 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.69189.log: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.69189.log.gz 2026-02-18T08:53:47.046 INFO:teuthology.orchestra.run.trial163.stderr: 95.7% -- replaced with /var/log/ceph/cluster2-client.mirror.105706.log.gz 2026-02-18T08:53:47.055 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.105684.log 2026-02-18T08:53:47.056 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.109857.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.109857.log.gz 2026-02-18T08:53:47.063 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.75458.log 2026-02-18T08:53:47.065 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.105684.log: 92.5% -- replaced with /var/log/ceph/cluster1-client.mirror.105684.log.gz 2026-02-18T08:53:47.074 INFO:teuthology.orchestra.run.trial163.stderr: 95.7% -- replaced with /var/log/ceph/cluster2-client.mirror.106304.log.gz 2026-02-18T08:53:47.075 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.32140.log 2026-02-18T08:53:47.076 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.75458.log: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.75458.log.gz 2026-02-18T08:53:47.079 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.94239.log 2026-02-18T08:53:47.081 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.32140.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.32140.log.gz 2026-02-18T08:53:47.081 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.63795.log 2026-02-18T08:53:47.082 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.94239.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.94239.log.gz 2026-02-18T08:53:47.082 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.50514.log 2026-02-18T08:53:47.083 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.63795.log: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.63795.log.gz 2026-02-18T08:53:47.083 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.66994.log 2026-02-18T08:53:47.084 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.50514.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.126695.log 2026-02-18T08:53:47.084 INFO:teuthology.orchestra.run.trial163.stderr: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.50514.log.gz 2026-02-18T08:53:47.085 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.66994.log: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.66994.log.gz 2026-02-18T08:53:47.085 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.84838.log 2026-02-18T08:53:47.086 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.126695.log: 90.8% -- replaced with /var/log/ceph/cluster2-client.mirror.126695.log.gz 2026-02-18T08:53:47.086 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.85665.log 2026-02-18T08:53:47.090 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.84838.log: 93.6% -- replaced with /var/log/ceph/cluster2-client.mirror.84838.log.gz 2026-02-18T08:53:47.091 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.65094.log 2026-02-18T08:53:47.091 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.85665.log: 84.7% -- replaced with /var/log/ceph/cluster2-client.mirror.85665.log.gz 2026-02-18T08:53:47.091 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.84185.log 2026-02-18T08:53:47.092 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.65094.log: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.65094.log.gz 2026-02-18T08:53:47.092 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.37863.log 2026-02-18T08:53:47.096 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.84185.log: 93.0% -- replaced with /var/log/ceph/cluster2-client.mirror.84185.log.gz 2026-02-18T08:53:47.096 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.94081.log 2026-02-18T08:53:47.097 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.37863.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.37863.log.gz 2026-02-18T08:53:47.097 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.123724.log 2026-02-18T08:53:47.098 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.94081.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.94081.log.gz 2026-02-18T08:53:47.098 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.21036.log 2026-02-18T08:53:47.099 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.123724.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.123724.log.gz 2026-02-18T08:53:47.099 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.56954.log 2026-02-18T08:53:47.100 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.21036.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.21036.log.gz 2026-02-18T08:53:47.100 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.129093.log 2026-02-18T08:53:47.102 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.56954.log: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.56954.log.gz 2026-02-18T08:53:47.102 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.21443.log 2026-02-18T08:53:47.102 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.129093.log: 1.3% -- replaced with /var/log/ceph/cluster1-client.mirror.129093.log.gz 2026-02-18T08:53:47.102 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.101186.log 2026-02-18T08:53:47.103 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.21443.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.21443.log.gz 2026-02-18T08:53:47.103 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.65059.log 2026-02-18T08:53:47.104 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.101186.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.101186.log.gz 2026-02-18T08:53:47.104 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.93411.log 2026-02-18T08:53:47.104 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.65059.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.65059.log.gz 2026-02-18T08:53:47.105 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.112684.log 2026-02-18T08:53:47.106 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.93411.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.93411.log.gz 2026-02-18T08:53:47.106 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.128450.log 2026-02-18T08:53:47.107 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.112684.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.112684.log.gz 2026-02-18T08:53:47.107 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.72934.log 2026-02-18T08:53:47.108 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.128450.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.128450.log.gz 2026-02-18T08:53:47.108 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.23082.log 2026-02-18T08:53:47.109 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.72934.log: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.72934.log.gz 2026-02-18T08:53:47.109 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.31175.log 2026-02-18T08:53:47.110 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.23082.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.72864.log 2026-02-18T08:53:47.110 INFO:teuthology.orchestra.run.trial163.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.23082.log.gz 2026-02-18T08:53:47.110 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.31175.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.43052.log 2026-02-18T08:53:47.111 INFO:teuthology.orchestra.run.trial163.stderr: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.31175.log.gz 2026-02-18T08:53:47.111 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.72864.log: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.72864.log.gz 2026-02-18T08:53:47.112 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.43052.log: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.43052.log.gz 2026-02-18T08:53:47.140 INFO:teuthology.orchestra.run.trial163.stderr: 95.7% -- replaced with /var/log/ceph/cluster2-client.mirror.114213.log.gz 2026-02-18T08:53:47.163 INFO:teuthology.orchestra.run.trial163.stderr: 95.6% -- replaced with /var/log/ceph/cluster2-client.mirror.131945.log.gz 2026-02-18T08:53:47.163 INFO:teuthology.orchestra.run.trial163.stderr: 95.7% -- replaced with /var/log/ceph/cluster2-client.mirror.101401.log.gz 2026-02-18T08:53:47.165 INFO:teuthology.orchestra.run.trial163.stderr: 95.8% -- replaced with /var/log/ceph/cluster2-client.mirror.90422.log.gz 2026-02-18T08:53:47.214 INFO:teuthology.orchestra.run.trial163.stderr: 95.7% -- replaced with /var/log/ceph/cluster2-client.mirror.88282.log.gz 2026-02-18T08:53:47.288 INFO:teuthology.orchestra.run.trial163.stderr: 95.7% -- replaced with /var/log/ceph/cluster2-client.mirror.42374.log.gz 2026-02-18T08:53:47.328 INFO:teuthology.orchestra.run.trial163.stderr: 95.7% -- replaced with /var/log/ceph/cluster2-client.mirror.66057.log.gz 2026-02-18T08:53:47.354 INFO:teuthology.orchestra.run.trial163.stderr: 95.8% -- replaced with /var/log/ceph/cluster2-client.mirror.88848.log.gz 2026-02-18T08:53:47.417 INFO:teuthology.orchestra.run.trial163.stderr: 95.7% -- replaced with /var/log/ceph/cluster2-client.mirror.133362.log.gz 2026-02-18T08:53:47.420 INFO:teuthology.orchestra.run.trial163.stderr: 95.8% -- replaced with /var/log/ceph/cluster2-client.mirror.87242.log.gz 2026-02-18T08:53:47.484 INFO:teuthology.orchestra.run.trial163.stderr: 95.7% -- replaced with /var/log/ceph/cluster2-client.mirror.134394.log.gz 2026-02-18T08:53:47.558 INFO:teuthology.orchestra.run.trial163.stderr: 95.8% -- replaced with /var/log/ceph/cluster2-client.mirror.76105.log.gz 2026-02-18T08:53:47.733 INFO:teuthology.orchestra.run.trial163.stderr: 95.8% -- replaced with /var/log/ceph/cluster2-client.mirror.87510.log.gz 2026-02-18T08:53:47.819 INFO:teuthology.orchestra.run.trial163.stderr: 95.8% -- replaced with /var/log/ceph/cluster2-client.mirror.86209.log.gz 2026-02-18T08:53:50.057 INFO:teuthology.orchestra.run.trial163.stderr: 94.9% -- replaced with /var/log/ceph/cluster1-client.mirror.0.14902.log.gz 2026-02-18T08:53:50.990 INFO:teuthology.orchestra.run.trial163.stderr: 94.5% -- replaced with /var/log/ceph/cluster1-client.mirror.2.26153.log.gz 2026-02-18T08:53:50.994 INFO:teuthology.orchestra.run.trial163.stderr: 94.4% -- replaced with /var/log/ceph/cluster1-client.mirror.0.86666.log.gz 2026-02-18T08:53:51.177 INFO:teuthology.orchestra.run.trial163.stderr: 94.4% -- replaced with /var/log/ceph/cluster1-client.mirror.3.88636.log.gz 2026-02-18T08:53:51.503 INFO:teuthology.orchestra.run.trial163.stderr: 94.5% -- replaced with /var/log/ceph/cluster1-client.mirror.1.37671.log.gz 2026-02-18T08:53:53.235 INFO:teuthology.orchestra.run.trial163.stderr: 94.7% -- replaced with /var/log/ceph/cluster1-client.mirror.3.14969.log.gz 2026-02-18T08:55:42.641 INFO:teuthology.orchestra.run.trial034.stderr: 93.8% -- replaced with /var/log/ceph/cluster1-osd.1.log.gz 2026-02-18T08:56:00.125 INFO:teuthology.orchestra.run.trial034.stderr: 93.9% -- replaced with /var/log/ceph/cluster1-osd.2.log.gz 2026-02-18T08:56:00.863 INFO:teuthology.orchestra.run.trial163.stderr: 94.1% -- replaced with /var/log/ceph/cluster2-osd.2.log.gz 2026-02-18T08:56:01.333 INFO:teuthology.orchestra.run.trial163.stderr: 94.1% -- replaced with /var/log/ceph/cluster2-osd.1.log.gz 2026-02-18T08:56:39.300 INFO:teuthology.orchestra.run.trial034.stderr: 93.8% -- replaced with /var/log/ceph/cluster1-osd.0.log.gz 2026-02-18T08:56:39.301 INFO:teuthology.orchestra.run.trial034.stderr: 2026-02-18T08:56:39.301 INFO:teuthology.orchestra.run.trial034.stderr:real 2m56.525s 2026-02-18T08:56:39.301 INFO:teuthology.orchestra.run.trial034.stderr:user 6m54.389s 2026-02-18T08:56:39.301 INFO:teuthology.orchestra.run.trial034.stderr:sys 0m20.202s 2026-02-18T08:57:10.005 INFO:teuthology.orchestra.run.trial163.stderr: 94.0% -- replaced with /var/log/ceph/cluster2-osd.0.log.gz 2026-02-18T08:57:10.006 INFO:teuthology.orchestra.run.trial163.stderr: 2026-02-18T08:57:10.006 INFO:teuthology.orchestra.run.trial163.stderr:real 3m27.225s 2026-02-18T08:57:10.006 INFO:teuthology.orchestra.run.trial163.stderr:user 9m27.335s 2026-02-18T08:57:10.007 INFO:teuthology.orchestra.run.trial163.stderr:sys 0m33.215s 2026-02-18T08:57:10.008 INFO:tasks.ceph:Archiving logs... 2026-02-18T08:57:10.008 DEBUG:teuthology.misc:Transferring archived files from trial034:/var/log/ceph to /home/teuthworker/mnt/teuthology/yuriw-2026-02-17_21:26:40-rbd-20.2.1_2.17.26-baseline_2-distro-default-trial/54948/remote/trial034/log 2026-02-18T08:57:10.009 DEBUG:teuthology.orchestra.run.trial034:> sudo tar c -f - -C /var/log/ceph -- . 2026-02-18T08:57:39.712 DEBUG:teuthology.misc:Transferring archived files from trial163:/var/log/ceph to /home/teuthworker/mnt/teuthology/yuriw-2026-02-17_21:26:40-rbd-20.2.1_2.17.26-baseline_2-distro-default-trial/54948/remote/trial163/log 2026-02-18T08:57:39.713 DEBUG:teuthology.orchestra.run.trial163:> sudo tar c -f - -C /var/log/ceph -- . 2026-02-18T08:58:18.895 DEBUG:teuthology.run_tasks:Unwinding manager install 2026-02-18T08:58:18.906 INFO:teuthology.task.install.util:Removing shipped files: /home/ubuntu/cephtest/valgrind.supp /usr/bin/daemon-helper /usr/bin/adjust-ulimits /usr/bin/stdin-killer... 2026-02-18T08:58:18.906 DEBUG:teuthology.orchestra.run.trial034:> sudo rm -f -- /home/ubuntu/cephtest/valgrind.supp /usr/bin/daemon-helper /usr/bin/adjust-ulimits /usr/bin/stdin-killer 2026-02-18T08:58:18.907 DEBUG:teuthology.orchestra.run.trial163:> sudo rm -f -- /home/ubuntu/cephtest/valgrind.supp /usr/bin/daemon-helper /usr/bin/adjust-ulimits /usr/bin/stdin-killer 2026-02-18T08:58:19.051 DEBUG:teuthology.run_tasks:Unwinding manager clock 2026-02-18T08:58:19.063 INFO:teuthology.task.clock:Checking final clock skew... 2026-02-18T08:58:19.063 DEBUG:teuthology.orchestra.run.trial034:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2026-02-18T08:58:19.064 DEBUG:teuthology.orchestra.run.trial163:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2026-02-18T08:58:19.127 INFO:teuthology.orchestra.run.trial163.stdout: remote refid st t when poll reach delay offset jitter 2026-02-18T08:58:19.127 INFO:teuthology.orchestra.run.trial163.stdout:============================================================================== 2026-02-18T08:58:19.127 INFO:teuthology.orchestra.run.trial163.stdout:-172-234-44-141. 80.192.165.246 2 u 28 128 377 7.622 -13.528 0.647 2026-02-18T08:58:19.127 INFO:teuthology.orchestra.run.trial163.stdout:+h96-60-247-29.m 69.89.207.199 2 u 109 128 377 28.109 +4.329 1.542 2026-02-18T08:58:19.127 INFO:teuthology.orchestra.run.trial163.stdout:*time.ecansol.ne .PPS. 1 u 113 128 377 29.897 -0.304 0.430 2026-02-18T08:58:19.127 INFO:teuthology.orchestra.run.trial163.stdout:+dns-e.ns4v.icu 242.71.143.169 2 u 36 128 377 58.307 +3.363 2.275 2026-02-18T08:58:19.141 INFO:teuthology.orchestra.run.trial034.stdout: remote refid st t when poll reach delay offset jitter 2026-02-18T08:58:19.141 INFO:teuthology.orchestra.run.trial034.stdout:============================================================================== 2026-02-18T08:58:19.141 INFO:teuthology.orchestra.run.trial034.stdout:*as393746.mci.tr 42.20.202.230 2 u 41 128 377 29.832 +0.561 1.309 2026-02-18T08:58:19.141 INFO:teuthology.orchestra.run.trial034.stdout:+dns-e.ns4v.icu 242.71.143.169 2 u 97 128 377 58.299 +1.564 1.613 2026-02-18T08:58:19.141 INFO:teuthology.orchestra.run.trial034.stdout:+clock.xmission. 132.163.96.2 2 u 104 128 377 65.803 +4.888 6.322 2026-02-18T08:58:19.141 INFO:teuthology.orchestra.run.trial034.stdout:-172-235-154-118 40.119.6.228 4 u 53 128 377 36.830 +6.411 1.664 2026-02-18T08:58:19.141 DEBUG:teuthology.run_tasks:Unwinding manager ansible.cephlab 2026-02-18T08:58:19.156 INFO:teuthology.task.ansible:Skipping ansible cleanup... 2026-02-18T08:58:19.156 DEBUG:teuthology.run_tasks:Unwinding manager selinux 2026-02-18T08:58:19.169 DEBUG:teuthology.run_tasks:Unwinding manager pcp 2026-02-18T08:58:19.183 DEBUG:teuthology.run_tasks:Unwinding manager internal.timer 2026-02-18T08:58:19.197 INFO:teuthology.task.internal:Duration was 3068.403420 seconds 2026-02-18T08:58:19.197 DEBUG:teuthology.run_tasks:Unwinding manager internal.syslog 2026-02-18T08:58:19.216 INFO:teuthology.task.internal.syslog:Shutting down syslog monitoring... 2026-02-18T08:58:19.216 DEBUG:teuthology.orchestra.run.trial034:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2026-02-18T08:58:19.218 DEBUG:teuthology.orchestra.run.trial163:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2026-02-18T08:58:19.246 INFO:teuthology.task.internal.syslog:Checking logs for errors... 2026-02-18T08:58:19.246 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@trial034.front.sepia.ceph.com 2026-02-18T08:58:19.246 DEBUG:teuthology.orchestra.run.trial034:> grep -E --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | grep -E -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | grep -E -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2026-02-18T08:58:19.292 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@trial163.front.sepia.ceph.com 2026-02-18T08:58:19.292 DEBUG:teuthology.orchestra.run.trial163:> grep -E --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | grep -E -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | grep -E -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2026-02-18T08:58:19.299 INFO:teuthology.task.internal.syslog:Gathering journactl... 2026-02-18T08:58:19.299 DEBUG:teuthology.orchestra.run.trial034:> sudo journalctl > /home/ubuntu/cephtest/archive/syslog/journalctl.log 2026-02-18T08:58:19.335 DEBUG:teuthology.orchestra.run.trial163:> sudo journalctl > /home/ubuntu/cephtest/archive/syslog/journalctl.log 2026-02-18T08:58:19.520 INFO:teuthology.task.internal.syslog:Compressing syslogs... 2026-02-18T08:58:19.520 DEBUG:teuthology.orchestra.run.trial034:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --max-args=1 --max-procs=0 --verbose --no-run-if-empty -- gzip -5 --verbose -- 2026-02-18T08:58:19.522 DEBUG:teuthology.orchestra.run.trial163:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --max-args=1 --max-procs=0 --verbose --no-run-if-empty -- gzip -5 --verbose -- 2026-02-18T08:58:19.529 INFO:teuthology.orchestra.run.trial034.stderr:gzip -5 --verbose -- /home/ubuntu/cephtest/archive/syslog/misc.log 2026-02-18T08:58:19.531 INFO:teuthology.orchestra.run.trial034.stderr:gzip -5 --verbose -- /home/ubuntu/cephtest/archive/syslog/journalctl.log 2026-02-18T08:58:19.531 INFO:teuthology.orchestra.run.trial034.stderr:/home/ubuntu/cephtest/archive/syslog/misc.log: 0.0% -- replaced with /home/ubuntu/cephtest/archive/syslog/misc.log.gz 2026-02-18T08:58:19.531 INFO:teuthology.orchestra.run.trial034.stderr:gzip -5 --verbose -- /home/ubuntu/cephtest/archive/syslog/kern.log 2026-02-18T08:58:19.531 INFO:teuthology.orchestra.run.trial034.stderr:/home/ubuntu/cephtest/archive/syslog/journalctl.log: /home/ubuntu/cephtest/archive/syslog/kern.log: 0.0% -- replaced with /home/ubuntu/cephtest/archive/syslog/kern.log.gz 2026-02-18T08:58:19.533 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /home/ubuntu/cephtest/archive/syslog/misc.log 2026-02-18T08:58:19.533 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /home/ubuntu/cephtest/archive/syslog/journalctl.log 2026-02-18T08:58:19.533 INFO:teuthology.orchestra.run.trial163.stderr:/home/ubuntu/cephtest/archive/syslog/misc.log: gzip -5 --verbose -- /home/ubuntu/cephtest/archive/syslog/kern.log 2026-02-18T08:58:19.533 INFO:teuthology.orchestra.run.trial163.stderr: 0.0% -- replaced with /home/ubuntu/cephtest/archive/syslog/misc.log.gz 2026-02-18T08:58:19.533 INFO:teuthology.orchestra.run.trial163.stderr:/home/ubuntu/cephtest/archive/syslog/journalctl.log: /home/ubuntu/cephtest/archive/syslog/kern.log: 0.0% -- replaced with /home/ubuntu/cephtest/archive/syslog/kern.log.gz 2026-02-18T08:58:19.579 INFO:teuthology.orchestra.run.trial034.stderr: 81.8% -- replaced with /home/ubuntu/cephtest/archive/syslog/journalctl.log.gz 2026-02-18T08:58:19.582 INFO:teuthology.orchestra.run.trial163.stderr: 81.8% -- replaced with /home/ubuntu/cephtest/archive/syslog/journalctl.log.gz 2026-02-18T08:58:19.584 DEBUG:teuthology.run_tasks:Unwinding manager internal.sudo 2026-02-18T08:58:19.605 INFO:teuthology.task.internal:Restoring /etc/sudoers... 2026-02-18T08:58:19.605 DEBUG:teuthology.orchestra.run.trial034:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2026-02-18T08:58:19.629 DEBUG:teuthology.orchestra.run.trial163:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2026-02-18T08:58:19.637 DEBUG:teuthology.run_tasks:Unwinding manager internal.coredump 2026-02-18T08:58:19.649 DEBUG:teuthology.orchestra.run.trial034:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2026-02-18T08:58:19.670 DEBUG:teuthology.orchestra.run.trial163:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2026-02-18T08:58:19.677 INFO:teuthology.orchestra.run.trial034.stdout:kernel.core_pattern = core 2026-02-18T08:58:19.686 INFO:teuthology.orchestra.run.trial163.stdout:kernel.core_pattern = core 2026-02-18T08:58:19.695 DEBUG:teuthology.orchestra.run.trial034:> test -e /home/ubuntu/cephtest/archive/coredump 2026-02-18T08:58:19.728 DEBUG:teuthology.orchestra.run:got remote process result: 1 2026-02-18T08:58:19.728 DEBUG:teuthology.orchestra.run.trial163:> test -e /home/ubuntu/cephtest/archive/coredump 2026-02-18T08:58:19.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2026-02-18T08:58:19.742 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive 2026-02-18T08:58:19.754 INFO:teuthology.task.internal:Transferring archived files... 2026-02-18T08:58:19.755 DEBUG:teuthology.misc:Transferring archived files from trial034:/home/ubuntu/cephtest/archive to /home/teuthworker/mnt/teuthology/yuriw-2026-02-17_21:26:40-rbd-20.2.1_2.17.26-baseline_2-distro-default-trial/54948/remote/trial034 2026-02-18T08:58:19.755 DEBUG:teuthology.orchestra.run.trial034:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2026-02-18T08:58:19.800 DEBUG:teuthology.misc:Transferring archived files from trial163:/home/ubuntu/cephtest/archive to /home/teuthworker/mnt/teuthology/yuriw-2026-02-17_21:26:40-rbd-20.2.1_2.17.26-baseline_2-distro-default-trial/54948/remote/trial163 2026-02-18T08:58:19.800 DEBUG:teuthology.orchestra.run.trial163:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2026-02-18T08:58:19.830 INFO:teuthology.task.internal:Removing archive directory... 2026-02-18T08:58:19.831 DEBUG:teuthology.orchestra.run.trial034:> rm -rf -- /home/ubuntu/cephtest/archive 2026-02-18T08:58:19.833 DEBUG:teuthology.orchestra.run.trial163:> rm -rf -- /home/ubuntu/cephtest/archive 2026-02-18T08:58:19.863 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive_upload 2026-02-18T08:58:19.886 INFO:teuthology.task.internal:Not uploading archives. 2026-02-18T08:58:19.886 DEBUG:teuthology.run_tasks:Unwinding manager internal.base 2026-02-18T08:58:19.908 INFO:teuthology.task.internal:Tidying up after the test... 2026-02-18T08:58:19.909 DEBUG:teuthology.orchestra.run.trial034:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2026-02-18T08:58:19.910 DEBUG:teuthology.orchestra.run.trial163:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2026-02-18T08:58:19.913 INFO:teuthology.orchestra.run.trial034.stdout: 1180220 4 drwxr-xr-x 2 ubuntu ubuntu 4096 Feb 18 08:58 /home/ubuntu/cephtest 2026-02-18T08:58:19.915 INFO:teuthology.orchestra.run.trial163.stdout: 1180220 4 drwxr-xr-x 2 ubuntu ubuntu 4096 Feb 18 08:58 /home/ubuntu/cephtest 2026-02-18T08:58:19.916 DEBUG:teuthology.run_tasks:Unwinding manager kernel 2026-02-18T08:58:19.935 DEBUG:teuthology.run_tasks:Unwinding manager console_log 2026-02-18T08:58:19.973 INFO:teuthology.run:Summary data: description: rbd/mirror-thrash/{base/install clients/mirror cluster/{2-node} conf/{disable-pool-app} msgr-failures/few objectstore/bluestore-hybrid policy/simple rbd-mirror/four-per-cluster supported-random-distro$/{ubuntu_latest} workloads/rbd-mirror-journal-stress-workunit} duration: 3068.403419971466 flavor: default owner: scheduled_yuriw@soko04.front.sepia.ceph.com success: true 2026-02-18T08:58:19.973 DEBUG:teuthology.report:Pushing job info to https://paddles-paddles.apps.pok.os.sepia.ceph.com 2026-02-18T08:58:20.096 INFO:teuthology.run:pass